diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index abd530b46..989ee18f0 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -489,12 +489,9 @@ cc_test( cc_library( name = "frame_buffer", - srcs = ["frame_buffer.cc"], hdrs = ["frame_buffer.h"], deps = [ "//mediapipe/framework/port:integral_types", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/log:check", ], ) diff --git a/mediapipe/framework/formats/frame_buffer.cc b/mediapipe/framework/formats/frame_buffer.cc deleted file mode 100644 index a86d3f2ad..000000000 --- a/mediapipe/framework/formats/frame_buffer.cc +++ /dev/null @@ -1,176 +0,0 @@ -/* Copyright 2022 The MediaPipe Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "mediapipe/framework/formats/frame_buffer.h" - -#include "absl/status/status.h" -#include "absl/status/statusor.h" - -namespace mediapipe { - -namespace { - -// Returns whether the input `format` is a supported YUV format. -bool IsSupportedYuvFormat(FrameBuffer::Format format) { - return format == FrameBuffer::Format::kNV21 || - format == FrameBuffer::Format::kNV12 || - format == FrameBuffer::Format::kYV12 || - format == FrameBuffer::Format::kYV21; -} - -// Returns supported 1-plane FrameBuffer in YuvData structure. -absl::StatusOr GetYuvDataFromOnePlaneFrameBuffer( - const FrameBuffer& source) { - if (!IsSupportedYuvFormat(source.format())) { - return absl::InvalidArgumentError( - "The source FrameBuffer format is not part of YUV420 family."); - } - - FrameBuffer::YuvData result; - const int y_buffer_size = - source.plane(0).stride.row_stride_bytes * source.dimension().height; - const int uv_buffer_size = - ((source.plane(0).stride.row_stride_bytes + 1) / 2) * - ((source.dimension().height + 1) / 2); - result.y_buffer = source.plane(0).buffer; - result.y_row_stride = source.plane(0).stride.row_stride_bytes; - result.uv_row_stride = result.y_row_stride; - - if (source.format() == FrameBuffer::Format::kNV21) { - result.v_buffer = result.y_buffer + y_buffer_size; - result.u_buffer = result.v_buffer + 1; - result.uv_pixel_stride = 2; - // If y_row_stride equals to the frame width and is an odd value, - // uv_row_stride = y_row_stride + 1, otherwise uv_row_stride = y_row_stride. - if (result.y_row_stride == source.dimension().width && - result.y_row_stride % 2 == 1) { - result.uv_row_stride = (result.y_row_stride + 1) / 2 * 2; - } - } else if (source.format() == FrameBuffer::Format::kNV12) { - result.u_buffer = result.y_buffer + y_buffer_size; - result.v_buffer = result.u_buffer + 1; - result.uv_pixel_stride = 2; - // If y_row_stride equals to the frame width and is an odd value, - // uv_row_stride = y_row_stride + 1, otherwise uv_row_stride = y_row_stride. - if (result.y_row_stride == source.dimension().width && - result.y_row_stride % 2 == 1) { - result.uv_row_stride = (result.y_row_stride + 1) / 2 * 2; - } - } else if (source.format() == FrameBuffer::Format::kYV21) { - result.u_buffer = result.y_buffer + y_buffer_size; - result.v_buffer = result.u_buffer + uv_buffer_size; - result.uv_pixel_stride = 1; - result.uv_row_stride = (result.y_row_stride + 1) / 2; - } else if (source.format() == FrameBuffer::Format::kYV12) { - result.v_buffer = result.y_buffer + y_buffer_size; - result.u_buffer = result.v_buffer + uv_buffer_size; - result.uv_pixel_stride = 1; - result.uv_row_stride = (result.y_row_stride + 1) / 2; - } - return result; -} - -// Returns supported 2-plane FrameBuffer in YuvData structure. -absl::StatusOr GetYuvDataFromTwoPlaneFrameBuffer( - const FrameBuffer& source) { - if (source.format() != FrameBuffer::Format::kNV12 && - source.format() != FrameBuffer::Format::kNV21) { - return absl::InvalidArgumentError("Unsupported YUV planar format."); - } - - FrameBuffer::YuvData result; - // Y plane - result.y_buffer = source.plane(0).buffer; - // All plane strides - result.y_row_stride = source.plane(0).stride.row_stride_bytes; - result.uv_row_stride = source.plane(1).stride.row_stride_bytes; - result.uv_pixel_stride = 2; - - if (source.format() == FrameBuffer::Format::kNV12) { - // Y and UV interleaved format - result.u_buffer = source.plane(1).buffer; - result.v_buffer = result.u_buffer + 1; - } else { - // Y and VU interleaved format - result.v_buffer = source.plane(1).buffer; - result.u_buffer = result.v_buffer + 1; - } - return result; -} - -// Returns supported 3-plane FrameBuffer in YuvData structure. Note that NV21 -// and NV12 are included in the supported Yuv formats. Technically, NV21 and -// NV12 should not be described by the 3-plane format. Historically, NV21 is -// used loosely such that it can also be used to describe YV21 format. For -// backwards compatibility, FrameBuffer supports NV21/NV12 with 3-plane format -// but such usage is discouraged -absl::StatusOr GetYuvDataFromThreePlaneFrameBuffer( - const FrameBuffer& source) { - if (!IsSupportedYuvFormat(source.format())) { - return absl::InvalidArgumentError( - "The source FrameBuffer format is not part of YUV420 family."); - } - - if (source.plane(1).stride.row_stride_bytes != - source.plane(2).stride.row_stride_bytes || - source.plane(1).stride.pixel_stride_bytes != - source.plane(2).stride.pixel_stride_bytes) { - return absl::InternalError("Unsupported YUV planar format."); - } - FrameBuffer::YuvData result; - if (source.format() == FrameBuffer::Format::kNV21 || - source.format() == FrameBuffer::Format::kYV12) { - // Y follow by VU order. The VU chroma planes can be interleaved or - // planar. - result.y_buffer = source.plane(0).buffer; - result.v_buffer = source.plane(1).buffer; - result.u_buffer = source.plane(2).buffer; - result.y_row_stride = source.plane(0).stride.row_stride_bytes; - result.uv_row_stride = source.plane(1).stride.row_stride_bytes; - result.uv_pixel_stride = source.plane(1).stride.pixel_stride_bytes; - } else { - // Y follow by UV order. The UV chroma planes can be interleaved or - // planar. - result.y_buffer = source.plane(0).buffer; - result.u_buffer = source.plane(1).buffer; - result.v_buffer = source.plane(2).buffer; - result.y_row_stride = source.plane(0).stride.row_stride_bytes; - result.uv_row_stride = source.plane(1).stride.row_stride_bytes; - result.uv_pixel_stride = source.plane(1).stride.pixel_stride_bytes; - } - return result; -} - -} // namespace - -absl::StatusOr FrameBuffer::GetYuvDataFromFrameBuffer( - const FrameBuffer& source) { - if (!IsSupportedYuvFormat(source.format())) { - return absl::InvalidArgumentError( - "The source FrameBuffer format is not part of YUV420 family."); - } - - if (source.plane_count() == 1) { - return GetYuvDataFromOnePlaneFrameBuffer(source); - } else if (source.plane_count() == 2) { - return GetYuvDataFromTwoPlaneFrameBuffer(source); - } else if (source.plane_count() == 3) { - return GetYuvDataFromThreePlaneFrameBuffer(source); - } - return absl::InvalidArgumentError( - "The source FrameBuffer must be consisted by 1, 2, or 3 planes"); -} - -} // namespace mediapipe diff --git a/mediapipe/framework/formats/frame_buffer.h b/mediapipe/framework/formats/frame_buffer.h index 7578a0121..ccc699724 100644 --- a/mediapipe/framework/formats/frame_buffer.h +++ b/mediapipe/framework/formats/frame_buffer.h @@ -16,14 +16,9 @@ limitations under the License. #ifndef MEDIAPIPE_FRAMEWORK_FORMATS_FRAME_BUFFER_H_ #define MEDIAPIPE_FRAMEWORK_FORMATS_FRAME_BUFFER_H_ -#include -#include -#include -#include #include -#include "absl/memory/memory.h" -#include "absl/status/statusor.h" +#include "absl/log/check.h" #include "mediapipe/framework/port/integral_types.h" namespace mediapipe { @@ -36,19 +31,16 @@ namespace mediapipe { // Examples: // // // Create an metadata instance with no backing buffer. -// auto buffer = FrameBuffer::Create(/*planes=*/{}, dimension, kRGBA, -// KTopLeft); +// FrameBuffer buffer{/*planes=*/{}, dimension, kRGBA}; // // // Create an RGBA instance with backing buffer on single plane. -// FrameBuffer::Plane plane = -// {rgba_buffer, /*stride=*/{dimension.width * 4, 4}}; -// auto buffer = FrameBuffer::Create({plane}, dimension, kRGBA, kTopLeft); +// FrameBuffer::Plane plane{rgba_buffer, /*stride=*/{dimension.width * 4, 4}}; +// FrameBuffer buffer{{plane}, dimension, kRGBA, kTopLeft)}; // // // Create an YUV instance with planar backing buffer. -// FrameBuffer::Plane y_plane = {y_buffer, /*stride=*/{dimension.width , 1}}; -// FrameBuffer::Plane uv_plane = {u_buffer, /*stride=*/{dimension.width, 2}}; -// auto buffer = FrameBuffer::Create({y_plane, uv_plane}, dimension, kNV21, -// kLeftTop); +// FrameBuffer::Plane y_plane{y_buffer, /*stride=*/{dimension.width , 1}}; +// FrameBuffer::Plane uv_plane{u_buffer, /*stride=*/{dimension.width, 2}}; +// FrameBuffer buffer{{y_plane, uv_plane}, dimension, kNV21}; class FrameBuffer { public: // Colorspace formats. @@ -81,39 +73,16 @@ class FrameBuffer { bool operator!=(const Stride& other) const { return !operator==(other); } }; - // YUV data structure. - struct YuvData { - const uint8* y_buffer; - const uint8* u_buffer; - const uint8* v_buffer; - // Y buffer row stride in bytes. - int y_row_stride; - // U/V buffer row stride in bytes. - int uv_row_stride; - // U/V pixel stride in bytes. This is the distance between two consecutive - // u/v pixel values in a row. - int uv_pixel_stride; - }; - - // FrameBuffer content orientation follows EXIF specification. The name of - // each enum value defines the position of the 0th row and the 0th column of - // the image content. See http://jpegclub.org/exif_orientation.html for - // details. - enum class Orientation { - kTopLeft = 1, - kTopRight = 2, - kBottomRight = 3, - kBottomLeft = 4, - kLeftTop = 5, - kRightTop = 6, - kRightBottom = 7, - kLeftBottom = 8 - }; - // Plane encapsulates buffer and stride information. struct Plane { - const uint8* buffer; - Stride stride; + Plane(uint8* buffer, Stride stride) : buffer_(buffer), stride_(stride) {} + const uint8* buffer() const { return buffer_; } + uint8* mutable_buffer() { return buffer_; } + Stride stride() const { return stride_; } + + private: + uint8* buffer_; + Stride stride_; }; // Dimension information for the whole frame or a cropped portion of it. @@ -149,80 +118,30 @@ class FrameBuffer { int Size() const { return width * height; } }; - // Factory method for creating a FrameBuffer object from row-major backing - // buffers. - static std::unique_ptr Create(const std::vector& planes, - Dimension dimension, Format format, - Orientation orientation) { - return absl::make_unique(planes, dimension, format, - orientation); - } - - // Factory method for creating a FrameBuffer object from row-major movable - // backing buffers. - static std::unique_ptr Create(std::vector&& planes, - Dimension dimension, Format format, - Orientation orientation) { - return absl::make_unique(std::move(planes), dimension, format, - orientation); - } - - // Returns YuvData which contains the Y, U, and V buffer and their - // stride info from the input `source` FrameBuffer which is in the YUV family - // formats (e.g NV12, NV21, YV12, and YV21). - static absl::StatusOr GetYuvDataFromFrameBuffer( - const FrameBuffer& source); - // Builds a FrameBuffer object from a row-major backing buffer. // - // The FrameBuffer does not take ownership of the backing buffer. The backing - // buffer is read-only and the caller is responsible for maintaining the - // backing buffer lifecycle for the lifetime of FrameBuffer. + // The FrameBuffer does not take ownership of the backing buffer. The caller + // is responsible for maintaining the backing buffer lifecycle for the + // lifetime of FrameBuffer. FrameBuffer(const std::vector& planes, Dimension dimension, - Format format, Orientation orientation) - : planes_(planes), - dimension_(dimension), - format_(format), - orientation_(orientation) {} - - // Builds a FrameBuffer object from a movable row-major backing buffer. - // - // The FrameBuffer does not take ownership of the backing buffer. The backing - // buffer is read-only and the caller is responsible for maintaining the - // backing buffer lifecycle for the lifetime of FrameBuffer. - FrameBuffer(std::vector&& planes, Dimension dimension, Format format, - Orientation orientation) - : planes_(std::move(planes)), - dimension_(dimension), - format_(format), - orientation_(orientation) {} - - // Copy constructor. - // - // FrameBuffer does not take ownership of the backing buffer. The copy - // constructor behaves the same way to only copy the buffer pointer and not - // take ownership of the backing buffer. - FrameBuffer(const FrameBuffer& frame_buffer) { - planes_.clear(); - for (int i = 0; i < frame_buffer.plane_count(); i++) { - planes_.push_back( - FrameBuffer::Plane{.buffer = frame_buffer.plane(i).buffer, - .stride = frame_buffer.plane(i).stride}); - } - dimension_ = frame_buffer.dimension(); - format_ = frame_buffer.format(); - orientation_ = frame_buffer.orientation(); - } + Format format) + : planes_(planes), dimension_(dimension), format_(format) {} // Returns number of planes. int plane_count() const { return planes_.size(); } // Returns plane indexed by the input `index`. - Plane plane(int index) const { - if (index > -1 && static_cast(index) < planes_.size()) { - return planes_[index]; - } - return {}; + const Plane& plane(int index) const { + CHECK_GE(index, 0); + CHECK_LT(static_cast(index), planes_.size()); + return planes_[index]; + } + + // Returns mutable plane indexed by the input `index`. + Plane mutable_plane(int index) { + CHECK_GE(index, 0); + CHECK_LT(static_cast(index), planes_.size()); + return planes_[index]; } // Returns FrameBuffer dimension. @@ -231,14 +150,10 @@ class FrameBuffer { // Returns FrameBuffer format. Format format() const { return format_; } - // Returns FrameBuffer orientation. - Orientation orientation() const { return orientation_; } - private: std::vector planes_; Dimension dimension_; Format format_; - Orientation orientation_; }; } // namespace mediapipe