diff --git a/docs/framework_concepts/building_graphs_cpp.md b/docs/framework_concepts/building_graphs_cpp.md index c7e11f131..49ee52991 100644 --- a/docs/framework_concepts/building_graphs_cpp.md +++ b/docs/framework_concepts/building_graphs_cpp.md @@ -12,8 +12,6 @@ nav_order: 1 {:toc} --- -## C++ Graph Builder - C++ graph builder is a powerful tool for: * Building complex graphs @@ -25,7 +23,7 @@ C++ graph builder is a powerful tool for: * Supporting optional graph inputs/outputs * Customizing graphs per platform -### Basic Usage +## Basic Usage Let's see how C++ graph builder can be used for a simple graph: @@ -95,9 +93,9 @@ Short summary: unleashing graph builder capabilities and improving your graphs readability. -### Advanced Usage +## Advanced Usage -#### Utility Functions +### Utility Functions Let's extract inference construction code into a dedicated utility function to help for readability and code reuse: @@ -162,7 +160,7 @@ graphs construction code and helps automatically pull in calculator dependencies (e.g. no need to manually add `:inference_calculator` dep, just let your IDE include `inference.h` and build cleaner pull in corresponding dependency). -#### Utility Classes +### Utility Classes And surely, it's not only about functions, in some cases it's beneficial to introduce utility classes which can help making your graph construction code @@ -277,3 +275,69 @@ Tip: the same as for the `RunInference` function, extracting `PassThroughNodeBuilder` and similar utility classes into dedicated modules enables reuse in graph construction code and helps to automatically pull in the corresponding calculator dependencies. + +## Dos and Don'ts + +### Define graph inputs at the very beginning if possible + +```c++ {.bad} +Stream RunSomething(Stream a, Stream b, Graph& graph) { + Stream c = graph.In(2).SetName("c").Cast(); // Bad. + // ... +} + +CalculatorGraphConfig BuildGraph() { + Graph graph; + + Stream a = graph.In(0).SetName("a").Cast(); + // 10/100/N lines of code. + Stream b = graph.In(1).SetName("b").Cast() // Bad. + Stream d = RunSomething(a, b, graph); + // ... +} + +``` + +In the above code: + +* It can be hard to guess how many inputs you have in the graph. +* Can be error prone overall and hard to maintain in future (e.g. is it a + correct index? name? what if some inputs are removed or made optional? + etc.). + +Instead, simply define your graph inputs at the very beginning of your graph +builder: + +```c++ {.good} +Stream RunSomething(Stream a, Stream b, Stream c, Graph& graph) { + // ... +} + +CalculatorGraphConfig BuildGraph() { + Graph graph; + + Stream a = graph.In(0).SetName("a").Cast(); + Stream b = graph.In(1).SetName("b").Cast(); + Stream c = graph.In(2).SetName("c").Cast(); + + // 10/100/N lines of code. + Stream d = RunSomething(a, b, c, graph); + // ... +} +``` + +And if you have an input stream or side packet that is not always defined - +simply use `std::optional` and put it at the very beginning as well: + +```c++ {.good} +std::optional> a; +if (needs_a) { + a = graph.In(0).SetName(a).Cast(); +} +``` + +Note: of course, there can be exceptions - for example, there can be a use case +where calling `RunSomething1(..., graph)`, ..., `RunSomethingN(..., graph)` is +**intended to add new inputs**, so afterwards you can iterate over them and feed +only added inputs into the graph. However, in any case, try to make it easy for +readers to find out what graph inputs it has or may have. diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index dd311fb46..26525b5da 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -489,9 +489,12 @@ cc_test( cc_library( name = "frame_buffer", + srcs = ["frame_buffer.cc"], hdrs = ["frame_buffer.h"], deps = [ "//mediapipe/framework/port:integral_types", "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", ], ) diff --git a/mediapipe/framework/formats/frame_buffer.cc b/mediapipe/framework/formats/frame_buffer.cc new file mode 100644 index 000000000..930a3651a --- /dev/null +++ b/mediapipe/framework/formats/frame_buffer.cc @@ -0,0 +1,176 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/framework/formats/frame_buffer.h" + +#include "absl/status/status.h" +#include "absl/status/statusor.h" + +namespace mediapipe { + +namespace { + +// Returns whether the input `format` is a supported YUV format. +bool IsSupportedYuvFormat(FrameBuffer::Format format) { + return format == FrameBuffer::Format::kNV21 || + format == FrameBuffer::Format::kNV12 || + format == FrameBuffer::Format::kYV12 || + format == FrameBuffer::Format::kYV21; +} + +// Returns supported 1-plane FrameBuffer in YuvData structure. +absl::StatusOr GetYuvDataFromOnePlaneFrameBuffer( + const FrameBuffer& source) { + if (!IsSupportedYuvFormat(source.format())) { + return absl::InvalidArgumentError( + "The source FrameBuffer format is not part of YUV420 family."); + } + + FrameBuffer::YuvData result; + const int y_buffer_size = + source.plane(0).stride().row_stride_bytes * source.dimension().height; + const int uv_buffer_size = + ((source.plane(0).stride().row_stride_bytes + 1) / 2) * + ((source.dimension().height + 1) / 2); + result.y_buffer = source.plane(0).buffer(); + result.y_row_stride = source.plane(0).stride().row_stride_bytes; + result.uv_row_stride = result.y_row_stride; + + if (source.format() == FrameBuffer::Format::kNV21) { + result.v_buffer = result.y_buffer + y_buffer_size; + result.u_buffer = result.v_buffer + 1; + result.uv_pixel_stride = 2; + // If y_row_stride equals to the frame width and is an odd value, + // uv_row_stride = y_row_stride + 1, otherwise uv_row_stride = y_row_stride. + if (result.y_row_stride == source.dimension().width && + result.y_row_stride % 2 == 1) { + result.uv_row_stride = (result.y_row_stride + 1) / 2 * 2; + } + } else if (source.format() == FrameBuffer::Format::kNV12) { + result.u_buffer = result.y_buffer + y_buffer_size; + result.v_buffer = result.u_buffer + 1; + result.uv_pixel_stride = 2; + // If y_row_stride equals to the frame width and is an odd value, + // uv_row_stride = y_row_stride + 1, otherwise uv_row_stride = y_row_stride. + if (result.y_row_stride == source.dimension().width && + result.y_row_stride % 2 == 1) { + result.uv_row_stride = (result.y_row_stride + 1) / 2 * 2; + } + } else if (source.format() == FrameBuffer::Format::kYV21) { + result.u_buffer = result.y_buffer + y_buffer_size; + result.v_buffer = result.u_buffer + uv_buffer_size; + result.uv_pixel_stride = 1; + result.uv_row_stride = (result.y_row_stride + 1) / 2; + } else if (source.format() == FrameBuffer::Format::kYV12) { + result.v_buffer = result.y_buffer + y_buffer_size; + result.u_buffer = result.v_buffer + uv_buffer_size; + result.uv_pixel_stride = 1; + result.uv_row_stride = (result.y_row_stride + 1) / 2; + } + return result; +} + +// Returns supported 2-plane FrameBuffer in YuvData structure. +absl::StatusOr GetYuvDataFromTwoPlaneFrameBuffer( + const FrameBuffer& source) { + if (source.format() != FrameBuffer::Format::kNV12 && + source.format() != FrameBuffer::Format::kNV21) { + return absl::InvalidArgumentError("Unsupported YUV planar format."); + } + + FrameBuffer::YuvData result; + // Y plane + result.y_buffer = source.plane(0).buffer(); + // All plane strides + result.y_row_stride = source.plane(0).stride().row_stride_bytes; + result.uv_row_stride = source.plane(1).stride().row_stride_bytes; + result.uv_pixel_stride = 2; + + if (source.format() == FrameBuffer::Format::kNV12) { + // Y and UV interleaved format + result.u_buffer = source.plane(1).buffer(); + result.v_buffer = result.u_buffer + 1; + } else { + // Y and VU interleaved format + result.v_buffer = source.plane(1).buffer(); + result.u_buffer = result.v_buffer + 1; + } + return result; +} + +// Returns supported 3-plane FrameBuffer in YuvData structure. Note that NV21 +// and NV12 are included in the supported Yuv formats. Technically, NV21 and +// NV12 should not be described by the 3-plane format. Historically, NV21 is +// used loosely such that it can also be used to describe YV21 format. For +// backwards compatibility, FrameBuffer supports NV21/NV12 with 3-plane format +// but such usage is discouraged +absl::StatusOr GetYuvDataFromThreePlaneFrameBuffer( + const FrameBuffer& source) { + if (!IsSupportedYuvFormat(source.format())) { + return absl::InvalidArgumentError( + "The source FrameBuffer format is not part of YUV420 family."); + } + + if (source.plane(1).stride().row_stride_bytes != + source.plane(2).stride().row_stride_bytes || + source.plane(1).stride().pixel_stride_bytes != + source.plane(2).stride().pixel_stride_bytes) { + return absl::InternalError("Unsupported YUV planar format."); + } + FrameBuffer::YuvData result; + if (source.format() == FrameBuffer::Format::kNV21 || + source.format() == FrameBuffer::Format::kYV12) { + // Y follow by VU order. The VU chroma planes can be interleaved or + // planar. + result.y_buffer = source.plane(0).buffer(); + result.v_buffer = source.plane(1).buffer(); + result.u_buffer = source.plane(2).buffer(); + result.y_row_stride = source.plane(0).stride().row_stride_bytes; + result.uv_row_stride = source.plane(1).stride().row_stride_bytes; + result.uv_pixel_stride = source.plane(1).stride().pixel_stride_bytes; + } else { + // Y follow by UV order. The UV chroma planes can be interleaved or + // planar. + result.y_buffer = source.plane(0).buffer(); + result.u_buffer = source.plane(1).buffer(); + result.v_buffer = source.plane(2).buffer(); + result.y_row_stride = source.plane(0).stride().row_stride_bytes; + result.uv_row_stride = source.plane(1).stride().row_stride_bytes; + result.uv_pixel_stride = source.plane(1).stride().pixel_stride_bytes; + } + return result; +} + +} // namespace + +absl::StatusOr FrameBuffer::GetYuvDataFromFrameBuffer( + const FrameBuffer& source) { + if (!IsSupportedYuvFormat(source.format())) { + return absl::InvalidArgumentError( + "The source FrameBuffer format is not part of YUV420 family."); + } + + if (source.plane_count() == 1) { + return GetYuvDataFromOnePlaneFrameBuffer(source); + } else if (source.plane_count() == 2) { + return GetYuvDataFromTwoPlaneFrameBuffer(source); + } else if (source.plane_count() == 3) { + return GetYuvDataFromThreePlaneFrameBuffer(source); + } + return absl::InvalidArgumentError( + "The source FrameBuffer must be consisted by 1, 2, or 3 planes"); +} + +} // namespace mediapipe diff --git a/mediapipe/framework/formats/frame_buffer.h b/mediapipe/framework/formats/frame_buffer.h index ccc699724..32ba41a2d 100644 --- a/mediapipe/framework/formats/frame_buffer.h +++ b/mediapipe/framework/formats/frame_buffer.h @@ -1,4 +1,4 @@ -/* Copyright 2022 The MediaPipe Authors. All Rights Reserved. +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ limitations under the License. #include #include "absl/log/check.h" +#include "absl/status/statusor.h" #include "mediapipe/framework/port/integral_types.h" namespace mediapipe { @@ -118,6 +119,20 @@ class FrameBuffer { int Size() const { return width * height; } }; + // YUV data structure. + struct YuvData { + const uint8* y_buffer; + const uint8* u_buffer; + const uint8* v_buffer; + // Y buffer row stride in bytes. + int y_row_stride; + // U/V buffer row stride in bytes. + int uv_row_stride; + // U/V pixel stride in bytes. This is the distance between two consecutive + // u/v pixel values in a row. + int uv_pixel_stride; + }; + // Builds a FrameBuffer object from a row-major backing buffer. // // The FrameBuffer does not take ownership of the backing buffer. The caller @@ -150,6 +165,12 @@ class FrameBuffer { // Returns FrameBuffer format. Format format() const { return format_; } + // Returns YuvData which contains the Y, U, and V buffer and their + // stride info from the input `source` FrameBuffer which is in the YUV family + // formats (e.g NV12, NV21, YV12, and YV21). + static absl::StatusOr GetYuvDataFromFrameBuffer( + const FrameBuffer& source); + private: std::vector planes_; Dimension dimension_; diff --git a/mediapipe/python/BUILD b/mediapipe/python/BUILD index 6e36ca1c2..f56e5b3d4 100644 --- a/mediapipe/python/BUILD +++ b/mediapipe/python/BUILD @@ -87,6 +87,8 @@ cc_library( cc_library( name = "builtin_task_graphs", deps = [ + "//mediapipe/tasks/cc/audio/audio_classifier:audio_classifier_graph", + "//mediapipe/tasks/cc/audio/audio_embedder:audio_embedder_graph", "//mediapipe/tasks/cc/vision/gesture_recognizer:gesture_recognizer_graph", "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", "//mediapipe/tasks/cc/vision/image_embedder:image_embedder_graph", @@ -94,11 +96,8 @@ cc_library( "//mediapipe/tasks/cc/vision/object_detector:object_detector_graph", ] + select({ # TODO: Build text_classifier_graph and text_embedder_graph on Windows. - # TODO: Build audio_classifier_graph and audio_embedder_graph on Windows. "//mediapipe:windows": [], "//conditions:default": [ - "//mediapipe/tasks/cc/audio/audio_classifier:audio_classifier_graph", - "//mediapipe/tasks/cc/audio/audio_embedder:audio_embedder_graph", "//mediapipe/tasks/cc/text/text_classifier:text_classifier_graph", "//mediapipe/tasks/cc/text/text_embedder:text_embedder_graph", ], diff --git a/mediapipe/tasks/ios/test/vision/core/BUILD b/mediapipe/tasks/ios/test/vision/core/BUILD new file mode 100644 index 000000000..ef95e4681 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/core/BUILD @@ -0,0 +1,58 @@ +load( + "@build_bazel_rules_apple//apple:ios.bzl", + "ios_unit_test", +) +load( + "//mediapipe/tasks:ios/ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPImageObjcTestLibrary", + testonly = 1, + srcs = ["MPPImageTests.m"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + ], + sdk_frameworks = [ + "CoreMedia", + "CoreVideo", + "CoreGraphics", + "UIKit", + "Accelerate", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/vision/core:MPPImage", + ], +) + +ios_unit_test( + name = "MPPImageObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPImageObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m b/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m new file mode 100644 index 000000000..a7fa97bfa --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m @@ -0,0 +1,358 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" + +#import +#import +#import +#import +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +static NSString *const kTestImageName = @"burger"; +static NSString *const kTestImageType = @"jpg"; +static CGFloat kTestImageWidthInPixels = 480.0f; +static CGFloat kTestImageHeightInPixels = 325.0f; +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertNotEqual( \ + [error.localizedDescription rangeOfString:expectedError.localizedDescription].location, \ + NSNotFound) + +/** Unit tests for `MPPImage`. */ +@interface MPPImageTests : XCTestCase + +/** Test image. */ +@property(nonatomic, nullable) UIImage *image; + +@end + +@implementation MPPImageTests + +#pragma mark - Tests + +- (void)setUp { + [super setUp]; + NSString *imageName = [[NSBundle bundleForClass:[self class]] pathForResource:kTestImageName + ofType:kTestImageType]; + self.image = [[UIImage alloc] initWithContentsOfFile:imageName]; +} + +- (void)tearDown { + self.image = nil; + [super tearDown]; +} + +- (void)assertMPPImage:(nullable MPPImage *)mppImage + hasSourceType:(MPPImageSourceType)sourceType + hasOrientation:(UIImageOrientation)expectedOrientation + width:(CGFloat)expectedWidth + height:(CGFloat)expectedHeight { + XCTAssertNotNil(mppImage); + XCTAssertEqual(mppImage.imageSourceType, sourceType); + XCTAssertEqual(mppImage.orientation, expectedOrientation); + XCTAssertEqualWithAccuracy(mppImage.width, expectedWidth, FLT_EPSILON); + XCTAssertEqualWithAccuracy(mppImage.height, expectedHeight, FLT_EPSILON); +} + +- (void)assertInitFailsWithImage:(nullable MPPImage *)mppImage + error:(NSError *)error + expectedError:(NSError *)expectedError { + XCTAssertNil(mppImage); + XCTAssertNotNil(error); + AssertEqualErrors(error, expectedError); +} + +- (void)testInitWithImageSuceeds { + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:self.image error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeImage + hasOrientation:self.image.imageOrientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithImageAndOrientation { + UIImageOrientation orientation = UIImageOrientationRight; + + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:self.image + orientation:orientation + error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeImage + hasOrientation:orientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithImage_nilImage { + NSError *error; + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wnonnull" + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:nil error:&error]; +#pragma clang diagnostic pop + + [self + assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"Image cannot be nil." + }]]; +} + +- (void)testInitWithImageAndOrientation_nilImage { + NSError *error; + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wnonnull" + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:nil + orientation:UIImageOrientationRight + error:&error]; +#pragma clang diagnostic pop + + [self + assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"Image cannot be nil." + }]]; +} + +- (void)testInitWithSampleBuffer { + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:sampleBuffer error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeSampleBuffer + hasOrientation:UIImageOrientationUp + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithSampleBufferAndOrientation { + UIImageOrientation orientation = UIImageOrientationRight; + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:sampleBuffer + orientation:orientation + error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeSampleBuffer + hasOrientation:orientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithSampleBuffer_nilImage { + NSError *error; + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wnonnull" + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:nil error:&error]; +#pragma clang diagnostic pop + + [self + assertInitFailsWithImage:mppImage + error:error + expectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Sample buffer is not valid. Invoking " + @"CMSampleBufferIsValid(sampleBuffer) must return true." + }]]; +} + +- (void)testInitWithSampleBufferAndOrientation_nilImage { + NSError *error; + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wnonnull" + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:nil + orientation:UIImageOrientationRight + error:&error]; +#pragma clang diagnostic pop + + [self + assertInitFailsWithImage:mppImage + error:error + expectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Sample buffer is not valid. Invoking " + @"CMSampleBufferIsValid(sampleBuffer) must return true." + }]]; +} + +- (void)testInitWithPixelBuffer { + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:pixelBuffer error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypePixelBuffer + hasOrientation:UIImageOrientationUp + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithPixelBufferAndOrientation { + UIImageOrientation orientation = UIImageOrientationRight; + + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:pixelBuffer + orientation:orientation + error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypePixelBuffer + hasOrientation:orientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithPixelBuffer_nilImage { + NSError *error; + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wnonnull" + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:nil error:&error]; +#pragma clang diagnostic pop + + [self assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Pixel Buffer cannot be nil." + }]]; +} + +- (void)testInitWithPixelBufferAndOrientation_nilImage { + NSError *error; + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wnonnull" + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:nil + orientation:UIImageOrientationRight + error:&error]; +#pragma clang diagnostic pop + + [self assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Pixel Buffer cannot be nil." + }]]; +} + +#pragma mark - Private + +/** + * Converts the input image in RGBA space into a `CMSampleBuffer`. + * + * @return `CMSampleBuffer` converted from the given `UIImage`. + */ +- (CMSampleBufferRef)sampleBuffer { + // Rotate the image and convert from RGBA to BGRA. + CGImageRef CGImage = self.image.CGImage; + size_t width = CGImageGetWidth(CGImage); + size_t height = CGImageGetHeight(CGImage); + size_t bpr = CGImageGetBytesPerRow(CGImage); + + CGDataProviderRef provider = CGImageGetDataProvider(CGImage); + NSData *imageRGBAData = (id)CFBridgingRelease(CGDataProviderCopyData(provider)); + const uint8_t order[4] = {2, 1, 0, 3}; + + NSData *imageBGRAData = nil; + unsigned char *bgraPixel = (unsigned char *)malloc([imageRGBAData length]); + if (bgraPixel) { + vImage_Buffer src; + src.height = height; + src.width = width; + src.rowBytes = bpr; + src.data = (void *)[imageRGBAData bytes]; + + vImage_Buffer dest; + dest.height = height; + dest.width = width; + dest.rowBytes = bpr; + dest.data = bgraPixel; + + // Specify ordering changes in map. + vImage_Error error = vImagePermuteChannels_ARGB8888(&src, &dest, order, kvImageNoFlags); + + // Package the result. + if (error == kvImageNoError) { + imageBGRAData = [NSData dataWithBytes:bgraPixel length:[imageRGBAData length]]; + } + + // Memory cleanup. + free(bgraPixel); + } + + if (imageBGRAData == nil) { + XCTFail(@"Failed to convert input image."); + } + + // Write data to `CMSampleBuffer`. + NSDictionary *options = @{ + (__bridge NSString *)kCVPixelBufferCGImageCompatibilityKey : @(YES), + (__bridge NSString *)kCVPixelBufferCGBitmapContextCompatibilityKey : @(YES) + }; + CVPixelBufferRef pixelBuffer; + CVReturn status = CVPixelBufferCreateWithBytes( + kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA, (void *)[imageBGRAData bytes], + bpr, NULL, nil, (__bridge CFDictionaryRef)options, &pixelBuffer); + + if (status != kCVReturnSuccess) { + XCTFail(@"Failed to create pixel buffer."); + } + + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + CMVideoFormatDescriptionRef videoInfo = NULL; + CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &videoInfo); + + CMSampleBufferRef buffer; + CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, true, NULL, NULL, videoInfo, + &kCMTimingInfoInvalid, &buffer); + + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + return buffer; +} + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index 1a6180968..b2ed63d9c 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -4,8 +4,12 @@ licenses(["notice"]) objc_library( name = "MPPImage", - srcs = ["sources/MPPImage.h"], - hdrs = ["sources/MPPImage.m"], + srcs = ["sources/MPPImage.m"], + hdrs = ["sources/MPPImage.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], module_name = "MPPImage", sdk_frameworks = [ "CoreMedia", @@ -13,6 +17,7 @@ objc_library( "UIKit", ], deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", ], copts = [