From 445876d96b852b05c6327e6979cb9fe7bc10958a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 8 Feb 2023 21:35:15 +0530 Subject: [PATCH 1/7] Updated formatting --- mediapipe/tasks/ios/vision/core/sources/MPPImage.h | 12 ++++++------ mediapipe/tasks/ios/vision/core/sources/MPPImage.m | 12 +++++++----- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPImage.h b/mediapipe/tasks/ios/vision/core/sources/MPPImage.h index 6faaa0d31..08364ad0b 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPImage.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPImage.h @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#import + #import #import #import @@ -72,8 +74,7 @@ NS_SWIFT_NAME(MPImage) * @return A new `MPPImage` instance with the given image as the source. `nil` if the given * `image` is `nil` or invalid. */ -- (nullable instancetype)initWithUIImage:(UIImage *)image - error:(NSError **)error; +- (nullable instancetype)initWithUIImage:(UIImage *)image error:(NSError **)error; /** * Initializes an `MPPImage` object with the given `UIImabe` and orientation. @@ -110,8 +111,7 @@ NS_SWIFT_NAME(MPImage) * @return A new `MPPImage` instance with the given pixel buffer as the source. `nil` if the * given pixel buffer is `nil` or invalid. */ -- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer - error:(NSError **)error; +- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error; /** * Initializes an `MPPImage` object with the given pixel buffer and orientation. @@ -130,7 +130,7 @@ NS_SWIFT_NAME(MPImage) */ - (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer orientation:(UIImageOrientation)orientation - NS_DESIGNATED_INITIALIZER; + error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** * Initializes an `MPPImage` object with the given sample buffer. @@ -166,7 +166,7 @@ NS_SWIFT_NAME(MPImage) */ - (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer orientation:(UIImageOrientation)orientation - NS_DESIGNATED_INITIALIZER; + error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** Unavailable. */ - (instancetype)init NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPImage.m b/mediapipe/tasks/ios/vision/core/sources/MPPImage.m index b8591953a..1f5104ef7 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPImage.m +++ b/mediapipe/tasks/ios/vision/core/sources/MPPImage.m @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" NS_ASSUME_NONNULL_BEGIN @@ -20,7 +21,7 @@ NS_ASSUME_NONNULL_BEGIN @implementation MPPImage - (nullable instancetype)initWithUIImage:(UIImage *)image error:(NSError **)error { - return [self initWithUIImage:image orientation:image.orientation error:error]; + return [self initWithUIImage:image orientation:image.imageOrientation error:error]; } - (nullable instancetype)initWithUIImage:(UIImage *)image @@ -30,6 +31,7 @@ NS_ASSUME_NONNULL_BEGIN [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError description:@"Image cannot be nil."]; + return nil; } if (image.CGImage == NULL) { [MPPCommonUtils createCustomError:error @@ -41,7 +43,7 @@ NS_ASSUME_NONNULL_BEGIN self = [super init]; if (self) { - _imageSourceType = MPPTaskImageSourceTypeImage; + _imageSourceType = MPPImageSourceTypeImage; _orientation = orientation; _image = image; _width = image.size.width * image.scale; @@ -66,7 +68,7 @@ NS_ASSUME_NONNULL_BEGIN self = [super init]; if (self != nil) { - _imageSourceType = MPPTaskImageSourceTypePixelBuffer; + _imageSourceType = MPPImageSourceTypePixelBuffer; _orientation = orientation; CVPixelBufferRetain(pixelBuffer); _pixelBuffer = pixelBuffer; @@ -78,7 +80,7 @@ NS_ASSUME_NONNULL_BEGIN - (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer error:(NSError **)error { - return [self initWithSampleBuffer:sampleBuffer orientation:UIImageOrientation error:error]; + return [self initWithSampleBuffer:sampleBuffer orientation:UIImageOrientationUp error:error]; } - (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer @@ -99,7 +101,7 @@ NS_ASSUME_NONNULL_BEGIN self = [super init]; if (self != nil) { - _imageSourceType = MPPTaskImageSourceTypeSampleBuffer; + _imageSourceType = MPPImageSourceTypeSampleBuffer; _orientation = orientation; CFRetain(sampleBuffer); _sampleBuffer = sampleBuffer; From a82ffcd97c244756a1585d83b667b3d355416c4e Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 8 Feb 2023 21:40:23 +0530 Subject: [PATCH 2/7] Updated mppimage target --- mediapipe/tasks/ios/vision/core/BUILD | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index 816486391..ca3b96169 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -4,12 +4,20 @@ licenses(["notice"]) objc_library( name = "MPPImage", - srcs = ["sources/MPPImage.h"], - hdrs = ["sources/MPPImage.m"], + srcs = ["sources/MPPImage.m"], + hdrs = ["sources/MPPImage.h"], module_name = "MPPImage", + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + ], sdk_frameworks = [ "CoreMedia", "CoreVideo", "UIKit", - ] + ], + copts = [ + "-ObjC++", + "-std=c++17", + ], ) From e1ca81c3525ca837fba0bc0f9202219c87994b77 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 8 Feb 2023 21:40:41 +0530 Subject: [PATCH 3/7] Added objective c tests for MPPImage --- mediapipe/tasks/ios/test/vision/core/BUILD | 62 ++++ .../ios/test/vision/core/MPPImageTests.m | 328 ++++++++++++++++++ 2 files changed, 390 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/core/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/core/MPPImageTests.m diff --git a/mediapipe/tasks/ios/test/vision/core/BUILD b/mediapipe/tasks/ios/test/vision/core/BUILD new file mode 100644 index 000000000..541ca4301 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/core/BUILD @@ -0,0 +1,62 @@ +load( + "@build_bazel_rules_apple//apple:ios.bzl", + "ios_unit_test", +) +load( + "@build_bazel_rules_swift//swift:swift.bzl", + "swift_library", +) +load( + "//mediapipe/tasks:ios/ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPImageObjcTestLibrary", + testonly = 1, + srcs = ["MPPImageTests.m"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/vision/core:MPPImage", + ], + sdk_frameworks = [ + "CoreMedia", + "CoreVideo", + "CoreGraphics", + "UIKit", + "Accelerate", + ], +) + +ios_unit_test( + name = "MPPImageObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPImageObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m b/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m new file mode 100644 index 000000000..a5986b555 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m @@ -0,0 +1,328 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" + +#import +#import +#import +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +static NSString *const kTestImageName = @"burger"; +static NSString *const kTestImageType = @"jpg"; +static CGFloat kTestImageWidthInPixels = 480.0f; +static CGFloat kTestImageHeightInPixels = 325.0f; +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertNotEqual( \ + [error.localizedDescription rangeOfString:expectedError.localizedDescription].location, \ + NSNotFound) + +/** Unit tests for `MPPImage`. */ +@interface MPPImageTests : XCTestCase + +/** Test image. */ +@property(nonatomic, nullable) UIImage *image; + +@end + +@implementation MPPImageTests + +#pragma mark - Tests + +- (void)setUp { + [super setUp]; + NSString *imageName = [[NSBundle bundleForClass:[self class]] pathForResource:kTestImageName + ofType:kTestImageType]; + self.image = [[UIImage alloc] initWithContentsOfFile:imageName]; +} + +- (void)tearDown { + self.image = nil; + [super tearDown]; +} + +- (void)assertMPPImage:(nullable MPPImage *)mppImage + hasSourceType:(MPPImageSourceType)sourceType + hasOrientation:(UIImageOrientation)expectedOrientation + width:(CGFloat)expectedWidth + height:(CGFloat)expectedHeight { + XCTAssertNotNil(mppImage); + XCTAssertEqual(mppImage.imageSourceType, sourceType); + XCTAssertEqual(mppImage.orientation, expectedOrientation); + XCTAssertEqualWithAccuracy(mppImage.width, expectedWidth, FLT_EPSILON); + XCTAssertEqualWithAccuracy(mppImage.height, expectedHeight, FLT_EPSILON); +} + +- (void)assertInitFailsWithImage:(nullable MPPImage *)mppImage + error:(NSError *)error + expectedError:(NSError *)expectedError { + XCTAssertNil(mppImage); + XCTAssertNotNil(error); + AssertEqualErrors(error, expectedError); +} + +- (void)testInitWithImageSuceeds { + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:self.image error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeImage + hasOrientation:self.image.imageOrientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithImageAndOrientation { + UIImageOrientation orientation = UIImageOrientationRight; + + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:self.image + orientation:orientation + error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeImage + hasOrientation:orientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithImage_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:nil error:&error]; + [self + assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"Image cannot be nil." + }]]; + NSLog(@"Errorrrr........%@", error.localizedDescription); +} + +- (void)testInitWithImageAndOrientation_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:nil + orientation:UIImageOrientationRight + error:&error]; + [self + assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"Image cannot be nil." + }]]; +} + +- (void)testInitWithSampleBuffer { + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:sampleBuffer error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeSampleBuffer + hasOrientation:UIImageOrientationUp + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithSampleBufferAndOrientation { + UIImageOrientation orientation = UIImageOrientationRight; + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:sampleBuffer + orientation:orientation + error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeSampleBuffer + hasOrientation:orientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithSampleBuffer_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:nil error:&error]; + [self + assertInitFailsWithImage:mppImage + error:error + expectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Sample buffer is not valid. Invoking " + @"CMSampleBufferIsValid(sampleBuffer) must return true." + }]]; +} + +- (void)testInitWithSampleBufferAndOrientation_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:nil + orientation:UIImageOrientationRight + error:&error]; + [self + assertInitFailsWithImage:mppImage + error:error + expectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Sample buffer is not valid. Invoking " + @"CMSampleBufferIsValid(sampleBuffer) must return true." + }]]; +} + +- (void)testInitWithPixelBuffer { + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:pixelBuffer error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypePixelBuffer + hasOrientation:UIImageOrientationUp + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithPixelBufferAndOrientation { + UIImageOrientation orientation = UIImageOrientationRight; + + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:pixelBuffer + orientation:orientation + error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypePixelBuffer + hasOrientation:orientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithPixelBuffer_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:nil error:&error]; + [self assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Pixel Buffer cannot be nil." + }]]; +} + +- (void)testInitWithPixelBufferAndOrientation_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:nil + orientation:UIImageOrientationRight + error:&error]; + [self assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Pixel Buffer cannot be nil." + }]]; +} + +#pragma mark - Private + +/** + * Converts the input image in RGBA space into a `CMSampleBuffer`. + * + * @return `CMSampleBuffer` converted from the given `UIImage`. + */ +- (CMSampleBufferRef)sampleBuffer { + // Rotate the image and convert from RGBA to BGRA. + CGImageRef CGImage = self.image.CGImage; + size_t width = CGImageGetWidth(CGImage); + size_t height = CGImageGetHeight(CGImage); + size_t bpr = CGImageGetBytesPerRow(CGImage); + + CGDataProviderRef provider = CGImageGetDataProvider(CGImage); + NSData *imageRGBAData = (id)CFBridgingRelease(CGDataProviderCopyData(provider)); + const uint8_t order[4] = {2, 1, 0, 3}; + + NSData *imageBGRAData = nil; + unsigned char *bgraPixel = (unsigned char *)malloc([imageRGBAData length]); + if (bgraPixel) { + vImage_Buffer src; + src.height = height; + src.width = width; + src.rowBytes = bpr; + src.data = (void *)[imageRGBAData bytes]; + + vImage_Buffer dest; + dest.height = height; + dest.width = width; + dest.rowBytes = bpr; + dest.data = bgraPixel; + + // Specify ordering changes in map. + vImage_Error error = vImagePermuteChannels_ARGB8888(&src, &dest, order, kvImageNoFlags); + + // Package the result. + if (error == kvImageNoError) { + imageBGRAData = [NSData dataWithBytes:bgraPixel length:[imageRGBAData length]]; + } + + // Memory cleanup. + free(bgraPixel); + } + + if (imageBGRAData == nil) { + XCTFail(@"Failed to convert input image."); + } + + // Write data to `CMSampleBuffer`. + NSDictionary *options = @{ + (__bridge NSString *)kCVPixelBufferCGImageCompatibilityKey : @(YES), + (__bridge NSString *)kCVPixelBufferCGBitmapContextCompatibilityKey : @(YES) + }; + CVPixelBufferRef pixelBuffer; + CVReturn status = CVPixelBufferCreateWithBytes( + kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA, (void *)[imageBGRAData bytes], + bpr, NULL, nil, (__bridge CFDictionaryRef)options, &pixelBuffer); + + if (status != kCVReturnSuccess) { + XCTFail(@"Failed to create pixel buffer."); + } + + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + CMVideoFormatDescriptionRef videoInfo = NULL; + CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &videoInfo); + + CMSampleBufferRef buffer; + CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, true, NULL, NULL, videoInfo, + &kCMTimingInfoInvalid, &buffer); + + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + return buffer; +} + +@end + +NS_ASSUME_NONNULL_END From deae714a5ccfc0a03a12846deb526f8134dfc40d Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 13 Feb 2023 09:03:08 -0800 Subject: [PATCH 4/7] Enable Python Audio Classifier & Embedder on Windows PiperOrigin-RevId: 509243198 --- mediapipe/python/BUILD | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mediapipe/python/BUILD b/mediapipe/python/BUILD index 6e36ca1c2..f56e5b3d4 100644 --- a/mediapipe/python/BUILD +++ b/mediapipe/python/BUILD @@ -87,6 +87,8 @@ cc_library( cc_library( name = "builtin_task_graphs", deps = [ + "//mediapipe/tasks/cc/audio/audio_classifier:audio_classifier_graph", + "//mediapipe/tasks/cc/audio/audio_embedder:audio_embedder_graph", "//mediapipe/tasks/cc/vision/gesture_recognizer:gesture_recognizer_graph", "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", "//mediapipe/tasks/cc/vision/image_embedder:image_embedder_graph", @@ -94,11 +96,8 @@ cc_library( "//mediapipe/tasks/cc/vision/object_detector:object_detector_graph", ] + select({ # TODO: Build text_classifier_graph and text_embedder_graph on Windows. - # TODO: Build audio_classifier_graph and audio_embedder_graph on Windows. "//mediapipe:windows": [], "//conditions:default": [ - "//mediapipe/tasks/cc/audio/audio_classifier:audio_classifier_graph", - "//mediapipe/tasks/cc/audio/audio_embedder:audio_embedder_graph", "//mediapipe/tasks/cc/text/text_classifier:text_classifier_graph", "//mediapipe/tasks/cc/text/text_embedder:text_embedder_graph", ], From 513ab7abba2233fead276dc220e824b5f98ca65b Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 13 Feb 2023 14:45:49 -0800 Subject: [PATCH 5/7] Internal change PiperOrigin-RevId: 509338497 --- mediapipe/framework/formats/BUILD | 3 + mediapipe/framework/formats/frame_buffer.cc | 176 ++++++++++++++++++++ mediapipe/framework/formats/frame_buffer.h | 23 ++- 3 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 mediapipe/framework/formats/frame_buffer.cc diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index dd311fb46..26525b5da 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -489,9 +489,12 @@ cc_test( cc_library( name = "frame_buffer", + srcs = ["frame_buffer.cc"], hdrs = ["frame_buffer.h"], deps = [ "//mediapipe/framework/port:integral_types", "@com_google_absl//absl/log:check", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", ], ) diff --git a/mediapipe/framework/formats/frame_buffer.cc b/mediapipe/framework/formats/frame_buffer.cc new file mode 100644 index 000000000..930a3651a --- /dev/null +++ b/mediapipe/framework/formats/frame_buffer.cc @@ -0,0 +1,176 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/framework/formats/frame_buffer.h" + +#include "absl/status/status.h" +#include "absl/status/statusor.h" + +namespace mediapipe { + +namespace { + +// Returns whether the input `format` is a supported YUV format. +bool IsSupportedYuvFormat(FrameBuffer::Format format) { + return format == FrameBuffer::Format::kNV21 || + format == FrameBuffer::Format::kNV12 || + format == FrameBuffer::Format::kYV12 || + format == FrameBuffer::Format::kYV21; +} + +// Returns supported 1-plane FrameBuffer in YuvData structure. +absl::StatusOr GetYuvDataFromOnePlaneFrameBuffer( + const FrameBuffer& source) { + if (!IsSupportedYuvFormat(source.format())) { + return absl::InvalidArgumentError( + "The source FrameBuffer format is not part of YUV420 family."); + } + + FrameBuffer::YuvData result; + const int y_buffer_size = + source.plane(0).stride().row_stride_bytes * source.dimension().height; + const int uv_buffer_size = + ((source.plane(0).stride().row_stride_bytes + 1) / 2) * + ((source.dimension().height + 1) / 2); + result.y_buffer = source.plane(0).buffer(); + result.y_row_stride = source.plane(0).stride().row_stride_bytes; + result.uv_row_stride = result.y_row_stride; + + if (source.format() == FrameBuffer::Format::kNV21) { + result.v_buffer = result.y_buffer + y_buffer_size; + result.u_buffer = result.v_buffer + 1; + result.uv_pixel_stride = 2; + // If y_row_stride equals to the frame width and is an odd value, + // uv_row_stride = y_row_stride + 1, otherwise uv_row_stride = y_row_stride. + if (result.y_row_stride == source.dimension().width && + result.y_row_stride % 2 == 1) { + result.uv_row_stride = (result.y_row_stride + 1) / 2 * 2; + } + } else if (source.format() == FrameBuffer::Format::kNV12) { + result.u_buffer = result.y_buffer + y_buffer_size; + result.v_buffer = result.u_buffer + 1; + result.uv_pixel_stride = 2; + // If y_row_stride equals to the frame width and is an odd value, + // uv_row_stride = y_row_stride + 1, otherwise uv_row_stride = y_row_stride. + if (result.y_row_stride == source.dimension().width && + result.y_row_stride % 2 == 1) { + result.uv_row_stride = (result.y_row_stride + 1) / 2 * 2; + } + } else if (source.format() == FrameBuffer::Format::kYV21) { + result.u_buffer = result.y_buffer + y_buffer_size; + result.v_buffer = result.u_buffer + uv_buffer_size; + result.uv_pixel_stride = 1; + result.uv_row_stride = (result.y_row_stride + 1) / 2; + } else if (source.format() == FrameBuffer::Format::kYV12) { + result.v_buffer = result.y_buffer + y_buffer_size; + result.u_buffer = result.v_buffer + uv_buffer_size; + result.uv_pixel_stride = 1; + result.uv_row_stride = (result.y_row_stride + 1) / 2; + } + return result; +} + +// Returns supported 2-plane FrameBuffer in YuvData structure. +absl::StatusOr GetYuvDataFromTwoPlaneFrameBuffer( + const FrameBuffer& source) { + if (source.format() != FrameBuffer::Format::kNV12 && + source.format() != FrameBuffer::Format::kNV21) { + return absl::InvalidArgumentError("Unsupported YUV planar format."); + } + + FrameBuffer::YuvData result; + // Y plane + result.y_buffer = source.plane(0).buffer(); + // All plane strides + result.y_row_stride = source.plane(0).stride().row_stride_bytes; + result.uv_row_stride = source.plane(1).stride().row_stride_bytes; + result.uv_pixel_stride = 2; + + if (source.format() == FrameBuffer::Format::kNV12) { + // Y and UV interleaved format + result.u_buffer = source.plane(1).buffer(); + result.v_buffer = result.u_buffer + 1; + } else { + // Y and VU interleaved format + result.v_buffer = source.plane(1).buffer(); + result.u_buffer = result.v_buffer + 1; + } + return result; +} + +// Returns supported 3-plane FrameBuffer in YuvData structure. Note that NV21 +// and NV12 are included in the supported Yuv formats. Technically, NV21 and +// NV12 should not be described by the 3-plane format. Historically, NV21 is +// used loosely such that it can also be used to describe YV21 format. For +// backwards compatibility, FrameBuffer supports NV21/NV12 with 3-plane format +// but such usage is discouraged +absl::StatusOr GetYuvDataFromThreePlaneFrameBuffer( + const FrameBuffer& source) { + if (!IsSupportedYuvFormat(source.format())) { + return absl::InvalidArgumentError( + "The source FrameBuffer format is not part of YUV420 family."); + } + + if (source.plane(1).stride().row_stride_bytes != + source.plane(2).stride().row_stride_bytes || + source.plane(1).stride().pixel_stride_bytes != + source.plane(2).stride().pixel_stride_bytes) { + return absl::InternalError("Unsupported YUV planar format."); + } + FrameBuffer::YuvData result; + if (source.format() == FrameBuffer::Format::kNV21 || + source.format() == FrameBuffer::Format::kYV12) { + // Y follow by VU order. The VU chroma planes can be interleaved or + // planar. + result.y_buffer = source.plane(0).buffer(); + result.v_buffer = source.plane(1).buffer(); + result.u_buffer = source.plane(2).buffer(); + result.y_row_stride = source.plane(0).stride().row_stride_bytes; + result.uv_row_stride = source.plane(1).stride().row_stride_bytes; + result.uv_pixel_stride = source.plane(1).stride().pixel_stride_bytes; + } else { + // Y follow by UV order. The UV chroma planes can be interleaved or + // planar. + result.y_buffer = source.plane(0).buffer(); + result.u_buffer = source.plane(1).buffer(); + result.v_buffer = source.plane(2).buffer(); + result.y_row_stride = source.plane(0).stride().row_stride_bytes; + result.uv_row_stride = source.plane(1).stride().row_stride_bytes; + result.uv_pixel_stride = source.plane(1).stride().pixel_stride_bytes; + } + return result; +} + +} // namespace + +absl::StatusOr FrameBuffer::GetYuvDataFromFrameBuffer( + const FrameBuffer& source) { + if (!IsSupportedYuvFormat(source.format())) { + return absl::InvalidArgumentError( + "The source FrameBuffer format is not part of YUV420 family."); + } + + if (source.plane_count() == 1) { + return GetYuvDataFromOnePlaneFrameBuffer(source); + } else if (source.plane_count() == 2) { + return GetYuvDataFromTwoPlaneFrameBuffer(source); + } else if (source.plane_count() == 3) { + return GetYuvDataFromThreePlaneFrameBuffer(source); + } + return absl::InvalidArgumentError( + "The source FrameBuffer must be consisted by 1, 2, or 3 planes"); +} + +} // namespace mediapipe diff --git a/mediapipe/framework/formats/frame_buffer.h b/mediapipe/framework/formats/frame_buffer.h index ccc699724..32ba41a2d 100644 --- a/mediapipe/framework/formats/frame_buffer.h +++ b/mediapipe/framework/formats/frame_buffer.h @@ -1,4 +1,4 @@ -/* Copyright 2022 The MediaPipe Authors. All Rights Reserved. +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ limitations under the License. #include #include "absl/log/check.h" +#include "absl/status/statusor.h" #include "mediapipe/framework/port/integral_types.h" namespace mediapipe { @@ -118,6 +119,20 @@ class FrameBuffer { int Size() const { return width * height; } }; + // YUV data structure. + struct YuvData { + const uint8* y_buffer; + const uint8* u_buffer; + const uint8* v_buffer; + // Y buffer row stride in bytes. + int y_row_stride; + // U/V buffer row stride in bytes. + int uv_row_stride; + // U/V pixel stride in bytes. This is the distance between two consecutive + // u/v pixel values in a row. + int uv_pixel_stride; + }; + // Builds a FrameBuffer object from a row-major backing buffer. // // The FrameBuffer does not take ownership of the backing buffer. The caller @@ -150,6 +165,12 @@ class FrameBuffer { // Returns FrameBuffer format. Format format() const { return format_; } + // Returns YuvData which contains the Y, U, and V buffer and their + // stride info from the input `source` FrameBuffer which is in the YUV family + // formats (e.g NV12, NV21, YV12, and YV21). + static absl::StatusOr GetYuvDataFromFrameBuffer( + const FrameBuffer& source); + private: std::vector planes_; Dimension dimension_; From caa7d85141324099275a0ebf18d3f711029b4ec6 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 13 Feb 2023 15:20:36 -0800 Subject: [PATCH 6/7] Improves page-contents navigation: adds Uility Functions/Classes to page-contents) PiperOrigin-RevId: 509347516 --- docs/framework_concepts/building_graphs_cpp.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/framework_concepts/building_graphs_cpp.md b/docs/framework_concepts/building_graphs_cpp.md index c7e11f131..29d0c25d5 100644 --- a/docs/framework_concepts/building_graphs_cpp.md +++ b/docs/framework_concepts/building_graphs_cpp.md @@ -12,8 +12,6 @@ nav_order: 1 {:toc} --- -## C++ Graph Builder - C++ graph builder is a powerful tool for: * Building complex graphs @@ -25,7 +23,7 @@ C++ graph builder is a powerful tool for: * Supporting optional graph inputs/outputs * Customizing graphs per platform -### Basic Usage +## Basic Usage Let's see how C++ graph builder can be used for a simple graph: @@ -95,9 +93,9 @@ Short summary: unleashing graph builder capabilities and improving your graphs readability. -### Advanced Usage +## Advanced Usage -#### Utility Functions +### Utility Functions Let's extract inference construction code into a dedicated utility function to help for readability and code reuse: @@ -162,7 +160,7 @@ graphs construction code and helps automatically pull in calculator dependencies (e.g. no need to manually add `:inference_calculator` dep, just let your IDE include `inference.h` and build cleaner pull in corresponding dependency). -#### Utility Classes +### Utility Classes And surely, it's not only about functions, in some cases it's beneficial to introduce utility classes which can help making your graph construction code From ef9e5f881bf80a3a9de893c92d9796b0c0588d8c Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 13 Feb 2023 15:21:12 -0800 Subject: [PATCH 7/7] Add Dos and Donts Section / Graph inputs PiperOrigin-RevId: 509347658 --- .../framework_concepts/building_graphs_cpp.md | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/docs/framework_concepts/building_graphs_cpp.md b/docs/framework_concepts/building_graphs_cpp.md index 29d0c25d5..49ee52991 100644 --- a/docs/framework_concepts/building_graphs_cpp.md +++ b/docs/framework_concepts/building_graphs_cpp.md @@ -275,3 +275,69 @@ Tip: the same as for the `RunInference` function, extracting `PassThroughNodeBuilder` and similar utility classes into dedicated modules enables reuse in graph construction code and helps to automatically pull in the corresponding calculator dependencies. + +## Dos and Don'ts + +### Define graph inputs at the very beginning if possible + +```c++ {.bad} +Stream RunSomething(Stream a, Stream b, Graph& graph) { + Stream c = graph.In(2).SetName("c").Cast(); // Bad. + // ... +} + +CalculatorGraphConfig BuildGraph() { + Graph graph; + + Stream a = graph.In(0).SetName("a").Cast(); + // 10/100/N lines of code. + Stream b = graph.In(1).SetName("b").Cast() // Bad. + Stream d = RunSomething(a, b, graph); + // ... +} + +``` + +In the above code: + +* It can be hard to guess how many inputs you have in the graph. +* Can be error prone overall and hard to maintain in future (e.g. is it a + correct index? name? what if some inputs are removed or made optional? + etc.). + +Instead, simply define your graph inputs at the very beginning of your graph +builder: + +```c++ {.good} +Stream RunSomething(Stream a, Stream b, Stream c, Graph& graph) { + // ... +} + +CalculatorGraphConfig BuildGraph() { + Graph graph; + + Stream a = graph.In(0).SetName("a").Cast(); + Stream b = graph.In(1).SetName("b").Cast(); + Stream c = graph.In(2).SetName("c").Cast(); + + // 10/100/N lines of code. + Stream d = RunSomething(a, b, c, graph); + // ... +} +``` + +And if you have an input stream or side packet that is not always defined - +simply use `std::optional` and put it at the very beginning as well: + +```c++ {.good} +std::optional> a; +if (needs_a) { + a = graph.In(0).SetName(a).Cast(); +} +``` + +Note: of course, there can be exceptions - for example, there can be a use case +where calling `RunSomething1(..., graph)`, ..., `RunSomethingN(..., graph)` is +**intended to add new inputs**, so afterwards you can iterate over them and feed +only added inputs into the graph. However, in any case, try to make it easy for +readers to find out what graph inputs it has or may have.