From 445876d96b852b05c6327e6979cb9fe7bc10958a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 8 Feb 2023 21:35:15 +0530 Subject: [PATCH 1/3] Updated formatting --- mediapipe/tasks/ios/vision/core/sources/MPPImage.h | 12 ++++++------ mediapipe/tasks/ios/vision/core/sources/MPPImage.m | 12 +++++++----- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPImage.h b/mediapipe/tasks/ios/vision/core/sources/MPPImage.h index 6faaa0d31..08364ad0b 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPImage.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPImage.h @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#import + #import #import #import @@ -72,8 +74,7 @@ NS_SWIFT_NAME(MPImage) * @return A new `MPPImage` instance with the given image as the source. `nil` if the given * `image` is `nil` or invalid. */ -- (nullable instancetype)initWithUIImage:(UIImage *)image - error:(NSError **)error; +- (nullable instancetype)initWithUIImage:(UIImage *)image error:(NSError **)error; /** * Initializes an `MPPImage` object with the given `UIImabe` and orientation. @@ -110,8 +111,7 @@ NS_SWIFT_NAME(MPImage) * @return A new `MPPImage` instance with the given pixel buffer as the source. `nil` if the * given pixel buffer is `nil` or invalid. */ -- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer - error:(NSError **)error; +- (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error; /** * Initializes an `MPPImage` object with the given pixel buffer and orientation. @@ -130,7 +130,7 @@ NS_SWIFT_NAME(MPImage) */ - (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer orientation:(UIImageOrientation)orientation - NS_DESIGNATED_INITIALIZER; + error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** * Initializes an `MPPImage` object with the given sample buffer. @@ -166,7 +166,7 @@ NS_SWIFT_NAME(MPImage) */ - (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer orientation:(UIImageOrientation)orientation - NS_DESIGNATED_INITIALIZER; + error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** Unavailable. */ - (instancetype)init NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPImage.m b/mediapipe/tasks/ios/vision/core/sources/MPPImage.m index b8591953a..1f5104ef7 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPImage.m +++ b/mediapipe/tasks/ios/vision/core/sources/MPPImage.m @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" NS_ASSUME_NONNULL_BEGIN @@ -20,7 +21,7 @@ NS_ASSUME_NONNULL_BEGIN @implementation MPPImage - (nullable instancetype)initWithUIImage:(UIImage *)image error:(NSError **)error { - return [self initWithUIImage:image orientation:image.orientation error:error]; + return [self initWithUIImage:image orientation:image.imageOrientation error:error]; } - (nullable instancetype)initWithUIImage:(UIImage *)image @@ -30,6 +31,7 @@ NS_ASSUME_NONNULL_BEGIN [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError description:@"Image cannot be nil."]; + return nil; } if (image.CGImage == NULL) { [MPPCommonUtils createCustomError:error @@ -41,7 +43,7 @@ NS_ASSUME_NONNULL_BEGIN self = [super init]; if (self) { - _imageSourceType = MPPTaskImageSourceTypeImage; + _imageSourceType = MPPImageSourceTypeImage; _orientation = orientation; _image = image; _width = image.size.width * image.scale; @@ -66,7 +68,7 @@ NS_ASSUME_NONNULL_BEGIN self = [super init]; if (self != nil) { - _imageSourceType = MPPTaskImageSourceTypePixelBuffer; + _imageSourceType = MPPImageSourceTypePixelBuffer; _orientation = orientation; CVPixelBufferRetain(pixelBuffer); _pixelBuffer = pixelBuffer; @@ -78,7 +80,7 @@ NS_ASSUME_NONNULL_BEGIN - (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer error:(NSError **)error { - return [self initWithSampleBuffer:sampleBuffer orientation:UIImageOrientation error:error]; + return [self initWithSampleBuffer:sampleBuffer orientation:UIImageOrientationUp error:error]; } - (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer @@ -99,7 +101,7 @@ NS_ASSUME_NONNULL_BEGIN self = [super init]; if (self != nil) { - _imageSourceType = MPPTaskImageSourceTypeSampleBuffer; + _imageSourceType = MPPImageSourceTypeSampleBuffer; _orientation = orientation; CFRetain(sampleBuffer); _sampleBuffer = sampleBuffer; From a82ffcd97c244756a1585d83b667b3d355416c4e Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 8 Feb 2023 21:40:23 +0530 Subject: [PATCH 2/3] Updated mppimage target --- mediapipe/tasks/ios/vision/core/BUILD | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index 816486391..ca3b96169 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -4,12 +4,20 @@ licenses(["notice"]) objc_library( name = "MPPImage", - srcs = ["sources/MPPImage.h"], - hdrs = ["sources/MPPImage.m"], + srcs = ["sources/MPPImage.m"], + hdrs = ["sources/MPPImage.h"], module_name = "MPPImage", + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + ], sdk_frameworks = [ "CoreMedia", "CoreVideo", "UIKit", - ] + ], + copts = [ + "-ObjC++", + "-std=c++17", + ], ) From e1ca81c3525ca837fba0bc0f9202219c87994b77 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 8 Feb 2023 21:40:41 +0530 Subject: [PATCH 3/3] Added objective c tests for MPPImage --- mediapipe/tasks/ios/test/vision/core/BUILD | 62 ++++ .../ios/test/vision/core/MPPImageTests.m | 328 ++++++++++++++++++ 2 files changed, 390 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/core/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/core/MPPImageTests.m diff --git a/mediapipe/tasks/ios/test/vision/core/BUILD b/mediapipe/tasks/ios/test/vision/core/BUILD new file mode 100644 index 000000000..541ca4301 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/core/BUILD @@ -0,0 +1,62 @@ +load( + "@build_bazel_rules_apple//apple:ios.bzl", + "ios_unit_test", +) +load( + "@build_bazel_rules_swift//swift:swift.bzl", + "swift_library", +) +load( + "//mediapipe/tasks:ios/ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPImageObjcTestLibrary", + testonly = 1, + srcs = ["MPPImageTests.m"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/vision/core:MPPImage", + ], + sdk_frameworks = [ + "CoreMedia", + "CoreVideo", + "CoreGraphics", + "UIKit", + "Accelerate", + ], +) + +ios_unit_test( + name = "MPPImageObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPImageObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m b/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m new file mode 100644 index 000000000..a5986b555 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/core/MPPImageTests.m @@ -0,0 +1,328 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" + +#import +#import +#import +#import +#import + +NS_ASSUME_NONNULL_BEGIN + +static NSString *const kTestImageName = @"burger"; +static NSString *const kTestImageType = @"jpg"; +static CGFloat kTestImageWidthInPixels = 480.0f; +static CGFloat kTestImageHeightInPixels = 325.0f; +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertNotEqual( \ + [error.localizedDescription rangeOfString:expectedError.localizedDescription].location, \ + NSNotFound) + +/** Unit tests for `MPPImage`. */ +@interface MPPImageTests : XCTestCase + +/** Test image. */ +@property(nonatomic, nullable) UIImage *image; + +@end + +@implementation MPPImageTests + +#pragma mark - Tests + +- (void)setUp { + [super setUp]; + NSString *imageName = [[NSBundle bundleForClass:[self class]] pathForResource:kTestImageName + ofType:kTestImageType]; + self.image = [[UIImage alloc] initWithContentsOfFile:imageName]; +} + +- (void)tearDown { + self.image = nil; + [super tearDown]; +} + +- (void)assertMPPImage:(nullable MPPImage *)mppImage + hasSourceType:(MPPImageSourceType)sourceType + hasOrientation:(UIImageOrientation)expectedOrientation + width:(CGFloat)expectedWidth + height:(CGFloat)expectedHeight { + XCTAssertNotNil(mppImage); + XCTAssertEqual(mppImage.imageSourceType, sourceType); + XCTAssertEqual(mppImage.orientation, expectedOrientation); + XCTAssertEqualWithAccuracy(mppImage.width, expectedWidth, FLT_EPSILON); + XCTAssertEqualWithAccuracy(mppImage.height, expectedHeight, FLT_EPSILON); +} + +- (void)assertInitFailsWithImage:(nullable MPPImage *)mppImage + error:(NSError *)error + expectedError:(NSError *)expectedError { + XCTAssertNil(mppImage); + XCTAssertNotNil(error); + AssertEqualErrors(error, expectedError); +} + +- (void)testInitWithImageSuceeds { + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:self.image error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeImage + hasOrientation:self.image.imageOrientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithImageAndOrientation { + UIImageOrientation orientation = UIImageOrientationRight; + + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:self.image + orientation:orientation + error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeImage + hasOrientation:orientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithImage_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:nil error:&error]; + [self + assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"Image cannot be nil." + }]]; + NSLog(@"Errorrrr........%@", error.localizedDescription); +} + +- (void)testInitWithImageAndOrientation_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithUIImage:nil + orientation:UIImageOrientationRight + error:&error]; + [self + assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"Image cannot be nil." + }]]; +} + +- (void)testInitWithSampleBuffer { + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:sampleBuffer error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeSampleBuffer + hasOrientation:UIImageOrientationUp + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithSampleBufferAndOrientation { + UIImageOrientation orientation = UIImageOrientationRight; + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:sampleBuffer + orientation:orientation + error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypeSampleBuffer + hasOrientation:orientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithSampleBuffer_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:nil error:&error]; + [self + assertInitFailsWithImage:mppImage + error:error + expectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Sample buffer is not valid. Invoking " + @"CMSampleBufferIsValid(sampleBuffer) must return true." + }]]; +} + +- (void)testInitWithSampleBufferAndOrientation_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithSampleBuffer:nil + orientation:UIImageOrientationRight + error:&error]; + [self + assertInitFailsWithImage:mppImage + error:error + expectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Sample buffer is not valid. Invoking " + @"CMSampleBufferIsValid(sampleBuffer) must return true." + }]]; +} + +- (void)testInitWithPixelBuffer { + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:pixelBuffer error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypePixelBuffer + hasOrientation:UIImageOrientationUp + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithPixelBufferAndOrientation { + UIImageOrientation orientation = UIImageOrientationRight; + + CMSampleBufferRef sampleBuffer = [self sampleBuffer]; + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:pixelBuffer + orientation:orientation + error:nil]; + [self assertMPPImage:mppImage + hasSourceType:MPPImageSourceTypePixelBuffer + hasOrientation:orientation + width:kTestImageWidthInPixels + height:kTestImageHeightInPixels]; +} + +- (void)testInitWithPixelBuffer_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:nil error:&error]; + [self assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Pixel Buffer cannot be nil." + }]]; +} + +- (void)testInitWithPixelBufferAndOrientation_nilImage { + NSError *error; + MPPImage *mppImage = [[MPPImage alloc] initWithPixelBuffer:nil + orientation:UIImageOrientationRight + error:&error]; + [self assertInitFailsWithImage:mppImage + error:error + expectedError:[NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"Pixel Buffer cannot be nil." + }]]; +} + +#pragma mark - Private + +/** + * Converts the input image in RGBA space into a `CMSampleBuffer`. + * + * @return `CMSampleBuffer` converted from the given `UIImage`. + */ +- (CMSampleBufferRef)sampleBuffer { + // Rotate the image and convert from RGBA to BGRA. + CGImageRef CGImage = self.image.CGImage; + size_t width = CGImageGetWidth(CGImage); + size_t height = CGImageGetHeight(CGImage); + size_t bpr = CGImageGetBytesPerRow(CGImage); + + CGDataProviderRef provider = CGImageGetDataProvider(CGImage); + NSData *imageRGBAData = (id)CFBridgingRelease(CGDataProviderCopyData(provider)); + const uint8_t order[4] = {2, 1, 0, 3}; + + NSData *imageBGRAData = nil; + unsigned char *bgraPixel = (unsigned char *)malloc([imageRGBAData length]); + if (bgraPixel) { + vImage_Buffer src; + src.height = height; + src.width = width; + src.rowBytes = bpr; + src.data = (void *)[imageRGBAData bytes]; + + vImage_Buffer dest; + dest.height = height; + dest.width = width; + dest.rowBytes = bpr; + dest.data = bgraPixel; + + // Specify ordering changes in map. + vImage_Error error = vImagePermuteChannels_ARGB8888(&src, &dest, order, kvImageNoFlags); + + // Package the result. + if (error == kvImageNoError) { + imageBGRAData = [NSData dataWithBytes:bgraPixel length:[imageRGBAData length]]; + } + + // Memory cleanup. + free(bgraPixel); + } + + if (imageBGRAData == nil) { + XCTFail(@"Failed to convert input image."); + } + + // Write data to `CMSampleBuffer`. + NSDictionary *options = @{ + (__bridge NSString *)kCVPixelBufferCGImageCompatibilityKey : @(YES), + (__bridge NSString *)kCVPixelBufferCGBitmapContextCompatibilityKey : @(YES) + }; + CVPixelBufferRef pixelBuffer; + CVReturn status = CVPixelBufferCreateWithBytes( + kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA, (void *)[imageBGRAData bytes], + bpr, NULL, nil, (__bridge CFDictionaryRef)options, &pixelBuffer); + + if (status != kCVReturnSuccess) { + XCTFail(@"Failed to create pixel buffer."); + } + + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + CMVideoFormatDescriptionRef videoInfo = NULL; + CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &videoInfo); + + CMSampleBufferRef buffer; + CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, true, NULL, NULL, videoInfo, + &kCMTimingInfoInvalid, &buffer); + + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + return buffer; +} + +@end + +NS_ASSUME_NONNULL_END