From acba3ab26d6abcd075558fffde1ad0ffdf6ec66a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 13 Feb 2023 19:41:15 +0530 Subject: [PATCH 01/11] Bug fixes in MPPImage --- mediapipe/tasks/ios/vision/core/sources/MPPImage.h | 6 ++++-- mediapipe/tasks/ios/vision/core/sources/MPPImage.m | 12 +++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPImage.h b/mediapipe/tasks/ios/vision/core/sources/MPPImage.h index 730b5a277..deffc97e2 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPImage.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPImage.h @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#import + #import #import #import @@ -128,7 +130,7 @@ NS_SWIFT_NAME(MPImage) */ - (nullable instancetype)initWithPixelBuffer:(CVPixelBufferRef)pixelBuffer orientation:(UIImageOrientation)orientation - NS_DESIGNATED_INITIALIZER; + error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** * Initializes an `MPPImage` object with the given sample buffer. @@ -164,7 +166,7 @@ NS_SWIFT_NAME(MPImage) */ - (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer orientation:(UIImageOrientation)orientation - NS_DESIGNATED_INITIALIZER; + error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** Unavailable. */ - (instancetype)init NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPImage.m b/mediapipe/tasks/ios/vision/core/sources/MPPImage.m index b8591953a..1f5104ef7 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPImage.m +++ b/mediapipe/tasks/ios/vision/core/sources/MPPImage.m @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" NS_ASSUME_NONNULL_BEGIN @@ -20,7 +21,7 @@ NS_ASSUME_NONNULL_BEGIN @implementation MPPImage - (nullable instancetype)initWithUIImage:(UIImage *)image error:(NSError **)error { - return [self initWithUIImage:image orientation:image.orientation error:error]; + return [self initWithUIImage:image orientation:image.imageOrientation error:error]; } - (nullable instancetype)initWithUIImage:(UIImage *)image @@ -30,6 +31,7 @@ NS_ASSUME_NONNULL_BEGIN [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError description:@"Image cannot be nil."]; + return nil; } if (image.CGImage == NULL) { [MPPCommonUtils createCustomError:error @@ -41,7 +43,7 @@ NS_ASSUME_NONNULL_BEGIN self = [super init]; if (self) { - _imageSourceType = MPPTaskImageSourceTypeImage; + _imageSourceType = MPPImageSourceTypeImage; _orientation = orientation; _image = image; _width = image.size.width * image.scale; @@ -66,7 +68,7 @@ NS_ASSUME_NONNULL_BEGIN self = [super init]; if (self != nil) { - _imageSourceType = MPPTaskImageSourceTypePixelBuffer; + _imageSourceType = MPPImageSourceTypePixelBuffer; _orientation = orientation; CVPixelBufferRetain(pixelBuffer); _pixelBuffer = pixelBuffer; @@ -78,7 +80,7 @@ NS_ASSUME_NONNULL_BEGIN - (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer error:(NSError **)error { - return [self initWithSampleBuffer:sampleBuffer orientation:UIImageOrientation error:error]; + return [self initWithSampleBuffer:sampleBuffer orientation:UIImageOrientationUp error:error]; } - (nullable instancetype)initWithSampleBuffer:(CMSampleBufferRef)sampleBuffer @@ -99,7 +101,7 @@ NS_ASSUME_NONNULL_BEGIN self = [super init]; if (self != nil) { - _imageSourceType = MPPTaskImageSourceTypeSampleBuffer; + _imageSourceType = MPPImageSourceTypeSampleBuffer; _orientation = orientation; CFRetain(sampleBuffer); _sampleBuffer = sampleBuffer; From 02734b3add81014f36bb40b3443dc1518b6412af Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 13 Feb 2023 19:43:49 +0530 Subject: [PATCH 02/11] Added MPPImageUtils --- mediapipe/tasks/ios/vision/core/utils/BUILD | 24 ++ .../core/utils/sources/MPPImage+Utils.h | 44 +++ .../core/utils/sources/MPPImage+Utils.m | 324 ++++++++++++++++++ 3 files changed, 392 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/core/utils/BUILD create mode 100644 mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h create mode 100644 mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.m diff --git a/mediapipe/tasks/ios/vision/core/utils/BUILD b/mediapipe/tasks/ios/vision/core/utils/BUILD new file mode 100644 index 000000000..ab9fc5383 --- /dev/null +++ b/mediapipe/tasks/ios/vision/core/utils/BUILD @@ -0,0 +1,24 @@ +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPImageUtils", + srcs = ["sources/MPPImage+Utils.m"], + hdrs = ["sources/MPPImage+Utils.h"], + module_name = "MPPImageUtils", + sdk_frameworks = [ + "Accelerate", + "CoreGraphics", + "CoreImage", + "CoreVideo", + ], + deps = [ + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/vision/core:MPPImage", + ], + copts = [ + "-ObjC++", + "-std=c++17", + ], +) \ No newline at end of file diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h new file mode 100644 index 000000000..69788e05e --- /dev/null +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h @@ -0,0 +1,44 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Helper utility for performing operations on MPPImage specific to the + * MediaPipe Vision library. + */ +@interface MPPImage (Utils) + +// /** Bitmap size of the image. */ +@property(nonatomic, readonly) CGSize bitmapSize; + +/** + * Returns the underlying uint8 pixel buffer of an `MPPImage`. + * Irrespective of whether the underlying buffer is grayscale, RGB, RGBA, BGRA etc., the pixel + * data is converted to an RGB format. In case of grayscale images, the mono channel is duplicated + * in the R, G, B channels. + * + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return The underlying pixel buffer of the `MPPImage` or nil in case of errors. + */ +- (nullable uint8_t *)rgbPixelDataWithError:(NSError **)error; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.m b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.m new file mode 100644 index 000000000..8cd2147c4 --- /dev/null +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.m @@ -0,0 +1,324 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h" + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" + +#import +#import +#import +#import + +@interface MPPPixelDataUtils : NSObject + ++ (uint8_t *)rgbPixelDatafromPixelData:(uint8_t *)pixelData + withWidth:(size_t)width + height:(size_t)height + stride:(size_t)stride + pixelBufferFormat:(OSType)pixelBufferFormatType + error:(NSError **)error; + +@end + +@interface MPPCVPixelBufferUtils : NSObject + ++ (uint8_t *)pixelDataFromCVPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error; + +@end + +@interface MPPCGImageUtils : NSObject + ++ (UInt8 *_Nullable)pixelDataFromCGImage:(CGImageRef)cgImage error:(NSError **)error; + +@end + +@interface UIImage (RawPixelDataUtils) + +@property(nonatomic, readonly) CGSize bitmapSize; + +- (uint8_t *)pixelDataWithError:(NSError **)error; + +@end + +@implementation MPPPixelDataUtils : NSObject + ++ (uint8_t *)rgbPixelDatafromPixelData:(uint8_t *)pixelData + withWidth:(size_t)width + height:(size_t)height + stride:(size_t)stride + pixelBufferFormat:(OSType)pixelBufferFormatType + error:(NSError **)error { + NSInteger destinationChannelCount = 3; + size_t destinationBytesPerRow = width * destinationChannelCount; + + uint8_t *destPixelBufferAddress = + (uint8_t *)[MPPCommonUtils mallocWithSize:sizeof(uint8_t) * height * destinationBytesPerRow + error:error]; + + if (!destPixelBufferAddress) { + return NULL; + } + + vImage_Buffer srcBuffer = {.data = pixelData, + .height = (vImagePixelCount)height, + .width = (vImagePixelCount)width, + .rowBytes = stride}; + + vImage_Buffer destBuffer = {.data = destPixelBufferAddress, + .height = (vImagePixelCount)height, + .width = (vImagePixelCount)width, + .rowBytes = destinationBytesPerRow}; + + vImage_Error convertError = kvImageNoError; + + switch (pixelBufferFormatType) { + case kCVPixelFormatType_32RGBA: { + convertError = vImageConvert_RGBA8888toRGB888(&srcBuffer, &destBuffer, kvImageNoFlags); + break; + } + case kCVPixelFormatType_32BGRA: { + convertError = vImageConvert_BGRA8888toRGB888(&srcBuffer, &destBuffer, kvImageNoFlags); + break; + } + default: { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"Invalid source pixel buffer format. Expecting one of " + @"kCVPixelFormatType_32RGBA, kCVPixelFormatType_32BGRA"]; + + free(destPixelBufferAddress); + return NULL; + } + } + + if (convertError != kvImageNoError) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInternalError + description:@"Image format conversion failed."]; + + free(destPixelBufferAddress); + return NULL; + } + + return destPixelBufferAddress; +} + +@end + +@implementation MPPCVPixelBufferUtils + ++ (uint8_t *)rgbPixelDatafromCVPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error { + CVPixelBufferLockBaseAddress(pixelBuffer, 0); + + uint8_t *rgbPixelData = [MPPPixelDataUtils + rgbPixelDatafromPixelData:(uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer) + withWidth:CVPixelBufferGetWidth(pixelBuffer) + height:CVPixelBufferGetHeight(pixelBuffer) + stride:CVPixelBufferGetBytesPerRow(pixelBuffer) + pixelBufferFormat:CVPixelBufferGetPixelFormatType(pixelBuffer) + error:error]; + + CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); + + return rgbPixelData; +} + ++ (nullable uint8_t *)pixelDataFromCVPixelBuffer:(CVPixelBufferRef)pixelBuffer + error:(NSError **)error { + uint8_t *pixelData = NULL; + + OSType pixelBufferFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); + + switch (pixelBufferFormat) { + case kCVPixelFormatType_32BGRA: { + pixelData = [MPPCVPixelBufferUtils rgbPixelDatafromCVPixelBuffer:pixelBuffer error:error]; + break; + } + default: { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"Unsupported pixel format for CVPixelBuffer. Supported " + @"pixel format types are kCVPixelFormatType_32BGRA"]; + } + } + + return pixelData; +} + +@end + +@implementation MPPCGImageUtils + ++ (UInt8 *_Nullable)pixelDataFromCGImage:(CGImageRef)cgImage error:(NSError **)error { + size_t width = CGImageGetWidth(cgImage); + size_t height = CGImageGetHeight(cgImage); + + NSInteger bitsPerComponent = 8; + NSInteger channelCount = 4; + UInt8 *pixel_data_to_return = NULL; + + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + size_t bytesPerRow = channelCount * width; + + // iOS infers bytesPerRow if it is set to 0. + // See https://developer.apple.com/documentation/coregraphics/1455939-cgbitmapcontextcreate + // But for segmentation test image, this was not the case. + // Hence setting it to the value of channelCount*width. + // kCGImageAlphaNoneSkipLast specifies that Alpha will always be next to B. + // kCGBitmapByteOrder32Big specifies that R will be stored before B. + // In combination they signify a pixelFormat of kCVPixelFormatType32RGBA. + CGBitmapInfo bitMapinfoFor32RGBA = kCGImageAlphaNoneSkipLast | kCGBitmapByteOrder32Big; + CGContextRef context = CGBitmapContextCreate(nil, width, height, bitsPerComponent, bytesPerRow, + colorSpace, bitMapinfoFor32RGBA); + + if (context) { + CGContextDrawImage(context, CGRectMake(0, 0, width, height), cgImage); + uint8_t *srcData = (uint8_t *)CGBitmapContextGetData(context); + + if (srcData) { + // We have drawn the image as an RGBA image with 8 bitsPerComponent and hence can safely input + // a pixel format of type kCVPixelFormatType_32RGBA for conversion by vImage. + pixel_data_to_return = [MPPPixelDataUtils rgbPixelDatafromPixelData:srcData + withWidth:width + height:height + stride:bytesPerRow + pixelBufferFormat:kCVPixelFormatType_32RGBA + error:error]; + } + + CGContextRelease(context); + } + + CGColorSpaceRelease(colorSpace); + + return pixel_data_to_return; +} + +@end + +@implementation UIImage (RawPixelDataUtils) + +- (uint8_t *)pixelDataFromCIImageWithError:(NSError **)error { + uint8_t *pixelData = NULL; + + if (self.CIImage.pixelBuffer) { + pixelData = [MPPCVPixelBufferUtils pixelDataFromCVPixelBuffer:self.CIImage.pixelBuffer + error:error]; + + } else if (self.CIImage.CGImage) { + pixelData = [MPPCGImageUtils pixelDataFromCGImage:self.CIImage.CGImage error:error]; + } else { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"CIImage should have CGImage or CVPixelBuffer info."]; + } + + return pixelData; +} + +- (uint8_t *)pixelDataWithError:(NSError **)error { + uint8_t *pixelData = nil; + + if (self.CGImage) { + pixelData = [MPPCGImageUtils pixelDataFromCGImage:self.CGImage error:error]; + } else if (self.CIImage) { + pixelData = [self pixelDataFromCIImageWithError:error]; + } else { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"UIImage should be initialized from" + " CIImage or CGImage."]; + } + + return pixelData; +} + +- (CGSize)bitmapSize { + CGFloat width = 0; + CGFloat height = 0; + + if (self.CGImage) { + width = CGImageGetWidth(self.CGImage); + height = CGImageGetHeight(self.CGImage); + } else if (self.CIImage.pixelBuffer) { + width = CVPixelBufferGetWidth(self.CIImage.pixelBuffer); + height = CVPixelBufferGetHeight(self.CIImage.pixelBuffer); + } else if (self.CIImage.CGImage) { + width = CGImageGetWidth(self.CIImage.CGImage); + height = CGImageGetHeight(self.CIImage.CGImage); + } + return CGSizeMake(width, height); +} +@end + +@implementation MPPImage (Utils) + +- (nullable uint8_t *)pixelDataWithError:(NSError **)error { + uint8_t *pixelData = NULL; + + switch (self.imageSourceType) { + case MPPImageSourceTypeSampleBuffer: { + CVPixelBufferRef sampleImagePixelBuffer = CMSampleBufferGetImageBuffer(self.sampleBuffer); + pixelData = [MPPCVPixelBufferUtils pixelDataFromCVPixelBuffer:sampleImagePixelBuffer + error:error]; + break; + } + case MPPImageSourceTypePixelBuffer: { + pixelData = [MPPCVPixelBufferUtils pixelDataFromCVPixelBuffer:self.pixelBuffer error:error]; + break; + } + case MPPImageSourceTypeImage: { + pixelData = [self.image pixelDataWithError:error]; + break; + } + default: + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"Invalid source type for MPPImage."]; + } + + return pixelData; +} + +- (CGSize)bitmapSize { + CGFloat width = 0; + CGFloat height = 0; + + switch (self.imageSourceType) { + case MPPImageSourceTypeSampleBuffer: { + CVPixelBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(self.sampleBuffer); + width = CVPixelBufferGetWidth(pixelBuffer); + height = CVPixelBufferGetHeight(pixelBuffer); + break; + } + case MPPImageSourceTypePixelBuffer: { + width = CVPixelBufferGetWidth(self.pixelBuffer); + height = CVPixelBufferGetHeight(self.pixelBuffer); + break; + } + case MPPImageSourceTypeImage: { + width = self.image.bitmapSize.width; + height = self.image.bitmapSize.height; + break; + } + default: + break; + } + + return CGSizeMake(width, height); +} + +@end From e2f1d07c8b84279a4292592b1a0bedc013d2ccc6 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 13 Feb 2023 19:44:09 +0530 Subject: [PATCH 03/11] Updated target definition for MPPImage --- mediapipe/tasks/ios/vision/core/BUILD | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index b9d0c84dd..53c335fa5 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -14,8 +14,9 @@ objc_library( ], deps = [ "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", - "//third_party/apple_frameworks:CoreMedia", - "//third_party/apple_frameworks:CoreVideo", - "//third_party/apple_frameworks:UIKit", + ], + copts = [ + "-ObjC++", + "-std=c++17", ], ) From 1623b591b7f6d5ee3ffea8a28a7f78615787c62b Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 13 Feb 2023 19:52:00 +0530 Subject: [PATCH 04/11] Added MPPRunningMode --- mediapipe/tasks/ios/vision/core/BUILD | 7 ++++ .../ios/vision/core/sources/MPPRunningMode.h | 40 +++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index 53c335fa5..cc232807a 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -20,3 +20,10 @@ objc_library( "-std=c++17", ], ) + +objc_library( + name = "MPPRunningMode", + hdrs = ["sources/MPPRunningMode.h"], + module_name = "MPPRunningMode", +) + diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h new file mode 100644 index 000000000..fb64c20e0 --- /dev/null +++ b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h @@ -0,0 +1,40 @@ +// Copyright 2023 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + * MediaPipe vision task running mode. A MediaPipe vision task can be run with three different + * modes: image, video and live stream. + */ +typedef NS_ENUM(NSUInteger, MPPRunningMode) { + + // Generic error codes. + + /** The mode for running a mediapipe vision task on single image inputs. */ + MPPRunningModeImage, + + /**mThe mode for running a mediapipe vision task on the decoded frames of a video. */ + MPPRunningModeVideo, + + /** The mode for running a mediapipe vision task on a live stream of input data, + * such as from camera. + */ + MPPRunningModeLiveStream, + +} NS_SWIFT_NAME(RunningMode); + +NS_ASSUME_NONNULL_END From 328fe4ed395b38e2fba2a899c6ddffcd92d61c2f Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 13 Feb 2023 19:55:12 +0530 Subject: [PATCH 05/11] Added vision task runner --- mediapipe/tasks/ios/vision/core/BUILD | 13 ++++ .../vision/core/sources/MPPVisionTaskRunner.h | 54 ++++++++++++++ .../core/sources/MPPVisionTaskRunner.mm | 72 +++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h create mode 100644 mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index cc232807a..1a6180968 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -27,3 +27,16 @@ objc_library( module_name = "MPPRunningMode", ) +objc_library( + name = "MPPVisionTaskRunner", + srcs = ["sources/MPPVisionTaskRunner.mm"], + hdrs = ["sources/MPPVisionTaskRunner.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + ":MPPRunningMode", + "//mediapipe/tasks/ios/core:MPPTaskRunner", + ], +) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h new file mode 100644 index 000000000..d8545bb64 --- /dev/null +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -0,0 +1,54 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, + * execute and terminate any MediaPipe vision task. + */ +@interface MPPVisionTaskRunner : MPPTaskRunner + +/** + * Initializes a new `MPPVisionTaskRunner` with the MediaPipe calculator config protol running mode and packetsCallback. + * + * @param graphConfig A MediaPipe calculator config proto. + * @param runningMode MediaPipe vision task running mode. + * @param packetsCallback An optional C++ callback function that takes a list of output packets as + * the input argument. If provided, the callback must in turn call the block provided by the user in + * the appropriate task options. + * @param error Pointer to the memory location where errors if any should be + * saved. If @c NULL, no error will be saved. + * + * @return An instance of `MPPVisionTaskRunner` initialized to the given MediaPipe calculator config + * proto, running mode and packets callback. + */ +- (nullable instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig + runningMode:(MPPRunningMode)runningMode + packetsCallback: + (mediapipe::tasks::core::PacketsCallback)packetsCallback + error:(NSError **)error; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm new file mode 100644 index 000000000..208e4a5c8 --- /dev/null +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -0,0 +1,72 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" + +namespace { +using ::mediapipe::CalculatorGraphConfig; +} // namespace + +@interface MPPVisionTaskRunner () { + MPPRunningMode _runningMode; +} +@end + +@implementation MPPVisionTaskRunner + +- (nullable instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig + runningMode:(MPPRunningMode)runningMode + packetsCallback: + (mediapipe::tasks::core::PacketsCallback)packetsCallback + error:(NSError **)error { + + switch (runningMode) { + case MPPRunningModeImage: + case MPPRunningModeVideo: { + if (packetsCallback) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"The vision task is in image or video mode, a user-defined result callback should not be provided."]; + return nil; + } + break; + } + case MPPRunningModeLiveStream: { + if (!packetsCallback) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"The vision task is in live stream mode, a user-defined result callback must be provided."]; + return nil; + } + break; + } + default: + break + } + + _runningMode = runningMode; + self = [super initWithCalculatorGraphConfig:graphConfig + packetsCallback:packetsCallback + error:error]; + return self; +} + +@end From 1cd3c05fc5a8c0de60ffffba04484a000a979cc0 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 13 Feb 2023 19:58:14 +0530 Subject: [PATCH 06/11] Added default case to running mode check --- .../ios/vision/core/sources/MPPVisionTaskRunner.mm | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index 208e4a5c8..1b14f2179 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -58,8 +58,14 @@ using ::mediapipe::CalculatorGraphConfig; } break; } - default: - break + default: { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"Unrecognized running mode"]; + return nil; + } } _runningMode = runningMode; From 17ff493a9d3cfb80349d41e59790810e84a5f2e7 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 14 Feb 2023 20:39:25 +0530 Subject: [PATCH 07/11] Updated formatting --- .../ios/vision/core/sources/MPPRunningMode.h | 8 ++-- .../vision/core/sources/MPPVisionTaskRunner.h | 11 +++--- .../core/sources/MPPVisionTaskRunner.mm | 39 ++++++++----------- 3 files changed, 27 insertions(+), 31 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h index fb64c20e0..a5a2cc3d0 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h @@ -27,12 +27,12 @@ typedef NS_ENUM(NSUInteger, MPPRunningMode) { /** The mode for running a mediapipe vision task on single image inputs. */ MPPRunningModeImage, - /**mThe mode for running a mediapipe vision task on the decoded frames of a video. */ + /** The mode for running a mediapipe vision task on the decoded frames of a video. */ MPPRunningModeVideo, - /** The mode for running a mediapipe vision task on a live stream of input data, - * such as from camera. - */ + /** The mode for running a mediapipe vision task on a live stream of input data, such as from + * camera. + */ MPPRunningModeLiveStream, } NS_SWIFT_NAME(RunningMode); diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index d8545bb64..b7f4af7d9 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -26,7 +26,8 @@ NS_ASSUME_NONNULL_BEGIN @interface MPPVisionTaskRunner : MPPTaskRunner /** - * Initializes a new `MPPVisionTaskRunner` with the MediaPipe calculator config protol running mode and packetsCallback. + * Initializes a new `MPPVisionTaskRunner` with the MediaPipe calculator config protol running mode + * and packetsCallback. * * @param graphConfig A MediaPipe calculator config proto. * @param runningMode MediaPipe vision task running mode. @@ -40,10 +41,10 @@ NS_ASSUME_NONNULL_BEGIN * proto, running mode and packets callback. */ - (nullable instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig - runningMode:(MPPRunningMode)runningMode - packetsCallback: - (mediapipe::tasks::core::PacketsCallback)packetsCallback - error:(NSError **)error; + runningMode:(MPPRunningMode)runningMode + packetsCallback: + (mediapipe::tasks::core::PacketsCallback)packetsCallback + error:(NSError **)error; - (instancetype)init NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index 1b14f2179..fddad964b 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -29,45 +29,40 @@ using ::mediapipe::CalculatorGraphConfig; @implementation MPPVisionTaskRunner - (nullable instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig - runningMode:(MPPRunningMode)runningMode - packetsCallback: - (mediapipe::tasks::core::PacketsCallback)packetsCallback - error:(NSError **)error { - + runningMode:(MPPRunningMode)runningMode + packetsCallback: + (mediapipe::tasks::core::PacketsCallback)packetsCallback + error:(NSError **)error { switch (runningMode) { case MPPRunningModeImage: case MPPRunningModeVideo: { if (packetsCallback) { - [MPPCommonUtils - createCustomError:error - withCode:MPPTasksErrorCodeInvalidArgumentError - description: - @"The vision task is in image or video mode, a user-defined result callback should not be provided."]; + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"The vision task is in image or video mode, a " + @"user-defined result callback should not be provided."]; return nil; } break; } case MPPRunningModeLiveStream: { if (!packetsCallback) { - [MPPCommonUtils - createCustomError:error - withCode:MPPTasksErrorCodeInvalidArgumentError - description: - @"The vision task is in live stream mode, a user-defined result callback must be provided."]; + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"The vision task is in live stream mode, a user-defined " + @"result callback must be provided."]; return nil; } break; } default: { - [MPPCommonUtils - createCustomError:error - withCode:MPPTasksErrorCodeInvalidArgumentError - description: - @"Unrecognized running mode"]; - return nil; + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"Unrecognized running mode"]; + return nil; } } - + _runningMode = runningMode; self = [super initWithCalculatorGraphConfig:graphConfig packetsCallback:packetsCallback From a490255c177e8de9b56b85197020eacda87cbf66 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 14 Feb 2023 20:40:06 +0530 Subject: [PATCH 08/11] Updated formatting --- mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h index a5a2cc3d0..4b7864597 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h @@ -31,7 +31,7 @@ typedef NS_ENUM(NSUInteger, MPPRunningMode) { MPPRunningModeVideo, /** The mode for running a mediapipe vision task on a live stream of input data, such as from - * camera. + * camera. */ MPPRunningModeLiveStream, From 4e1fa82d50e335be5c47ff5b6d70db859b623385 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 14 Feb 2023 20:40:21 +0530 Subject: [PATCH 09/11] Added designated initializer in vision task runner --- mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index b7f4af7d9..325e06228 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -44,7 +44,7 @@ NS_ASSUME_NONNULL_BEGIN runningMode:(MPPRunningMode)runningMode packetsCallback: (mediapipe::tasks::core::PacketsCallback)packetsCallback - error:(NSError **)error; + error:(NSError **)error NS_DESIGNATED_INITIALIZER; - (instancetype)init NS_UNAVAILABLE; From 3a7aa29226d63e8e4ddd792616f0d5c72046fa47 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 14 Feb 2023 20:49:14 +0530 Subject: [PATCH 10/11] Updated note about packets callback in vision task runner --- .../ios/vision/core/sources/MPPVisionTaskRunner.h | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index 325e06228..5c5a835b2 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -26,14 +26,22 @@ NS_ASSUME_NONNULL_BEGIN @interface MPPVisionTaskRunner : MPPTaskRunner /** - * Initializes a new `MPPVisionTaskRunner` with the MediaPipe calculator config protol running mode + * Initializes a new `MPPVisionTaskRunner` with the MediaPipe calculator config proto running mode * and packetsCallback. + * Make sure that the packets callback is set properly based on the vision task's running mode. + * In case of live stream running mode, a C++ packets callback that is intended to deliver inference + * results must be provided. In case of image or video running mode, packets callback must be set to + * nil. * * @param graphConfig A MediaPipe calculator config proto. * @param runningMode MediaPipe vision task running mode. * @param packetsCallback An optional C++ callback function that takes a list of output packets as * the input argument. If provided, the callback must in turn call the block provided by the user in - * the appropriate task options. + * the appropriate task options. Make sure that the packets callback is set properly based on the + * vision task's running mode. In case of live stream running mode, a C++ packets callback that is + * intended to deliver inference results must be provided. In case of image or video running mode, + * packets callback must be set to nil. + * * @param error Pointer to the memory location where errors if any should be * saved. If @c NULL, no error will be saved. * From 55e5377c08413aca775d67cad55591b1eb3896f4 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 14 Feb 2023 20:51:51 +0530 Subject: [PATCH 11/11] Updated formatting --- .../ios/vision/core/utils/sources/MPPImage+Utils.h | 6 +++--- .../ios/vision/core/utils/sources/MPPImage+Utils.m | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h index 69788e05e..65771cb4d 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h @@ -18,12 +18,12 @@ NS_ASSUME_NONNULL_BEGIN -/** Helper utility for performing operations on MPPImage specific to the - * MediaPipe Vision library. +/** + * Helper utility for performing operations on MPPImage specific to the MediaPipe Vision library. */ @interface MPPImage (Utils) -// /** Bitmap size of the image. */ +/** Bitmap size of the image. */ @property(nonatomic, readonly) CGSize bitmapSize; /** diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.m b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.m index 8cd2147c4..ba71c63c8 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.m +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.m @@ -24,7 +24,7 @@ @interface MPPPixelDataUtils : NSObject -+ (uint8_t *)rgbPixelDatafromPixelData:(uint8_t *)pixelData ++ (uint8_t *)rgbPixelDataFromPixelData:(uint8_t *)pixelData withWidth:(size_t)width height:(size_t)height stride:(size_t)stride @@ -55,7 +55,7 @@ @implementation MPPPixelDataUtils : NSObject -+ (uint8_t *)rgbPixelDatafromPixelData:(uint8_t *)pixelData ++ (uint8_t *)rgbPixelDataFromPixelData:(uint8_t *)pixelData withWidth:(size_t)width height:(size_t)height stride:(size_t)stride @@ -120,11 +120,11 @@ @implementation MPPCVPixelBufferUtils -+ (uint8_t *)rgbPixelDatafromCVPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error { ++ (uint8_t *)rgbPixelDataFromCVPixelBuffer:(CVPixelBufferRef)pixelBuffer error:(NSError **)error { CVPixelBufferLockBaseAddress(pixelBuffer, 0); uint8_t *rgbPixelData = [MPPPixelDataUtils - rgbPixelDatafromPixelData:(uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer) + rgbPixelDataFromPixelData:(uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer) withWidth:CVPixelBufferGetWidth(pixelBuffer) height:CVPixelBufferGetHeight(pixelBuffer) stride:CVPixelBufferGetBytesPerRow(pixelBuffer) @@ -144,7 +144,7 @@ switch (pixelBufferFormat) { case kCVPixelFormatType_32BGRA: { - pixelData = [MPPCVPixelBufferUtils rgbPixelDatafromCVPixelBuffer:pixelBuffer error:error]; + pixelData = [MPPCVPixelBufferUtils rgbPixelDataFromCVPixelBuffer:pixelBuffer error:error]; break; } default: { @@ -191,7 +191,7 @@ if (srcData) { // We have drawn the image as an RGBA image with 8 bitsPerComponent and hence can safely input // a pixel format of type kCVPixelFormatType_32RGBA for conversion by vImage. - pixel_data_to_return = [MPPPixelDataUtils rgbPixelDatafromPixelData:srcData + pixel_data_to_return = [MPPPixelDataUtils rgbPixelDataFromPixelData:srcData withWidth:width height:height stride:bytesPerRow