From a21c08bf4d5a2e75c6f70579987a18102d78241a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 4 May 2023 17:00:12 +0530 Subject: [PATCH 1/9] Added method for creating unique dispatch queue names in MPPVisionTaskRunner --- .../common/utils/sources/NSString+Helpers.h | 1 + .../common/utils/sources/NSString+Helpers.mm | 4 ++++ mediapipe/tasks/ios/vision/core/BUILD | 1 + .../vision/core/sources/MPPVisionTaskRunner.h | 14 +++++++++++ .../core/sources/MPPVisionTaskRunner.mm | 23 ++++++++++++++----- 5 files changed, 37 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h index 8433baaaf..1697a28e4 100644 --- a/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h +++ b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h @@ -24,6 +24,7 @@ NS_ASSUME_NONNULL_BEGIN + (NSString *)stringWithCppString:(std::string)text; ++ (NSString *)uuidString; @end NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm index b7d486e80..dfc7749be 100644 --- a/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm +++ b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm @@ -24,4 +24,8 @@ return [NSString stringWithCString:text.c_str() encoding:[NSString defaultCStringEncoding]]; } ++ (NSString *)uuidString{ + return [[NSUUID UUID] UUIDString]; +} + @end diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index 4b72fc91d..328d9e892 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -58,6 +58,7 @@ objc_library( "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/tasks/ios/common:MPPCommon", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", "//mediapipe/tasks/ios/core:MPPTaskRunner", "//third_party/apple_frameworks:UIKit", "@com_google_absl//absl/status:statusor", diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index 92b5563ef..318b24051 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -141,6 +141,20 @@ NS_ASSUME_NONNULL_BEGIN (mediapipe::tasks::core::PacketsCallback)packetsCallback error:(NSError **)error NS_UNAVAILABLE; +/** + * This method returns a unique dispatch queue name by adding the given suffix and a `UUID` to the + * pre-defined queue name prefix for vision tasks. The vision tasks can use this method to get + * unique dispatch queue names which are consistent with other vision tasks. + * Dispatch queue names need not be unique, but for easy debugging we ensure that the queue names + * are unique. + * + * @param suffix A suffix that identifies a dispatch queue's functionality. + * + * @return A unique dispatch queue name by adding the given suffix and a `UUID` to the pre-defined + * queue name prefix for vision tasks. + */ ++ (const char *)uniqueDispatchQueueNameWithSuffix:(NSString *)suffix; + - (instancetype)init NS_UNAVAILABLE; + (instancetype)new NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm index 40b68a211..0089e516f 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.mm @@ -16,6 +16,7 @@ #import "mediapipe/tasks/ios/common/sources/MPPCommon.h" #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #include "absl/status/statusor.h" @@ -37,6 +38,8 @@ static const NSInteger kMPPOrientationDegreesDown = -180; /** Rotation degrees for a 90 degree rotation to the left. */ static const NSInteger kMPPOrientationDegreesLeft = -270; +static NSString *const kTaskPrefix = @"com.mediapipe.tasks.vision"; + @interface MPPVisionTaskRunner () { MPPRunningMode _runningMode; } @@ -54,18 +57,21 @@ static const NSInteger kMPPOrientationDegreesLeft = -270; if (packetsCallback) { [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError - description:@"The vision task is in image or video mode, a " - @"user-defined result callback should not be provided."]; + description:@"The vision task is in image or video mode. The " + @"delegate must not be set in the task's options."]; return nil; } break; } case MPPRunningModeLiveStream: { if (!packetsCallback) { - [MPPCommonUtils createCustomError:error - withCode:MPPTasksErrorCodeInvalidArgumentError - description:@"The vision task is in live stream mode, a user-defined " - @"result callback must be provided."]; + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"The vision task is in live stream mode. An object must be set as the " + @"delegate of the task in its options to ensure asynchronous delivery of " + @"results."]; return nil; } break; @@ -197,4 +203,9 @@ static const NSInteger kMPPOrientationDegreesLeft = -270; return [self sendPacketMap:packetMap error:error]; } ++ (const char *)uniqueDispatchQueueNameWithSuffix:(NSString *)suffix { + return [NSString stringWithFormat:@"%@.%@_%@", kTaskPrefix, suffix, [NSString uuidString]] + .UTF8String; +} + @end From ab135190e586b0928ff8fa9d2ca101da20e2cdb1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 4 May 2023 17:03:40 +0530 Subject: [PATCH 2/9] Updated iOS object detector to use delegates instead of callbacks for async calls --- .../object_detector/MPPObjectDetectorTests.m | 94 +++++++++++-------- .../sources/MPPObjectDetector.h | 3 + .../sources/MPPObjectDetector.mm | 43 +++++++-- .../sources/MPPObjectDetectorOptions.h | 61 +++++++++++- .../sources/MPPObjectDetectorOptions.m | 2 +- 5 files changed, 148 insertions(+), 55 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m index fd9466b7d..9ccfece91 100644 --- a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m +++ b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m @@ -25,6 +25,8 @@ static NSDictionary *const kCatsAndDogsRotatedImage = static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; static const float pixelDifferenceTolerance = 10.0f; static const float scoreDifferenceTolerance = 0.02f; +static NSString *const kLiveStreamTestsDictObjectDetectorKey = @"object_detector"; +static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; #define AssertEqualErrors(error, expectedError) \ XCTAssertNotNil(error); \ @@ -58,7 +60,10 @@ static const float scoreDifferenceTolerance = 0.02f; XCTAssertEqualWithAccuracy(boundingBox.size.height, expectedBoundingBox.size.height, \ pixelDifferenceTolerance, @"index i = %d", idx); -@interface MPPObjectDetectorTests : XCTestCase +@interface MPPObjectDetectorTests : XCTestCase { + NSDictionary *liveStreamSucceedsTestDict; + NSDictionary *outOfOrderTimestampTestDict; +} @end @implementation MPPObjectDetectorTests @@ -446,31 +451,28 @@ static const float scoreDifferenceTolerance = 0.02f; #pragma mark Running Mode Tests -- (void)testCreateObjectDetectorFailsWithResultListenerInNonLiveStreamMode { +- (void)testCreateObjectDetectorFailsWithDelegateInNonLiveStreamMode { MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo}; for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) { MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; options.runningMode = runningModesToTest[i]; - options.completion = - ^(MPPObjectDetectionResult *result, NSInteger timestampInMilliseconds, NSError *error) { - }; + options.objectDetectorLiveStreamDelegate = self; [self assertCreateObjectDetectorWithOptions:options failsWithExpectedError: - [NSError - errorWithDomain:kExpectedErrorDomain - code:MPPTasksErrorCodeInvalidArgumentError - userInfo:@{ - NSLocalizedDescriptionKey : - @"The vision task is in image or video mode, a " - @"user-defined result callback should not be provided." - }]]; + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in image or video mode. The " + @"delegate must not be set in the task's options." + }]]; } } -- (void)testCreateObjectDetectorFailsWithMissingResultListenerInLiveStreamMode { +- (void)testCreateObjectDetectorFailsWithMissingDelegateInLiveStreamMode { MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; options.runningMode = MPPRunningModeLiveStream; @@ -481,8 +483,11 @@ static const float scoreDifferenceTolerance = 0.02f; code:MPPTasksErrorCodeInvalidArgumentError userInfo:@{ NSLocalizedDescriptionKey : - @"The vision task is in live stream mode, a " - @"user-defined result callback must be provided." + @"The vision task is in live stream mode. An " + @"object must be set as the " + @"delegate of the task in its options to ensure " + @"asynchronous delivery of " + @"results." }]]; } @@ -563,10 +568,7 @@ static const float scoreDifferenceTolerance = 0.02f; MPPObjectDetectorOptions *options = [self objectDetectorOptionsWithModelName:kModelName]; options.runningMode = MPPRunningModeLiveStream; - options.completion = - ^(MPPObjectDetectionResult *result, NSInteger timestampInMilliseconds, NSError *error) { - - }; + options.objectDetectorLiveStreamDelegate = self; MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; @@ -631,23 +633,17 @@ static const float scoreDifferenceTolerance = 0.02f; options.maxResults = maxResults; options.runningMode = MPPRunningModeLiveStream; + options.objectDetectorLiveStreamDelegate = self; XCTestExpectation *expectation = [[XCTestExpectation alloc] initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; expectation.expectedFulfillmentCount = 1; - options.completion = - ^(MPPObjectDetectionResult *result, NSInteger timestampInMilliseconds, NSError *error) { - [self assertObjectDetectionResult:result - isEqualToExpectedResult: - [MPPObjectDetectorTests - expectedDetectionResultForCatsAndDogsImageWithTimestampInMilliseconds: - timestampInMilliseconds] - expectedDetectionsCount:maxResults]; - [expectation fulfill]; - }; - MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + liveStreamSucceedsTestDict = @{ + kLiveStreamTestsDictObjectDetectorKey : objectDetector, + kLiveStreamTestsDictExpectationKey : expectation + }; MPPImage *image = [self imageWithFileInfo:kCatsAndDogsImage]; @@ -693,19 +689,15 @@ static const float scoreDifferenceTolerance = 0.02f; expectation.expectedFulfillmentCount = iterationCount + 1; expectation.inverted = YES; - options.completion = - ^(MPPObjectDetectionResult *result, NSInteger timestampInMilliseconds, NSError *error) { - [self assertObjectDetectionResult:result - isEqualToExpectedResult: - [MPPObjectDetectorTests - expectedDetectionResultForCatsAndDogsImageWithTimestampInMilliseconds: - timestampInMilliseconds] - expectedDetectionsCount:maxResults]; - [expectation fulfill]; - }; + options.objectDetectorLiveStreamDelegate = self; MPPObjectDetector *objectDetector = [self objectDetectorWithOptionsSucceeds:options]; + liveStreamSucceedsTestDict = @{ + kLiveStreamTestsDictObjectDetectorKey : objectDetector, + kLiveStreamTestsDictExpectationKey : expectation + }; + // TODO: Mimic initialization from CMSampleBuffer as live stream mode is most likely to be used // with the iOS camera. AVCaptureVideoDataOutput sample buffer delegates provide frames of type // `CMSampleBuffer`. @@ -718,4 +710,24 @@ static const float scoreDifferenceTolerance = 0.02f; [self waitForExpectations:@[ expectation ] timeout:0.5]; } +#pragma mark MPPObjectDetectorLiveStreamDelegate Methods +- (void)objectDetector:(MPPObjectDetector *)objectDetector + didFinishDetectionWithResult:(MPPObjectDetectionResult *)objectDetectionResult + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError *)error { + NSInteger maxResults = 4; + [self assertObjectDetectionResult:objectDetectionResult + isEqualToExpectedResult: + [MPPObjectDetectorTests + expectedDetectionResultForCatsAndDogsImageWithTimestampInMilliseconds: + timestampInMilliseconds] + expectedDetectionsCount:maxResults]; + + if (objectDetector == outOfOrderTimestampTestDict[kLiveStreamTestsDictObjectDetectorKey]) { + [outOfOrderTimestampTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } else if (objectDetector == liveStreamSucceedsTestDict[kLiveStreamTestsDictObjectDetectorKey]) { + [liveStreamSucceedsTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } +} + @end diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h index 4443f56d1..249ee0434 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h @@ -137,6 +137,9 @@ NS_SWIFT_NAME(ObjectDetector) * the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with * `MPPRunningModeLiveStream`. Results are provided asynchronously via the `completion` callback * provided in the `MPPObjectDetectorOptions`. + * The object which needs to be continuously notified of the available results of object + * detection must confirm to `MPPObjectDetectorLiveStreamDelegate` protocol and implement the + * `objectDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` delegate method. * * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent * to the object detector. The input timestamps must be monotonically increasing. diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm index f0914cdb1..5dfbfdab8 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm @@ -37,8 +37,8 @@ static NSString *const kImageOutStreamName = @"image_out"; static NSString *const kImageTag = @"IMAGE"; static NSString *const kNormRectStreamName = @"norm_rect_in"; static NSString *const kNormRectTag = @"NORM_RECT"; - static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorGraph"; +static NSString *const kTaskName = @"objectDetector"; #define InputPacketMap(imagePacket, normalizedRectPacket) \ { \ @@ -51,6 +51,7 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; } +@property(nonatomic, weak) id objectDetectorLiveStreamDelegate; @end @implementation MPPObjectDetector @@ -78,11 +79,32 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG PacketsCallback packetsCallback = nullptr; - if (options.completion) { + if (options.objectDetectorLiveStreamDelegate) { + _objectDetectorLiveStreamDelegate = options.objectDetectorLiveStreamDelegate; + + // Capturing `self` as weak in order to avoid `self` being kept in memory + // and cause a retain cycle, after self is set to `nil`. + MPPObjectDetector *__weak weakSelf = self; + dispatch_queue_t callbackQueue = + dispatch_queue_create([MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName], NULL); packetsCallback = [=](absl::StatusOr statusOrPackets) { + if (!weakSelf) { + return; + } + if (![weakSelf.objectDetectorLiveStreamDelegate + respondsToSelector:@selector + (objectDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { + return; + } + NSError *callbackError = nil; if (![MPPCommonUtils checkCppError:statusOrPackets.status() toError:&callbackError]) { - options.completion(nil, Timestamp::Unset().Value(), callbackError); + dispatch_async(callbackQueue, ^{ + [weakSelf.objectDetectorLiveStreamDelegate objectDetector:weakSelf + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); return; } @@ -95,10 +117,15 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG objectDetectionResultWithDetectionsPacket:statusOrPackets.value()[kDetectionsStreamName .cppString]]; - options.completion(result, - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicroSecondsPerMilliSecond, - callbackError); + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond; + dispatch_async(callbackQueue, ^{ + [weakSelf.objectDetectorLiveStreamDelegate objectDetector:weakSelf + didFinishDetectionWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); }; } @@ -112,6 +139,7 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG return nil; } } + return self; } @@ -224,5 +252,4 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.ObjectDetectorG return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; } - @end diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h index 79bc9baa6..c60c8acac 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h @@ -20,19 +20,70 @@ NS_ASSUME_NONNULL_BEGIN +@class MPPObjectDetector; + +/** + * This protocol defines an interface for the delegates of `MPPObjectDetector` object to receive + * results of performing asynchronous object detection on images + * (i.e, when `runningMode` = `MPPRunningModeLiveStream`). + * + * The delegate of `MPPObjectDetector` must adopt `MPPObjectDetectorLiveStreamDelegate` protocol. + * The methods in this protocol are optional. + */ +NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) +@protocol MPPObjectDetectorLiveStreamDelegate + +@optional + +/** + * This method notifies a delegate that the results of asynchronous object detection of + * an image submitted to the `MPPObjectDetector` is available. + * + * This method is called on a private serial dispatch queue created by the `MPPObjectDetector` + * for performing the asynchronous delegates calls. + * + * @param objectDetector The object detector which performed the object detection. + * This is useful to test equality when there are multiple instances of `MPPObjectDetector`. + * @param result The `MPPObjectDetectionResult` object that contains a list of detections, each + * detection has a bounding box that is expressed in the unrotated input frame of reference + * coordinates system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the + * underlying image data. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image was sent to the object detector. + * @param error An optional error parameter populated when there is an error in performing object + * detection on the input live stream image data. + * + */ +- (void)objectDetector:(MPPObjectDetector *)objectDetector + didFinishDetectionWithResult:(nullable MPPObjectDetectionResult *)result + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(nullable NSError *)error + NS_SWIFT_NAME(objectDetector(_:didFinishDetection:timestampInMilliseconds:error:)); +@end + /** Options for setting up a `MPPObjectDetector`. */ NS_SWIFT_NAME(ObjectDetectorOptions) @interface MPPObjectDetectorOptions : MPPTaskOptions +/** + * Running mode of the object detector task. Defaults to `MPPRunningModeImage`. + * `MPPImageClassifier` can be created with one of the following running modes: + * 1. `MPPRunningModeImage`: The mode for performing object detection on single image inputs. + * 2. `MPPRunningModeVideo`: The mode for performing object detection on the decoded frames of a + * video. + * 3. `MPPRunningModeLiveStream`: The mode for performing object detection on a live stream of + * input data, such as from the camera. + */ @property(nonatomic) MPPRunningMode runningMode; /** - * The user-defined result callback for processing live stream data. The result callback should only - * be specified when the running mode is set to the live stream mode. - * TODO: Add parameter `MPPImage` in the callback. + * An object that confirms to `MPPObjectDetectorLiveStreamDelegate` protocol. This object must + * implement `objectDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` to receive + * the results of performing asynchronous object detection on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). */ -@property(nonatomic, copy) void (^completion) - (MPPObjectDetectionResult *__nullable result, NSInteger timestampMs, NSError *error); +@property(nonatomic, weak, nullable) id + objectDetectorLiveStreamDelegate; /** * The locale to use for display names specified through the TFLite Model Metadata, if any. Defaults diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.m b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.m index 73f8ce5b5..b93a6b30b 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.m +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.m @@ -33,7 +33,7 @@ objectDetectorOptions.categoryDenylist = self.categoryDenylist; objectDetectorOptions.categoryAllowlist = self.categoryAllowlist; objectDetectorOptions.displayNamesLocale = self.displayNamesLocale; - objectDetectorOptions.completion = self.completion; + objectDetectorOptions.objectDetectorLiveStreamDelegate = self.objectDetectorLiveStreamDelegate; return objectDetectorOptions; } From 87593a2aded4b8907db4d551e77840d88e179a60 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 4 May 2023 17:05:17 +0530 Subject: [PATCH 3/9] Updated docuemntation of MPPObjectDetector --- .../ios/vision/object_detector/sources/MPPObjectDetector.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h index 249ee0434..4ac8cc773 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h @@ -135,8 +135,7 @@ NS_SWIFT_NAME(ObjectDetector) * Sends live stream image data of type `MPPImage` to perform object detection using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of * the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with - * `MPPRunningModeLiveStream`. Results are provided asynchronously via the `completion` callback - * provided in the `MPPObjectDetectorOptions`. + * `MPPRunningModeLiveStream`. * The object which needs to be continuously notified of the available results of object * detection must confirm to `MPPObjectDetectorLiveStreamDelegate` protocol and implement the * `objectDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` delegate method. From 381ffcb474c6df8cb7f17f0beced787727a7cd1c Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 4 May 2023 17:10:07 +0530 Subject: [PATCH 4/9] Added hash implementation for iOS normalized keypoint --- .../tasks/ios/components/containers/sources/MPPDetection.m | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPDetection.m b/mediapipe/tasks/ios/components/containers/sources/MPPDetection.m index c245478db..c61cf0b39 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPDetection.m +++ b/mediapipe/tasks/ios/components/containers/sources/MPPDetection.m @@ -28,7 +28,12 @@ return self; } -// TODO: Implement hash +- (NSUInteger)hash { + NSUInteger nonNullPropertiesHash = + @(self.location.x).hash ^ @(self.location.y).hash ^ @(self.score).hash; + + return self.label ? nonNullPropertiesHash ^ self.label.hash : nonNullPropertiesHash; +} - (BOOL)isEqual:(nullable id)object { if (!object) { From 8ec0724b65c03dddddc2e13c09f6ea9238cdfc1d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 4 May 2023 19:44:11 +0530 Subject: [PATCH 5/9] Updated documentation to include note about rgba images --- .../sources/MPPObjectDetector.h | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h index 4ac8cc773..82404849b 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h @@ -95,6 +95,15 @@ NS_SWIFT_NAME(ObjectDetector) * interest. Rotation will be applied according to the `orientation` property of the provided * `MPPImage`. Only use this method when the `MPPObjectDetector` is created with * `MPPRunningModeImage`. + * + * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. * * @param image The `MPPImage` on which object detection is to be performed. * @param error An optional error parameter populated when there is an error in performing object @@ -114,6 +123,15 @@ NS_SWIFT_NAME(ObjectDetector) * image as region of interest. Rotation will be applied according to the `orientation` property of * the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with * `MPPRunningModeVideo`. + * + * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. * * @param image The `MPPImage` on which object detection is to be performed. * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input @@ -136,6 +154,7 @@ NS_SWIFT_NAME(ObjectDetector) * image as region of interest. Rotation will be applied according to the `orientation` property of * the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with * `MPPRunningModeLiveStream`. + * * The object which needs to be continuously notified of the available results of object * detection must confirm to `MPPObjectDetectorLiveStreamDelegate` protocol and implement the * `objectDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` delegate method. @@ -143,6 +162,19 @@ NS_SWIFT_NAME(ObjectDetector) * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent * to the object detector. The input timestamps must be monotonically increasing. * + * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color + * space is RGB with an Alpha channel. + * + * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you + * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its + * `videoSettings` property. + * * @param image A live stream image data of type `MPPImage` on which object detection is to be * performed. * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input From 00712d727ea1fc9a238096fdc1344a0b23c016d1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 4 May 2023 19:53:06 +0530 Subject: [PATCH 6/9] Updated wait time for object detector tests --- .../ios/test/vision/object_detector/MPPObjectDetectorTests.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m index 9ccfece91..bc4e24af8 100644 --- a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m +++ b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m @@ -660,7 +660,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; @"INVALID_ARGUMENT: Input timestamp must be monotonically increasing." }]; AssertEqualErrors(error, expectedError); - [self waitForExpectations:@[ expectation ] timeout:1.0]; + [self waitForExpectations:@[ expectation ] timeout:0.5f]; } - (void)testDetectWithLiveStreamModeSucceeds { @@ -707,7 +707,7 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; XCTAssertTrue([objectDetector detectAsyncInImage:image timestampInMilliseconds:i error:nil]); } - [self waitForExpectations:@[ expectation ] timeout:0.5]; + [self waitForExpectations:@[ expectation ] timeout:0.5f]; } #pragma mark MPPObjectDetectorLiveStreamDelegate Methods From a4e11eac7851394e125a07693763c4b12b0dae95 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 4 May 2023 19:55:19 +0530 Subject: [PATCH 7/9] Added constants for time out --- .../test/vision/object_detector/MPPObjectDetectorTests.m | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m index bc4e24af8..daa984ecf 100644 --- a/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m +++ b/mediapipe/tasks/ios/test/vision/object_detector/MPPObjectDetectorTests.m @@ -660,7 +660,9 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; @"INVALID_ARGUMENT: Input timestamp must be monotonically increasing." }]; AssertEqualErrors(error, expectedError); - [self waitForExpectations:@[ expectation ] timeout:0.5f]; + + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; } - (void)testDetectWithLiveStreamModeSucceeds { @@ -707,7 +709,8 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; XCTAssertTrue([objectDetector detectAsyncInImage:image timestampInMilliseconds:i error:nil]); } - [self waitForExpectations:@[ expectation ] timeout:0.5f]; + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; } #pragma mark MPPObjectDetectorLiveStreamDelegate Methods From 253662149e367105896b65e2ba7a6a3bbffbdfe9 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 4 May 2023 23:03:03 +0530 Subject: [PATCH 8/9] Updated pixel format types in object detector --- .../tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm index 75dfcc650..667279b9f 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm @@ -162,6 +162,7 @@ using ::mediapipe::ImageFrame; OSType pixelBufferFormat = CVPixelBufferGetPixelFormatType(pixelBuffer); switch (pixelBufferFormat) { + case kCVPixelFormatType_32RGBA: case kCVPixelFormatType_32BGRA: { return [MPPCVPixelBufferUtils rgbImageFrameFromCVPixelBuffer:pixelBuffer error:error]; } @@ -169,7 +170,8 @@ using ::mediapipe::ImageFrame; [MPPCommonUtils createCustomError:error withCode:MPPTasksErrorCodeInvalidArgumentError description:@"Unsupported pixel format for CVPixelBuffer. Supported " - @"pixel format types are kCVPixelFormatType_32BGRA"]; + @"pixel format types are kCVPixelFormatType_32BGRA and " + @"kCVPixelFormatType_32RGBA"]; } } From 1db1c29f5016cafb1934eadf15de2ce8e6241588 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 5 May 2023 01:04:04 +0530 Subject: [PATCH 9/9] Added code comments --- .../ios/vision/object_detector/sources/MPPObjectDetector.mm | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm index 5dfbfdab8..6daebcf90 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm @@ -85,6 +85,11 @@ static NSString *const kTaskName = @"objectDetector"; // Capturing `self` as weak in order to avoid `self` being kept in memory // and cause a retain cycle, after self is set to `nil`. MPPObjectDetector *__weak weakSelf = self; + + // Create a private serial dispatch queue in which the deleagte method will be called + // asynchronously. This is to ensure that if the client performs a long running operation in + // the delegate method, the queue on which the C++ callbacks is invoked is not blocked and is + // freed up to continue with its operations. dispatch_queue_t callbackQueue = dispatch_queue_create([MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName], NULL); packetsCallback = [=](absl::StatusOr statusOrPackets) {