diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm index 6da599fd7..ef292bcac 100644 --- a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm @@ -179,13 +179,13 @@ static NSString *const kTaskName = @"faceDetector"; MPPFaceDetectorResult *result = FaceDetectorResultWithOutputPacketMap(liveStreamResult.value()); - NSInteger timeStampInMilliseconds = + NSInteger timestampInMilliseconds = outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / kMicrosecondsPerMillisecond; dispatch_async(_callbackQueue, ^{ [self.faceDetectorLiveStreamDelegate faceDetector:self didFinishDetectionWithResult:result - timestampInMilliseconds:timeStampInMilliseconds + timestampInMilliseconds:timestampInMilliseconds error:callbackError]; }); } diff --git a/mediapipe/tasks/ios/vision/face_landmarker/BUILD b/mediapipe/tasks/ios/vision/face_landmarker/BUILD index 24a656b98..0ff88c6c8 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/BUILD +++ b/mediapipe/tasks/ios/vision/face_landmarker/BUILD @@ -70,7 +70,7 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskInfo", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", - "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunnerRefactored", "//mediapipe/tasks/ios/vision/face_landmarker/utils:MPPFaceLandmarkerOptionsHelpers", "//mediapipe/tasks/ios/vision/face_landmarker/utils:MPPFaceLandmarkerResultHelpers", ], diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm index eec0d66ee..16b2bf94f 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm @@ -19,7 +19,7 @@ #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" -#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" #import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarksConnections.h" #import "mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerOptions+Helpers.h" #import "mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerResult+Helpers.h" @@ -56,6 +56,13 @@ static NSString *const kTaskName = @"faceLandmarker"; } \ } +#define FaceLandmarkerResultWithOutputPacketMap(outputPacketMap) \ + ([MPPFaceLandmarkerResult \ + faceLandmarkerResultWithLandmarksPacket:outputPacketMap[kLandmarksOutStreamName.cppString] \ + blendshapesPacket:outputPacketMap[kBlendshapesOutStreamName.cppString] \ + transformationMatrixesPacket:outputPacketMap[kFaceGeometryOutStreamName \ + .cppString]]) + @interface MPPFaceLandmarker () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; @@ -71,6 +78,8 @@ static NSString *const kTaskName = @"faceLandmarker"; @implementation MPPFaceLandmarker +#pragma mark - Public + - (instancetype)initWithOptions:(MPPFaceLandmarkerOptions *)options error:(NSError **)error { self = [super init]; if (self) { @@ -124,12 +133,13 @@ static NSString *const kTaskName = @"faceLandmarker"; }; } - _visionTaskRunner = - [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] - runningMode:options.runningMode - packetsCallback:std::move(packetsCallback) - error:error]; - + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:NO + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectStreamName + error:error]; if (!_visionTaskRunner) { return nil; } @@ -144,138 +154,29 @@ static NSString *const kTaskName = @"faceLandmarker"; return [self initWithOptions:options error:error]; } -- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return std::nullopt; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (imagePacket.IsEmpty()) { - return std::nullopt; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:*rect - timestampInMilliseconds:timestampInMilliseconds]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - return inputPacketMap; -} - - (nullable MPPFaceLandmarkerResult *)detectInImage:(MPPImage *)image error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return nil; - } + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = [MPPVisionPacketCreator createPacketWithNormalizedRect:*rect]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - - std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap - error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } - - return [MPPFaceLandmarkerResult - faceLandmarkerResultWithLandmarksPacket:outputPacketMap - .value()[kLandmarksOutStreamName.cppString] - blendshapesPacket:outputPacketMap - .value()[kBlendshapesOutStreamName.cppString] - transformationMatrixesPacket:outputPacketMap - .value()[kFaceGeometryOutStreamName.cppString]]; + return [MPPFaceLandmarker faceLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (nullable MPPFaceLandmarkerResult *)detectInVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return nil; - } - std::optional outputPacketMap = - [_visionTaskRunner processVideoFramePacketMap:*inputPacketMap error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; - return [MPPFaceLandmarkerResult - faceLandmarkerResultWithLandmarksPacket:outputPacketMap - .value()[kLandmarksOutStreamName.cppString] - blendshapesPacket:outputPacketMap - .value()[kBlendshapesOutStreamName.cppString] - transformationMatrixesPacket:outputPacketMap - .value()[kFaceGeometryOutStreamName.cppString]]; + return [MPPFaceLandmarker faceLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (BOOL)detectAsyncInImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return NO; - } - - return [_visionTaskRunner processLiveStreamPacketMap:*inputPacketMap error:error]; -} - -- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { - NSError *callbackError; - if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { - dispatch_async(_callbackQueue, ^{ - [_faceLandmarkerLiveStreamDelegate faceLandmarker:self - didFinishDetectionWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = *liveStreamResult; - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - // The graph did not return a result. We therefore do not raise the user callback. This mirrors - // returning `nil` in the other methods and is acceptable for the live stream delegate since - // it is expected that we drop frames and don't return results for every input. - return; - } - - MPPFaceLandmarkerResult *result = [MPPFaceLandmarkerResult - faceLandmarkerResultWithLandmarksPacket:outputPacketMap[kLandmarksOutStreamName.cppString] - blendshapesPacket:outputPacketMap[kBlendshapesOutStreamName.cppString] - transformationMatrixesPacket:outputPacketMap[kFaceGeometryOutStreamName - .cppString]]; - - NSInteger timeStampInMilliseconds = - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicrosecondsPerMillisecond; - dispatch_async(_callbackQueue, ^{ - [_faceLandmarkerLiveStreamDelegate faceLandmarker:self - didFinishDetectionWithResult:result - timestampInMilliseconds:timeStampInMilliseconds - error:callbackError]; - }); + return [_visionTaskRunner processLiveStreamImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; } + (NSArray *)lipsConnections { @@ -322,4 +223,48 @@ static NSString *const kTaskName = @"faceLandmarker"; return MPPFaceConnections; } +#pragma mark - Private + ++ (nullable MPPFaceLandmarkerResult *)faceLandmarkerResultWithOptionalOutputPacketMap: + (std::optional)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + + return FaceLandmarkerResultWithOutputPacketMap(outputPacketMap.value()); +} + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + NSError *callbackError; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [_faceLandmarkerLiveStreamDelegate faceLandmarker:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = *liveStreamResult; + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + // The graph did not return a result. We therefore do not raise the user callback. This mirrors + // returning `nil` in the other methods and is acceptable for the live stream delegate since + // it is expected that we drop frames and don't return results for every input. + return; + } + + MPPFaceLandmarkerResult *result = FaceLandmarkerResultWithOutputPacketMap(outputPacketMap); + + NSInteger timestampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicrosecondsPerMillisecond; + dispatch_async(_callbackQueue, ^{ + [_faceLandmarkerLiveStreamDelegate faceLandmarker:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timestampInMilliseconds + error:callbackError]; + }); +} + @end diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD index d9a76afde..e5b46d344 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD @@ -58,7 +58,7 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskInfo", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", - "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunnerRefactored", "//mediapipe/tasks/ios/vision/gesture_recognizer/utils:MPPGestureRecognizerOptionsHelpers", "//mediapipe/tasks/ios/vision/gesture_recognizer/utils:MPPGestureRecognizerResultHelpers", ], diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.mm index b722163b2..06ad11afc 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.mm +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.mm @@ -18,7 +18,7 @@ #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" -#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" #import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h" #import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h" @@ -54,6 +54,17 @@ static NSString *const kTaskName = @"gestureRecognizer"; } \ } +#define GestureRecognizerResultWithOutputPacketMap(outputPacketMap) \ + ([MPPGestureRecognizerResult \ + gestureRecognizerResultWithHandGesturesPacket:outputPacketMap[kHandGesturesOutStreamName \ + .cppString] \ + handednessPacket:outputPacketMap[kHandednessOutStreamName \ + .cppString] \ + handLandmarksPacket:outputPacketMap[kLandmarksOutStreamName \ + .cppString] \ + worldLandmarksPacket:outputPacketMap[kWorldLandmarksOutStreamName \ + .cppString]]) + @interface MPPGestureRecognizer () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; @@ -65,56 +76,6 @@ static NSString *const kTaskName = @"gestureRecognizer"; @implementation MPPGestureRecognizer -- (nullable MPPGestureRecognizerResult *)gestureRecognizerResultWithOutputPacketMap: - (PacketMap &)outputPacketMap { - return [MPPGestureRecognizerResult - gestureRecognizerResultWithHandGesturesPacket:outputPacketMap[kHandGesturesOutStreamName - .cppString] - handednessPacket:outputPacketMap[kHandednessOutStreamName - .cppString] - handLandmarksPacket:outputPacketMap[kLandmarksOutStreamName - .cppString] - worldLandmarksPacket:outputPacketMap[kWorldLandmarksOutStreamName - .cppString]]; -} - -- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { - if (![self.gestureRecognizerLiveStreamDelegate - respondsToSelector:@selector(gestureRecognizer: - didFinishRecognitionWithResult:timestampInMilliseconds:error:)]) { - return; - } - - NSError *callbackError = nil; - if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { - dispatch_async(_callbackQueue, ^{ - [self.gestureRecognizerLiveStreamDelegate gestureRecognizer:self - didFinishRecognitionWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = liveStreamResult.value(); - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - return; - } - - MPPGestureRecognizerResult *result = - [self gestureRecognizerResultWithOutputPacketMap:outputPacketMap]; - - NSInteger timeStampInMilliseconds = - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicroSecondsPerMilliSecond; - dispatch_async(_callbackQueue, ^{ - [self.gestureRecognizerLiveStreamDelegate gestureRecognizer:self - didFinishRecognitionWithResult:result - timestampInMilliseconds:timeStampInMilliseconds - error:callbackError]; - }); -} - - (instancetype)initWithOptions:(MPPGestureRecognizerOptions *)options error:(NSError **)error { self = [super init]; if (self) { @@ -161,11 +122,13 @@ static NSString *const kTaskName = @"gestureRecognizer"; }; } - _visionTaskRunner = - [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] - runningMode:options.runningMode - packetsCallback:std::move(packetsCallback) - error:error]; + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:NO + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectInStreamName + error:error]; if (!_visionTaskRunner) { return nil; } @@ -181,93 +144,76 @@ static NSString *const kTaskName = @"gestureRecognizer"; return [self initWithOptions:options error:error]; } -- (nullable MPPGestureRecognizerResult *)gestureRecognizerResultWithOptionalOutputPacketMap: - (std::optional &)outputPacketMap { - if (!outputPacketMap.has_value()) { - return nil; - } - MPPGestureRecognizerResult *result = - [self gestureRecognizerResultWithOutputPacketMap:outputPacketMap.value()]; - return result; -} - - (nullable MPPGestureRecognizerResult *)recognizeImage:(MPPImage *)image error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return nil; - } + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - - std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap - error:error]; - return [self gestureRecognizerResultWithOptionalOutputPacketMap:outputPacketMap]; -} - -- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return std::nullopt; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (imagePacket.IsEmpty()) { - return std::nullopt; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() - timestampInMilliseconds:timestampInMilliseconds]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - return inputPacketMap; + return [MPPGestureRecognizer gestureRecognizerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (nullable MPPGestureRecognizerResult *)recognizeVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return nil; - } - std::optional outputPacketMap = - [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; - return [self gestureRecognizerResultWithOptionalOutputPacketMap:outputPacketMap]; + return [MPPGestureRecognizer gestureRecognizerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (BOOL)recognizeAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return NO; + return [_visionTaskRunner processLiveStreamImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; +} + +#pragma mark - Private + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.gestureRecognizerLiveStreamDelegate + respondsToSelector:@selector(gestureRecognizer: + didFinishRecognitionWithResult:timestampInMilliseconds:error:)]) { + return; } - return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.gestureRecognizerLiveStreamDelegate gestureRecognizer:self + didFinishRecognitionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPGestureRecognizerResult *result = GestureRecognizerResultWithOutputPacketMap(outputPacketMap); + + NSInteger timestampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond; + dispatch_async(_callbackQueue, ^{ + [self.gestureRecognizerLiveStreamDelegate gestureRecognizer:self + didFinishRecognitionWithResult:result + timestampInMilliseconds:timestampInMilliseconds + error:callbackError]; + }); +} + ++ (nullable MPPGestureRecognizerResult *)gestureRecognizerResultWithOptionalOutputPacketMap: + (std::optional &)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + + return GestureRecognizerResultWithOutputPacketMap(outputPacketMap.value()); } @end diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/BUILD b/mediapipe/tasks/ios/vision/hand_landmarker/BUILD index 677caf061..d228b8e0a 100644 --- a/mediapipe/tasks/ios/vision/hand_landmarker/BUILD +++ b/mediapipe/tasks/ios/vision/hand_landmarker/BUILD @@ -66,7 +66,7 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskInfo", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", - "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunnerRefactored", "//mediapipe/tasks/ios/vision/hand_landmarker/utils:MPPHandLandmarkerOptionsHelpers", "//mediapipe/tasks/ios/vision/hand_landmarker/utils:MPPHandLandmarkerResultHelpers", ], diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm index 46a6d1f33..3e2a7060d 100644 --- a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm +++ b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm @@ -18,7 +18,7 @@ #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" -#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" #import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarksConnections.h" #import "mediapipe/tasks/ios/vision/hand_landmarker/utils/sources/MPPHandLandmarkerOptions+Helpers.h" #import "mediapipe/tasks/ios/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+Helpers.h" @@ -53,6 +53,14 @@ static NSString *const kTaskName = @"handLandmarker"; } \ } +#define HandLandmarkerResultWithOutputPacketMap(outputPacketMap) \ + ([MPPHandLandmarkerResult \ + handLandmarkerResultWithLandmarksPacket:outputPacketMap[kLandmarksOutStreamName.cppString] \ + worldLandmarksPacket:outputPacketMap[kWorldLandmarksOutStreamName \ + .cppString] \ + handednessPacket:outputPacketMap[kHandednessOutStreamName \ + .cppString]]) + @interface MPPHandLandmarker () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; @@ -63,50 +71,7 @@ static NSString *const kTaskName = @"handLandmarker"; @implementation MPPHandLandmarker -- (nullable MPPHandLandmarkerResult *)handLandmarkerResultWithOutputPacketMap: - (PacketMap &)outputPacketMap { - return [MPPHandLandmarkerResult - handLandmarkerResultWithLandmarksPacket:outputPacketMap[kLandmarksOutStreamName.cppString] - worldLandmarksPacket:outputPacketMap[kWorldLandmarksOutStreamName - .cppString] - handednessPacket:outputPacketMap[kHandednessOutStreamName.cppString]]; -} - -- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { - if (![self.handLandmarkerLiveStreamDelegate - respondsToSelector:@selector(handLandmarker: - didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { - return; - } - - NSError *callbackError = nil; - if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { - dispatch_async(_callbackQueue, ^{ - [self.handLandmarkerLiveStreamDelegate handLandmarker:self - didFinishDetectionWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = liveStreamResult.value(); - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - return; - } - - MPPHandLandmarkerResult *result = [self handLandmarkerResultWithOutputPacketMap:outputPacketMap]; - - NSInteger timeStampInMilliseconds = - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicroSecondsPerMilliSecond; - dispatch_async(_callbackQueue, ^{ - [self.handLandmarkerLiveStreamDelegate handLandmarker:self - didFinishDetectionWithResult:result - timestampInMilliseconds:timeStampInMilliseconds - error:callbackError]; - }); -} +#pragma mark - Public - (instancetype)initWithOptions:(MPPHandLandmarkerOptions *)options error:(NSError **)error { self = [super init]; @@ -152,11 +117,14 @@ static NSString *const kTaskName = @"handLandmarker"; }; } - _visionTaskRunner = - [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] - runningMode:options.runningMode - packetsCallback:std::move(packetsCallback) - error:error]; + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:NO + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectInStreamName + error:error]; + if (!_visionTaskRunner) { return nil; } @@ -172,93 +140,29 @@ static NSString *const kTaskName = @"handLandmarker"; return [self initWithOptions:options error:error]; } -- (nullable MPPHandLandmarkerResult *)handLandmarkerResultWithOptionalOutputPacketMap: - (std::optional &)outputPacketMap { - if (!outputPacketMap.has_value()) { - return nil; - } - MPPHandLandmarkerResult *result = - [self handLandmarkerResultWithOutputPacketMap:outputPacketMap.value()]; - return result; -} - - (nullable MPPHandLandmarkerResult *)detectInImage:(MPPImage *)image error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return nil; - } + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - - std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap - error:error]; - return [self handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; -} - -- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return std::nullopt; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (imagePacket.IsEmpty()) { - return std::nullopt; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() - timestampInMilliseconds:timestampInMilliseconds]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - return inputPacketMap; + return [MPPHandLandmarker handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (nullable MPPHandLandmarkerResult *)detectInVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return nil; - } - std::optional outputPacketMap = - [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; - return [self handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; + return [MPPHandLandmarker handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (BOOL)detectAsyncInImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return NO; - } - - return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; + return [_visionTaskRunner processLiveStreamImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; } + (NSArray *)handPalmConnections { @@ -285,4 +189,51 @@ static NSString *const kTaskName = @"handLandmarker"; return MPPHandConnections; } +#pragma mark - Private + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.handLandmarkerLiveStreamDelegate + respondsToSelector:@selector(handLandmarker: + didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { + return; + } + + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.handLandmarkerLiveStreamDelegate handLandmarker:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPHandLandmarkerResult *result = HandLandmarkerResultWithOutputPacketMap(outputPacketMap); + + NSInteger timestampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond; + dispatch_async(_callbackQueue, ^{ + [self.handLandmarkerLiveStreamDelegate handLandmarker:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timestampInMilliseconds + error:callbackError]; + }); +} + ++ (nullable MPPHandLandmarkerResult *)handLandmarkerResultWithOptionalOutputPacketMap: + (std::optional &)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + + return HandLandmarkerResultWithOutputPacketMap(outputPacketMap.value()); +} + @end diff --git a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm index 3e1592e11..91abaaeb6 100644 --- a/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm +++ b/mediapipe/tasks/ios/vision/image_classifier/sources/MPPImageClassifier.mm @@ -219,13 +219,13 @@ static const int kMicroSecondsPerMilliSecond = 1000; MPPImageClassifierResult *result = ImageClassifierResultWithOutputPacketMap(outputPacketMap); - NSInteger timeStampInMilliseconds = + NSInteger timestampInMilliseconds = outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / kMicroSecondsPerMilliSecond; dispatch_async(_callbackQueue, ^{ [self.imageClassifierLiveStreamDelegate imageClassifier:self didFinishClassificationWithResult:result - timestampInMilliseconds:timeStampInMilliseconds + timestampInMilliseconds:timestampInMilliseconds error:callbackError]; }); } diff --git a/mediapipe/tasks/ios/vision/object_detector/BUILD b/mediapipe/tasks/ios/vision/object_detector/BUILD index 002a59920..5777112d4 100644 --- a/mediapipe/tasks/ios/vision/object_detector/BUILD +++ b/mediapipe/tasks/ios/vision/object_detector/BUILD @@ -55,7 +55,7 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskInfo", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", - "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunnerRefactored", "//mediapipe/tasks/ios/vision/object_detector/utils:MPPObjectDetectorOptionsHelpers", "//mediapipe/tasks/ios/vision/object_detector/utils:MPPObjectDetectorResultHelpers", ], diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm index e4704fbbc..b340225b3 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm @@ -18,7 +18,7 @@ #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" -#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" #import "mediapipe/tasks/ios/vision/object_detector/utils/sources/MPPObjectDetectorOptions+Helpers.h" #import "mediapipe/tasks/ios/vision/object_detector/utils/sources/MPPObjectDetectorResult+Helpers.h" @@ -47,6 +47,10 @@ static NSString *const kTaskName = @"objectDetector"; } \ } +#define ObjectDetectorResultWithOutputPacketMap(outputPacketMap) \ + ([MPPObjectDetectorResult \ + objectDetectorResultWithDetectionsPacket:outputPacketMap[kDetectionsStreamName.cppString]]) + @interface MPPObjectDetector () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; @@ -57,42 +61,7 @@ static NSString *const kTaskName = @"objectDetector"; @implementation MPPObjectDetector -- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { - if (![self.objectDetectorLiveStreamDelegate - respondsToSelector:@selector(objectDetector: - didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { - return; - } - - NSError *callbackError = nil; - if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { - dispatch_async(_callbackQueue, ^{ - [self.objectDetectorLiveStreamDelegate objectDetector:self - didFinishDetectionWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = liveStreamResult.value(); - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - return; - } - - MPPObjectDetectorResult *result = [MPPObjectDetectorResult - objectDetectorResultWithDetectionsPacket:outputPacketMap[kDetectionsStreamName.cppString]]; - - NSInteger timeStampInMilliseconds = - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicroSecondsPerMilliSecond; - dispatch_async(_callbackQueue, ^{ - [self.objectDetectorLiveStreamDelegate objectDetector:self - didFinishDetectionWithResult:result - timestampInMilliseconds:timeStampInMilliseconds - error:callbackError]; - }); -} +#pragma mark - Public - (instancetype)initWithOptions:(MPPObjectDetectorOptions *)options error:(NSError **)error { self = [super init]; @@ -135,11 +104,13 @@ static NSString *const kTaskName = @"objectDetector"; }; } - _visionTaskRunner = - [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] - runningMode:options.runningMode - packetsCallback:std::move(packetsCallback) - error:error]; + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:NO + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectStreamName + error:error]; if (!_visionTaskRunner) { return nil; @@ -157,101 +128,76 @@ static NSString *const kTaskName = @"objectDetector"; return [self initWithOptions:options error:error]; } -- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return std::nullopt; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (imagePacket.IsEmpty()) { - return std::nullopt; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() - timestampInMilliseconds:timestampInMilliseconds]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - return inputPacketMap; -} - -- (nullable MPPObjectDetectorResult *)detectInImage:(MPPImage *)image - regionOfInterest:(CGRect)roi - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return nil; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - - std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap - error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } - - return [MPPObjectDetectorResult - objectDetectorResultWithDetectionsPacket:outputPacketMap - .value()[kDetectionsStreamName.cppString]]; -} - - (nullable MPPObjectDetectorResult *)detectInImage:(MPPImage *)image error:(NSError **)error { - return [self detectInImage:image regionOfInterest:CGRectZero error:error]; + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; + + return [MPPObjectDetector objectDetectorResultWithOptionalOutputPacketMap:outputPacketMap]; } - (nullable MPPObjectDetectorResult *)detectInVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return nil; - } - std::optional outputPacketMap = - [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } - - return [MPPObjectDetectorResult - objectDetectorResultWithDetectionsPacket:outputPacketMap - .value()[kDetectionsStreamName.cppString]]; + return [MPPObjectDetector objectDetectorResultWithOptionalOutputPacketMap:outputPacketMap]; } - (BOOL)detectAsyncInImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return NO; + return [_visionTaskRunner processLiveStreamImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; +} + +#pragma mark - Private + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.objectDetectorLiveStreamDelegate + respondsToSelector:@selector(objectDetector: + didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { + return; } - return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.objectDetectorLiveStreamDelegate objectDetector:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPObjectDetectorResult *result = ObjectDetectorResultWithOutputPacketMap(outputPacketMap); + + NSInteger timestampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond; + dispatch_async(_callbackQueue, ^{ + [self.objectDetectorLiveStreamDelegate objectDetector:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timestampInMilliseconds + error:callbackError]; + }); +} + ++ (nullable MPPObjectDetectorResult *)objectDetectorResultWithOptionalOutputPacketMap: + (std::optional &)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + + return ObjectDetectorResultWithOutputPacketMap(outputPacketMap.value()); } @end