From fe9c7a47e95ca5272b2289320c701a97b51ec760 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 1 Sep 2023 14:06:50 +0530 Subject: [PATCH 01/10] Updated iOS face landmarker to use refactored vision task runner --- .../tasks/ios/vision/face_landmarker/BUILD | 2 +- .../sources/MPPFaceLandmarker.mm | 198 +++++++----------- 2 files changed, 74 insertions(+), 126 deletions(-) diff --git a/mediapipe/tasks/ios/vision/face_landmarker/BUILD b/mediapipe/tasks/ios/vision/face_landmarker/BUILD index 24a656b98..0ff88c6c8 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/BUILD +++ b/mediapipe/tasks/ios/vision/face_landmarker/BUILD @@ -70,7 +70,7 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskInfo", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", - "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunnerRefactored", "//mediapipe/tasks/ios/vision/face_landmarker/utils:MPPFaceLandmarkerOptionsHelpers", "//mediapipe/tasks/ios/vision/face_landmarker/utils:MPPFaceLandmarkerResultHelpers", ], diff --git a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm index eec0d66ee..aada4bfab 100644 --- a/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm +++ b/mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarker.mm @@ -19,7 +19,7 @@ #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" -#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" #import "mediapipe/tasks/ios/vision/face_landmarker/sources/MPPFaceLandmarksConnections.h" #import "mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerOptions+Helpers.h" #import "mediapipe/tasks/ios/vision/face_landmarker/utils/sources/MPPFaceLandmarkerResult+Helpers.h" @@ -56,6 +56,16 @@ static NSString *const kTaskName = @"faceLandmarker"; } \ } +#define FaceLandmarkerResultWithOutputPacketMap(outputPacketMap) \ + { \ + [MPPFaceLandmarkerResult \ + faceLandmarkerResultWithLandmarksPacket:outputPacketMap[kLandmarksOutStreamName.cppString] \ + blendshapesPacket:outputPacketMap[kBlendshapesOutStreamName \ + .cppString] \ + transformationMatrixesPacket:outputPacketMap[kFaceGeometryOutStreamName \ + .cppString]] \ + } + @interface MPPFaceLandmarker () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; @@ -71,6 +81,8 @@ static NSString *const kTaskName = @"faceLandmarker"; @implementation MPPFaceLandmarker +#pragma mark - Public + - (instancetype)initWithOptions:(MPPFaceLandmarkerOptions *)options error:(NSError **)error { self = [super init]; if (self) { @@ -124,12 +136,13 @@ static NSString *const kTaskName = @"faceLandmarker"; }; } - _visionTaskRunner = - [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] - runningMode:options.runningMode - packetsCallback:std::move(packetsCallback) - error:error]; - + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:NO + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectStreamName + error:error]; if (!_visionTaskRunner) { return nil; } @@ -144,138 +157,29 @@ static NSString *const kTaskName = @"faceLandmarker"; return [self initWithOptions:options error:error]; } -- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return std::nullopt; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (imagePacket.IsEmpty()) { - return std::nullopt; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:*rect - timestampInMilliseconds:timestampInMilliseconds]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - return inputPacketMap; -} - - (nullable MPPFaceLandmarkerResult *)detectInImage:(MPPImage *)image error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return nil; - } + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = [MPPVisionPacketCreator createPacketWithNormalizedRect:*rect]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - - std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap - error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } - - return [MPPFaceLandmarkerResult - faceLandmarkerResultWithLandmarksPacket:outputPacketMap - .value()[kLandmarksOutStreamName.cppString] - blendshapesPacket:outputPacketMap - .value()[kBlendshapesOutStreamName.cppString] - transformationMatrixesPacket:outputPacketMap - .value()[kFaceGeometryOutStreamName.cppString]]; + return [MPPFaceLandmarker faceLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (nullable MPPFaceLandmarkerResult *)detectInVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return nil; - } - std::optional outputPacketMap = - [_visionTaskRunner processVideoFramePacketMap:*inputPacketMap error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; - return [MPPFaceLandmarkerResult - faceLandmarkerResultWithLandmarksPacket:outputPacketMap - .value()[kLandmarksOutStreamName.cppString] - blendshapesPacket:outputPacketMap - .value()[kBlendshapesOutStreamName.cppString] - transformationMatrixesPacket:outputPacketMap - .value()[kFaceGeometryOutStreamName.cppString]]; + return [MPPFaceLandmarker faceLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (BOOL)detectAsyncInImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return NO; - } - - return [_visionTaskRunner processLiveStreamPacketMap:*inputPacketMap error:error]; -} - -- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { - NSError *callbackError; - if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { - dispatch_async(_callbackQueue, ^{ - [_faceLandmarkerLiveStreamDelegate faceLandmarker:self - didFinishDetectionWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = *liveStreamResult; - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - // The graph did not return a result. We therefore do not raise the user callback. This mirrors - // returning `nil` in the other methods and is acceptable for the live stream delegate since - // it is expected that we drop frames and don't return results for every input. - return; - } - - MPPFaceLandmarkerResult *result = [MPPFaceLandmarkerResult - faceLandmarkerResultWithLandmarksPacket:outputPacketMap[kLandmarksOutStreamName.cppString] - blendshapesPacket:outputPacketMap[kBlendshapesOutStreamName.cppString] - transformationMatrixesPacket:outputPacketMap[kFaceGeometryOutStreamName - .cppString]]; - - NSInteger timeStampInMilliseconds = - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicrosecondsPerMillisecond; - dispatch_async(_callbackQueue, ^{ - [_faceLandmarkerLiveStreamDelegate faceLandmarker:self - didFinishDetectionWithResult:result - timestampInMilliseconds:timeStampInMilliseconds - error:callbackError]; - }); + return [_visionTaskRunner processLiveStreamImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; } + (NSArray *)lipsConnections { @@ -322,4 +226,48 @@ static NSString *const kTaskName = @"faceLandmarker"; return MPPFaceConnections; } +#pragma mark - Private + ++ (nullable MPPFaceLandmarkerResult *)faceLandmarkerResultWithOptionalOutputPacketMap: + (std::optional)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + + return FaceLandmarkerResultWithOutputPacketMap(outputPacketMap.value()); +} + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + NSError *callbackError; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [_faceLandmarkerLiveStreamDelegate faceLandmarker:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = *liveStreamResult; + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + // The graph did not return a result. We therefore do not raise the user callback. This mirrors + // returning `nil` in the other methods and is acceptable for the live stream delegate since + // it is expected that we drop frames and don't return results for every input. + return; + } + + MPPFaceLandmarkerResult *result = FaceLandmarkerResultWithOutputPacketMap(outputPacketMap); + + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicrosecondsPerMillisecond; + dispatch_async(_callbackQueue, ^{ + [_faceLandmarkerLiveStreamDelegate faceLandmarker:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); +} + @end From 020ca5eb7758868f6bb4efa4bd91d3bf3ac7dfc8 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 1 Sep 2023 14:07:07 +0530 Subject: [PATCH 02/10] Updated iOS gesture recognizer to use refactored vision task runner --- .../tasks/ios/vision/gesture_recognizer/BUILD | 2 +- .../sources/MPPGestureRecognizer.mm | 202 +++++++----------- 2 files changed, 76 insertions(+), 128 deletions(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD index d9a76afde..e5b46d344 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD @@ -58,7 +58,7 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskInfo", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", - "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunnerRefactored", "//mediapipe/tasks/ios/vision/gesture_recognizer/utils:MPPGestureRecognizerOptionsHelpers", "//mediapipe/tasks/ios/vision/gesture_recognizer/utils:MPPGestureRecognizerResultHelpers", ], diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.mm index b722163b2..3853f6b02 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.mm +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.mm @@ -18,7 +18,7 @@ #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" -#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" #import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h" #import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h" @@ -54,6 +54,19 @@ static NSString *const kTaskName = @"gestureRecognizer"; } \ } +#define GestureRecognizerResultWithOutputPacketMap(outputPacketMap) \ + { \ + [MPPGestureRecognizerResult \ + gestureRecognizerResultWithHandGesturesPacket:outputPacketMap[kHandGesturesOutStreamName \ + .cppString] \ + handednessPacket:outputPacketMap[kHandednessOutStreamName \ + .cppString] \ + handLandmarksPacket:outputPacketMap[kLandmarksOutStreamName \ + .cppString] \ + worldLandmarksPacket:outputPacketMap[kWorldLandmarksOutStreamName \ + .cppString]] \ + } + @interface MPPGestureRecognizer () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; @@ -65,56 +78,6 @@ static NSString *const kTaskName = @"gestureRecognizer"; @implementation MPPGestureRecognizer -- (nullable MPPGestureRecognizerResult *)gestureRecognizerResultWithOutputPacketMap: - (PacketMap &)outputPacketMap { - return [MPPGestureRecognizerResult - gestureRecognizerResultWithHandGesturesPacket:outputPacketMap[kHandGesturesOutStreamName - .cppString] - handednessPacket:outputPacketMap[kHandednessOutStreamName - .cppString] - handLandmarksPacket:outputPacketMap[kLandmarksOutStreamName - .cppString] - worldLandmarksPacket:outputPacketMap[kWorldLandmarksOutStreamName - .cppString]]; -} - -- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { - if (![self.gestureRecognizerLiveStreamDelegate - respondsToSelector:@selector(gestureRecognizer: - didFinishRecognitionWithResult:timestampInMilliseconds:error:)]) { - return; - } - - NSError *callbackError = nil; - if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { - dispatch_async(_callbackQueue, ^{ - [self.gestureRecognizerLiveStreamDelegate gestureRecognizer:self - didFinishRecognitionWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = liveStreamResult.value(); - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - return; - } - - MPPGestureRecognizerResult *result = - [self gestureRecognizerResultWithOutputPacketMap:outputPacketMap]; - - NSInteger timeStampInMilliseconds = - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicroSecondsPerMilliSecond; - dispatch_async(_callbackQueue, ^{ - [self.gestureRecognizerLiveStreamDelegate gestureRecognizer:self - didFinishRecognitionWithResult:result - timestampInMilliseconds:timeStampInMilliseconds - error:callbackError]; - }); -} - - (instancetype)initWithOptions:(MPPGestureRecognizerOptions *)options error:(NSError **)error { self = [super init]; if (self) { @@ -161,11 +124,13 @@ static NSString *const kTaskName = @"gestureRecognizer"; }; } - _visionTaskRunner = - [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] - runningMode:options.runningMode - packetsCallback:std::move(packetsCallback) - error:error]; + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:NO + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectInStreamName + error:error]; if (!_visionTaskRunner) { return nil; } @@ -181,93 +146,76 @@ static NSString *const kTaskName = @"gestureRecognizer"; return [self initWithOptions:options error:error]; } -- (nullable MPPGestureRecognizerResult *)gestureRecognizerResultWithOptionalOutputPacketMap: - (std::optional &)outputPacketMap { - if (!outputPacketMap.has_value()) { - return nil; - } - MPPGestureRecognizerResult *result = - [self gestureRecognizerResultWithOutputPacketMap:outputPacketMap.value()]; - return result; -} - - (nullable MPPGestureRecognizerResult *)recognizeImage:(MPPImage *)image error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return nil; - } + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - - std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap - error:error]; - return [self gestureRecognizerResultWithOptionalOutputPacketMap:outputPacketMap]; -} - -- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return std::nullopt; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (imagePacket.IsEmpty()) { - return std::nullopt; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() - timestampInMilliseconds:timestampInMilliseconds]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - return inputPacketMap; + return [MPPGestureRecognizer gestureRecognizerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (nullable MPPGestureRecognizerResult *)recognizeVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return nil; - } - std::optional outputPacketMap = - [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; - return [self gestureRecognizerResultWithOptionalOutputPacketMap:outputPacketMap]; + return [MPPGestureRecognizer gestureRecognizerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (BOOL)recognizeAsyncImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return NO; + return [_visionTaskRunner processLiveStreamImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; +} + +#pragma mark - Private + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.gestureRecognizerLiveStreamDelegate + respondsToSelector:@selector(gestureRecognizer: + didFinishRecognitionWithResult:timestampInMilliseconds:error:)]) { + return; } - return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.gestureRecognizerLiveStreamDelegate gestureRecognizer:self + didFinishRecognitionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPGestureRecognizerResult *result = GestureRecognizerResultWithOutputPacketMap(outputPacketMap); + + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond; + dispatch_async(_callbackQueue, ^{ + [self.gestureRecognizerLiveStreamDelegate gestureRecognizer:self + didFinishRecognitionWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); +} + ++ (nullable MPPGestureRecognizerResult *)gestureRecognizerResultWithOptionalOutputPacketMap: + (std::optional &)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + + return GestureRecognizerResultWithOutputPacketMap(outputPacketMap.value()); } @end From 188321ace46f48848749fec82ffa3d286acea421 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 1 Sep 2023 14:07:25 +0530 Subject: [PATCH 03/10] Updated iOS hand landmarker to use refactored vision task runner --- .../tasks/ios/vision/hand_landmarker/BUILD | 2 +- .../sources/MPPHandLandmarker.mm | 199 +++++++----------- 2 files changed, 77 insertions(+), 124 deletions(-) diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/BUILD b/mediapipe/tasks/ios/vision/hand_landmarker/BUILD index 677caf061..d228b8e0a 100644 --- a/mediapipe/tasks/ios/vision/hand_landmarker/BUILD +++ b/mediapipe/tasks/ios/vision/hand_landmarker/BUILD @@ -66,7 +66,7 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskInfo", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", - "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunnerRefactored", "//mediapipe/tasks/ios/vision/hand_landmarker/utils:MPPHandLandmarkerOptionsHelpers", "//mediapipe/tasks/ios/vision/hand_landmarker/utils:MPPHandLandmarkerResultHelpers", ], diff --git a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm index 46a6d1f33..29a8f09a4 100644 --- a/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm +++ b/mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.mm @@ -18,7 +18,7 @@ #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" -#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" #import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarksConnections.h" #import "mediapipe/tasks/ios/vision/hand_landmarker/utils/sources/MPPHandLandmarkerOptions+Helpers.h" #import "mediapipe/tasks/ios/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+Helpers.h" @@ -53,6 +53,16 @@ static NSString *const kTaskName = @"handLandmarker"; } \ } +#define HandLandmarkerResultWithOutputPacketMap(outputPacketMap) \ + { \ + [MPPHandLandmarkerResult \ + handLandmarkerResultWithLandmarksPacket:outputPacketMap[kLandmarksOutStreamName.cppString] \ + worldLandmarksPacket:outputPacketMap[kWorldLandmarksOutStreamName \ + .cppString] \ + handednessPacket:outputPacketMap[kHandednessOutStreamName \ + .cppString]] \ + } + @interface MPPHandLandmarker () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; @@ -63,50 +73,7 @@ static NSString *const kTaskName = @"handLandmarker"; @implementation MPPHandLandmarker -- (nullable MPPHandLandmarkerResult *)handLandmarkerResultWithOutputPacketMap: - (PacketMap &)outputPacketMap { - return [MPPHandLandmarkerResult - handLandmarkerResultWithLandmarksPacket:outputPacketMap[kLandmarksOutStreamName.cppString] - worldLandmarksPacket:outputPacketMap[kWorldLandmarksOutStreamName - .cppString] - handednessPacket:outputPacketMap[kHandednessOutStreamName.cppString]]; -} - -- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { - if (![self.handLandmarkerLiveStreamDelegate - respondsToSelector:@selector(handLandmarker: - didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { - return; - } - - NSError *callbackError = nil; - if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { - dispatch_async(_callbackQueue, ^{ - [self.handLandmarkerLiveStreamDelegate handLandmarker:self - didFinishDetectionWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = liveStreamResult.value(); - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - return; - } - - MPPHandLandmarkerResult *result = [self handLandmarkerResultWithOutputPacketMap:outputPacketMap]; - - NSInteger timeStampInMilliseconds = - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicroSecondsPerMilliSecond; - dispatch_async(_callbackQueue, ^{ - [self.handLandmarkerLiveStreamDelegate handLandmarker:self - didFinishDetectionWithResult:result - timestampInMilliseconds:timeStampInMilliseconds - error:callbackError]; - }); -} +#pragma mark - Public - (instancetype)initWithOptions:(MPPHandLandmarkerOptions *)options error:(NSError **)error { self = [super init]; @@ -152,11 +119,14 @@ static NSString *const kTaskName = @"handLandmarker"; }; } - _visionTaskRunner = - [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] - runningMode:options.runningMode - packetsCallback:std::move(packetsCallback) - error:error]; + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:NO + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectInStreamName + error:error]; + if (!_visionTaskRunner) { return nil; } @@ -172,93 +142,29 @@ static NSString *const kTaskName = @"handLandmarker"; return [self initWithOptions:options error:error]; } -- (nullable MPPHandLandmarkerResult *)handLandmarkerResultWithOptionalOutputPacketMap: - (std::optional &)outputPacketMap { - if (!outputPacketMap.has_value()) { - return nil; - } - MPPHandLandmarkerResult *result = - [self handLandmarkerResultWithOutputPacketMap:outputPacketMap.value()]; - return result; -} - - (nullable MPPHandLandmarkerResult *)detectInImage:(MPPImage *)image error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return nil; - } + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - - std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap - error:error]; - return [self handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; -} - -- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return std::nullopt; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (imagePacket.IsEmpty()) { - return std::nullopt; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() - timestampInMilliseconds:timestampInMilliseconds]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - return inputPacketMap; + return [MPPHandLandmarker handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (nullable MPPHandLandmarkerResult *)detectInVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return nil; - } - std::optional outputPacketMap = - [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; - return [self handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; + return [MPPHandLandmarker handLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; } - (BOOL)detectAsyncInImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return NO; - } - - return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; + return [_visionTaskRunner processLiveStreamImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; } + (NSArray *)handPalmConnections { @@ -285,4 +191,51 @@ static NSString *const kTaskName = @"handLandmarker"; return MPPHandConnections; } +#pragma mark - Private + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.handLandmarkerLiveStreamDelegate + respondsToSelector:@selector(handLandmarker: + didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { + return; + } + + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.handLandmarkerLiveStreamDelegate handLandmarker:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPHandLandmarkerResult *result = HandLandmarkerResultWithOutputPacketMap(outputPacketMap); + + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond; + dispatch_async(_callbackQueue, ^{ + [self.handLandmarkerLiveStreamDelegate handLandmarker:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); +} + ++ (nullable MPPHandLandmarkerResult *)handLandmarkerResultWithOptionalOutputPacketMap: + (std::optional &)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + + return HandLandmarkerResultWithOutputPacketMap(outputPacketMap.value()); +} + @end From 40da111ba7531757381b862f1e2fe1f5113b889e Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 1 Sep 2023 14:07:42 +0530 Subject: [PATCH 04/10] Updated iOS object detector to use refactored vision task runner --- .../tasks/ios/vision/object_detector/BUILD | 2 +- .../sources/MPPObjectDetector.mm | 192 +++++++----------- 2 files changed, 71 insertions(+), 123 deletions(-) diff --git a/mediapipe/tasks/ios/vision/object_detector/BUILD b/mediapipe/tasks/ios/vision/object_detector/BUILD index 002a59920..5777112d4 100644 --- a/mediapipe/tasks/ios/vision/object_detector/BUILD +++ b/mediapipe/tasks/ios/vision/object_detector/BUILD @@ -55,7 +55,7 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskInfo", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", - "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunnerRefactored", "//mediapipe/tasks/ios/vision/object_detector/utils:MPPObjectDetectorOptionsHelpers", "//mediapipe/tasks/ios/vision/object_detector/utils:MPPObjectDetectorResultHelpers", ], diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm index e4704fbbc..3de8af889 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.mm @@ -18,7 +18,7 @@ #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" -#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunnerRefactored.h" #import "mediapipe/tasks/ios/vision/object_detector/utils/sources/MPPObjectDetectorOptions+Helpers.h" #import "mediapipe/tasks/ios/vision/object_detector/utils/sources/MPPObjectDetectorResult+Helpers.h" @@ -47,6 +47,12 @@ static NSString *const kTaskName = @"objectDetector"; } \ } +#define ObjectDetectorResultWithOutputPacketMap(outputPacketMap) \ + { \ + [MPPObjectDetectorResult \ + objectDetectorResultWithDetectionsPacket:outputPacketMap[kDetectionsStreamName.cppString]] \ + } + @interface MPPObjectDetector () { /** iOS Vision Task Runner */ MPPVisionTaskRunner *_visionTaskRunner; @@ -57,42 +63,7 @@ static NSString *const kTaskName = @"objectDetector"; @implementation MPPObjectDetector -- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { - if (![self.objectDetectorLiveStreamDelegate - respondsToSelector:@selector(objectDetector: - didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { - return; - } - - NSError *callbackError = nil; - if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { - dispatch_async(_callbackQueue, ^{ - [self.objectDetectorLiveStreamDelegate objectDetector:self - didFinishDetectionWithResult:nil - timestampInMilliseconds:Timestamp::Unset().Value() - error:callbackError]; - }); - return; - } - - PacketMap &outputPacketMap = liveStreamResult.value(); - if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { - return; - } - - MPPObjectDetectorResult *result = [MPPObjectDetectorResult - objectDetectorResultWithDetectionsPacket:outputPacketMap[kDetectionsStreamName.cppString]]; - - NSInteger timeStampInMilliseconds = - outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / - kMicroSecondsPerMilliSecond; - dispatch_async(_callbackQueue, ^{ - [self.objectDetectorLiveStreamDelegate objectDetector:self - didFinishDetectionWithResult:result - timestampInMilliseconds:timeStampInMilliseconds - error:callbackError]; - }); -} +#pragma mark - Public - (instancetype)initWithOptions:(MPPObjectDetectorOptions *)options error:(NSError **)error { self = [super init]; @@ -135,11 +106,13 @@ static NSString *const kTaskName = @"objectDetector"; }; } - _visionTaskRunner = - [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] - runningMode:options.runningMode - packetsCallback:std::move(packetsCallback) - error:error]; + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:NO + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectStreamName + error:error]; if (!_visionTaskRunner) { return nil; @@ -157,101 +130,76 @@ static NSString *const kTaskName = @"objectDetector"; return [self initWithOptions:options error:error]; } -- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return std::nullopt; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (imagePacket.IsEmpty()) { - return std::nullopt; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() - timestampInMilliseconds:timestampInMilliseconds]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - return inputPacketMap; -} - -- (nullable MPPObjectDetectorResult *)detectInImage:(MPPImage *)image - regionOfInterest:(CGRect)roi - error:(NSError **)error { - std::optional rect = - [_visionTaskRunner normalizedRectWithImageOrientation:image.orientation - imageSize:CGSizeMake(image.width, image.height) - error:error]; - if (!rect.has_value()) { - return nil; - } - - Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; - if (imagePacket.IsEmpty()) { - return nil; - } - - Packet normalizedRectPacket = - [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; - - PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); - - std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap - error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } - - return [MPPObjectDetectorResult - objectDetectorResultWithDetectionsPacket:outputPacketMap - .value()[kDetectionsStreamName.cppString]]; -} - - (nullable MPPObjectDetectorResult *)detectInImage:(MPPImage *)image error:(NSError **)error { - return [self detectInImage:image regionOfInterest:CGRectZero error:error]; + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; + + return [MPPObjectDetector objectDetectorResultWithOptionalOutputPacketMap:outputPacketMap]; } - (nullable MPPObjectDetectorResult *)detectInVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return nil; - } - std::optional outputPacketMap = - [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; - if (!outputPacketMap.has_value()) { - return nil; - } - - return [MPPObjectDetectorResult - objectDetectorResultWithDetectionsPacket:outputPacketMap - .value()[kDetectionsStreamName.cppString]]; + return [MPPObjectDetector objectDetectorResultWithOptionalOutputPacketMap:outputPacketMap]; } - (BOOL)detectAsyncInImage:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds error:(NSError **)error { - std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image - timestampInMilliseconds:timestampInMilliseconds - error:error]; - if (!inputPacketMap.has_value()) { - return NO; + return [_visionTaskRunner processLiveStreamImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; +} + +#pragma mark - Private + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.objectDetectorLiveStreamDelegate + respondsToSelector:@selector(objectDetector: + didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { + return; } - return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.objectDetectorLiveStreamDelegate objectDetector:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPObjectDetectorResult *result = ObjectDetectorResultWithOutputPacketMap(outputPacketMap); + + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMilliSecond; + dispatch_async(_callbackQueue, ^{ + [self.objectDetectorLiveStreamDelegate objectDetector:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); +} + ++ (nullable MPPObjectDetectorResult *)objectDetectorResultWithOptionalOutputPacketMap: + (std::optional &)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + + return ObjectDetectorResultWithOutputPacketMap(outputPacketMap.value()); } @end From be0cde8c2e430f4519e8d630b1639f2ff7c73af0 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 5 Sep 2023 03:37:16 -0700 Subject: [PATCH 05/10] No public description PiperOrigin-RevId: 562724334 --- mediapipe/gpu/BUILD | 3 +++ mediapipe/gpu/shader_util.cc | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 74c7e2d05..bdedb85f2 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -768,6 +768,9 @@ cc_library( ":gl_base", "//mediapipe/framework/port:logging", "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/log:check", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", ], ) diff --git a/mediapipe/gpu/shader_util.cc b/mediapipe/gpu/shader_util.cc index 3e3eb462d..83dc33df2 100644 --- a/mediapipe/gpu/shader_util.cc +++ b/mediapipe/gpu/shader_util.cc @@ -16,7 +16,15 @@ #include +#include +#include +#include + #include "absl/log/absl_log.h" +#include "absl/log/check.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" #include "mediapipe/framework/port/logging.h" #if DEBUG @@ -48,9 +56,26 @@ } while (0) namespace mediapipe { +namespace { constexpr int kMaxShaderInfoLength = 1024; +std::string AddLineNumbers(const GLchar* source) { + // Use format "%ni %s", with n=1 for 1..9 lines, n=2 for 10..99 lines etc. + // Note that StrFormat needs either a constexpr format or a ParsedFormat. + std::vector lines = absl::StrSplit(source, '\n'); + std::string format = absl::StrFormat( + "%%%ii %%s", static_cast(ceilf(log10(1 + lines.size())))); + auto parsed_format = absl::ParsedFormat<'i', 's'>::New(format); + CHECK(parsed_format); + for (int n = 0; n < lines.size(); n++) { + lines[n] = absl::StrFormat(*parsed_format, n + 1, lines[n]); + } + return absl::StrJoin(lines, "\n"); +} + +} // namespace + GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader, bool force_log_errors) { *shader = glCreateShader(target); @@ -72,7 +97,7 @@ GLint GlhCompileShader(GLenum target, const GLchar* source, GLuint* shader, glGetShaderiv(*shader, GL_COMPILE_STATUS, &status); ABSL_LOG_IF(ERROR, status == GL_FALSE) << "Failed to compile shader:\n" - << source; + << AddLineNumbers(source); if (status == GL_FALSE) { int length = 0; @@ -145,6 +170,9 @@ GLint GlhCreateProgram(const GLchar* vert_src, const GLchar* frag_src, } ok = GlhLinkProgram(*program, force_log_errors); + + glDetachShader(*program, frag_shader); + glDetachShader(*program, vert_shader); } if (vert_shader) glDeleteShader(vert_shader); @@ -172,7 +200,8 @@ bool CompileShader(GLenum shader_type, const std::string& shader_source, GLint compiled; glGetShaderiv(*shader, GL_COMPILE_STATUS, &compiled); if (!compiled) { - VLOG(2) << "Unable to compile shader:\n" << shader_source; + VLOG(2) << "Unable to compile shader:\n" + << AddLineNumbers(shader_source_cstr); GL_ERROR_LOG(Shader, *shader, "compile"); glDeleteShader(*shader); *shader = 0; From a19da255656d5309ff5f43cd8b7d90dcd28b3626 Mon Sep 17 00:00:00 2001 From: Fergus Henderson Date: Tue, 5 Sep 2023 09:26:15 -0700 Subject: [PATCH 06/10] Some spelling and grammar fixes in the comments. PiperOrigin-RevId: 562802023 --- mediapipe/framework/calculator_contract.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mediapipe/framework/calculator_contract.h b/mediapipe/framework/calculator_contract.h index 7726065a7..1bc5c5aed 100644 --- a/mediapipe/framework/calculator_contract.h +++ b/mediapipe/framework/calculator_contract.h @@ -169,12 +169,12 @@ class CalculatorContract { // For services which allow default initialization: // - `CalculatorGraph` will try to create corresponding service object by // default even if request is made optional - // (`GraphServiceRequest::Optional()`) + // (`GraphServiceRequest::Optional()`). // // For services which disallow default initialization: // - `CalculatorGraph` requires client to set corresponding service object and - // otherwise fails, unles request is mad optional - // (`GraphServiceRequest::Optional()`) + // otherwise fails, unless request is made optional + // (`GraphServiceRequest::Optional()`). GraphServiceRequest& UseService(const GraphServiceBase& service) { auto it = service_requests_.emplace(service.key, service).first; return it->second; From 2aefa2308b628e96f11a7c5a41be14e1dc12c7e3 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 5 Sep 2023 11:08:10 -0700 Subject: [PATCH 07/10] Internal PiperOrigin-RevId: 562834129 --- .../java/com/google/mediapipe/tasks/vision/imagegenerator/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/BUILD index 6c5b2400b..6a3bd27e3 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagegenerator/BUILD @@ -44,7 +44,7 @@ cc_binary( ) cc_library( - name = "mediapipe_tasks_vision_image_generator_jni_lib", + name = "libmediapipe_tasks_vision_image_generator_jni_lib", srcs = [ ":libimagegenerator_gpu.so", ":libmediapipe_tasks_vision_image_generator_jni.so", From 4e52e969732f0e629efc56644b32b526d4e1031f Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 5 Sep 2023 13:00:48 -0700 Subject: [PATCH 08/10] No public description PiperOrigin-RevId: 562865700 --- mediapipe/framework/testdata/BUILD | 6 ++ .../framework/testdata/proto3_options.proto | 25 ++++++ mediapipe/framework/tool/BUILD | 3 +- mediapipe/framework/tool/options_map.h | 14 +-- mediapipe/framework/tool/options_map_test.cc | 87 +++++++++++++++++-- 5 files changed, 122 insertions(+), 13 deletions(-) create mode 100644 mediapipe/framework/testdata/proto3_options.proto diff --git a/mediapipe/framework/testdata/BUILD b/mediapipe/framework/testdata/BUILD index 8720e39ee..93e416eaa 100644 --- a/mediapipe/framework/testdata/BUILD +++ b/mediapipe/framework/testdata/BUILD @@ -35,6 +35,12 @@ mediapipe_proto_library( ], ) +mediapipe_proto_library( + name = "proto3_options_proto", + srcs = ["proto3_options.proto"], + visibility = ["//visibility:public"], +) + mediapipe_proto_library( name = "zoo_mutator_proto", srcs = ["zoo_mutator.proto"], diff --git a/mediapipe/framework/testdata/proto3_options.proto b/mediapipe/framework/testdata/proto3_options.proto new file mode 100644 index 000000000..c76894819 --- /dev/null +++ b/mediapipe/framework/testdata/proto3_options.proto @@ -0,0 +1,25 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Forked from mediapipe/framework/tool/source.proto. +// The forked proto must remain identical to the original proto and should be +// ONLY used by mediapipe open source project. + +syntax = "proto3"; + +package mediapipe; + +message Proto3Options { + double test_value = 1; +} diff --git a/mediapipe/framework/tool/BUILD b/mediapipe/framework/tool/BUILD index b13dba9b9..77e3ab16d 100644 --- a/mediapipe/framework/tool/BUILD +++ b/mediapipe/framework/tool/BUILD @@ -192,9 +192,8 @@ cc_test( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", - "//mediapipe/framework/port:status", "//mediapipe/framework/testdata:night_light_calculator_cc_proto", - "//mediapipe/framework/testdata:night_light_calculator_options_lib", + "//mediapipe/framework/testdata:proto3_options_cc_proto", ], ) diff --git a/mediapipe/framework/tool/options_map.h b/mediapipe/framework/tool/options_map.h index 2b69f4fb6..4950669c6 100644 --- a/mediapipe/framework/tool/options_map.h +++ b/mediapipe/framework/tool/options_map.h @@ -128,7 +128,8 @@ class OptionsMap { return *options_.Get(); } T* result = options_.Get(); - if (node_config_->has_options()) { + if (node_config_->has_options() && + HasExtension(node_config_->options())) { GetExtension(node_config_->options(), result); } else { GetNodeOptions(*node_config_, result); @@ -141,8 +142,9 @@ class OptionsMap { if (options_.Has()) { return true; } - if (node_config_->has_options()) { - return HasExtension(node_config_->options()); + if (node_config_->has_options() && + HasExtension(node_config_->options())) { + return true; } #if defined(MEDIAPIPE_PROTO_LITE) && defined(MEDIAPIPE_PROTO_THIRD_PARTY) // protobuf::Any is unavailable with third_party/protobuf:protobuf-lite. @@ -170,7 +172,8 @@ class MutableOptionsMap : public OptionsMap { template void Set(const T& value) const { *options_.Get() = value; - if (node_config_->has_options()) { + if (node_config_->has_options() && + HasExtension(node_config_->options())) { *GetExtension(*node_config_->mutable_options()) = value; } else { SetNodeOptions(*node_config_, value); @@ -182,7 +185,8 @@ class MutableOptionsMap : public OptionsMap { if (options_.Has()) { return options_.Get(); } - if (node_config_->has_options()) { + if (node_config_->has_options() && + HasExtension(node_config_->options())) { return GetExtension(*node_config_->mutable_options()); } T* result = options_.Get(); diff --git a/mediapipe/framework/tool/options_map_test.cc b/mediapipe/framework/tool/options_map_test.cc index 529fd5770..8efd1cb94 100644 --- a/mediapipe/framework/tool/options_map_test.cc +++ b/mediapipe/framework/tool/options_map_test.cc @@ -17,14 +17,11 @@ #include -#include - #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/parse_text_proto.h" -#include "mediapipe/framework/port/status.h" -#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/framework/testdata/night_light_calculator.pb.h" +#include "mediapipe/framework/testdata/proto3_options.pb.h" namespace mediapipe { namespace tool { @@ -40,9 +37,10 @@ TEST(OptionsMapTest, QueryNotFound) { OptionsMap options; options.Initialize(node); EXPECT_FALSE(options.Has()); + EXPECT_FALSE(options.Has()); } -TEST(OptionsMapTest, QueryFound) { +TEST(OptionsMapTest, Proto2QueryFound) { CalculatorGraphConfig::Node node = ParseTextProtoOrDie(R"pb( calculator: "NightLightCalculator" @@ -64,7 +62,7 @@ TEST(OptionsMapTest, QueryFound) { 123); } -TEST(MutableOptionsMapTest, InsertAndQueryFound) { +TEST(MutableOptionsMapTest, InsertProto2AndQueryFound) { CalculatorGraphConfig::Node node = ParseTextProtoOrDie(R"pb( calculator: "NightLightCalculator" @@ -83,6 +81,83 @@ TEST(MutableOptionsMapTest, InsertAndQueryFound) { 123); } +TEST(OptionsMapTest, Proto3QueryFound) { + CalculatorGraphConfig::Node node = + ParseTextProtoOrDie(R"pb( + calculator: "NightLightCalculator" + input_side_packet: "input_value" + output_stream: "values" + node_options { + [type.googleapis.com/mediapipe.Proto3Options] { test_value: 123 } + } + )pb"); + OptionsMap options; + options.Initialize(node); + EXPECT_TRUE(options.Has()); + EXPECT_EQ(options.Get().test_value(), 123); +} + +TEST(MutableOptionsMapTest, InsertProto3AndQueryFound) { + CalculatorGraphConfig::Node node = + ParseTextProtoOrDie(R"pb( + calculator: "NightLightCalculator" + input_side_packet: "input_value" + output_stream: "values" + )pb"); + MutableOptionsMap options; + options.Initialize(node); + EXPECT_FALSE(options.Has()); + mediapipe::Proto3Options proto3_options; + proto3_options.set_test_value(123); + options.Set(proto3_options); + EXPECT_TRUE(options.Has()); + EXPECT_EQ(options.Get().test_value(), 123); +} + +TEST(OptionsMapTest, BothProto2AndProto3QueriesFound) { + CalculatorGraphConfig::Node node = + ParseTextProtoOrDie(R"pb( + calculator: "NightLightCalculator" + input_side_packet: "input_value" + output_stream: "values" + options { + [mediapipe.NightLightCalculatorOptions.ext] { jitter: 321 } + } + node_options { + [type.googleapis.com/mediapipe.Proto3Options] { test_value: 123 } + } + )pb"); + OptionsMap options; + options.Initialize(node); + EXPECT_TRUE(options.Has()); + EXPECT_EQ(options.Get().test_value(), 123); + EXPECT_TRUE(options.Has()); + EXPECT_EQ(options.Get().jitter(), + 321); +} + +TEST(OptionsMapTest, PrefersOptionsOverNodeOptions) { + CalculatorGraphConfig::Node node = + ParseTextProtoOrDie(R"pb( + calculator: "NightLightCalculator" + input_side_packet: "input_value" + output_stream: "values" + options { + [mediapipe.NightLightCalculatorOptions.ext] { jitter: 111 } + } + node_options { + [type.googleapis.com/mediapipe.NightLightCalculatorOptions] { + jitter: 222 + } + } + )pb"); + OptionsMap options; + options.Initialize(node); + EXPECT_TRUE(options.Has()); + EXPECT_EQ(options.Get().jitter(), + 111); +} + } // namespace } // namespace tool } // namespace mediapipe From bf32d1acb2aafd23239ebc157ef62e487522b1fc Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 5 Sep 2023 15:49:59 -0700 Subject: [PATCH 09/10] No public description PiperOrigin-RevId: 562915674 --- mediapipe/calculators/tensorflow/BUILD | 1 - .../pack_media_sequence_calculator.cc | 14 +++- .../pack_media_sequence_calculator_test.cc | 78 +++++++++++++++++++ 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD index 995ba6a7f..5f5f51657 100644 --- a/mediapipe/calculators/tensorflow/BUILD +++ b/mediapipe/calculators/tensorflow/BUILD @@ -379,7 +379,6 @@ cc_library( "//mediapipe/util/sequence:media_sequence", "//mediapipe/util/sequence:media_sequence_util", "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/log", "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/core:protos_all_cc", diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc index fcb640ff2..d87029143 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc @@ -287,6 +287,13 @@ class PackMediaSequenceCalculator : public CalculatorBase { mpms::ClearClipLabelString(key, sequence_.get()); mpms::ClearClipLabelConfidence(key, sequence_.get()); } + if (absl::StartsWith(tag, kFloatContextFeaturePrefixTag)) { + const std::string& key = + tag.substr(sizeof(kFloatContextFeaturePrefixTag) / + sizeof(*kFloatContextFeaturePrefixTag) - + 1); + mpms::ClearContextFeatureFloats(key, sequence_.get()); + } if (absl::StartsWith(tag, kIntsContextFeaturePrefixTag)) { const std::string& key = tag.substr(sizeof(kIntsContextFeaturePrefixTag) / @@ -536,9 +543,10 @@ class PackMediaSequenceCalculator : public CalculatorBase { sizeof(*kFloatContextFeaturePrefixTag) - 1); RET_CHECK_EQ(cc->InputTimestamp(), Timestamp::PostStream()); - mpms::SetContextFeatureFloats( - key, cc->Inputs().Tag(tag).Get>(), - sequence_.get()); + for (const auto& value : + cc->Inputs().Tag(tag).Get>()) { + mpms::AddContextFeatureFloats(key, value, sequence_.get()); + } } if (absl::StartsWith(tag, kIntsContextFeaturePrefixTag) && !cc->Inputs().Tag(tag).IsEmpty()) { diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc index 277e2d63c..d9dc56e9c 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc @@ -13,6 +13,7 @@ // limitations under the License. #include +#include #include #include @@ -455,6 +456,83 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoContextFloatLists) { testing::ElementsAre(4, 4)); } +TEST_F(PackMediaSequenceCalculatorTest, ReplaceTwoContextFloatLists) { + SetUpCalculator( + /*input_streams=*/{"FLOAT_CONTEXT_FEATURE_TEST:test", + "FLOAT_CONTEXT_FEATURE_OTHER:test2"}, + /*features=*/{}, + /*output_only_if_all_present=*/false, /*replace_instead_of_append=*/true); + auto input_sequence = std::make_unique(); + mpms::SetContextFeatureFloats("TEST", {2, 3}, input_sequence.get()); + mpms::SetContextFeatureFloats("OTHER", {2, 4}, input_sequence.get()); + + const std::vector vf_1 = {5, 6}; + runner_->MutableInputs() + ->Tag(kFloatContextFeatureTestTag) + .packets.push_back( + MakePacket>(vf_1).At(Timestamp::PostStream())); + const std::vector vf_2 = {7, 8}; + runner_->MutableInputs() + ->Tag(kFloatContextFeatureOtherTag) + .packets.push_back( + MakePacket>(vf_2).At(Timestamp::PostStream())); + + runner_->MutableSidePackets()->Tag(kSequenceExampleTag) = + Adopt(input_sequence.release()); + + MP_ASSERT_OK(runner_->Run()); + + const std::vector& output_packets = + runner_->Outputs().Tag(kSequenceExampleTag).packets; + ASSERT_EQ(1, output_packets.size()); + const tf::SequenceExample& output_sequence = + output_packets[0].Get(); + + ASSERT_THAT(mpms::GetContextFeatureFloats("TEST", output_sequence), + testing::ElementsAre(5, 6)); + ASSERT_THAT(mpms::GetContextFeatureFloats("OTHER", output_sequence), + testing::ElementsAre(7, 8)); +} + +TEST_F(PackMediaSequenceCalculatorTest, AppendTwoContextFloatLists) { + SetUpCalculator( + /*input_streams=*/{"FLOAT_CONTEXT_FEATURE_TEST:test", + "FLOAT_CONTEXT_FEATURE_OTHER:test2"}, + /*features=*/{}, + /*output_only_if_all_present=*/false, + /*replace_instead_of_append=*/false); + auto input_sequence = std::make_unique(); + mpms::SetContextFeatureFloats("TEST", {2, 3}, input_sequence.get()); + mpms::SetContextFeatureFloats("OTHER", {2, 4}, input_sequence.get()); + + const std::vector vf_1 = {5, 6}; + runner_->MutableInputs() + ->Tag(kFloatContextFeatureTestTag) + .packets.push_back( + MakePacket>(vf_1).At(Timestamp::PostStream())); + const std::vector vf_2 = {7, 8}; + runner_->MutableInputs() + ->Tag(kFloatContextFeatureOtherTag) + .packets.push_back( + MakePacket>(vf_2).At(Timestamp::PostStream())); + + runner_->MutableSidePackets()->Tag(kSequenceExampleTag) = + Adopt(input_sequence.release()); + + MP_ASSERT_OK(runner_->Run()); + + const std::vector& output_packets = + runner_->Outputs().Tag(kSequenceExampleTag).packets; + ASSERT_EQ(1, output_packets.size()); + const tf::SequenceExample& output_sequence = + output_packets[0].Get(); + + EXPECT_THAT(mpms::GetContextFeatureFloats("TEST", output_sequence), + testing::ElementsAre(2, 3, 5, 6)); + EXPECT_THAT(mpms::GetContextFeatureFloats("OTHER", output_sequence), + testing::ElementsAre(2, 4, 7, 8)); +} + TEST_F(PackMediaSequenceCalculatorTest, PackTwoContextIntLists) { SetUpCalculator( /*input_streams=*/{"INTS_CONTEXT_FEATURE_TEST:test", From e39119ae533edb86066e44df464bd24024ab86ef Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 5 Sep 2023 20:48:57 -0700 Subject: [PATCH 10/10] Add missing cache writing implementation in InferenceCalculatorAdvancedGL Add missing implementation for absl::Status SaveGpuCachesBasedOnBehavior(tflite::gpu::TFLiteGPURunner* gpu_runner) const for non android/chromeos. PiperOrigin-RevId: 562973338 --- mediapipe/calculators/tensor/BUILD | 1 + .../calculators/tensor/inference_calculator_gl_advanced.cc | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index 017ab4f39..e74ecffe6 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -448,6 +448,7 @@ cc_library( "//mediapipe/framework/deps:file_path", "//mediapipe/gpu:gl_calculator_helper", "//mediapipe/util/tflite:tflite_gpu_runner", + "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc b/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc index 77e6eeafe..28ea45de0 100644 --- a/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc +++ b/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc @@ -355,6 +355,12 @@ absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init( return absl::OkStatus(); } +absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper:: + SaveGpuCachesBasedOnBehavior( + tflite::gpu::TFLiteGPURunner* gpu_runner) const { + return absl::OkStatus(); +} + absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::ReadGpuCaches( tflite::gpu::TFLiteGPURunner* gpu_runner) const {