Merge pull request #4541 from priankakariatyml:ios-hand-landmarker-tests

PiperOrigin-RevId: 540995916

mediapipe/framework/tool/ios.bzl

@@ -14,7 +14,7 @@
 
 """MediaPipe Task Library Helper Rules for iOS"""
 
-MPP_TASK_MINIMUM_OS_VERSION = "11.0"
+MPP_TASK_MINIMUM_OS_VERSION = "12.0"
 
 # When the static framework is built with bazel, the all header files are moved
 # to the "Headers" directory with no header path prefixes. This auxiliary rule

mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m

@@ -98,18 +98,17 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation";
       [MPPGestureRecognizerTests filePathWithFileInfo:kExpectedThumbUpLandmarksFile];
 
   return [MPPGestureRecognizerResult
-      gestureRecognizerResultsFromTextEncodedProtobufFileWithName:filePath
+      gestureRecognizerResultsFromProtobufFileWithName:filePath
-                                                     gestureLabel:kExpectedThumbUpLabel
+                                          gestureLabel:kExpectedThumbUpLabel
-                                            shouldRemoveZPosition:YES];
+                                 shouldRemoveZPosition:YES];
 }
 
 + (MPPGestureRecognizerResult *)fistGestureRecognizerResultWithLabel:(NSString *)gestureLabel {
   NSString *filePath = [MPPGestureRecognizerTests filePathWithFileInfo:kExpectedFistLandmarksFile];
 
-  return [MPPGestureRecognizerResult
+  return [MPPGestureRecognizerResult gestureRecognizerResultsFromProtobufFileWithName:filePath
-      gestureRecognizerResultsFromTextEncodedProtobufFileWithName:filePath
+                                                                         gestureLabel:gestureLabel
-                                                     gestureLabel:gestureLabel
+                                                                shouldRemoveZPosition:YES];
-                                            shouldRemoveZPosition:YES];
 }
 
 #pragma mark Assert Gesture Recognizer Results

mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.h

@@ -1,4 +1,4 @@
-// Copyright 2022 The MediaPipe Authors.
+// Copyright 2023 The MediaPipe Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -19,9 +19,9 @@ NS_ASSUME_NONNULL_BEGIN
 @interface MPPGestureRecognizerResult (ProtobufHelpers)
 
 + (MPPGestureRecognizerResult *)
-    gestureRecognizerResultsFromTextEncodedProtobufFileWithName:(NSString *)fileName
+    gestureRecognizerResultsFromProtobufFileWithName:(NSString *)fileName
-                                                   gestureLabel:(NSString *)gestureLabel
+                                        gestureLabel:(NSString *)gestureLabel
-                                          shouldRemoveZPosition:(BOOL)removeZPosition;
+                               shouldRemoveZPosition:(BOOL)removeZPosition;
 
 @end
 

mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm

@@ -32,9 +32,9 @@ using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt;
 @implementation MPPGestureRecognizerResult (ProtobufHelpers)
 
 + (MPPGestureRecognizerResult *)
-    gestureRecognizerResultsFromTextEncodedProtobufFileWithName:(NSString *)fileName
+    gestureRecognizerResultsFromProtobufFileWithName:(NSString *)fileName
-                                                   gestureLabel:(NSString *)gestureLabel
+                                        gestureLabel:(NSString *)gestureLabel
-                                          shouldRemoveZPosition:(BOOL)removeZPosition {
+                               shouldRemoveZPosition:(BOOL)removeZPosition {
   LandmarksDetectionResultProto landmarkDetectionResultProto;
 
   if (!get_proto_from_pbtxt(fileName.cppString, landmarkDetectionResultProto).ok()) {

mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m

@@ -40,6 +40,9 @@ static ResourceFileInfo *const kExpectedPointingUpRotatedLandmarksFile =
 static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks";
 static const float kLandmarksErrorTolerance = 0.03f;
 
+static NSString *const kLiveStreamTestsDictHandLandmarkerKey = @"gesture_recognizer";
+static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation";
+
 #define AssertEqualErrors(error, expectedError)              \
   XCTAssertNotNil(error);                                    \
   XCTAssertEqualObjects(error.domain, expectedError.domain); \
@@ -57,7 +60,10 @@ static const float kLandmarksErrorTolerance = 0.03f;
   XCTAssertTrue(handLandmarkerResult.landmarks.count == 0);     \
   XCTAssertTrue(handLandmarkerResult.worldLandmarks.count == 0);
 
-@interface MPPHandLandmarkerTests : XCTestCase
+@interface MPPHandLandmarkerTests : XCTestCase <MPPHandLandmarkerLiveStreamDelegate> {
+  NSDictionary<NSString *, id> *_liveStreamSucceedsTestDict;
+  NSDictionary<NSString *, id> *_outOfOrderTimestampTestDict;
+}
 @end
 
 @implementation MPPHandLandmarkerTests
@@ -161,7 +167,7 @@ static const float kLandmarksErrorTolerance = 0.03f;
 
 - (MPPHandLandmarker *)createHandLandmarkerWithOptionsSucceeds:
     (MPPHandLandmarkerOptions *)handLandmarkerOptions {
-  NSError* error;
+  NSError *error;
   MPPHandLandmarker *handLandmarker =
       [[MPPHandLandmarker alloc] initWithOptions:handLandmarkerOptions error:&error];
   XCTAssertNotNil(handLandmarker);
@@ -281,4 +287,271 @@ static const float kLandmarksErrorTolerance = 0.03f;
                                                pointingUpRotatedHandLandmarkerResult]];
 }
 
+#pragma mark Running Mode Tests
+
+- (void)testCreateHandLandmarkerFailsWithDelegateInNonLiveStreamMode {
+  MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo};
+  for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) {
+    MPPHandLandmarkerOptions *options =
+        [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
+
+    options.runningMode = runningModesToTest[i];
+    options.handLandmarkerLiveStreamDelegate = self;
+
+    [self
+        assertCreateHandLandmarkerWithOptions:options
+                       failsWithExpectedError:
+                           [NSError errorWithDomain:kExpectedErrorDomain
+                                               code:MPPTasksErrorCodeInvalidArgumentError
+                                           userInfo:@{
+                                             NSLocalizedDescriptionKey :
+                                                 @"The vision task is in image or video mode. The "
+                                                 @"delegate must not be set in the task's options."
+                                           }]];
+  }
+}
+
+- (void)testCreateHandLandmarkerFailsWithMissingDelegateInLiveStreamMode {
+  MPPHandLandmarkerOptions *options =
+      [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
+
+  options.runningMode = MPPRunningModeLiveStream;
+
+  [self assertCreateHandLandmarkerWithOptions:options
+                       failsWithExpectedError:
+                           [NSError errorWithDomain:kExpectedErrorDomain
+                                               code:MPPTasksErrorCodeInvalidArgumentError
+                                           userInfo:@{
+                                             NSLocalizedDescriptionKey :
+                                                 @"The vision task is in live stream mode. An "
+                                                 @"object must be set as the delegate of the task "
+                                                 @"in its options to ensure asynchronous delivery "
+                                                 @"of results."
+                                           }]];
+}
+
+- (void)testDetectFailsWithCallingWrongApiInImageMode {
+  MPPHandLandmarkerOptions *options =
+      [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
+
+  MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options];
+
+  MPPImage *image = [self imageWithFileInfo:kThumbUpImage];
+
+  NSError *liveStreamApiCallError;
+  XCTAssertFalse([handLandmarker detectAsyncInImage:image
+                            timestampInMilliseconds:0
+                                              error:&liveStreamApiCallError]);
+
+  NSError *expectedLiveStreamApiCallError =
+      [NSError errorWithDomain:kExpectedErrorDomain
+                          code:MPPTasksErrorCodeInvalidArgumentError
+                      userInfo:@{
+                        NSLocalizedDescriptionKey : @"The vision task is not initialized with live "
+                                                    @"stream mode. Current Running Mode: Image"
+                      }];
+
+  AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError);
+
+  NSError *videoApiCallError;
+  XCTAssertFalse([handLandmarker detectInVideoFrame:image
+                            timestampInMilliseconds:0
+                                              error:&videoApiCallError]);
+
+  NSError *expectedVideoApiCallError =
+      [NSError errorWithDomain:kExpectedErrorDomain
+                          code:MPPTasksErrorCodeInvalidArgumentError
+                      userInfo:@{
+                        NSLocalizedDescriptionKey : @"The vision task is not initialized with "
+                                                    @"video mode. Current Running Mode: Image"
+                      }];
+  AssertEqualErrors(videoApiCallError, expectedVideoApiCallError);
+}
+
+- (void)testDetectFailsWithCallingWrongApiInVideoMode {
+  MPPHandLandmarkerOptions *options =
+      [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
+  options.runningMode = MPPRunningModeVideo;
+
+  MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options];
+
+  MPPImage *image = [self imageWithFileInfo:kThumbUpImage];
+
+  NSError *liveStreamApiCallError;
+  XCTAssertFalse([handLandmarker detectAsyncInImage:image
+                            timestampInMilliseconds:0
+                                              error:&liveStreamApiCallError]);
+
+  NSError *expectedLiveStreamApiCallError =
+      [NSError errorWithDomain:kExpectedErrorDomain
+                          code:MPPTasksErrorCodeInvalidArgumentError
+                      userInfo:@{
+                        NSLocalizedDescriptionKey : @"The vision task is not initialized with live "
+                                                    @"stream mode. Current Running Mode: Video"
+                      }];
+
+  AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError);
+
+  NSError *imageApiCallError;
+  XCTAssertFalse([handLandmarker detectInImage:image error:&imageApiCallError]);
+
+  NSError *expectedImageApiCallError =
+      [NSError errorWithDomain:kExpectedErrorDomain
+                          code:MPPTasksErrorCodeInvalidArgumentError
+                      userInfo:@{
+                        NSLocalizedDescriptionKey : @"The vision task is not initialized with "
+                                                    @"image mode. Current Running Mode: Video"
+                      }];
+  AssertEqualErrors(imageApiCallError, expectedImageApiCallError);
+}
+
+- (void)testDetectFailsWithCallingWrongApiInLiveStreamMode {
+  MPPHandLandmarkerOptions *options =
+      [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
+  options.runningMode = MPPRunningModeLiveStream;
+  options.handLandmarkerLiveStreamDelegate = self;
+
+  MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options];
+
+  MPPImage *image = [self imageWithFileInfo:kThumbUpImage];
+
+  NSError *imageApiCallError;
+  XCTAssertFalse([handLandmarker detectInImage:image error:&imageApiCallError]);
+
+  NSError *expectedImageApiCallError =
+      [NSError errorWithDomain:kExpectedErrorDomain
+                          code:MPPTasksErrorCodeInvalidArgumentError
+                      userInfo:@{
+                        NSLocalizedDescriptionKey : @"The vision task is not initialized with "
+                                                    @"image mode. Current Running Mode: Live Stream"
+                      }];
+  AssertEqualErrors(imageApiCallError, expectedImageApiCallError);
+
+  NSError *videoApiCallError;
+  XCTAssertFalse([handLandmarker detectInVideoFrame:image
+                            timestampInMilliseconds:0
+                                              error:&videoApiCallError]);
+
+  NSError *expectedVideoApiCallError =
+      [NSError errorWithDomain:kExpectedErrorDomain
+                          code:MPPTasksErrorCodeInvalidArgumentError
+                      userInfo:@{
+                        NSLocalizedDescriptionKey : @"The vision task is not initialized with "
+                                                    @"video mode. Current Running Mode: Live Stream"
+                      }];
+  AssertEqualErrors(videoApiCallError, expectedVideoApiCallError);
+}
+
+- (void)testDetectWithVideoModeSucceeds {
+  MPPHandLandmarkerOptions *options =
+      [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
+  options.runningMode = MPPRunningModeVideo;
+
+  MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options];
+
+  MPPImage *image = [self imageWithFileInfo:kThumbUpImage];
+
+  for (int i = 0; i < 3; i++) {
+    MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectInVideoFrame:image
+                                                               timestampInMilliseconds:i
+                                                                                 error:nil];
+    [self assertHandLandmarkerResult:handLandmarkerResult
+        isApproximatelyEqualToExpectedResult:[MPPHandLandmarkerTests thumbUpHandLandmarkerResult]];
+  }
+}
+
+- (void)testDetectWithOutOfOrderTimestampsAndLiveStreamModeFails {
+  MPPHandLandmarkerOptions *options =
+      [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
+  options.runningMode = MPPRunningModeLiveStream;
+  options.handLandmarkerLiveStreamDelegate = self;
+
+  XCTestExpectation *expectation = [[XCTestExpectation alloc]
+      initWithDescription:@"detectWiththOutOfOrderTimestampsAndLiveStream"];
+
+  expectation.expectedFulfillmentCount = 1;
+
+  MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options];
+
+  _outOfOrderTimestampTestDict = @{
+    kLiveStreamTestsDictHandLandmarkerKey : handLandmarker,
+    kLiveStreamTestsDictExpectationKey : expectation
+  };
+
+  MPPImage *image = [self imageWithFileInfo:kThumbUpImage];
+
+  XCTAssertTrue([handLandmarker detectAsyncInImage:image timestampInMilliseconds:1 error:nil]);
+
+  NSError *error;
+  XCTAssertFalse([handLandmarker detectAsyncInImage:image timestampInMilliseconds:0 error:&error]);
+
+  NSError *expectedError =
+      [NSError errorWithDomain:kExpectedErrorDomain
+                          code:MPPTasksErrorCodeInvalidArgumentError
+                      userInfo:@{
+                        NSLocalizedDescriptionKey :
+                            @"INVALID_ARGUMENT: Input timestamp must be monotonically increasing."
+                      }];
+  AssertEqualErrors(error, expectedError);
+
+  NSTimeInterval timeout = 0.5f;
+  [self waitForExpectations:@[ expectation ] timeout:timeout];
+}
+
+- (void)testDetectWithLiveStreamModeSucceeds {
+  MPPHandLandmarkerOptions *options =
+      [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile];
+  options.runningMode = MPPRunningModeLiveStream;
+  options.handLandmarkerLiveStreamDelegate = self;
+
+  NSInteger iterationCount = 100;
+
+  // Because of flow limiting, we cannot ensure that the callback will be invoked `iterationCount`
+  // times. An normal expectation will fail if expectation.fulfill() is not called
+  // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will
+  // only succeed if expectation is not fulfilled for the specified `expectedFulfillmentCount`.
+  // Since in our case we cannot predict how many times the expectation is supposed to be fullfilled
+  // setting, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and
+  // `expectation.isInverted = true` ensures that test succeeds ifexpectation is fullfilled <=
+  // `iterationCount` times.
+  XCTestExpectation *expectation =
+      [[XCTestExpectation alloc] initWithDescription:@"detectWithLiveStream"];
+
+  expectation.expectedFulfillmentCount = iterationCount + 1;
+  expectation.inverted = YES;
+
+  MPPHandLandmarker *handLandmarker = [self createHandLandmarkerWithOptionsSucceeds:options];
+
+  _liveStreamSucceedsTestDict = @{
+    kLiveStreamTestsDictHandLandmarkerKey : handLandmarker,
+    kLiveStreamTestsDictExpectationKey : expectation
+  };
+
+  // TODO: Mimic initialization from CMSampleBuffer as live stream mode is most likely to be used
+  // with the iOS camera. AVCaptureVideoDataOutput sample buffer delegates provide frames of type
+  // `CMSampleBuffer`.
+  MPPImage *image = [self imageWithFileInfo:kThumbUpImage];
+
+  for (int i = 0; i < iterationCount; i++) {
+    XCTAssertTrue([handLandmarker detectAsyncInImage:image timestampInMilliseconds:i error:nil]);
+  }
+
+  NSTimeInterval timeout = 0.5f;
+  [self waitForExpectations:@[ expectation ] timeout:timeout];
+}
+
+- (void)handLandmarker:(MPPHandLandmarker *)handLandmarker
+    didFinishDetectionWithResult:(MPPHandLandmarkerResult *)handLandmarkerResult
+         timestampInMilliseconds:(NSInteger)timestampInMilliseconds
+                           error:(NSError *)error {
+  [self assertHandLandmarkerResult:handLandmarkerResult
+      isApproximatelyEqualToExpectedResult:[MPPHandLandmarkerTests thumbUpHandLandmarkerResult]];
+
+  if (handLandmarker == _outOfOrderTimestampTestDict[kLiveStreamTestsDictHandLandmarkerKey]) {
+    [_outOfOrderTimestampTestDict[kLiveStreamTestsDictExpectationKey] fulfill];
+  } else if (handLandmarker == _liveStreamSucceedsTestDict[kLiveStreamTestsDictHandLandmarkerKey]) {
+    [_liveStreamSucceedsTestDict[kLiveStreamTestsDictExpectationKey] fulfill];
+  }
+}
+
 @end