From 746f466c3a55a4947fc1a32bdbbf38d2f0110f1c Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 31 May 2023 11:28:50 +0530 Subject: [PATCH 01/18] Added iOS Gesture Recognizer Protobuf utils --- .../vision/gesture_recognizer/utils/BUILD | 21 ++++++ .../MPPGestureRecognizerResult+ProtoHelpers.h | 28 ++++++++ ...MPPGestureRecognizerResult+ProtoHelpers.mm | 66 +++++++++++++++++++ 3 files changed, 115 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h create mode 100644 mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.mm diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/BUILD b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/BUILD new file mode 100644 index 000000000..ddac21ed2 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/BUILD @@ -0,0 +1,21 @@ +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPGestureRecognizerResultProtoHelpers", + srcs = ["sources/MPPGestureRecognizerResult+ProtoHelpers.mm"], + hdrs = ["sources/MPPGestureRecognizerResult+ProtoHelpers.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + "//mediapipe/tasks/ios/test/vision/utils:parse_proto_utils", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/tasks/cc/components/containers/proto:landmarks_detection_result_cc_proto", + "//mediapipe/tasks/ios/vision/gesture_recognizer/utils:MPPGestureRecognizerResultHelpers", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h new file mode 100644 index 000000000..6bb2e5182 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h @@ -0,0 +1,28 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" + +NS_ASSUME_NONNULL_BEGIN +@interface MPPGestureRecognizerResult (ProtoHelpers) + ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultsFromTextEncodedProtobufFileWithName:(NSString *)fileName + gestureLabel:(NSString *)gestureLabel + shouldRemoveZPosition:(BOOL)removeZPosition; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.mm b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.mm new file mode 100644 index 000000000..ce3a262b8 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.mm @@ -0,0 +1,66 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h" + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/tasks/cc/components/containers/proto/landmarks_detection_result.pb.h" +#include "mediapipe/tasks/ios/test/vision/utils/sources/parse_proto_utils.h" + +namespace { +using ClassificationListProto = ::mediapipe::ClassificationList; +using ClassificationProto = ::mediapipe::Classification; +using LandmarksDetectionResultProto = + ::mediapipe::tasks::containers::proto::LandmarksDetectionResult; +using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt; +} // anonymous namespace + +@implementation MPPGestureRecognizerResult (ProtoHelpers) + ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultsFromTextEncodedProtobufFileWithName:(NSString *)fileName + gestureLabel:(NSString *)gestureLabel + shouldRemoveZPosition:(BOOL)removeZPosition { + LandmarksDetectionResultProto landmarkDetectionResultProto; + + if (!get_proto_from_pbtxt(fileName.cppString, landmarkDetectionResultProto).ok()) { + return nil; + } + + if (removeZPosition) { + // Remove z position of landmarks, because they are not used in correctness + // testing. For video or live stream mode, the z positions varies a lot during + // tracking from frame to frame. + for (int i = 0; i < landmarkDetectionResultProto.landmarks().landmark().size(); i++) { + auto &landmark = *landmarkDetectionResultProto.mutable_landmarks()->mutable_landmark(i); + landmark.clear_z(); + } + } + + ClassificationListProto gesturesProto; + ClassificationProto *classificationProto = gesturesProto.add_classification(); + classificationProto->set_label([gestureLabel UTF8String]); + + return [MPPGestureRecognizerResult + gestureRecognizerResultWithHandGesturesProto:{gesturesProto} + handednessroto:{landmarkDetectionResultProto.classifications()} + handLandmarksPacket:{landmarkDetectionResultProto.landmarks()} + worldLandmarksPacket:{landmarkDetectionResultProto.world_landmarks()} + timestampInMilliSeconds:0]; +} + +@end From 1e77468eec6fa005b0982b2a4fb758709a7ce41a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 31 May 2023 11:29:37 +0530 Subject: [PATCH 02/18] Added iOS Gesture Recognizer ObjC Test for simple recognition --- .../ios/test/vision/gesture_recognizer/BUILD | 62 ++++ .../MPPGestureRecognizerTests.m | 287 ++++++++++++++++++ 2 files changed, 349 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/gesture_recognizer/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/BUILD b/mediapipe/tasks/ios/test/vision/gesture_recognizer/BUILD new file mode 100644 index 000000000..5be17a26c --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/BUILD @@ -0,0 +1,62 @@ +load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "//mediapipe/framework/tool:ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPGestureRecognizerObjcTestLibrary", + testonly = 1, + srcs = ["MPPGestureRecognizerTests.m"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:gesture_recognizer.task", + "//mediapipe/tasks/testdata/vision:test_protos", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + "//mediapipe/tasks/ios/vision/gesture_recognizer:MPPGestureRecognizer", + "//mediapipe/tasks/ios/test/vision/gesture_recognizer/utils:MPPGestureRecognizerResultProtoHelpers", + ] + select({ + "//third_party:opencv_ios_sim_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_x86_64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//conditions:default": ["@ios_opencv//:OpencvFramework"], + }), +) + +ios_unit_test( + name = "MPPGestureRecognizerObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPGestureRecognizerObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m new file mode 100644 index 000000000..1a48322b4 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -0,0 +1,287 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h" + +static NSDictionary *const kGestureRecognizerBundleAssetFile = + @{@"name" : @"gesture_recognizer", @"type" : @"task"}; + +static NSDictionary *const kTwoHandsImage = @{@"name" : @"right_hands", @"type" : @"jpg"}; +static NSDictionary *const kFistImage = @{@"name" : @"fist", @"type" : @"jpg"}; +static NSDictionary *const kNoHandsImage = @{@"name" : @"cats_and_dogs", @"type" : @"jpg"}; +static NSDictionary *const kThumbUpImage = @{@"name" : @"thumb_up", @"type" : @"jpg"}; +static NSDictionary *const kPointingUpRotatedImage = + @{@"name" : @"pointing_up_rotated", @"type" : @"jpg"}; + +static NSDictionary *const kExpectedFistLandmarksFile = + @{@"name" : @"fist_landmarks", @"type" : @"pbtxt"}; +static NSDictionary *const kExpectedThumbUpLandmarksFile = + @{@"name" : @"thumb_up_landmarks", @"type" : @"pbtxt"}; + +static NSString *const kFistLabel = @"Closed_Fist"; +static NSString *const kExpectedThumbUpLabel = @"Thumb_Up"; +static NSString *const kExpectedPointingUpLabel = @"Pointing_Up"; +static NSString *const kRockLabel = @"Rock"; + +static const NSInteger kGestureExpectedIndex = -1; + +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; +static const float kLandmarksErrorTolerance = 0.03f; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertNotEqual( \ + [error.localizedDescription rangeOfString:expectedError.localizedDescription].location, \ + NSNotFound) + +#define AssertEqualGestures(gesture, expectedGesture, handIndex, gestureIndex) \ + XCTAssertEqual(gesture.index, kGestureExpectedIndex, @"hand index = %d gesture index j = %d", \ + handIndex, gestureIndex); \ + XCTAssertEqualObjects(gesture.categoryName, expectedGesture.categoryName, \ + @"hand index = %d gesture index j = %d", handIndex, gestureIndex); + +#define AssertApproximatelyEqualLandmarks(landmark, expectedLandmark, handIndex, landmarkIndex) \ + XCTAssertEqualWithAccuracy(landmark.x, expectedLandmark.x, kLandmarksErrorTolerance, \ + @"hand index = %d landmark index j = %d", handIndex, landmarkIndex); \ + XCTAssertEqualWithAccuracy(landmark.y, expectedLandmark.y, kLandmarksErrorTolerance, \ + @"hand index = %d landmark index j = %d", handIndex, landmarkIndex); + +#define AssertApproximatelyEqualMultiHandLandmarks(multiHandLandmarks, expectedMultiHandLandmars) \ + XCTAssertEqual(multiHandLandmarks.count, expectedMultiHandLandmars.count) \ + XCTAssertEqualWithAccuracy(landmark.x, expectedLandmark.x, kLandmarksErrorTolerance, \ + @"hand index = %d landmark index j = %d", handIndex, handIndex); \ + XCTAssertEqualWithAccuracy(landmark.y, expectedLandmark.y, kLandmarksErrorTolerance, \ + @"hand index = %d landmark index j = %d", handIndex, handIndex); + +#define AssertGestureRecognizerResultIsEmpty(gestureRecognizerResult) \ + XCTAssertTrue(gestureRecognizerResult.gestures.count == 0); \ + XCTAssertTrue(gestureRecognizerResult.handedness.count == 0); \ + XCTAssertTrue(gestureRecognizerResult.landmarks.count == 0); \ + XCTAssertTrue(gestureRecognizerResult.worldLandmarks.count == 0); + +@interface MPPGestureRecognizerTests : XCTestCase +@end + +@implementation MPPGestureRecognizerTests + +#pragma mark Results + ++ (MPPGestureRecognizerResult *)emptyGestureRecognizerResult { + return [[MPPGestureRecognizerResult alloc] initWithGestures:@[] + handedness:@[] + landmarks:@[] + worldLandmarks:@[] + timestampInMilliseconds:0]; +} + ++ (MPPGestureRecognizerResult *)thumbUpGestureRecognizerResult { + NSString *filePath = + [MPPGestureRecognizerTests filePathWithFileInfo:kExpectedThumbUpLandmarksFile]; + + return [MPPGestureRecognizerResult + gestureRecognizerResultsFromTextEncodedProtobufFileWithName:filePath + gestureLabel:kExpectedThumbUpLabel + shouldRemoveZPosition:YES]; +} + ++ (MPPGestureRecognizerResult *)fistGestureRecognizerResultWithLabel:(NSString *)gestureLabel { + NSString *filePath = [MPPGestureRecognizerTests filePathWithFileInfo:kExpectedFistLandmarksFile]; + + return [MPPGestureRecognizerResult + gestureRecognizerResultsFromTextEncodedProtobufFileWithName:filePath + gestureLabel:gestureLabel + shouldRemoveZPosition:YES]; +} + +- (void)assertMultiHandLandmarks:(NSArray *> *)multiHandLandmarks + isApproximatelyEqualToExpectedMultiHandLandmarks: + (NSArray *> *)expectedMultiHandLandmarks { + XCTAssertEqual(multiHandLandmarks.count, expectedMultiHandLandmarks.count); + if (multiHandLandmarks.count == 0) { + return; + } + + NSArray *topHandLandmarks = multiHandLandmarks[0]; + NSArray *expectedTopHandLandmarks = expectedMultiHandLandmarks[0]; + + XCTAssertEqual(topHandLandmarks.count, expectedTopHandLandmarks.count); + for (int i = 0; i < expectedTopHandLandmarks.count; i++) { + MPPNormalizedLandmark *landmark = topHandLandmarks[i]; + XCTAssertNotNil(landmark); + AssertApproximatelyEqualLandmarks(landmark, expectedTopHandLandmarks[i], 0, i); + } +} + +- (void)assertMultiHandWorldLandmarks:(NSArray *> *)multiHandWorldLandmarks + isApproximatelyEqualToExpectedMultiHandWorldLandmarks: + (NSArray *> *)expectedMultiHandWorldLandmarks { + XCTAssertEqual(multiHandWorldLandmarks.count, expectedMultiHandWorldLandmarks.count); + if (expectedMultiHandWorldLandmarks.count == 0) { + return; + } + + NSArray *topHandWorldLandmarks = multiHandWorldLandmarks[0]; + NSArray *expectedTopHandWorldLandmarks = expectedMultiHandWorldLandmarks[0]; + + XCTAssertEqual(topHandWorldLandmarks.count, expectedTopHandWorldLandmarks.count); + for (int i = 0; i < expectedTopHandWorldLandmarks.count; i++) { + MPPLandmark *landmark = topHandWorldLandmarks[i]; + XCTAssertNotNil(landmark); + AssertApproximatelyEqualLandmarks(landmark, expectedTopHandWorldLandmarks[i], 0, i); + } +} + +- (void)assertMultiHandGestures:(NSArray *> *)multiHandGestures + isApproximatelyEqualToExpectedMultiHandGestures: + (NSArray *> *)expectedMultiHandGestures { + XCTAssertEqual(multiHandGestures.count, expectedMultiHandGestures.count); + if (multiHandGestures.count == 0) { + return; + } + + NSArray *topHandGestures = multiHandGestures[0]; + NSArray *expectedTopHandGestures = expectedMultiHandGestures[0]; + + XCTAssertEqual(topHandGestures.count, expectedTopHandGestures.count); + for (int i = 0; i < expectedTopHandGestures.count; i++) { + MPPCategory *gesture = topHandGestures[i]; + XCTAssertNotNil(gesture); + AssertEqualGestures(gesture, expectedTopHandGestures[i], 0, i); + } +} + +- (void)assertGestureRecognizerResult:(MPPGestureRecognizerResult *)gestureRecognizerResult + isApproximatelyEqualToExpectedResult: + (MPPGestureRecognizerResult *)expectedGestureRecognizerResult { + [self assertMultiHandLandmarks:gestureRecognizerResult.landmarks + isApproximatelyEqualToExpectedMultiHandLandmarks:expectedGestureRecognizerResult.landmarks]; + [self assertMultiHandWorldLandmarks:gestureRecognizerResult.worldLandmarks + isApproximatelyEqualToExpectedMultiHandWorldLandmarks:expectedGestureRecognizerResult + .worldLandmarks]; + [self assertMultiHandGestures:gestureRecognizerResult.gestures + isApproximatelyEqualToExpectedMultiHandGestures:expectedGestureRecognizerResult.gestures]; +} + +#pragma mark File + ++ (NSString *)filePathWithFileInfo:(NSDictionary *)fileInfo { + NSString *filePath = [MPPGestureRecognizerTests filePathWithName:fileInfo[@"name"] + extension:fileInfo[@"type"]]; + return filePath; +} + ++ (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension { + NSString *filePath = [[NSBundle bundleForClass:self.class] pathForResource:fileName + ofType:extension]; + return filePath; +} + +#pragma mark Gesture Recognizer Initializers + +- (MPPGestureRecognizerOptions *)gestureRecognizerOptionsWithModelFileInfo: + (NSDictionary *)modelFileInfo { + NSString *modelPath = [MPPGestureRecognizerTests filePathWithFileInfo:modelFileInfo]; + MPPGestureRecognizerOptions *gestureRecognizerOptions = + [[MPPGestureRecognizerOptions alloc] init]; + gestureRecognizerOptions.baseOptions.modelAssetPath = modelPath; + + return gestureRecognizerOptions; +} + +- (MPPGestureRecognizer *)createGestureRecognizerWithOptionsSucceeds: + (MPPGestureRecognizerOptions *)gestureRecognizerOptions { + MPPGestureRecognizer *gestureRecognizer = + [[MPPGestureRecognizer alloc] initWithOptions:gestureRecognizerOptions error:nil]; + XCTAssertNotNil(gestureRecognizer); + + return gestureRecognizer; +} + +- (void)assertCreateGestureRecognizerWithOptions: + (MPPGestureRecognizerOptions *)gestureRecognizerOptions + failsWithExpectedError:(NSError *)expectedError { + NSError *error = nil; + MPPGestureRecognizer *gestureRecognizer = + [[MPPGestureRecognizer alloc] initWithOptions:gestureRecognizerOptions error:&error]; + + XCTAssertNil(gestureRecognizer); + AssertEqualErrors(error, expectedError); +} + +#pragma mark Assert Gesture Recognizer Results + +- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo { + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPGestureRecognizerTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"]]; + XCTAssertNotNil(image); + + return image; +} + +- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo + orientation:(UIImageOrientation)orientation { + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPGestureRecognizerTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"] + orientation:orientation]; + XCTAssertNotNil(image); + + return image; +} + +- (MPPGestureRecognizerResult *)recognizeImageWithFileInfo:(NSDictionary *)imageFileInfo + usingGestureRecognizer: + (MPPGestureRecognizer *)gestureRecognizer { + MPPImage *mppImage = [self imageWithFileInfo:imageFileInfo]; + MPPGestureRecognizerResult *gestureRecognizerResult = [gestureRecognizer recognizeImage:mppImage + error:nil]; + XCTAssertNotNil(gestureRecognizerResult); + + return gestureRecognizerResult; +} + +- (void)assertResultsOfRecognizeImageWithFileInfo:(NSDictionary *)fileInfo + usingGestureRecognizer:(MPPGestureRecognizer *)gestureRecognizer + approximatelyEqualsGestureRecognizerResult: + (MPPGestureRecognizerResult *)expectedGestureRecognizerResult { + MPPGestureRecognizerResult *gestureRecognizerResult = + [self recognizeImageWithFileInfo:fileInfo usingGestureRecognizer:gestureRecognizer]; + [self assertGestureRecognizerResult:gestureRecognizerResult + isApproximatelyEqualToExpectedResult:expectedGestureRecognizerResult]; +} + +#pragma mark General Tests + +- (void)testRecognizeWithModelPathSucceeds { + NSString *modelPath = + [MPPGestureRecognizerTests filePathWithFileInfo:kGestureRecognizerBundleAssetFile]; + MPPGestureRecognizer *gestureRecognizer = + [[MPPGestureRecognizer alloc] initWithModelPath:modelPath error:nil]; + XCTAssertNotNil(gestureRecognizer); + + [self assertResultsOfRecognizeImageWithFileInfo:kThumbUpImage + usingGestureRecognizer:gestureRecognizer + approximatelyEqualsGestureRecognizerResult:[MPPGestureRecognizerTests + thumbUpGestureRecognizerResult]]; +} + +@end From 84560f3e7db5026fb762f570b1ee13cd5c159442 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 31 May 2023 11:36:29 +0530 Subject: [PATCH 03/18] Added more recognize tests to iOS Gesture Recognizer Objective C tests --- .../MPPGestureRecognizerTests.m | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m index 1a48322b4..94600a83f 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -284,4 +284,128 @@ static const float kLandmarksErrorTolerance = 0.03f; thumbUpGestureRecognizerResult]]; } +- (void)testRecognizeWithEmptyResultsSucceeds { + MPPGestureRecognizerOptions *gestureRecognizerOptions = + [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; + + MPPGestureRecognizer *gestureRecognizer = + [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; + + MPPGestureRecognizerResult *gestureRecognizerResult = + [self recognizeImageWithFileInfo:kNoHandsImage usingGestureRecognizer:gestureRecognizer]; + AssertGestureRecognizerResultIsEmpty(gestureRecognizerResult); +} + +- (void)testRecognizeWithScoreThresholdSucceeds { + MPPGestureRecognizerOptions *gestureRecognizerOptions = + [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; + gestureRecognizerOptions.cannedGesturesClassifierOptions = [[MPPClassifierOptions alloc] init]; + gestureRecognizerOptions.cannedGesturesClassifierOptions.scoreThreshold = 0.5f; + + MPPGestureRecognizer *gestureRecognizer = + [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; + + MPPGestureRecognizerResult *gestureRecognizerResult = + [self recognizeImageWithFileInfo:kThumbUpImage usingGestureRecognizer:gestureRecognizer]; + + MPPGestureRecognizerResult *expectedGestureRecognizerResult = + [MPPGestureRecognizerTests thumbUpGestureRecognizerResult]; + + XCTAssertTrue(gestureRecognizerResult.gestures.count == 1); + AssertEqualGestures(gestureRecognizerResult.gestures[0][0], + expectedGestureRecognizerResult.gestures[0][0], 0, 0); +} + +- (void)testRecognizeWithNumHandsSucceeds { + MPPGestureRecognizerOptions *gestureRecognizerOptions = + [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; + + const NSInteger numberOfHands = 2; + gestureRecognizerOptions.numberOfHands = numberOfHands; + + MPPGestureRecognizer *gestureRecognizer = + [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; + + MPPGestureRecognizerResult *gestureRecognizerResult = + [self recognizeImageWithFileInfo:kTwoHandsImage usingGestureRecognizer:gestureRecognizer]; + + XCTAssertTrue(gestureRecognizerResult.handedness.count == numberOfHands); +} + +- (void)testRecognizeWithRotationSucceeds { + MPPGestureRecognizerOptions *gestureRecognizerOptions = + [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; + + const NSInteger numberOfHands = 2; + gestureRecognizerOptions.numberOfHands = numberOfHands; + + MPPGestureRecognizer *gestureRecognizer = + [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; + MPPImage *mppImage = [self imageWithFileInfo:kPointingUpRotatedImage + orientation:UIImageOrientationRight]; + + MPPGestureRecognizerResult *gestureRecognizerResult = [gestureRecognizer recognizeImage:mppImage + error:nil]; + + XCTAssertNotNil(gestureRecognizerResult); + + const NSInteger expectedGesturesCount = 1; + + XCTAssertEqual(gestureRecognizerResult.gestures.count, expectedGesturesCount); + XCTAssertEqualObjects(gestureRecognizerResult.gestures[0][0].categoryName, + kExpectedPointingUpLabel); +} + +- (void)testRecognizeWithCannedGestureFistSucceeds { + MPPGestureRecognizerOptions *gestureRecognizerOptions = + [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; + + const NSInteger numberOfHands = 1; + gestureRecognizerOptions.numberOfHands = numberOfHands; + + MPPGestureRecognizer *gestureRecognizer = + [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; + + [self assertResultsOfRecognizeImageWithFileInfo:kFistImage + usingGestureRecognizer:gestureRecognizer + approximatelyEqualsGestureRecognizerResult: + [MPPGestureRecognizerTests fistGestureRecognizerResultWithLabel:kFistLabel]]; +} + +- (void)testRecognizeWithAllowGestureFistSucceeds { + MPPGestureRecognizerOptions *gestureRecognizerOptions = + [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; + gestureRecognizerOptions.cannedGesturesClassifierOptions = [[MPPClassifierOptions alloc] init]; + gestureRecognizerOptions.cannedGesturesClassifierOptions.scoreThreshold = 0.5f; + gestureRecognizerOptions.cannedGesturesClassifierOptions.categoryAllowlist = @[ kFistLabel ]; + + const NSInteger numberOfHands = 1; + gestureRecognizerOptions.numberOfHands = numberOfHands; + + MPPGestureRecognizer *gestureRecognizer = + [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; + + [self assertResultsOfRecognizeImageWithFileInfo:kFistImage + usingGestureRecognizer:gestureRecognizer + approximatelyEqualsGestureRecognizerResult: + [MPPGestureRecognizerTests fistGestureRecognizerResultWithLabel:kFistLabel]]; +} + +- (void)testRecognizeWithDenyGestureFistSucceeds { + MPPGestureRecognizerOptions *gestureRecognizerOptions = + [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; + gestureRecognizerOptions.cannedGesturesClassifierOptions = [[MPPClassifierOptions alloc] init]; + gestureRecognizerOptions.cannedGesturesClassifierOptions.scoreThreshold = 0.5f; + gestureRecognizerOptions.cannedGesturesClassifierOptions.categoryDenylist = @[ kFistLabel ]; + + const NSInteger numberOfHands = 1; + gestureRecognizerOptions.numberOfHands = numberOfHands; + + MPPGestureRecognizer *gestureRecognizer = + [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; + MPPGestureRecognizerResult *gestureRecognizerResult = + [self recognizeImageWithFileInfo:kFistImage usingGestureRecognizer:gestureRecognizer]; + AssertGestureRecognizerResultIsEmpty(gestureRecognizerResult); +} + @end From f3f664300cac55b34ab478b26b9780c0ccd30bcc Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 31 May 2023 11:50:39 +0530 Subject: [PATCH 04/18] Added convenience method for creating results for tests in MPPGestureRecognizerResult Helpers --- .../MPPGestureRecognizerResult+Helpers.h | 24 +++ .../MPPGestureRecognizerResult+Helpers.mm | 162 ++++++++++-------- 2 files changed, 110 insertions(+), 76 deletions(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h index 6b0f8bf81..5e75febf3 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h @@ -43,6 +43,30 @@ static const int kMicroSecondsPerMilliSecond = 1000; handLandmarksPacket:(const mediapipe::Packet &)handLandmarksPacket worldLandmarksPacket:(const mediapipe::Packet &)worldLandmarksPacket; +/** + * Creates an `MPPGestureRecognizerResult` from hand gestures, handedness, hand landmarks and world + * landmarks proto vectors. + * + * @param handGesturesProto A vector of protos of type `std::vector`. + * @param handednessPacket A vector of protos of type `std::vector`. + * @param handLandmarksPacket A vector of protos of type `std::vector`. + * @param handLandmarksPacket A vector of protos of type `std::vector`. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultWithHandGesturesProto: + (const std::vector &)handGesturesProto + handednessroto: + (const std::vector &) + handednessProto + handLandmarksPacket: + (const std::vector &) + handLandmarksProto + worldLandmarksPacket: + (const std::vector &)worldLandmarksProto + timestampInMilliSeconds:(NSInteger)timestampInMilliseconds; @end NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm index 8eed2a923..5162dd891 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm @@ -17,12 +17,6 @@ #import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h" #import "mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h" -#include "mediapipe/framework/formats/classification.pb.h" -#include "mediapipe/framework/formats/landmark.pb.h" -#include "mediapipe/framework/packet.h" - -static const NSInteger kDefaultGestureIndex = -1; - namespace { using ClassificationListProto = ::mediapipe::ClassificationList; using LandmarkListProto = ::mediapipe::LandmarkList; @@ -30,6 +24,8 @@ using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList; using ::mediapipe::Packet; } // namespace +static const NSInteger kDefaultGestureIndex = -1; + @implementation MPPGestureRecognizerResult (Helpers) + (MPPGestureRecognizerResult *)emptyGestureRecognizerResultWithTimestampInMilliseconds: @@ -41,6 +37,80 @@ using ::mediapipe::Packet; timestampInMilliseconds:timestampInMilliseconds]; } ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultWithHandGesturesProto: + (const std::vector &)handGesturesProto + handednessroto: + (const std::vector &)handednessProto + handLandmarksPacket:(const std::vector &) + handLandmarksProto + worldLandmarksPacket: + (const std::vector &)worldLandmarksProto + timestampInMilliSeconds:(NSInteger)timestampInMilliseconds { + NSMutableArray *> *multiHandGestures = + [NSMutableArray arrayWithCapacity:(NSUInteger)handGesturesProto.size()]; + + for (const auto &classificationListProto : handGesturesProto) { + NSMutableArray *gestures = [NSMutableArray + arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; + for (const auto &classificationProto : classificationListProto.classification()) { + MPPCategory *category = [MPPCategory categoryWithProto:classificationProto + index:kDefaultGestureIndex]; + [gestures addObject:category]; + } + [multiHandGestures addObject:gestures]; + } + + NSMutableArray *> *multiHandHandedness = + [NSMutableArray arrayWithCapacity:(NSUInteger)handednessProto.size()]; + + for (const auto &classificationListProto : handednessProto) { + NSMutableArray *handedness = [NSMutableArray + arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; + for (const auto &classificationProto : classificationListProto.classification()) { + MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; + [handedness addObject:category]; + } + [multiHandHandedness addObject:handedness]; + } + + NSMutableArray *> *multiHandLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarksProto.size()]; + + for (const auto &handLandmarkListProto : handLandmarksProto) { + NSMutableArray *handLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProto.landmark().size()]; + for (const auto &normalizedLandmarkProto : handLandmarkListProto.landmark()) { + MPPNormalizedLandmark *normalizedLandmark = + [MPPNormalizedLandmark normalizedLandmarkWithProto:normalizedLandmarkProto]; + [handLandmarks addObject:normalizedLandmark]; + } + [multiHandLandmarks addObject:handLandmarks]; + } + + NSMutableArray *> *multiHandWorldLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarksProto.size()]; + + for (const auto &worldLandmarkListProto : worldLandmarksProto) { + NSMutableArray *worldLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProto.landmark().size()]; + for (const auto &landmarkProto : worldLandmarkListProto.landmark()) { + MPPLandmark *landmark = [MPPLandmark landmarkWithProto:landmarkProto]; + [worldLandmarks addObject:landmark]; + } + [multiHandWorldLandmarks addObject:worldLandmarks]; + } + + MPPGestureRecognizerResult *gestureRecognizerResult = + [[MPPGestureRecognizerResult alloc] initWithGestures:multiHandGestures + handedness:multiHandHandedness + landmarks:multiHandLandmarks + worldLandmarks:multiHandWorldLandmarks + timestampInMilliseconds:timestampInMilliseconds]; + + return gestureRecognizerResult; +} + + (MPPGestureRecognizerResult *) gestureRecognizerResultWithHandGesturesPacket:(const Packet &)handGesturesPacket handednessPacket:(const Packet &)handednessPacket @@ -62,76 +132,16 @@ using ::mediapipe::Packet; emptyGestureRecognizerResultWithTimestampInMilliseconds:timestampInMilliseconds]; } - const std::vector &handGesturesClassificationListProtos = - handGesturesPacket.Get>(); - NSMutableArray *> *multiHandGestures = - [NSMutableArray arrayWithCapacity:(NSUInteger)handGesturesClassificationListProtos.size()]; - - for (const auto &classificationListProto : handGesturesClassificationListProtos) { - NSMutableArray *gestures = [NSMutableArray - arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; - for (const auto &classificationProto : classificationListProto.classification()) { - MPPCategory *category = [MPPCategory categoryWithProto:classificationProto - index:kDefaultGestureIndex]; - [gestures addObject:category]; - } - [multiHandGestures addObject:gestures]; - } - - const std::vector &handednessClassificationListProtos = - handednessPacket.Get>(); - NSMutableArray *> *multiHandHandedness = - [NSMutableArray arrayWithCapacity:(NSUInteger)handednessClassificationListProtos.size()]; - - for (const auto &classificationListProto : handednessClassificationListProtos) { - NSMutableArray *handedness = [NSMutableArray - arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; - for (const auto &classificationProto : classificationListProto.classification()) { - MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; - [handedness addObject:category]; - } - [multiHandHandedness addObject:handedness]; - } - - const std::vector &handLandmarkListProtos = - handLandmarksPacket.Get>(); - NSMutableArray *> *multiHandLandmarks = - [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProtos.size()]; - - for (const auto &handLandmarkListProto : handLandmarkListProtos) { - NSMutableArray *handLandmarks = - [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProto.landmark().size()]; - for (const auto &normalizedLandmarkProto : handLandmarkListProto.landmark()) { - MPPNormalizedLandmark *normalizedLandmark = - [MPPNormalizedLandmark normalizedLandmarkWithProto:normalizedLandmarkProto]; - [handLandmarks addObject:normalizedLandmark]; - } - [multiHandLandmarks addObject:handLandmarks]; - } - - const std::vector &worldLandmarkListProtos = - worldLandmarksPacket.Get>(); - NSMutableArray *> *multiHandWorldLandmarks = - [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProtos.size()]; - - for (const auto &worldLandmarkListProto : worldLandmarkListProtos) { - NSMutableArray *worldLandmarks = - [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProto.landmark().size()]; - for (const auto &landmarkProto : worldLandmarkListProto.landmark()) { - MPPLandmark *landmark = [MPPLandmark landmarkWithProto:landmarkProto]; - [worldLandmarks addObject:landmark]; - } - [multiHandWorldLandmarks addObject:worldLandmarks]; - } - - MPPGestureRecognizerResult *gestureRecognizerResult = - [[MPPGestureRecognizerResult alloc] initWithGestures:multiHandGestures - handedness:multiHandHandedness - landmarks:multiHandLandmarks - worldLandmarks:multiHandWorldLandmarks - timestampInMilliseconds:timestampInMilliseconds]; - - return gestureRecognizerResult; + return [MPPGestureRecognizerResult + gestureRecognizerResultWithHandGesturesProto:handGesturesPacket + .Get>() + handednessroto:handednessPacket + .Get>() + handLandmarksPacket:handLandmarksPacket + .Get>() + worldLandmarksPacket:worldLandmarksPacket + .Get>() + timestampInMilliSeconds:timestampInMilliseconds]; } @end From 9546596b5af9151152d47e158db2e3e59c26d695 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 31 May 2023 11:51:00 +0530 Subject: [PATCH 05/18] Updated variable name in MPPGestureRecognizerTests.m --- .../MPPGestureRecognizerTests.m | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m index 94600a83f..c1504da00 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -320,8 +320,8 @@ static const float kLandmarksErrorTolerance = 0.03f; MPPGestureRecognizerOptions *gestureRecognizerOptions = [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; - const NSInteger numberOfHands = 2; - gestureRecognizerOptions.numberOfHands = numberOfHands; + const NSInteger numHands = 2; + gestureRecognizerOptions.numHands = numHands; MPPGestureRecognizer *gestureRecognizer = [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; @@ -329,15 +329,15 @@ static const float kLandmarksErrorTolerance = 0.03f; MPPGestureRecognizerResult *gestureRecognizerResult = [self recognizeImageWithFileInfo:kTwoHandsImage usingGestureRecognizer:gestureRecognizer]; - XCTAssertTrue(gestureRecognizerResult.handedness.count == numberOfHands); + XCTAssertTrue(gestureRecognizerResult.handedness.count == numHands); } - (void)testRecognizeWithRotationSucceeds { MPPGestureRecognizerOptions *gestureRecognizerOptions = [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; - const NSInteger numberOfHands = 2; - gestureRecognizerOptions.numberOfHands = numberOfHands; + const NSInteger numHands = 2; + gestureRecognizerOptions.numHands = numHands; MPPGestureRecognizer *gestureRecognizer = [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; @@ -360,8 +360,8 @@ static const float kLandmarksErrorTolerance = 0.03f; MPPGestureRecognizerOptions *gestureRecognizerOptions = [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; - const NSInteger numberOfHands = 1; - gestureRecognizerOptions.numberOfHands = numberOfHands; + const NSInteger numHands = 1; + gestureRecognizerOptions.numHands = numHands; MPPGestureRecognizer *gestureRecognizer = [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; @@ -379,8 +379,8 @@ static const float kLandmarksErrorTolerance = 0.03f; gestureRecognizerOptions.cannedGesturesClassifierOptions.scoreThreshold = 0.5f; gestureRecognizerOptions.cannedGesturesClassifierOptions.categoryAllowlist = @[ kFistLabel ]; - const NSInteger numberOfHands = 1; - gestureRecognizerOptions.numberOfHands = numberOfHands; + const NSInteger numHands = 1; + gestureRecognizerOptions.numHands = numHands; MPPGestureRecognizer *gestureRecognizer = [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; @@ -398,8 +398,8 @@ static const float kLandmarksErrorTolerance = 0.03f; gestureRecognizerOptions.cannedGesturesClassifierOptions.scoreThreshold = 0.5f; gestureRecognizerOptions.cannedGesturesClassifierOptions.categoryDenylist = @[ kFistLabel ]; - const NSInteger numberOfHands = 1; - gestureRecognizerOptions.numberOfHands = numberOfHands; + const NSInteger numHands = 1; + gestureRecognizerOptions.numHands = numHands; MPPGestureRecognizer *gestureRecognizer = [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; From 365956807db10c434ea2e16b23ae43055a4a3c89 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 31 May 2023 11:52:29 +0530 Subject: [PATCH 06/18] Added gesture_recognizer.task to vision tasks test data --- mediapipe/tasks/testdata/vision/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/testdata/vision/BUILD b/mediapipe/tasks/testdata/vision/BUILD index 632e8aa4e..f6153dd12 100644 --- a/mediapipe/tasks/testdata/vision/BUILD +++ b/mediapipe/tasks/testdata/vision/BUILD @@ -54,6 +54,7 @@ mediapipe_files(srcs = [ "hand_landmark_full.tflite", "hand_landmark_lite.tflite", "hand_landmarker.task", + "gesture_recognizer.task", "left_hands.jpg", "left_hands_rotated.jpg", "mobilenet_v1_0.25_192_quantized_1_default_1.tflite", @@ -104,7 +105,6 @@ exports_files( "expected_right_down_hand_landmarks.prototxt", "expected_right_up_hand_landmarks.prototxt", "face_geometry_expected_out.pbtxt", - "gesture_recognizer.task", "portrait_expected_detection.pbtxt", "portrait_expected_face_geometry.pbtxt", "portrait_rotated_expected_detection.pbtxt", From c87e21206a630d43d8ed0961d22d2d7fb58ed1de Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 31 May 2023 11:53:41 +0530 Subject: [PATCH 07/18] Removed few test from MPPGestureRecognizerTests.m --- .../MPPGestureRecognizerTests.m | 112 ------------------ 1 file changed, 112 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m index c1504da00..3b9d6367f 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -296,116 +296,4 @@ static const float kLandmarksErrorTolerance = 0.03f; AssertGestureRecognizerResultIsEmpty(gestureRecognizerResult); } -- (void)testRecognizeWithScoreThresholdSucceeds { - MPPGestureRecognizerOptions *gestureRecognizerOptions = - [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; - gestureRecognizerOptions.cannedGesturesClassifierOptions = [[MPPClassifierOptions alloc] init]; - gestureRecognizerOptions.cannedGesturesClassifierOptions.scoreThreshold = 0.5f; - - MPPGestureRecognizer *gestureRecognizer = - [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; - - MPPGestureRecognizerResult *gestureRecognizerResult = - [self recognizeImageWithFileInfo:kThumbUpImage usingGestureRecognizer:gestureRecognizer]; - - MPPGestureRecognizerResult *expectedGestureRecognizerResult = - [MPPGestureRecognizerTests thumbUpGestureRecognizerResult]; - - XCTAssertTrue(gestureRecognizerResult.gestures.count == 1); - AssertEqualGestures(gestureRecognizerResult.gestures[0][0], - expectedGestureRecognizerResult.gestures[0][0], 0, 0); -} - -- (void)testRecognizeWithNumHandsSucceeds { - MPPGestureRecognizerOptions *gestureRecognizerOptions = - [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; - - const NSInteger numHands = 2; - gestureRecognizerOptions.numHands = numHands; - - MPPGestureRecognizer *gestureRecognizer = - [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; - - MPPGestureRecognizerResult *gestureRecognizerResult = - [self recognizeImageWithFileInfo:kTwoHandsImage usingGestureRecognizer:gestureRecognizer]; - - XCTAssertTrue(gestureRecognizerResult.handedness.count == numHands); -} - -- (void)testRecognizeWithRotationSucceeds { - MPPGestureRecognizerOptions *gestureRecognizerOptions = - [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; - - const NSInteger numHands = 2; - gestureRecognizerOptions.numHands = numHands; - - MPPGestureRecognizer *gestureRecognizer = - [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; - MPPImage *mppImage = [self imageWithFileInfo:kPointingUpRotatedImage - orientation:UIImageOrientationRight]; - - MPPGestureRecognizerResult *gestureRecognizerResult = [gestureRecognizer recognizeImage:mppImage - error:nil]; - - XCTAssertNotNil(gestureRecognizerResult); - - const NSInteger expectedGesturesCount = 1; - - XCTAssertEqual(gestureRecognizerResult.gestures.count, expectedGesturesCount); - XCTAssertEqualObjects(gestureRecognizerResult.gestures[0][0].categoryName, - kExpectedPointingUpLabel); -} - -- (void)testRecognizeWithCannedGestureFistSucceeds { - MPPGestureRecognizerOptions *gestureRecognizerOptions = - [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; - - const NSInteger numHands = 1; - gestureRecognizerOptions.numHands = numHands; - - MPPGestureRecognizer *gestureRecognizer = - [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; - - [self assertResultsOfRecognizeImageWithFileInfo:kFistImage - usingGestureRecognizer:gestureRecognizer - approximatelyEqualsGestureRecognizerResult: - [MPPGestureRecognizerTests fistGestureRecognizerResultWithLabel:kFistLabel]]; -} - -- (void)testRecognizeWithAllowGestureFistSucceeds { - MPPGestureRecognizerOptions *gestureRecognizerOptions = - [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; - gestureRecognizerOptions.cannedGesturesClassifierOptions = [[MPPClassifierOptions alloc] init]; - gestureRecognizerOptions.cannedGesturesClassifierOptions.scoreThreshold = 0.5f; - gestureRecognizerOptions.cannedGesturesClassifierOptions.categoryAllowlist = @[ kFistLabel ]; - - const NSInteger numHands = 1; - gestureRecognizerOptions.numHands = numHands; - - MPPGestureRecognizer *gestureRecognizer = - [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; - - [self assertResultsOfRecognizeImageWithFileInfo:kFistImage - usingGestureRecognizer:gestureRecognizer - approximatelyEqualsGestureRecognizerResult: - [MPPGestureRecognizerTests fistGestureRecognizerResultWithLabel:kFistLabel]]; -} - -- (void)testRecognizeWithDenyGestureFistSucceeds { - MPPGestureRecognizerOptions *gestureRecognizerOptions = - [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; - gestureRecognizerOptions.cannedGesturesClassifierOptions = [[MPPClassifierOptions alloc] init]; - gestureRecognizerOptions.cannedGesturesClassifierOptions.scoreThreshold = 0.5f; - gestureRecognizerOptions.cannedGesturesClassifierOptions.categoryDenylist = @[ kFistLabel ]; - - const NSInteger numHands = 1; - gestureRecognizerOptions.numHands = numHands; - - MPPGestureRecognizer *gestureRecognizer = - [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; - MPPGestureRecognizerResult *gestureRecognizerResult = - [self recognizeImageWithFileInfo:kFistImage usingGestureRecognizer:gestureRecognizer]; - AssertGestureRecognizerResultIsEmpty(gestureRecognizerResult); -} - @end From f77e685ff97bbc5e8a8950e054d83fce7d13a706 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 31 May 2023 20:34:02 +0530 Subject: [PATCH 08/18] Removed unwanted header import from MPPGestureRecognizer.h --- mediapipe/tasks/ios/vision/gesture_recognizer/BUILD | 1 - .../ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h | 1 - 2 files changed, 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD index 78a07e17d..d9a76afde 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD @@ -56,7 +56,6 @@ objc_library( "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", "//mediapipe/tasks/ios/common/utils:NSStringHelpers", "//mediapipe/tasks/ios/core:MPPTaskInfo", - "//mediapipe/tasks/ios/core:MPPTaskOptions", "//mediapipe/tasks/ios/vision/core:MPPImage", "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h index ed8ff30f9..65136dc83 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h @@ -14,7 +14,6 @@ #import -#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" #import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h" #import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" From 955489d71dce7a18795aa0f1dcbc421744fbd225 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 31 May 2023 20:39:01 +0530 Subject: [PATCH 09/18] Removed a test from iOS ObjC Gesture Recognizer tests --- .../gesture_recognizer/MPPGestureRecognizerTests.m | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m index 3b9d6367f..1a48322b4 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -284,16 +284,4 @@ static const float kLandmarksErrorTolerance = 0.03f; thumbUpGestureRecognizerResult]]; } -- (void)testRecognizeWithEmptyResultsSucceeds { - MPPGestureRecognizerOptions *gestureRecognizerOptions = - [self gestureRecognizerOptionsWithModelFileInfo:kGestureRecognizerBundleAssetFile]; - - MPPGestureRecognizer *gestureRecognizer = - [self createGestureRecognizerWithOptionsSucceeds:gestureRecognizerOptions]; - - MPPGestureRecognizerResult *gestureRecognizerResult = - [self recognizeImageWithFileInfo:kNoHandsImage usingGestureRecognizer:gestureRecognizer]; - AssertGestureRecognizerResultIsEmpty(gestureRecognizerResult); -} - @end From ebeffc27eb082ebed06520d3a72951c80f21f06d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 31 May 2023 20:51:43 +0530 Subject: [PATCH 10/18] Renamed iOS gesture recognizer protobuf utils --- mediapipe/tasks/ios/test/vision/gesture_recognizer/BUILD | 2 +- .../gesture_recognizer/MPPGestureRecognizerTests.m | 9 +-------- .../tasks/ios/test/vision/gesture_recognizer/utils/BUILD | 6 +++--- ...rs.h => MPPGestureRecognizerResult+ProtobufHelpers.h} | 2 +- ....mm => MPPGestureRecognizerResult+ProtobufHelpers.mm} | 2 +- 5 files changed, 7 insertions(+), 14 deletions(-) rename mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/{MPPGestureRecognizerResult+ProtoHelpers.h => MPPGestureRecognizerResult+ProtobufHelpers.h} (95%) rename mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/{MPPGestureRecognizerResult+ProtoHelpers.mm => MPPGestureRecognizerResult+ProtobufHelpers.mm} (98%) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/BUILD b/mediapipe/tasks/ios/test/vision/gesture_recognizer/BUILD index 5be17a26c..6b53c9005 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/BUILD @@ -42,7 +42,7 @@ objc_library( "//mediapipe/tasks/ios/common:MPPCommon", "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", "//mediapipe/tasks/ios/vision/gesture_recognizer:MPPGestureRecognizer", - "//mediapipe/tasks/ios/test/vision/gesture_recognizer/utils:MPPGestureRecognizerResultProtoHelpers", + "//mediapipe/tasks/ios/test/vision/gesture_recognizer/utils:MPPGestureRecognizerResultProtobufHelpers", ] + select({ "//third_party:opencv_ios_sim_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], "//third_party:opencv_ios_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m index 1a48322b4..2fb2c8a5b 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -15,7 +15,7 @@ #import #import "mediapipe/tasks/ios/common/sources/MPPCommon.h" -#import "mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h" +#import "mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.h" #import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" #import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h" @@ -64,13 +64,6 @@ static const float kLandmarksErrorTolerance = 0.03f; XCTAssertEqualWithAccuracy(landmark.y, expectedLandmark.y, kLandmarksErrorTolerance, \ @"hand index = %d landmark index j = %d", handIndex, landmarkIndex); -#define AssertApproximatelyEqualMultiHandLandmarks(multiHandLandmarks, expectedMultiHandLandmars) \ - XCTAssertEqual(multiHandLandmarks.count, expectedMultiHandLandmars.count) \ - XCTAssertEqualWithAccuracy(landmark.x, expectedLandmark.x, kLandmarksErrorTolerance, \ - @"hand index = %d landmark index j = %d", handIndex, handIndex); \ - XCTAssertEqualWithAccuracy(landmark.y, expectedLandmark.y, kLandmarksErrorTolerance, \ - @"hand index = %d landmark index j = %d", handIndex, handIndex); - #define AssertGestureRecognizerResultIsEmpty(gestureRecognizerResult) \ XCTAssertTrue(gestureRecognizerResult.gestures.count == 0); \ XCTAssertTrue(gestureRecognizerResult.handedness.count == 0); \ diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/BUILD b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/BUILD index ddac21ed2..584d3e441 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/BUILD +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/BUILD @@ -3,9 +3,9 @@ package(default_visibility = ["//mediapipe/tasks:internal"]) licenses(["notice"]) objc_library( - name = "MPPGestureRecognizerResultProtoHelpers", - srcs = ["sources/MPPGestureRecognizerResult+ProtoHelpers.mm"], - hdrs = ["sources/MPPGestureRecognizerResult+ProtoHelpers.h"], + name = "MPPGestureRecognizerResultProtobufHelpers", + srcs = ["sources/MPPGestureRecognizerResult+ProtobufHelpers.mm"], + hdrs = ["sources/MPPGestureRecognizerResult+ProtobufHelpers.h"], copts = [ "-ObjC++", "-std=c++17", diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.h similarity index 95% rename from mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h rename to mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.h index 6bb2e5182..cfa0a5e53 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.h @@ -16,7 +16,7 @@ #import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" NS_ASSUME_NONNULL_BEGIN -@interface MPPGestureRecognizerResult (ProtoHelpers) +@interface MPPGestureRecognizerResult (ProtobufHelpers) + (MPPGestureRecognizerResult *) gestureRecognizerResultsFromTextEncodedProtobufFileWithName:(NSString *)fileName diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.mm b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm similarity index 98% rename from mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.mm rename to mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm index ce3a262b8..a4f6de5e1 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.mm +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm @@ -29,7 +29,7 @@ using LandmarksDetectionResultProto = using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt; } // anonymous namespace -@implementation MPPGestureRecognizerResult (ProtoHelpers) +@implementation MPPGestureRecognizerResult (ProtobufHelpers) + (MPPGestureRecognizerResult *) gestureRecognizerResultsFromTextEncodedProtobufFileWithName:(NSString *)fileName From 77bb5e72024f5906b5d1e1eeb3b7202aade93b48 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 31 May 2023 20:53:36 +0530 Subject: [PATCH 11/18] Fixed import in iOS gesture recognizer test utils --- .../utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm index a4f6de5e1..7cb1de12d 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#import "mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h" +#import "mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.h" #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h" From 0c2a7bee09b4d03b95af8a3ebc457c5042663542 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 5 Jun 2023 13:19:39 +0530 Subject: [PATCH 12/18] Update iOS Gesture Recognizer error assertion --- .../vision/gesture_recognizer/MPPGestureRecognizerTests.m | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m index 2fb2c8a5b..6bde8166f 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -48,10 +48,8 @@ static const float kLandmarksErrorTolerance = 0.03f; XCTAssertNotNil(error); \ XCTAssertEqualObjects(error.domain, expectedError.domain); \ XCTAssertEqual(error.code, expectedError.code); \ - XCTAssertNotEqual( \ - [error.localizedDescription rangeOfString:expectedError.localizedDescription].location, \ - NSNotFound) - + XCTAssertEqualObjects( error.localizedDescription, expectedError.localizedDescription) + #define AssertEqualGestures(gesture, expectedGesture, handIndex, gestureIndex) \ XCTAssertEqual(gesture.index, kGestureExpectedIndex, @"hand index = %d gesture index j = %d", \ handIndex, gestureIndex); \ From 56a035cb1bfddf1e123be05f90cf17976bdbf94b Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 5 Jun 2023 13:21:39 +0530 Subject: [PATCH 13/18] Updated method names in MPPGestureRecognizer --- .../gesture_recognizer/MPPGestureRecognizerTests.m | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m index 6bde8166f..43d4b6e72 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -49,7 +49,7 @@ static const float kLandmarksErrorTolerance = 0.03f; XCTAssertEqualObjects(error.domain, expectedError.domain); \ XCTAssertEqual(error.code, expectedError.code); \ XCTAssertEqualObjects( error.localizedDescription, expectedError.localizedDescription) - + #define AssertEqualGestures(gesture, expectedGesture, handIndex, gestureIndex) \ XCTAssertEqual(gesture.index, kGestureExpectedIndex, @"hand index = %d gesture index j = %d", \ handIndex, gestureIndex); \ @@ -103,7 +103,7 @@ static const float kLandmarksErrorTolerance = 0.03f; } - (void)assertMultiHandLandmarks:(NSArray *> *)multiHandLandmarks - isApproximatelyEqualToExpectedMultiHandLandmarks: + areApproximatelyEqualToExpectedMultiHandLandmarks: (NSArray *> *)expectedMultiHandLandmarks { XCTAssertEqual(multiHandLandmarks.count, expectedMultiHandLandmarks.count); if (multiHandLandmarks.count == 0) { @@ -122,7 +122,7 @@ static const float kLandmarksErrorTolerance = 0.03f; } - (void)assertMultiHandWorldLandmarks:(NSArray *> *)multiHandWorldLandmarks - isApproximatelyEqualToExpectedMultiHandWorldLandmarks: + areApproximatelyEqualToExpectedMultiHandWorldLandmarks: (NSArray *> *)expectedMultiHandWorldLandmarks { XCTAssertEqual(multiHandWorldLandmarks.count, expectedMultiHandWorldLandmarks.count); if (expectedMultiHandWorldLandmarks.count == 0) { @@ -141,7 +141,7 @@ static const float kLandmarksErrorTolerance = 0.03f; } - (void)assertMultiHandGestures:(NSArray *> *)multiHandGestures - isApproximatelyEqualToExpectedMultiHandGestures: + areApproximatelyEqualToExpectedMultiHandGestures: (NSArray *> *)expectedMultiHandGestures { XCTAssertEqual(multiHandGestures.count, expectedMultiHandGestures.count); if (multiHandGestures.count == 0) { @@ -163,12 +163,12 @@ static const float kLandmarksErrorTolerance = 0.03f; isApproximatelyEqualToExpectedResult: (MPPGestureRecognizerResult *)expectedGestureRecognizerResult { [self assertMultiHandLandmarks:gestureRecognizerResult.landmarks - isApproximatelyEqualToExpectedMultiHandLandmarks:expectedGestureRecognizerResult.landmarks]; + areApproximatelyEqualToExpectedMultiHandLandmarks:expectedGestureRecognizerResult.landmarks]; [self assertMultiHandWorldLandmarks:gestureRecognizerResult.worldLandmarks - isApproximatelyEqualToExpectedMultiHandWorldLandmarks:expectedGestureRecognizerResult + areApproximatelyEqualToExpectedMultiHandWorldLandmarks:expectedGestureRecognizerResult .worldLandmarks]; [self assertMultiHandGestures:gestureRecognizerResult.gestures - isApproximatelyEqualToExpectedMultiHandGestures:expectedGestureRecognizerResult.gestures]; + areApproximatelyEqualToExpectedMultiHandGestures:expectedGestureRecognizerResult.gestures]; } #pragma mark File From db0da30f187be47ad736d7b8c26d2947cbd91753 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 5 Jun 2023 13:26:17 +0530 Subject: [PATCH 14/18] Updated comments --- .../sources/MPPGestureRecognizerResult+ProtobufHelpers.mm | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm index 7cb1de12d..6bbade941 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtobufHelpers.mm @@ -42,9 +42,8 @@ using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt; } if (removeZPosition) { - // Remove z position of landmarks, because they are not used in correctness - // testing. For video or live stream mode, the z positions varies a lot during - // tracking from frame to frame. + // Remove z position of landmarks, because they are not used in correctness testing. For video + // or live stream mode, the z positions varies a lot during tracking from frame to frame. for (int i = 0; i < landmarkDetectionResultProto.landmarks().landmark().size(); i++) { auto &landmark = *landmarkDetectionResultProto.mutable_landmarks()->mutable_landmark(i); landmark.clear_z(); From 32195e6a8332ac426358355db28aeee9e1052186 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 5 Jun 2023 13:39:44 +0530 Subject: [PATCH 15/18] Updated MPPGestureRecognizerTests to use generics for file path dicts --- .../MPPGestureRecognizerTests.m | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m index 43d4b6e72..4331c5561 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -19,20 +19,25 @@ #import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" #import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h" + +static NSString *const kPbFileExtension = @"pbtxt"; + +typedef NSDictionary ResourceFileInfo; + static NSDictionary *const kGestureRecognizerBundleAssetFile = @{@"name" : @"gesture_recognizer", @"type" : @"task"}; -static NSDictionary *const kTwoHandsImage = @{@"name" : @"right_hands", @"type" : @"jpg"}; -static NSDictionary *const kFistImage = @{@"name" : @"fist", @"type" : @"jpg"}; -static NSDictionary *const kNoHandsImage = @{@"name" : @"cats_and_dogs", @"type" : @"jpg"}; -static NSDictionary *const kThumbUpImage = @{@"name" : @"thumb_up", @"type" : @"jpg"}; -static NSDictionary *const kPointingUpRotatedImage = +static ResourceFileInfo *const kTwoHandsImage = @{@"name" : @"right_hands", @"type" : @"jpg"}; +static ResourceFileInfo *const kFistImage = @{@"name" : @"fist", @"type" : @"jpg"}; +static ResourceFileInfo *const kNoHandsImage = @{@"name" : @"cats_and_dogs", @"type" : @"jpg"}; +static ResourceFileInfo *const kThumbUpImage = @{@"name" : @"thumb_up", @"type" : @"jpg"}; +static ResourceFileInfo *const kPointingUpRotatedImage = @{@"name" : @"pointing_up_rotated", @"type" : @"jpg"}; -static NSDictionary *const kExpectedFistLandmarksFile = - @{@"name" : @"fist_landmarks", @"type" : @"pbtxt"}; -static NSDictionary *const kExpectedThumbUpLandmarksFile = - @{@"name" : @"thumb_up_landmarks", @"type" : @"pbtxt"}; +static ResourceFileInfo *const kExpectedFistLandmarksFile = + @{@"name" : @"fist_landmarks", @"type" : kPbFileExtension}; +static ResourceFileInfo *const kExpectedThumbUpLandmarksFile = + @{@"name" : @"thumb_up_landmarks", @"type" : kPbFileExtension}; static NSString *const kFistLabel = @"Closed_Fist"; static NSString *const kExpectedThumbUpLabel = @"Thumb_Up"; From 1496b7c2d447297bc754c24d0b250659083e48f9 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 5 Jun 2023 13:40:56 +0530 Subject: [PATCH 16/18] Updated MPPGestureRecognizer tests to use generics --- .../gesture_recognizer/MPPGestureRecognizerTests.m | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m index 4331c5561..15b285b82 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -178,7 +178,7 @@ static const float kLandmarksErrorTolerance = 0.03f; #pragma mark File -+ (NSString *)filePathWithFileInfo:(NSDictionary *)fileInfo { ++ (NSString *)filePathWithFileInfo:(ResourceFileInfo *)fileInfo { NSString *filePath = [MPPGestureRecognizerTests filePathWithName:fileInfo[@"name"] extension:fileInfo[@"type"]]; return filePath; @@ -193,7 +193,7 @@ static const float kLandmarksErrorTolerance = 0.03f; #pragma mark Gesture Recognizer Initializers - (MPPGestureRecognizerOptions *)gestureRecognizerOptionsWithModelFileInfo: - (NSDictionary *)modelFileInfo { + (ResourceFileInfo *)modelFileInfo { NSString *modelPath = [MPPGestureRecognizerTests filePathWithFileInfo:modelFileInfo]; MPPGestureRecognizerOptions *gestureRecognizerOptions = [[MPPGestureRecognizerOptions alloc] init]; @@ -224,7 +224,7 @@ static const float kLandmarksErrorTolerance = 0.03f; #pragma mark Assert Gesture Recognizer Results -- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo { +- (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo { MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPGestureRecognizerTests class] fileName:fileInfo[@"name"] ofType:fileInfo[@"type"]]; @@ -233,7 +233,7 @@ static const float kLandmarksErrorTolerance = 0.03f; return image; } -- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo +- (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo orientation:(UIImageOrientation)orientation { MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPGestureRecognizerTests class] fileName:fileInfo[@"name"] @@ -244,7 +244,7 @@ static const float kLandmarksErrorTolerance = 0.03f; return image; } -- (MPPGestureRecognizerResult *)recognizeImageWithFileInfo:(NSDictionary *)imageFileInfo +- (MPPGestureRecognizerResult *)recognizeImageWithFileInfo:(ResourceFileInfo *)imageFileInfo usingGestureRecognizer: (MPPGestureRecognizer *)gestureRecognizer { MPPImage *mppImage = [self imageWithFileInfo:imageFileInfo]; @@ -255,7 +255,7 @@ static const float kLandmarksErrorTolerance = 0.03f; return gestureRecognizerResult; } -- (void)assertResultsOfRecognizeImageWithFileInfo:(NSDictionary *)fileInfo +- (void)assertResultsOfRecognizeImageWithFileInfo:(ResourceFileInfo *)fileInfo usingGestureRecognizer:(MPPGestureRecognizer *)gestureRecognizer approximatelyEqualsGestureRecognizerResult: (MPPGestureRecognizerResult *)expectedGestureRecognizerResult { From f213e0a6f3618c86acf49c83dbe85c306f656280 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 5 Jun 2023 13:47:11 +0530 Subject: [PATCH 17/18] Fixed typos --- .../MPPGestureRecognizerResult+Helpers.h | 16 +++++++-------- .../MPPGestureRecognizerResult+Helpers.mm | 20 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h index 5e75febf3..231bb16d8 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h @@ -58,14 +58,14 @@ static const int kMicroSecondsPerMilliSecond = 1000; + (MPPGestureRecognizerResult *) gestureRecognizerResultWithHandGesturesProto: (const std::vector &)handGesturesProto - handednessroto: - (const std::vector &) - handednessProto - handLandmarksPacket: - (const std::vector &) - handLandmarksProto - worldLandmarksPacket: - (const std::vector &)worldLandmarksProto + handednessProto: + (const std::vector &) + handednessProto + handLandmarksProto: + (const std::vector &) + handLandmarksProto + worldLandmarksProto: + (const std::vector &)worldLandmarksProto timestampInMilliSeconds:(NSInteger)timestampInMilliseconds; @end diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm index 5162dd891..4f3411c66 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm @@ -40,12 +40,12 @@ static const NSInteger kDefaultGestureIndex = -1; + (MPPGestureRecognizerResult *) gestureRecognizerResultWithHandGesturesProto: (const std::vector &)handGesturesProto - handednessroto: - (const std::vector &)handednessProto - handLandmarksPacket:(const std::vector &) + handednessProto: + (const std::vector &)handednessProto + handLandmarksroto:(const std::vector &) handLandmarksProto - worldLandmarksPacket: - (const std::vector &)worldLandmarksProto + worldLandmarksProto: + (const std::vector &)worldLandmarksProto timestampInMilliSeconds:(NSInteger)timestampInMilliseconds { NSMutableArray *> *multiHandGestures = [NSMutableArray arrayWithCapacity:(NSUInteger)handGesturesProto.size()]; @@ -55,7 +55,7 @@ static const NSInteger kDefaultGestureIndex = -1; arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; for (const auto &classificationProto : classificationListProto.classification()) { MPPCategory *category = [MPPCategory categoryWithProto:classificationProto - index:kDefaultGestureIndex]; + index:kDefaultGestureIndex]; [gestures addObject:category]; } [multiHandGestures addObject:gestures]; @@ -135,11 +135,11 @@ static const NSInteger kDefaultGestureIndex = -1; return [MPPGestureRecognizerResult gestureRecognizerResultWithHandGesturesProto:handGesturesPacket .Get>() - handednessroto:handednessPacket + handednessProto:handednessPacket .Get>() - handLandmarksPacket:handLandmarksPacket - .Get>() - worldLandmarksPacket:worldLandmarksPacket + handLandmarksProto:handLandmarksPacket.Get< + std::vector>() + worldLandmarksProto:worldLandmarksPacket .Get>() timestampInMilliSeconds:timestampInMilliseconds]; } From d256a3e67074f09562c54040da54216f39c04ada Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 5 Jun 2023 21:13:04 +0530 Subject: [PATCH 18/18] Updated dictionary to generics in iOS gesture recognizer tests --- .../test/vision/gesture_recognizer/MPPGestureRecognizerTests.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m index 15b285b82..55180f477 100644 --- a/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m +++ b/mediapipe/tasks/ios/test/vision/gesture_recognizer/MPPGestureRecognizerTests.m @@ -24,7 +24,7 @@ static NSString *const kPbFileExtension = @"pbtxt"; typedef NSDictionary ResourceFileInfo; -static NSDictionary *const kGestureRecognizerBundleAssetFile = +static ResourceFileInfo *const kGestureRecognizerBundleAssetFile = @{@"name" : @"gesture_recognizer", @"type" : @"task"}; static ResourceFileInfo *const kTwoHandsImage = @{@"name" : @"right_hands", @"type" : @"jpg"};