From eff56045e40198c569cd43b13c77b3711b2c5579 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 12 Jun 2023 19:56:20 +0530 Subject: [PATCH 1/2] Added hand landmarker protobuf utils --- .../test/vision/hand_landmarker/utils/BUILD | 21 +++++++ .../MPPHandLandmarkerResult+ProtobufHelpers.h | 27 +++++++++ ...MPPHandLandmarkerResult+ProtobufHelpers.mm | 59 +++++++++++++++++++ 3 files changed, 107 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/hand_landmarker/utils/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.h create mode 100644 mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.mm diff --git a/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/BUILD b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/BUILD new file mode 100644 index 000000000..b5b215785 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/BUILD @@ -0,0 +1,21 @@ +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPHandLandmarkerResultProtobufHelpers", + srcs = ["sources/MPPHandLandmarkerResult+ProtobufHelpers.mm"], + hdrs = ["sources/MPPHandLandmarkerResult+ProtobufHelpers.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + "//mediapipe/tasks/ios/test/vision/utils:parse_proto_utils", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/tasks/cc/components/containers/proto:landmarks_detection_result_cc_proto", + "//mediapipe/tasks/ios/vision/hand_landmarker/utils:MPPHandLandmarkerResultHelpers", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.h b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.h new file mode 100644 index 000000000..2e979e1fb --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.h @@ -0,0 +1,27 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarkerResult.h" + +NS_ASSUME_NONNULL_BEGIN +@interface MPPHandLandmarkerResult (ProtobufHelpers) + ++ (MPPHandLandmarkerResult *) + handLandmarkerResultFromTextEncodedProtobufFileWithName:(NSString *)fileName + shouldRemoveZPosition:(BOOL)removeZPosition; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.mm b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.mm new file mode 100644 index 000000000..36bf6f08c --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.mm @@ -0,0 +1,59 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+Helpers.h" + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/tasks/cc/components/containers/proto/landmarks_detection_result.pb.h" +#include "mediapipe/tasks/ios/test/vision/utils/sources/parse_proto_utils.h" + +namespace { +using ClassificationListProto = ::mediapipe::ClassificationList; +using ClassificationProto = ::mediapipe::Classification; +using LandmarksDetectionResultProto = + ::mediapipe::tasks::containers::proto::LandmarksDetectionResult; +using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt; +} // anonymous namespace + +@implementation MPPHandLandmarkerResult (ProtobufHelpers) + ++ (MPPHandLandmarkerResult *) + handLandmarkerResultFromTextEncodedProtobufFileWithName:(NSString *)fileName + shouldRemoveZPosition:(BOOL)removeZPosition { + LandmarksDetectionResultProto landmarkDetectionResultProto; + + if (!get_proto_from_pbtxt(fileName.cppString, landmarkDetectionResultProto).ok()) { + return nil; + } + + if (removeZPosition) { + // Remove z position of landmarks, because they are not used in correctness testing. For video + // or live stream mode, the z positions varies a lot during tracking from frame to frame. + for (int i = 0; i < landmarkDetectionResultProto.landmarks().landmark().size(); i++) { + auto &landmark = *landmarkDetectionResultProto.mutable_landmarks()->mutable_landmark(i); + landmark.clear_z(); + } + } + + return [MPPHandLandmarkerResult + handLandmarkerResultWithLandmarksProto:{landmarkDetectionResultProto.landmarks()} + worldLandmarksProto:{landmarkDetectionResultProto.world_landmarks()} + handednessProto:{landmarkDetectionResultProto.classifications()} + timestampInMilliSeconds:0]; +} + +@end From baa79046b93757ee044f8daecc0fe846d7a461ff Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 12 Jun 2023 19:56:34 +0530 Subject: [PATCH 2/2] Added iOS Objective C hand landmarker tests --- .../ios/test/vision/hand_landmarker/BUILD | 62 ++++ .../hand_landmarker/MPPHandLandmarkerTests.m | 292 ++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/hand_landmarker/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m diff --git a/mediapipe/tasks/ios/test/vision/hand_landmarker/BUILD b/mediapipe/tasks/ios/test/vision/hand_landmarker/BUILD new file mode 100644 index 000000000..eaa2f0642 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/hand_landmarker/BUILD @@ -0,0 +1,62 @@ +load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "//mediapipe/framework/tool:ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPHandLandmarkerObjcTestLibrary", + testonly = 1, + srcs = ["MPPHandLandmarkerTests.m"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + data = [ + "//mediapipe/tasks/testdata/vision:test_models", + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_protos", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/test/vision/hand_landmarker/utils:MPPHandLandmarkerResultProtobufHelpers", + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + "//mediapipe/tasks/ios/vision/hand_landmarker:MPPHandLandmarker", + ] + select({ + "//third_party:opencv_ios_sim_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_x86_64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//conditions:default": ["@ios_opencv//:OpencvFramework"], + }), +) + +ios_unit_test( + name = "MPPHandLandmarkerObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPHandLandmarkerObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m b/mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m new file mode 100644 index 000000000..f9bdeb150 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/hand_landmarker/MPPHandLandmarkerTests.m @@ -0,0 +1,292 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/test/vision/hand_landmarker/utils/sources/MPPHandLandmarkerResult+ProtobufHelpers.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +#import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarker.h" + +static NSString *const kPbFileExtension = @"pbtxt"; + +typedef NSDictionary ResourceFileInfo; + +static ResourceFileInfo *const kHandLandmarkerBundleAssetFile = + @{@"name" : @"hand_landmarker", @"type" : @"task"}; + +static ResourceFileInfo *const kTwoHandsImage = @{@"name" : @"right_hands", @"type" : @"jpg"}; +static ResourceFileInfo *const kNoHandsImage = @{@"name" : @"cats_and_dogs", @"type" : @"jpg"}; +static ResourceFileInfo *const kThumbUpImage = @{@"name" : @"thumb_up", @"type" : @"jpg"}; +static ResourceFileInfo *const kPointingUpRotatedImage = + @{@"name" : @"pointing_up_rotated", @"type" : @"jpg"}; + +static ResourceFileInfo *const kExpectedThumbUpLandmarksFile = + @{@"name" : @"thumb_up_landmarks", @"type" : kPbFileExtension}; +static ResourceFileInfo *const kExpectedPointingUpRotatedLandmarksFile = + @{@"name" : @"pointing_up_rotated_landmarks", @"type" : kPbFileExtension}; + + +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; +static const float kLandmarksErrorTolerance = 0.03f; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertEqualObjects(error.localizedDescription, expectedError.localizedDescription) + +#define AssertApproximatelyEqualLandmarks(landmark, expectedLandmark, handIndex, landmarkIndex) \ + XCTAssertEqualWithAccuracy(landmark.x, expectedLandmark.x, kLandmarksErrorTolerance, \ + @"hand index = %d landmark index j = %d", handIndex, landmarkIndex); \ + XCTAssertEqualWithAccuracy(landmark.y, expectedLandmark.y, kLandmarksErrorTolerance, \ + @"hand index = %d landmark index j = %d", handIndex, landmarkIndex); + +#define AssertHandLandmarkerResultIsEmpty(handLandmarkerResult) \ + XCTAssertTrue(handLandmarkerResult.handedness.count == 0); \ + XCTAssertTrue(handLandmarkerResult.landmarks.count == 0); \ + XCTAssertTrue(handLandmarkerResult.worldLandmarks.count == 0); + +@interface MPPHandLandmarkerTests : XCTestCase +@end + +@implementation MPPHandLandmarkerTests + +#pragma mark Results + ++ (MPPHandLandmarkerResult *)emptyHandLandmarkerResult { + return [[MPPHandLandmarkerResult alloc] initWithLandmarks:@[] + worldLandmarks:@[] + handedness:@[] + + timestampInMilliseconds:0]; +} + ++ (MPPHandLandmarkerResult *)thumbUpHandLandmarkerResult { + NSString *filePath = + [MPPHandLandmarkerTests filePathWithFileInfo:kExpectedThumbUpLandmarksFile]; + + return [MPPHandLandmarkerResult + handLandmarkerResultFromTextEncodedProtobufFileWithName:filePath + shouldRemoveZPosition:YES]; +} + ++ (MPPHandLandmarkerResult *)pointingUpRotatedHandLandmarkerResult { + NSString *filePath = + [MPPHandLandmarkerTests filePathWithFileInfo:kExpectedPointingUpRotatedLandmarksFile]; + + return [MPPHandLandmarkerResult + handLandmarkerResultFromTextEncodedProtobufFileWithName:filePath + shouldRemoveZPosition:YES]; +} + +- (void)assertMultiHandLandmarks:(NSArray *> *)multiHandLandmarks + areApproximatelyEqualToExpectedMultiHandLandmarks: + (NSArray *> *)expectedMultiHandLandmarks { + XCTAssertEqual(multiHandLandmarks.count, expectedMultiHandLandmarks.count); + if (multiHandLandmarks.count == 0) { + return; + } + + NSArray *topHandLandmarks = multiHandLandmarks[0]; + NSArray *expectedTopHandLandmarks = expectedMultiHandLandmarks[0]; + + XCTAssertEqual(topHandLandmarks.count, expectedTopHandLandmarks.count); + for (int i = 0; i < expectedTopHandLandmarks.count; i++) { + MPPNormalizedLandmark *landmark = topHandLandmarks[i]; + XCTAssertNotNil(landmark); + AssertApproximatelyEqualLandmarks(landmark, expectedTopHandLandmarks[i], 0, i); + } +} + +- (void)assertMultiHandWorldLandmarks:(NSArray *> *)multiHandWorldLandmarks + areApproximatelyEqualToExpectedMultiHandWorldLandmarks: + (NSArray *> *)expectedMultiHandWorldLandmarks { + XCTAssertEqual(multiHandWorldLandmarks.count, expectedMultiHandWorldLandmarks.count); + if (expectedMultiHandWorldLandmarks.count == 0) { + return; + } + + NSArray *topHandWorldLandmarks = multiHandWorldLandmarks[0]; + NSArray *expectedTopHandWorldLandmarks = expectedMultiHandWorldLandmarks[0]; + + XCTAssertEqual(topHandWorldLandmarks.count, expectedTopHandWorldLandmarks.count); + for (int i = 0; i < expectedTopHandWorldLandmarks.count; i++) { + MPPLandmark *landmark = topHandWorldLandmarks[i]; + XCTAssertNotNil(landmark); + AssertApproximatelyEqualLandmarks(landmark, expectedTopHandWorldLandmarks[i], 0, i); + } +} + +- (void)assertHandLandmarkerResult:(MPPHandLandmarkerResult *)handLandmarkerResult + isApproximatelyEqualToExpectedResult: + (MPPHandLandmarkerResult *)expectedHandLandmarkerResult { + [self assertMultiHandLandmarks:handLandmarkerResult.landmarks + areApproximatelyEqualToExpectedMultiHandLandmarks:expectedHandLandmarkerResult.landmarks]; + [self assertMultiHandWorldLandmarks:handLandmarkerResult.worldLandmarks + areApproximatelyEqualToExpectedMultiHandWorldLandmarks:expectedHandLandmarkerResult + .worldLandmarks]; +} + +#pragma mark File + ++ (NSString *)filePathWithFileInfo:(ResourceFileInfo *)fileInfo { + NSString *filePath = [MPPHandLandmarkerTests filePathWithName:fileInfo[@"name"] + extension:fileInfo[@"type"]]; + return filePath; +} + ++ (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension { + NSString *filePath = [[NSBundle bundleForClass:self.class] pathForResource:fileName + ofType:extension]; + return filePath; +} + +#pragma mark Gesture Recognizer Initializers + +- (MPPHandLandmarkerOptions *)handLandmarkerOptionsWithModelFileInfo: + (ResourceFileInfo *)modelFileInfo { + NSString *modelPath = [MPPHandLandmarkerTests filePathWithFileInfo:modelFileInfo]; + MPPHandLandmarkerOptions *handLandmarkerOptions = + [[MPPHandLandmarkerOptions alloc] init]; + handLandmarkerOptions.baseOptions.modelAssetPath = modelPath; + + return handLandmarkerOptions; +} + +- (MPPHandLandmarker *)createHandLandmarkerWithOptionsSucceeds: + (MPPHandLandmarkerOptions *)handLandmarkerOptions { + MPPHandLandmarker *handLandmarker = + [[MPPHandLandmarker alloc] initWithOptions:handLandmarkerOptions error:nil]; + XCTAssertNotNil(handLandmarker); + + return handLandmarker; +} + +- (void)assertCreateHandLandmarkerWithOptions: + (MPPHandLandmarkerOptions *)handLandmarkerOptions + failsWithExpectedError:(NSError *)expectedError { + NSError *error = nil; + MPPHandLandmarker *handLandmarker = + [[MPPHandLandmarker alloc] initWithOptions:handLandmarkerOptions error:&error]; + + XCTAssertNil(handLandmarkerOptions); + AssertEqualErrors(error, expectedError); +} + +#pragma mark Assert Gesture Recognizer Results + +- (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo { + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPHandLandmarkerTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"]]; + XCTAssertNotNil(image); + + return image; +} + +- (MPPImage *)imageWithFileInfo:(ResourceFileInfo *)fileInfo + orientation:(UIImageOrientation)orientation { + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPHandLandmarkerTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"] + orientation:orientation]; + XCTAssertNotNil(image); + + return image; +} + +- (MPPHandLandmarkerResult *)detectInImageWithFileInfo:(ResourceFileInfo *)imageFileInfo + usingHandLandmarker: + (MPPHandLandmarker *)handLandmarker { + MPPImage *mppImage = [self imageWithFileInfo:imageFileInfo]; + MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectInImage:mppImage + error:nil]; + XCTAssertNotNil(handLandmarkerResult); + + return handLandmarkerResult; +} + +- (void)assertResultsOfDetectInImageWithFileInfo:(ResourceFileInfo *)fileInfo + usingHandLandmarker:(MPPHandLandmarker *)handLandmarker + approximatelyEqualsHandLandmarkerResult: + (MPPHandLandmarkerResult *)expectedHandLandmarkerResult { + MPPHandLandmarkerResult *handLandmarkerResult = + [self detectInImageWithFileInfo:fileInfo usingHandLandmarker:handLandmarker]; + [self assertHandLandmarkerResult:handLandmarkerResult + isApproximatelyEqualToExpectedResult:expectedHandLandmarkerResult]; +} + +#pragma mark General Tests + +- (void)testDetectWithModelPathSucceeds { + NSString *modelPath = + [MPPHandLandmarkerTests filePathWithFileInfo:kHandLandmarkerBundleAssetFile]; + MPPHandLandmarker *handLandmarker = + [[MPPHandLandmarker alloc] initWithModelPath:modelPath error:nil]; + XCTAssertNotNil(handLandmarker); + + [self assertResultsOfDetectInImageWithFileInfo:kThumbUpImage + usingHandLandmarker:handLandmarker + approximatelyEqualsHandLandmarkerResult:[MPPHandLandmarkerTests + thumbUpHandLandmarkerResult]]; +} + +- (void)testDetectWithEmptyResultsSucceeds { + MPPHandLandmarkerOptions *handLandmarkerOptions = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + + MPPHandLandmarker *handLandmarker = + [self createHandLandmarkerWithOptionsSucceeds:handLandmarkerOptions]; + + MPPHandLandmarkerResult *handLandmarkerResult = + [self detectInImageWithFileInfo:kNoHandsImage usingHandLandmarker:handLandmarker]; + AssertHandLandmarkerResultIsEmpty(handLandmarkerResult); +} + +- (void)testDetectWithNumHandsSucceeds { + MPPHandLandmarkerOptions *handLandmarkerOptions = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + + const NSInteger numHands = 2; + handLandmarkerOptions.numHands = numHands; + + MPPHandLandmarker *handLandmarker = + [self createHandLandmarkerWithOptionsSucceeds:handLandmarkerOptions]; + + MPPHandLandmarkerResult *handLandmarkerResult = + [self detectInImageWithFileInfo:kTwoHandsImage usingHandLandmarker:handLandmarker]; + + XCTAssertTrue(handLandmarkerResult.handedness.count == numHands); +} + +- (void)testDetectWithRotationSucceeds { + MPPHandLandmarkerOptions *handLandmarkerOptions = + [self handLandmarkerOptionsWithModelFileInfo:kHandLandmarkerBundleAssetFile]; + + MPPHandLandmarker *handLandmarker = + [self createHandLandmarkerWithOptionsSucceeds:handLandmarkerOptions]; + + MPPImage *mppImage = [self imageWithFileInfo:kPointingUpRotatedImage + orientation:UIImageOrientationRight]; + + MPPHandLandmarkerResult *handLandmarkerResult = [handLandmarker detectInImage:mppImage + error:nil]; + + [self assertHandLandmarkerResult:handLandmarkerResult + isApproximatelyEqualToExpectedResult:[MPPHandLandmarkerTests pointingUpRotatedHandLandmarkerResult]]; + +} + +@end