Merge bece7c1abc into 759e9fd56e
This commit is contained in:
priankakariatyml
GitHub
commit
023ce50ff2
|
|
@ -19,8 +19,30 @@ NS_ASSUME_NONNULL_BEGIN
|
|||
|
||||
@interface MPPCategory (Helpers)
|
||||
|
||||
/**
|
||||
* Creates an `MPPCategory` with the given MediaPipe `Classification` proto.
|
||||
*
|
||||
* @param classificationProto A MediaPipe `Classification` proto.
|
||||
* @return An `MPPCategory` object that with the given MediaPipe `Classification` proto.
|
||||
*/
|
||||
+ (MPPCategory *)categoryWithProto:(const ::mediapipe::Classification &)classificationProto;
|
||||
|
||||
/**
|
||||
* Creates an `MPPCategory` with the given MediaPipe `Classification` proto and the given category
|
||||
* index. The resulting `MPPCategory` is created with the given category index instead of the
|
||||
* category index specified in the `Classification` proto. This method is useful for tasks like
|
||||
* gesture recognizer which always returns a default index for the recognized gestures.
|
||||
*
|
||||
* @param classificationProto A MediaPipe `Classification` proto.
|
||||
* @param index The index to be used for creating the `MPPCategory` instead of the category index
|
||||
* specified in the `Classification` proto.
|
||||
*
|
||||
* @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition
|
||||
* results.
|
||||
*/
|
||||
+ (MPPCategory *)categoryWithProto:(const ::mediapipe::Classification &)classificationProto
|
||||
index:(NSInteger)index;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
|
|||
|
|
@ -21,7 +21,8 @@ using ClassificationProto = ::mediapipe::Classification;
|
|||
|
||||
@implementation MPPCategory (Helpers)
|
||||
|
||||
+ (MPPCategory *)categoryWithProto:(const ClassificationProto &)classificationProto {
|
||||
+ (MPPCategory *)categoryWithProto:(const ClassificationProto &)classificationProto
|
||||
index:(NSInteger)index {
|
||||
NSString *categoryName;
|
||||
NSString *displayName;
|
||||
|
||||
|
|
@ -33,10 +34,14 @@ using ClassificationProto = ::mediapipe::Classification;
|
|||
displayName = [NSString stringWithCppString:classificationProto.display_name()];
|
||||
}
|
||||
|
||||
return [[MPPCategory alloc] initWithIndex:classificationProto.index()
|
||||
return [[MPPCategory alloc] initWithIndex:index
|
||||
score:classificationProto.score()
|
||||
categoryName:categoryName
|
||||
displayName:displayName];
|
||||
}
|
||||
|
||||
+ (MPPCategory *)categoryWithProto:(const ClassificationProto &)classificationProto {
|
||||
return [MPPCategory categoryWithProto:classificationProto index:classificationProto.index()];
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
|
|||
62
mediapipe/tasks/ios/test/vision/gesture_recognizer/BUILD
Normal file
62
mediapipe/tasks/ios/test/vision/gesture_recognizer/BUILD
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test")
|
||||
load(
|
||||
"//mediapipe/framework/tool:ios.bzl",
|
||||
"MPP_TASK_MINIMUM_OS_VERSION",
|
||||
)
|
||||
load(
|
||||
"@org_tensorflow//tensorflow/lite:special_rules.bzl",
|
||||
"tflite_ios_lab_runner",
|
||||
)
|
||||
|
||||
package(default_visibility = ["//mediapipe/tasks:internal"])
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
# Default tags for filtering iOS targets. Targets are restricted to Apple platforms.
|
||||
TFL_DEFAULT_TAGS = [
|
||||
"apple",
|
||||
]
|
||||
|
||||
# Following sanitizer tests are not supported by iOS test targets.
|
||||
TFL_DISABLED_SANITIZER_TAGS = [
|
||||
"noasan",
|
||||
"nomsan",
|
||||
"notsan",
|
||||
]
|
||||
|
||||
objc_library(
|
||||
name = "MPPGestureRecognizerObjcTestLibrary",
|
||||
testonly = 1,
|
||||
srcs = ["MPPGestureRecognizerTests.m"],
|
||||
copts = [
|
||||
"-ObjC++",
|
||||
"-std=c++17",
|
||||
"-x objective-c++",
|
||||
],
|
||||
data = [
|
||||
"//mediapipe/tasks/testdata/vision:test_images",
|
||||
"//mediapipe/tasks/testdata/vision:gesture_recognizer.task",
|
||||
"//mediapipe/tasks/testdata/vision:test_protos",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/tasks/ios/common:MPPCommon",
|
||||
"//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils",
|
||||
"//mediapipe/tasks/ios/vision/gesture_recognizer:MPPGestureRecognizer",
|
||||
"//mediapipe/tasks/ios/test/vision/gesture_recognizer/utils:MPPGestureRecognizerResultProtoHelpers",
|
||||
] + select({
|
||||
"//third_party:opencv_ios_sim_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"],
|
||||
"//third_party:opencv_ios_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"],
|
||||
"//third_party:opencv_ios_x86_64_source_build": ["@ios_opencv_source//:opencv_xcframework"],
|
||||
"//conditions:default": ["@ios_opencv//:OpencvFramework"],
|
||||
}),
|
||||
)
|
||||
|
||||
ios_unit_test(
|
||||
name = "MPPGestureRecognizerObjcTest",
|
||||
minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION,
|
||||
runner = tflite_ios_lab_runner("IOS_LATEST"),
|
||||
tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS,
|
||||
deps = [
|
||||
":MPPGestureRecognizerObjcTestLibrary",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,287 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <XCTest/XCTest.h>
|
||||
|
||||
#import "mediapipe/tasks/ios/common/sources/MPPCommon.h"
|
||||
#import "mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h"
|
||||
#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h"
|
||||
#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h"
|
||||
|
||||
static NSDictionary *const kGestureRecognizerBundleAssetFile =
|
||||
@{@"name" : @"gesture_recognizer", @"type" : @"task"};
|
||||
|
||||
static NSDictionary *const kTwoHandsImage = @{@"name" : @"right_hands", @"type" : @"jpg"};
|
||||
static NSDictionary *const kFistImage = @{@"name" : @"fist", @"type" : @"jpg"};
|
||||
static NSDictionary *const kNoHandsImage = @{@"name" : @"cats_and_dogs", @"type" : @"jpg"};
|
||||
static NSDictionary *const kThumbUpImage = @{@"name" : @"thumb_up", @"type" : @"jpg"};
|
||||
static NSDictionary *const kPointingUpRotatedImage =
|
||||
@{@"name" : @"pointing_up_rotated", @"type" : @"jpg"};
|
||||
|
||||
static NSDictionary *const kExpectedFistLandmarksFile =
|
||||
@{@"name" : @"fist_landmarks", @"type" : @"pbtxt"};
|
||||
static NSDictionary *const kExpectedThumbUpLandmarksFile =
|
||||
@{@"name" : @"thumb_up_landmarks", @"type" : @"pbtxt"};
|
||||
|
||||
static NSString *const kFistLabel = @"Closed_Fist";
|
||||
static NSString *const kExpectedThumbUpLabel = @"Thumb_Up";
|
||||
static NSString *const kExpectedPointingUpLabel = @"Pointing_Up";
|
||||
static NSString *const kRockLabel = @"Rock";
|
||||
|
||||
static const NSInteger kGestureExpectedIndex = -1;
|
||||
|
||||
static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks";
|
||||
static const float kLandmarksErrorTolerance = 0.03f;
|
||||
|
||||
#define AssertEqualErrors(error, expectedError) \
|
||||
XCTAssertNotNil(error); \
|
||||
XCTAssertEqualObjects(error.domain, expectedError.domain); \
|
||||
XCTAssertEqual(error.code, expectedError.code); \
|
||||
XCTAssertNotEqual( \
|
||||
[error.localizedDescription rangeOfString:expectedError.localizedDescription].location, \
|
||||
NSNotFound)
|
||||
|
||||
#define AssertEqualGestures(gesture, expectedGesture, handIndex, gestureIndex) \
|
||||
XCTAssertEqual(gesture.index, kGestureExpectedIndex, @"hand index = %d gesture index j = %d", \
|
||||
handIndex, gestureIndex); \
|
||||
XCTAssertEqualObjects(gesture.categoryName, expectedGesture.categoryName, \
|
||||
@"hand index = %d gesture index j = %d", handIndex, gestureIndex);
|
||||
|
||||
#define AssertApproximatelyEqualLandmarks(landmark, expectedLandmark, handIndex, landmarkIndex) \
|
||||
XCTAssertEqualWithAccuracy(landmark.x, expectedLandmark.x, kLandmarksErrorTolerance, \
|
||||
@"hand index = %d landmark index j = %d", handIndex, landmarkIndex); \
|
||||
XCTAssertEqualWithAccuracy(landmark.y, expectedLandmark.y, kLandmarksErrorTolerance, \
|
||||
@"hand index = %d landmark index j = %d", handIndex, landmarkIndex);
|
||||
|
||||
#define AssertApproximatelyEqualMultiHandLandmarks(multiHandLandmarks, expectedMultiHandLandmars) \
|
||||
XCTAssertEqual(multiHandLandmarks.count, expectedMultiHandLandmars.count) \
|
||||
XCTAssertEqualWithAccuracy(landmark.x, expectedLandmark.x, kLandmarksErrorTolerance, \
|
||||
@"hand index = %d landmark index j = %d", handIndex, handIndex); \
|
||||
XCTAssertEqualWithAccuracy(landmark.y, expectedLandmark.y, kLandmarksErrorTolerance, \
|
||||
@"hand index = %d landmark index j = %d", handIndex, handIndex);
|
||||
|
||||
#define AssertGestureRecognizerResultIsEmpty(gestureRecognizerResult) \
|
||||
XCTAssertTrue(gestureRecognizerResult.gestures.count == 0); \
|
||||
XCTAssertTrue(gestureRecognizerResult.handedness.count == 0); \
|
||||
XCTAssertTrue(gestureRecognizerResult.landmarks.count == 0); \
|
||||
XCTAssertTrue(gestureRecognizerResult.worldLandmarks.count == 0);
|
||||
|
||||
@interface MPPGestureRecognizerTests : XCTestCase
|
||||
@end
|
||||
|
||||
@implementation MPPGestureRecognizerTests
|
||||
|
||||
#pragma mark Results
|
||||
|
||||
+ (MPPGestureRecognizerResult *)emptyGestureRecognizerResult {
|
||||
return [[MPPGestureRecognizerResult alloc] initWithGestures:@[]
|
||||
handedness:@[]
|
||||
landmarks:@[]
|
||||
worldLandmarks:@[]
|
||||
timestampInMilliseconds:0];
|
||||
}
|
||||
|
||||
+ (MPPGestureRecognizerResult *)thumbUpGestureRecognizerResult {
|
||||
NSString *filePath =
|
||||
[MPPGestureRecognizerTests filePathWithFileInfo:kExpectedThumbUpLandmarksFile];
|
||||
|
||||
return [MPPGestureRecognizerResult
|
||||
gestureRecognizerResultsFromTextEncodedProtobufFileWithName:filePath
|
||||
gestureLabel:kExpectedThumbUpLabel
|
||||
shouldRemoveZPosition:YES];
|
||||
}
|
||||
|
||||
+ (MPPGestureRecognizerResult *)fistGestureRecognizerResultWithLabel:(NSString *)gestureLabel {
|
||||
NSString *filePath = [MPPGestureRecognizerTests filePathWithFileInfo:kExpectedFistLandmarksFile];
|
||||
|
||||
return [MPPGestureRecognizerResult
|
||||
gestureRecognizerResultsFromTextEncodedProtobufFileWithName:filePath
|
||||
gestureLabel:gestureLabel
|
||||
shouldRemoveZPosition:YES];
|
||||
}
|
||||
|
||||
- (void)assertMultiHandLandmarks:(NSArray<NSArray<MPPNormalizedLandmark *> *> *)multiHandLandmarks
|
||||
isApproximatelyEqualToExpectedMultiHandLandmarks:
|
||||
(NSArray<NSArray<MPPNormalizedLandmark *> *> *)expectedMultiHandLandmarks {
|
||||
XCTAssertEqual(multiHandLandmarks.count, expectedMultiHandLandmarks.count);
|
||||
if (multiHandLandmarks.count == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
NSArray<MPPNormalizedLandmark *> *topHandLandmarks = multiHandLandmarks[0];
|
||||
NSArray<MPPNormalizedLandmark *> *expectedTopHandLandmarks = expectedMultiHandLandmarks[0];
|
||||
|
||||
XCTAssertEqual(topHandLandmarks.count, expectedTopHandLandmarks.count);
|
||||
for (int i = 0; i < expectedTopHandLandmarks.count; i++) {
|
||||
MPPNormalizedLandmark *landmark = topHandLandmarks[i];
|
||||
XCTAssertNotNil(landmark);
|
||||
AssertApproximatelyEqualLandmarks(landmark, expectedTopHandLandmarks[i], 0, i);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)assertMultiHandWorldLandmarks:(NSArray<NSArray<MPPLandmark *> *> *)multiHandWorldLandmarks
|
||||
isApproximatelyEqualToExpectedMultiHandWorldLandmarks:
|
||||
(NSArray<NSArray<MPPLandmark *> *> *)expectedMultiHandWorldLandmarks {
|
||||
XCTAssertEqual(multiHandWorldLandmarks.count, expectedMultiHandWorldLandmarks.count);
|
||||
if (expectedMultiHandWorldLandmarks.count == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
NSArray<MPPLandmark *> *topHandWorldLandmarks = multiHandWorldLandmarks[0];
|
||||
NSArray<MPPLandmark *> *expectedTopHandWorldLandmarks = expectedMultiHandWorldLandmarks[0];
|
||||
|
||||
XCTAssertEqual(topHandWorldLandmarks.count, expectedTopHandWorldLandmarks.count);
|
||||
for (int i = 0; i < expectedTopHandWorldLandmarks.count; i++) {
|
||||
MPPLandmark *landmark = topHandWorldLandmarks[i];
|
||||
XCTAssertNotNil(landmark);
|
||||
AssertApproximatelyEqualLandmarks(landmark, expectedTopHandWorldLandmarks[i], 0, i);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)assertMultiHandGestures:(NSArray<NSArray<MPPCategory *> *> *)multiHandGestures
|
||||
isApproximatelyEqualToExpectedMultiHandGestures:
|
||||
(NSArray<NSArray<MPPCategory *> *> *)expectedMultiHandGestures {
|
||||
XCTAssertEqual(multiHandGestures.count, expectedMultiHandGestures.count);
|
||||
if (multiHandGestures.count == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
NSArray<MPPCategory *> *topHandGestures = multiHandGestures[0];
|
||||
NSArray<MPPCategory *> *expectedTopHandGestures = expectedMultiHandGestures[0];
|
||||
|
||||
XCTAssertEqual(topHandGestures.count, expectedTopHandGestures.count);
|
||||
for (int i = 0; i < expectedTopHandGestures.count; i++) {
|
||||
MPPCategory *gesture = topHandGestures[i];
|
||||
XCTAssertNotNil(gesture);
|
||||
AssertEqualGestures(gesture, expectedTopHandGestures[i], 0, i);
|
||||
}
|
||||
}
|
||||
|
||||
- (void)assertGestureRecognizerResult:(MPPGestureRecognizerResult *)gestureRecognizerResult
|
||||
isApproximatelyEqualToExpectedResult:
|
||||
(MPPGestureRecognizerResult *)expectedGestureRecognizerResult {
|
||||
[self assertMultiHandLandmarks:gestureRecognizerResult.landmarks
|
||||
isApproximatelyEqualToExpectedMultiHandLandmarks:expectedGestureRecognizerResult.landmarks];
|
||||
[self assertMultiHandWorldLandmarks:gestureRecognizerResult.worldLandmarks
|
||||
isApproximatelyEqualToExpectedMultiHandWorldLandmarks:expectedGestureRecognizerResult
|
||||
.worldLandmarks];
|
||||
[self assertMultiHandGestures:gestureRecognizerResult.gestures
|
||||
isApproximatelyEqualToExpectedMultiHandGestures:expectedGestureRecognizerResult.gestures];
|
||||
}
|
||||
|
||||
#pragma mark File
|
||||
|
||||
+ (NSString *)filePathWithFileInfo:(NSDictionary *)fileInfo {
|
||||
NSString *filePath = [MPPGestureRecognizerTests filePathWithName:fileInfo[@"name"]
|
||||
extension:fileInfo[@"type"]];
|
||||
return filePath;
|
||||
}
|
||||
|
||||
+ (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension {
|
||||
NSString *filePath = [[NSBundle bundleForClass:self.class] pathForResource:fileName
|
||||
ofType:extension];
|
||||
return filePath;
|
||||
}
|
||||
|
||||
#pragma mark Gesture Recognizer Initializers
|
||||
|
||||
- (MPPGestureRecognizerOptions *)gestureRecognizerOptionsWithModelFileInfo:
|
||||
(NSDictionary *)modelFileInfo {
|
||||
NSString *modelPath = [MPPGestureRecognizerTests filePathWithFileInfo:modelFileInfo];
|
||||
MPPGestureRecognizerOptions *gestureRecognizerOptions =
|
||||
[[MPPGestureRecognizerOptions alloc] init];
|
||||
gestureRecognizerOptions.baseOptions.modelAssetPath = modelPath;
|
||||
|
||||
return gestureRecognizerOptions;
|
||||
}
|
||||
|
||||
- (MPPGestureRecognizer *)createGestureRecognizerWithOptionsSucceeds:
|
||||
(MPPGestureRecognizerOptions *)gestureRecognizerOptions {
|
||||
MPPGestureRecognizer *gestureRecognizer =
|
||||
[[MPPGestureRecognizer alloc] initWithOptions:gestureRecognizerOptions error:nil];
|
||||
XCTAssertNotNil(gestureRecognizer);
|
||||
|
||||
return gestureRecognizer;
|
||||
}
|
||||
|
||||
- (void)assertCreateGestureRecognizerWithOptions:
|
||||
(MPPGestureRecognizerOptions *)gestureRecognizerOptions
|
||||
failsWithExpectedError:(NSError *)expectedError {
|
||||
NSError *error = nil;
|
||||
MPPGestureRecognizer *gestureRecognizer =
|
||||
[[MPPGestureRecognizer alloc] initWithOptions:gestureRecognizerOptions error:&error];
|
||||
|
||||
XCTAssertNil(gestureRecognizer);
|
||||
AssertEqualErrors(error, expectedError);
|
||||
}
|
||||
|
||||
#pragma mark Assert Gesture Recognizer Results
|
||||
|
||||
- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo {
|
||||
MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPGestureRecognizerTests class]
|
||||
fileName:fileInfo[@"name"]
|
||||
ofType:fileInfo[@"type"]];
|
||||
XCTAssertNotNil(image);
|
||||
|
||||
return image;
|
||||
}
|
||||
|
||||
- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo
|
||||
orientation:(UIImageOrientation)orientation {
|
||||
MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPGestureRecognizerTests class]
|
||||
fileName:fileInfo[@"name"]
|
||||
ofType:fileInfo[@"type"]
|
||||
orientation:orientation];
|
||||
XCTAssertNotNil(image);
|
||||
|
||||
return image;
|
||||
}
|
||||
|
||||
- (MPPGestureRecognizerResult *)recognizeImageWithFileInfo:(NSDictionary *)imageFileInfo
|
||||
usingGestureRecognizer:
|
||||
(MPPGestureRecognizer *)gestureRecognizer {
|
||||
MPPImage *mppImage = [self imageWithFileInfo:imageFileInfo];
|
||||
MPPGestureRecognizerResult *gestureRecognizerResult = [gestureRecognizer recognizeImage:mppImage
|
||||
error:nil];
|
||||
XCTAssertNotNil(gestureRecognizerResult);
|
||||
|
||||
return gestureRecognizerResult;
|
||||
}
|
||||
|
||||
- (void)assertResultsOfRecognizeImageWithFileInfo:(NSDictionary *)fileInfo
|
||||
usingGestureRecognizer:(MPPGestureRecognizer *)gestureRecognizer
|
||||
approximatelyEqualsGestureRecognizerResult:
|
||||
(MPPGestureRecognizerResult *)expectedGestureRecognizerResult {
|
||||
MPPGestureRecognizerResult *gestureRecognizerResult =
|
||||
[self recognizeImageWithFileInfo:fileInfo usingGestureRecognizer:gestureRecognizer];
|
||||
[self assertGestureRecognizerResult:gestureRecognizerResult
|
||||
isApproximatelyEqualToExpectedResult:expectedGestureRecognizerResult];
|
||||
}
|
||||
|
||||
#pragma mark General Tests
|
||||
|
||||
- (void)testRecognizeWithModelPathSucceeds {
|
||||
NSString *modelPath =
|
||||
[MPPGestureRecognizerTests filePathWithFileInfo:kGestureRecognizerBundleAssetFile];
|
||||
MPPGestureRecognizer *gestureRecognizer =
|
||||
[[MPPGestureRecognizer alloc] initWithModelPath:modelPath error:nil];
|
||||
XCTAssertNotNil(gestureRecognizer);
|
||||
|
||||
[self assertResultsOfRecognizeImageWithFileInfo:kThumbUpImage
|
||||
usingGestureRecognizer:gestureRecognizer
|
||||
approximatelyEqualsGestureRecognizerResult:[MPPGestureRecognizerTests
|
||||
thumbUpGestureRecognizerResult]];
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
package(default_visibility = ["//mediapipe/tasks:internal"])
|
||||
|
||||
licenses(["notice"])
|
||||
|
||||
objc_library(
|
||||
name = "MPPGestureRecognizerResultProtoHelpers",
|
||||
srcs = ["sources/MPPGestureRecognizerResult+ProtoHelpers.mm"],
|
||||
hdrs = ["sources/MPPGestureRecognizerResult+ProtoHelpers.h"],
|
||||
copts = [
|
||||
"-ObjC++",
|
||||
"-std=c++17",
|
||||
"-x objective-c++",
|
||||
],
|
||||
deps = [
|
||||
"//mediapipe/tasks/ios/test/vision/utils:parse_proto_utils",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/tasks/cc/components/containers/proto:landmarks_detection_result_cc_proto",
|
||||
"//mediapipe/tasks/ios/vision/gesture_recognizer/utils:MPPGestureRecognizerResultHelpers",
|
||||
"//mediapipe/tasks/ios/common/utils:NSStringHelpers",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
// Copyright 2022 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h"
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
@interface MPPGestureRecognizerResult (ProtoHelpers)
|
||||
|
||||
+ (MPPGestureRecognizerResult *)
|
||||
gestureRecognizerResultsFromTextEncodedProtobufFileWithName:(NSString *)fileName
|
||||
gestureLabel:(NSString *)gestureLabel
|
||||
shouldRemoveZPosition:(BOOL)removeZPosition;
|
||||
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "mediapipe/tasks/ios/test/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+ProtoHelpers.h"
|
||||
|
||||
#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h"
|
||||
#import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h"
|
||||
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/tasks/cc/components/containers/proto/landmarks_detection_result.pb.h"
|
||||
#include "mediapipe/tasks/ios/test/vision/utils/sources/parse_proto_utils.h"
|
||||
|
||||
namespace {
|
||||
using ClassificationListProto = ::mediapipe::ClassificationList;
|
||||
using ClassificationProto = ::mediapipe::Classification;
|
||||
using LandmarksDetectionResultProto =
|
||||
::mediapipe::tasks::containers::proto::LandmarksDetectionResult;
|
||||
using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt;
|
||||
} // namespace
|
||||
|
||||
@implementation MPPGestureRecognizerResult (ProtoHelpers)
|
||||
|
||||
+ (MPPGestureRecognizerResult *)
|
||||
gestureRecognizerResultsFromTextEncodedProtobufFileWithName:(NSString *)fileName
|
||||
gestureLabel:(NSString *)gestureLabel
|
||||
shouldRemoveZPosition:(BOOL)removeZPosition {
|
||||
LandmarksDetectionResultProto landmarkDetectionResultProto;
|
||||
|
||||
if (!get_proto_from_pbtxt(fileName.cppString, landmarkDetectionResultProto).ok()) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
if (removeZPosition) {
|
||||
// Remove z position of landmarks, because they are not used in correctness
|
||||
// testing. For video or live stream mode, the z positions varies a lot during
|
||||
// tracking from frame to frame.
|
||||
for (int i = 0; i < landmarkDetectionResultProto.landmarks().landmark().size(); i++) {
|
||||
auto &landmark = *landmarkDetectionResultProto.mutable_landmarks()->mutable_landmark(i);
|
||||
landmark.clear_z();
|
||||
}
|
||||
}
|
||||
|
||||
ClassificationListProto gesturesProto;
|
||||
ClassificationProto *classificationProto = gesturesProto.add_classification();
|
||||
classificationProto->set_label([gestureLabel UTF8String]);
|
||||
|
||||
return [MPPGestureRecognizerResult
|
||||
gestureRecognizerResultWithHandGesturesProto:{gesturesProto}
|
||||
handednessroto:{landmarkDetectionResultProto.classifications()}
|
||||
handLandmarksPacket:{landmarkDetectionResultProto.landmarks()}
|
||||
worldLandmarksPacket:{landmarkDetectionResultProto.world_landmarks()}
|
||||
timestampInMilliSeconds:0];
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -58,38 +58,57 @@ NS_ASSUME_NONNULL_BEGIN
|
|||
error:(NSError **)error NS_DESIGNATED_INITIALIZER;
|
||||
|
||||
/**
|
||||
* Creates a `NormalizedRect` from a region of interest and an image orientation, performing
|
||||
* sanity checks on-the-fly.
|
||||
* If the input region of interest equals `CGRectZero`, returns a default `NormalizedRect` covering
|
||||
* the whole image with rotation set according `imageOrientation`. If `ROIAllowed` is NO, an error
|
||||
* will be returned if the input region of interest is not equal to `CGRectZero`. Mirrored
|
||||
* orientations (`UIImageOrientationUpMirrored`,`UIImageOrientationDownMirrored`,
|
||||
* Creates a `NormalizedRect` from image orientation for a task which does not support roi,
|
||||
* performing sanity checks on-the-fly. Mirrored orientations
|
||||
* (`UIImageOrientationUpMirrored`,`UIImageOrientationDownMirrored`,
|
||||
* `UIImageOrientationLeftMirrored`,`UIImageOrientationRightMirrored`) are not supported. An error
|
||||
* will be returned if `imageOrientation` is equal to any one of them.
|
||||
*
|
||||
* @param roi A `CGRect` specifying the region of interest. If the input region of interest equals
|
||||
* `CGRectZero`, the returned `NormalizedRect` covers the whole image. Make sure that `roi` equals
|
||||
* `CGRectZero` if `ROIAllowed` is NO. Otherwise, an error will be returned.
|
||||
* @param imageSize A `CGSize` specifying the size of the image within which normalized rect is
|
||||
* calculated.
|
||||
* @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the
|
||||
* image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise.
|
||||
* Mirrored orientations (`UIImageOrientationUpMirrored`, `UIImageOrientationDownMirrored`,
|
||||
* `UIImageOrientationLeftMirrored`, `UIImageOrientationRightMirrored`) are not supported. An error
|
||||
* will be returned if `imageOrientation` is equal to any one of them.
|
||||
* @param ROIAllowed Indicates if the `roi` field is allowed to be a value other than `CGRectZero`.
|
||||
* @param imageSize A `CGSize` specifying the size of the image within which normalized rect is
|
||||
* calculated.
|
||||
* @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no
|
||||
* error will be saved.
|
||||
*
|
||||
* @return An optional `NormalizedRect` from the given region of interest and image orientation.
|
||||
*/
|
||||
- (std::optional<mediapipe::NormalizedRect>)normalizedRectWithImageOrientation:
|
||||
(UIImageOrientation)imageOrientation
|
||||
imageSize:(CGSize)imageSize
|
||||
error:(NSError **)error;
|
||||
|
||||
/**
|
||||
* Creates a `NormalizedRect` from roi and image orientation for a task which supports roi,
|
||||
* performing sanity checks on-the-fly. If the input region of interest equals `CGRectZero`, returns
|
||||
* a default `NormalizedRect` covering the whole image with rotation set according
|
||||
* `imageOrientation`. Mirrored orientations
|
||||
* (`UIImageOrientationUpMirrored`,`UIImageOrientationDownMirrored`,
|
||||
* `UIImageOrientationLeftMirrored`,`UIImageOrientationRightMirrored`) are not supported. An error
|
||||
* will be returned if `imageOrientation` is equal to any one of them.
|
||||
*
|
||||
* @param roi A `CGRect` specifying the region of interest. If the input region of interest equals
|
||||
* `CGRectZero`, the returned `NormalizedRect` covers the whole image.
|
||||
* @param imageOrientation A `UIImageOrientation` indicating the rotation to be applied to the
|
||||
* image. The resulting `NormalizedRect` will convert the `imageOrientation` to degrees clockwise.
|
||||
* Mirrored orientations (`UIImageOrientationUpMirrored`, `UIImageOrientationDownMirrored`,
|
||||
* `UIImageOrientationLeftMirrored`, `UIImageOrientationRightMirrored`) are not supported. An error
|
||||
* will be returned if `imageOrientation` is equal to any one of them.
|
||||
* @param imageSize A `CGSize` specifying the size of the image within which normalized rect is
|
||||
* calculated.
|
||||
* @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no
|
||||
* error will be saved.
|
||||
*
|
||||
* @return An optional `NormalizedRect` from the given region of interest and image orientation.
|
||||
*/
|
||||
- (std::optional<mediapipe::NormalizedRect>)
|
||||
normalizedRectFromRegionOfInterest:(CGRect)roi
|
||||
imageSize:(CGSize)imageSize
|
||||
normalizedRectWithRegionOfInterest:(CGRect)roi
|
||||
imageOrientation:(UIImageOrientation)imageOrientation
|
||||
ROIAllowed:(BOOL)ROIAllowed
|
||||
imageSize:(CGSize)imageSize
|
||||
error:(NSError **)error;
|
||||
|
||||
/**
|
||||
* A synchronous method to invoke the C++ task runner to process single image inputs. The call
|
||||
* blocks the current thread until a failure status or a successful result is returned.
|
||||
|
|
|
|||
|
|
@ -91,7 +91,30 @@ static NSString *const kTaskPrefix = @"com.mediapipe.tasks.vision";
|
|||
return self;
|
||||
}
|
||||
|
||||
- (std::optional<NormalizedRect>)normalizedRectFromRegionOfInterest:(CGRect)roi
|
||||
- (std::optional<NormalizedRect>)normalizedRectWithRegionOfInterest:(CGRect)roi
|
||||
imageOrientation:
|
||||
(UIImageOrientation)imageOrientation
|
||||
imageSize:(CGSize)imageSize
|
||||
error:(NSError **)error {
|
||||
return [self normalizedRectWithRegionOfInterest:roi
|
||||
imageSize:imageSize
|
||||
imageOrientation:imageOrientation
|
||||
ROIAllowed:YES
|
||||
error:error];
|
||||
}
|
||||
|
||||
- (std::optional<NormalizedRect>)normalizedRectWithImageOrientation:
|
||||
(UIImageOrientation)imageOrientation
|
||||
imageSize:(CGSize)imageSize
|
||||
error:(NSError **)error {
|
||||
return [self normalizedRectWithRegionOfInterest:CGRectZero
|
||||
imageSize:imageSize
|
||||
imageOrientation:imageOrientation
|
||||
ROIAllowed:NO
|
||||
error:error];
|
||||
}
|
||||
|
||||
- (std::optional<NormalizedRect>)normalizedRectWithRegionOfInterest:(CGRect)roi
|
||||
imageSize:(CGSize)imageSize
|
||||
imageOrientation:
|
||||
(UIImageOrientation)imageOrientation
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ objc_library(
|
|||
|
||||
objc_library(
|
||||
name = "MPPGestureRecognizer",
|
||||
srcs = ["sources/MPPGestureRecognizer.mm"],
|
||||
hdrs = ["sources/MPPGestureRecognizer.h"],
|
||||
copts = [
|
||||
"-ObjC++",
|
||||
|
|
@ -51,7 +52,15 @@ objc_library(
|
|||
deps = [
|
||||
":MPPGestureRecognizerOptions",
|
||||
":MPPGestureRecognizerResult",
|
||||
"//mediapipe/tasks/cc/vision/gesture_recognizer:gesture_recognizer_graph",
|
||||
"//mediapipe/tasks/ios/common/utils:MPPCommonUtils",
|
||||
"//mediapipe/tasks/ios/common/utils:NSStringHelpers",
|
||||
"//mediapipe/tasks/ios/core:MPPTaskInfo",
|
||||
"//mediapipe/tasks/ios/core:MPPTaskOptions",
|
||||
"//mediapipe/tasks/ios/vision/core:MPPImage",
|
||||
"//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator",
|
||||
"//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner",
|
||||
"//mediapipe/tasks/ios/vision/gesture_recognizer/utils:MPPGestureRecognizerOptionsHelpers",
|
||||
"//mediapipe/tasks/ios/vision/gesture_recognizer/utils:MPPGestureRecognizerResultHelpers",
|
||||
],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,273 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h"
|
||||
|
||||
#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h"
|
||||
#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h"
|
||||
#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h"
|
||||
#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h"
|
||||
#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h"
|
||||
#import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h"
|
||||
#import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h"
|
||||
|
||||
namespace {
|
||||
using ::mediapipe::NormalizedRect;
|
||||
using ::mediapipe::Packet;
|
||||
using ::mediapipe::Timestamp;
|
||||
using ::mediapipe::tasks::core::PacketMap;
|
||||
using ::mediapipe::tasks::core::PacketsCallback;
|
||||
} // namespace
|
||||
|
||||
static NSString *const kImageTag = @"IMAGE";
|
||||
static NSString *const kImageInStreamName = @"image_in";
|
||||
static NSString *const kNormRectTag = @"NORM_RECT";
|
||||
static NSString *const kNormRectInStreamName = @"norm_rect_in";
|
||||
static NSString *const kImageOutStreamName = @"image_out";
|
||||
static NSString *const kLandmarksTag = @"LANDMARKS";
|
||||
static NSString *const kLandmarksOutStreamName = @"hand_landmarks";
|
||||
static NSString *const kWorldLandmarksTag = @"WORLD_LANDMARKS";
|
||||
static NSString *const kWorldLandmarksOutStreamName = @"world_hand_landmarks";
|
||||
static NSString *const kHandednessTag = @"HANDEDNESS";
|
||||
static NSString *const kHandednessOutStreamName = @"handedness";
|
||||
static NSString *const kHandGesturesTag = @"HAND_GESTURES";
|
||||
static NSString *const kHandGesturesOutStreamName = @"hand_gestures";
|
||||
static NSString *const kTaskGraphName =
|
||||
@"mediapipe.tasks.vision.gesture_recognizer.GestureRecognizerGraph";
|
||||
static NSString *const kTaskName = @"gestureRecognizer";
|
||||
|
||||
#define InputPacketMap(imagePacket, normalizedRectPacket) \
|
||||
{ \
|
||||
{kImageInStreamName.cppString, imagePacket}, { \
|
||||
kNormRectInStreamName.cppString, normalizedRectPacket \
|
||||
} \
|
||||
}
|
||||
|
||||
@interface MPPGestureRecognizer () {
|
||||
/** iOS Vision Task Runner */
|
||||
MPPVisionTaskRunner *_visionTaskRunner;
|
||||
dispatch_queue_t _callbackQueue;
|
||||
}
|
||||
@property(nonatomic, weak) id<MPPGestureRecognizerLiveStreamDelegate>
|
||||
gestureRecognizerLiveStreamDelegate;
|
||||
@end
|
||||
|
||||
@implementation MPPGestureRecognizer
|
||||
|
||||
- (nullable MPPGestureRecognizerResult *)gestureRecognizerResultWithOutputPacketMap:
|
||||
(PacketMap &)outputPacketMap {
|
||||
return [MPPGestureRecognizerResult
|
||||
gestureRecognizerResultWithHandGesturesPacket:outputPacketMap[kHandGesturesOutStreamName
|
||||
.cppString]
|
||||
handednessPacket:outputPacketMap[kHandednessOutStreamName
|
||||
.cppString]
|
||||
handLandmarksPacket:outputPacketMap[kLandmarksOutStreamName
|
||||
.cppString]
|
||||
worldLandmarksPacket:outputPacketMap[kWorldLandmarksOutStreamName
|
||||
.cppString]];
|
||||
}
|
||||
|
||||
- (void)processLiveStreamResult:(absl::StatusOr<PacketMap>)liveStreamResult {
|
||||
if (![self.gestureRecognizerLiveStreamDelegate
|
||||
respondsToSelector:@selector(gestureRecognizer:
|
||||
didFinishRecognitionWithResult:timestampInMilliseconds:error:)]) {
|
||||
return;
|
||||
}
|
||||
|
||||
NSError *callbackError = nil;
|
||||
if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) {
|
||||
dispatch_async(_callbackQueue, ^{
|
||||
[self.gestureRecognizerLiveStreamDelegate gestureRecognizer:self
|
||||
didFinishRecognitionWithResult:nil
|
||||
timestampInMilliseconds:Timestamp::Unset().Value()
|
||||
error:callbackError];
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
PacketMap &outputPacketMap = liveStreamResult.value();
|
||||
if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
MPPGestureRecognizerResult *result =
|
||||
[self gestureRecognizerResultWithOutputPacketMap:outputPacketMap];
|
||||
|
||||
NSInteger timeStampInMilliseconds =
|
||||
outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() /
|
||||
kMicroSecondsPerMilliSecond;
|
||||
dispatch_async(_callbackQueue, ^{
|
||||
[self.gestureRecognizerLiveStreamDelegate gestureRecognizer:self
|
||||
didFinishRecognitionWithResult:result
|
||||
timestampInMilliseconds:timeStampInMilliseconds
|
||||
error:callbackError];
|
||||
});
|
||||
}
|
||||
|
||||
- (instancetype)initWithOptions:(MPPGestureRecognizerOptions *)options error:(NSError **)error {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc]
|
||||
initWithTaskGraphName:kTaskGraphName
|
||||
inputStreams:@[
|
||||
[NSString stringWithFormat:@"%@:%@", kImageTag, kImageInStreamName],
|
||||
[NSString stringWithFormat:@"%@:%@", kNormRectTag, kNormRectInStreamName]
|
||||
]
|
||||
outputStreams:@[
|
||||
[NSString stringWithFormat:@"%@:%@", kLandmarksTag, kLandmarksOutStreamName],
|
||||
[NSString
|
||||
stringWithFormat:@"%@:%@", kWorldLandmarksTag, kWorldLandmarksOutStreamName],
|
||||
[NSString stringWithFormat:@"%@:%@", kHandednessTag, kHandednessOutStreamName],
|
||||
[NSString
|
||||
stringWithFormat:@"%@:%@", kHandGesturesTag, kHandGesturesOutStreamName],
|
||||
[NSString stringWithFormat:@"%@:%@", kImageTag, kImageOutStreamName]
|
||||
]
|
||||
taskOptions:options
|
||||
enableFlowLimiting:options.runningMode == MPPRunningModeLiveStream
|
||||
error:error];
|
||||
|
||||
if (!taskInfo) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
PacketsCallback packetsCallback = nullptr;
|
||||
|
||||
if (options.gestureRecognizerLiveStreamDelegate) {
|
||||
_gestureRecognizerLiveStreamDelegate = options.gestureRecognizerLiveStreamDelegate;
|
||||
|
||||
// Create a private serial dispatch queue in which the deleagte method will be called
|
||||
// asynchronously. This is to ensure that if the client performs a long running operation in
|
||||
// the delegate method, the queue on which the C++ callbacks is invoked is not blocked and is
|
||||
// freed up to continue with its operations.
|
||||
_callbackQueue = dispatch_queue_create(
|
||||
[MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName], NULL);
|
||||
|
||||
// Capturing `self` as weak in order to avoid `self` being kept in memory
|
||||
// and cause a retain cycle, after self is set to `nil`.
|
||||
MPPGestureRecognizer *__weak weakSelf = self;
|
||||
packetsCallback = [=](absl::StatusOr<PacketMap> liveStreamResult) {
|
||||
[weakSelf processLiveStreamResult:liveStreamResult];
|
||||
};
|
||||
}
|
||||
|
||||
_visionTaskRunner =
|
||||
[[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig]
|
||||
runningMode:options.runningMode
|
||||
packetsCallback:std::move(packetsCallback)
|
||||
error:error];
|
||||
if (!_visionTaskRunner) {
|
||||
return nil;
|
||||
}
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error {
|
||||
MPPGestureRecognizerOptions *options = [[MPPGestureRecognizerOptions alloc] init];
|
||||
|
||||
options.baseOptions.modelAssetPath = modelPath;
|
||||
|
||||
return [self initWithOptions:options error:error];
|
||||
}
|
||||
|
||||
- (nullable MPPGestureRecognizerResult *)gestureRecognizerResultWithOptionalOutputPacketMap:
|
||||
(std::optional<PacketMap> &)outputPacketMap {
|
||||
if (!outputPacketMap.has_value()) {
|
||||
return nil;
|
||||
}
|
||||
MPPGestureRecognizerResult *result =
|
||||
[self gestureRecognizerResultWithOutputPacketMap:outputPacketMap.value()];
|
||||
return result;
|
||||
}
|
||||
|
||||
- (nullable MPPGestureRecognizerResult *)recognizeImage:(MPPImage *)image error:(NSError **)error {
|
||||
std::optional<NormalizedRect> rect =
|
||||
[_visionTaskRunner normalizedRectWithImageOrientation:image.orientation
|
||||
imageSize:CGSizeMake(image.width, image.height)
|
||||
error:error];
|
||||
if (!rect.has_value()) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error];
|
||||
if (imagePacket.IsEmpty()) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
Packet normalizedRectPacket =
|
||||
[MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()];
|
||||
|
||||
PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket);
|
||||
|
||||
std::optional<PacketMap> outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap
|
||||
error:error];
|
||||
return [self gestureRecognizerResultWithOptionalOutputPacketMap:outputPacketMap];
|
||||
}
|
||||
|
||||
- (std::optional<PacketMap>)inputPacketMapWithMPPImage:(MPPImage *)image
|
||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
||||
error:(NSError **)error {
|
||||
std::optional<NormalizedRect> rect =
|
||||
[_visionTaskRunner normalizedRectWithImageOrientation:image.orientation
|
||||
imageSize:CGSizeMake(image.width, image.height)
|
||||
error:error];
|
||||
if (!rect.has_value()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image
|
||||
timestampInMilliseconds:timestampInMilliseconds
|
||||
error:error];
|
||||
if (imagePacket.IsEmpty()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
Packet normalizedRectPacket =
|
||||
[MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()
|
||||
timestampInMilliseconds:timestampInMilliseconds];
|
||||
|
||||
PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket);
|
||||
return inputPacketMap;
|
||||
}
|
||||
|
||||
- (nullable MPPGestureRecognizerResult *)recognizeVideoFrame:(MPPImage *)image
|
||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
||||
error:(NSError **)error {
|
||||
std::optional<PacketMap> inputPacketMap = [self inputPacketMapWithMPPImage:image
|
||||
timestampInMilliseconds:timestampInMilliseconds
|
||||
error:error];
|
||||
if (!inputPacketMap.has_value()) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
std::optional<PacketMap> outputPacketMap =
|
||||
[_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error];
|
||||
|
||||
return [self gestureRecognizerResultWithOptionalOutputPacketMap:outputPacketMap];
|
||||
}
|
||||
|
||||
- (BOOL)recognizeAsyncImage:(MPPImage *)image
|
||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
||||
error:(NSError **)error {
|
||||
std::optional<PacketMap> inputPacketMap = [self inputPacketMapWithMPPImage:image
|
||||
timestampInMilliseconds:timestampInMilliseconds
|
||||
error:error];
|
||||
if (!inputPacketMap.has_value()) {
|
||||
return NO;
|
||||
}
|
||||
|
||||
return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error];
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
@ -14,6 +14,8 @@
|
|||
|
||||
#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h"
|
||||
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
|
|
@ -26,11 +28,11 @@ static const int kMicroSecondsPerMilliSecond = 1000;
|
|||
* Creates an `MPPGestureRecognizerResult` from hand gestures, handedness, hand landmarks and world
|
||||
* landmarks packets.
|
||||
*
|
||||
* @param handGesturesPacket a MediaPipe packet wrapping a`std::vector<ClassificationListProto>`.
|
||||
* @param handednessPacket a MediaPipe packet wrapping a`std::vector<ClassificationListProto>`.
|
||||
* @param handLandmarksPacket a MediaPipe packet wrapping
|
||||
* @param handGesturesPacket A MediaPipe packet wrapping a`std::vector<ClassificationListProto>`.
|
||||
* @param handednessPacket A MediaPipe packet wrapping a`std::vector<ClassificationListProto>`.
|
||||
* @param handLandmarksPacket A MediaPipe packet wrapping
|
||||
* a`std::vector<NormalizedlandmarkListProto>`.
|
||||
* @param handLandmarksPacket a MediaPipe packet wrapping a`std::vector<LandmarkListProto>`.
|
||||
* @param worldLandmarksPacket A MediaPipe packet wrapping a`std::vector<LandmarkListProto>`.
|
||||
*
|
||||
* @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition
|
||||
* results.
|
||||
|
|
@ -41,6 +43,30 @@ static const int kMicroSecondsPerMilliSecond = 1000;
|
|||
handLandmarksPacket:(const mediapipe::Packet &)handLandmarksPacket
|
||||
worldLandmarksPacket:(const mediapipe::Packet &)worldLandmarksPacket;
|
||||
|
||||
/**
|
||||
* Creates an `MPPGestureRecognizerResult` from hand gestures, handedness, hand landmarks and world
|
||||
* landmarks proto vectors.
|
||||
*
|
||||
* @param handGesturesProto A vector of protos of type `std::vector<ClassificationListProto>`.
|
||||
* @param handednessPacket A vector of protos of type `std::vector<ClassificationListProto>`.
|
||||
* @param handLandmarksPacket A vector of protos of type `std::vector<NormalizedlandmarkListProto>`.
|
||||
* @param handLandmarksPacket A vector of protos of type `std::vector<LandmarkListProto>`.
|
||||
*
|
||||
* @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition
|
||||
* results.
|
||||
*/
|
||||
+ (MPPGestureRecognizerResult *)
|
||||
gestureRecognizerResultWithHandGesturesProto:
|
||||
(const std::vector<mediapipe::ClassificationList> &)handGesturesProto
|
||||
handednessroto:
|
||||
(const std::vector<mediapipe::ClassificationList> &)
|
||||
handednessProto
|
||||
handLandmarksPacket:
|
||||
(const std::vector<mediapipe::NormalizedLandmarkList> &)
|
||||
handLandmarksProto
|
||||
worldLandmarksPacket:
|
||||
(const std::vector<mediapipe::LandmarkList> &)worldLandmarksProto
|
||||
timestampInMilliSeconds:(NSInteger)timestampInMilliseconds;
|
||||
@end
|
||||
|
||||
NS_ASSUME_NONNULL_END
|
||||
|
|
|
|||
|
|
@ -17,10 +17,6 @@
|
|||
#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h"
|
||||
#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h"
|
||||
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
|
||||
namespace {
|
||||
using ClassificationListProto = ::mediapipe::ClassificationList;
|
||||
using LandmarkListProto = ::mediapipe::LandmarkList;
|
||||
|
|
@ -28,17 +24,12 @@ using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList;
|
|||
using ::mediapipe::Packet;
|
||||
} // namespace
|
||||
|
||||
static const NSInteger kDefaultGestureIndex = -1;
|
||||
|
||||
@implementation MPPGestureRecognizerResult (Helpers)
|
||||
|
||||
+ (MPPGestureRecognizerResult *)
|
||||
gestureRecognizerResultWithHandGesturesPacket:(const Packet &)handGesturesPacket
|
||||
handednessPacket:(const Packet &)handednessPacket
|
||||
handLandmarksPacket:(const Packet &)handLandmarksPacket
|
||||
worldLandmarksPacket:(const Packet &)worldLandmarksPacket {
|
||||
NSInteger timestampInMilliseconds =
|
||||
(NSInteger)(handGesturesPacket.Timestamp().Value() / kMicroSecondsPerMilliSecond);
|
||||
|
||||
if (handGesturesPacket.IsEmpty()) {
|
||||
+ (MPPGestureRecognizerResult *)emptyGestureRecognizerResultWithTimestampInMilliseconds:
|
||||
(NSInteger)timestampInMilliseconds {
|
||||
return [[MPPGestureRecognizerResult alloc] initWithGestures:@[]
|
||||
handedness:@[]
|
||||
landmarks:@[]
|
||||
|
|
@ -46,34 +37,34 @@ using ::mediapipe::Packet;
|
|||
timestampInMilliseconds:timestampInMilliseconds];
|
||||
}
|
||||
|
||||
if (!handGesturesPacket.ValidateAsType<std::vector<ClassificationListProto>>().ok() ||
|
||||
!handednessPacket.ValidateAsType<std::vector<ClassificationListProto>>().ok() ||
|
||||
!handLandmarksPacket.ValidateAsType<std::vector<NormalizedLandmarkListProto>>().ok() ||
|
||||
!worldLandmarksPacket.ValidateAsType<std::vector<LandmarkListProto>>().ok()) {
|
||||
return nil;
|
||||
}
|
||||
|
||||
const std::vector<ClassificationListProto> &handGesturesClassificationListProtos =
|
||||
handGesturesPacket.Get<std::vector<ClassificationListProto>>();
|
||||
+ (MPPGestureRecognizerResult *)
|
||||
gestureRecognizerResultWithHandGesturesProto:
|
||||
(const std::vector<ClassificationListProto> &)handGesturesProto
|
||||
handednessroto:
|
||||
(const std::vector<ClassificationListProto> &)handednessProto
|
||||
handLandmarksPacket:(const std::vector<NormalizedLandmarkListProto> &)
|
||||
handLandmarksProto
|
||||
worldLandmarksPacket:
|
||||
(const std::vector<LandmarkListProto> &)worldLandmarksProto
|
||||
timestampInMilliSeconds:(NSInteger)timestampInMilliseconds {
|
||||
NSMutableArray<NSMutableArray<MPPCategory *> *> *multiHandGestures =
|
||||
[NSMutableArray arrayWithCapacity:(NSUInteger)handGesturesClassificationListProtos.size()];
|
||||
[NSMutableArray arrayWithCapacity:(NSUInteger)handGesturesProto.size()];
|
||||
|
||||
for (const auto &classificationListProto : handGesturesClassificationListProtos) {
|
||||
for (const auto &classificationListProto : handGesturesProto) {
|
||||
NSMutableArray<MPPCategory *> *gestures = [NSMutableArray
|
||||
arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()];
|
||||
for (const auto &classificationProto : classificationListProto.classification()) {
|
||||
MPPCategory *category = [MPPCategory categoryWithProto:classificationProto];
|
||||
MPPCategory *category = [MPPCategory categoryWithProto:classificationProto
|
||||
index:kDefaultGestureIndex];
|
||||
[gestures addObject:category];
|
||||
}
|
||||
[multiHandGestures addObject:gestures];
|
||||
}
|
||||
|
||||
const std::vector<ClassificationListProto> &handednessClassificationListProtos =
|
||||
handednessPacket.Get<std::vector<ClassificationListProto>>();
|
||||
NSMutableArray<NSMutableArray<MPPCategory *> *> *multiHandHandedness =
|
||||
[NSMutableArray arrayWithCapacity:(NSUInteger)handednessClassificationListProtos.size()];
|
||||
[NSMutableArray arrayWithCapacity:(NSUInteger)handednessProto.size()];
|
||||
|
||||
for (const auto &classificationListProto : handednessClassificationListProtos) {
|
||||
for (const auto &classificationListProto : handednessProto) {
|
||||
NSMutableArray<MPPCategory *> *handedness = [NSMutableArray
|
||||
arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()];
|
||||
for (const auto &classificationProto : classificationListProto.classification()) {
|
||||
|
|
@ -83,12 +74,10 @@ using ::mediapipe::Packet;
|
|||
[multiHandHandedness addObject:handedness];
|
||||
}
|
||||
|
||||
const std::vector<NormalizedLandmarkListProto> &handLandmarkListProtos =
|
||||
handLandmarksPacket.Get<std::vector<NormalizedLandmarkListProto>>();
|
||||
NSMutableArray<NSMutableArray<MPPNormalizedLandmark *> *> *multiHandLandmarks =
|
||||
[NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProtos.size()];
|
||||
[NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarksProto.size()];
|
||||
|
||||
for (const auto &handLandmarkListProto : handLandmarkListProtos) {
|
||||
for (const auto &handLandmarkListProto : handLandmarksProto) {
|
||||
NSMutableArray<MPPNormalizedLandmark *> *handLandmarks =
|
||||
[NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProto.landmark().size()];
|
||||
for (const auto &normalizedLandmarkProto : handLandmarkListProto.landmark()) {
|
||||
|
|
@ -99,12 +88,10 @@ using ::mediapipe::Packet;
|
|||
[multiHandLandmarks addObject:handLandmarks];
|
||||
}
|
||||
|
||||
const std::vector<LandmarkListProto> &worldLandmarkListProtos =
|
||||
worldLandmarksPacket.Get<std::vector<LandmarkListProto>>();
|
||||
NSMutableArray<NSMutableArray<MPPLandmark *> *> *multiHandWorldLandmarks =
|
||||
[NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProtos.size()];
|
||||
[NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarksProto.size()];
|
||||
|
||||
for (const auto &worldLandmarkListProto : worldLandmarkListProtos) {
|
||||
for (const auto &worldLandmarkListProto : worldLandmarksProto) {
|
||||
NSMutableArray<MPPLandmark *> *worldLandmarks =
|
||||
[NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProto.landmark().size()];
|
||||
for (const auto &landmarkProto : worldLandmarkListProto.landmark()) {
|
||||
|
|
@ -124,4 +111,37 @@ using ::mediapipe::Packet;
|
|||
return gestureRecognizerResult;
|
||||
}
|
||||
|
||||
+ (MPPGestureRecognizerResult *)
|
||||
gestureRecognizerResultWithHandGesturesPacket:(const Packet &)handGesturesPacket
|
||||
handednessPacket:(const Packet &)handednessPacket
|
||||
handLandmarksPacket:(const Packet &)handLandmarksPacket
|
||||
worldLandmarksPacket:(const Packet &)worldLandmarksPacket {
|
||||
NSInteger timestampInMilliseconds =
|
||||
(NSInteger)(handGesturesPacket.Timestamp().Value() / kMicroSecondsPerMilliSecond);
|
||||
|
||||
if (handGesturesPacket.IsEmpty()) {
|
||||
return [MPPGestureRecognizerResult
|
||||
emptyGestureRecognizerResultWithTimestampInMilliseconds:timestampInMilliseconds];
|
||||
}
|
||||
|
||||
if (!handGesturesPacket.ValidateAsType<std::vector<ClassificationListProto>>().ok() ||
|
||||
!handednessPacket.ValidateAsType<std::vector<ClassificationListProto>>().ok() ||
|
||||
!handLandmarksPacket.ValidateAsType<std::vector<NormalizedLandmarkListProto>>().ok() ||
|
||||
!worldLandmarksPacket.ValidateAsType<std::vector<LandmarkListProto>>().ok()) {
|
||||
return [MPPGestureRecognizerResult
|
||||
emptyGestureRecognizerResultWithTimestampInMilliseconds:timestampInMilliseconds];
|
||||
}
|
||||
|
||||
return [MPPGestureRecognizerResult
|
||||
gestureRecognizerResultWithHandGesturesProto:handGesturesPacket
|
||||
.Get<std::vector<ClassificationListProto>>()
|
||||
handednessroto:handednessPacket
|
||||
.Get<std::vector<ClassificationListProto>>()
|
||||
handLandmarksPacket:handLandmarksPacket.Get<
|
||||
std::vector<NormalizedLandmarkListProto>>()
|
||||
worldLandmarksPacket:worldLandmarksPacket
|
||||
.Get<std::vector<LandmarkListProto>>()
|
||||
timestampInMilliSeconds:timestampInMilliseconds];
|
||||
}
|
||||
|
||||
@end
|
||||
|
|
|
|||
|
|
@ -166,10 +166,9 @@ static const int kMicroSecondsPerMilliSecond = 1000;
|
|||
regionOfInterest:(CGRect)roi
|
||||
error:(NSError **)error {
|
||||
std::optional<NormalizedRect> rect =
|
||||
[_visionTaskRunner normalizedRectFromRegionOfInterest:roi
|
||||
imageSize:CGSizeMake(image.width, image.height)
|
||||
[_visionTaskRunner normalizedRectWithRegionOfInterest:roi
|
||||
imageOrientation:image.orientation
|
||||
ROIAllowed:YES
|
||||
imageSize:CGSizeMake(image.width, image.height)
|
||||
error:error];
|
||||
if (!rect.has_value()) {
|
||||
return nil;
|
||||
|
|
@ -196,15 +195,18 @@ static const int kMicroSecondsPerMilliSecond = 1000;
|
|||
outputPacketMap.value()[kClassificationsStreamName.cppString]];
|
||||
}
|
||||
|
||||
- (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image error:(NSError **)error {
|
||||
return [self classifyImage:image regionOfInterest:CGRectZero error:error];
|
||||
}
|
||||
|
||||
- (std::optional<PacketMap>)inputPacketMapWithMPPImage:(MPPImage *)image
|
||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
||||
regionOfInterest:(CGRect)roi
|
||||
error:(NSError **)error {
|
||||
std::optional<NormalizedRect> rect =
|
||||
[_visionTaskRunner normalizedRectFromRegionOfInterest:roi
|
||||
imageSize:CGSizeMake(image.width, image.height)
|
||||
[_visionTaskRunner normalizedRectWithRegionOfInterest:roi
|
||||
imageOrientation:image.orientation
|
||||
ROIAllowed:YES
|
||||
imageSize:CGSizeMake(image.width, image.height)
|
||||
error:error];
|
||||
if (!rect.has_value()) {
|
||||
return std::nullopt;
|
||||
|
|
@ -225,10 +227,6 @@ static const int kMicroSecondsPerMilliSecond = 1000;
|
|||
return inputPacketMap;
|
||||
}
|
||||
|
||||
- (nullable MPPImageClassifierResult *)classifyImage:(MPPImage *)image error:(NSError **)error {
|
||||
return [self classifyImage:image regionOfInterest:CGRectZero error:error];
|
||||
}
|
||||
|
||||
- (nullable MPPImageClassifierResult *)classifyVideoFrame:(MPPImage *)image
|
||||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
||||
regionOfInterest:(CGRect)roi
|
||||
|
|
|
|||
|
|
@ -160,10 +160,8 @@ static NSString *const kTaskName = @"objectDetector";
|
|||
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
|
||||
error:(NSError **)error {
|
||||
std::optional<NormalizedRect> rect =
|
||||
[_visionTaskRunner normalizedRectFromRegionOfInterest:CGRectZero
|
||||
[_visionTaskRunner normalizedRectWithImageOrientation:image.orientation
|
||||
imageSize:CGSizeMake(image.width, image.height)
|
||||
imageOrientation:image.orientation
|
||||
ROIAllowed:NO
|
||||
error:error];
|
||||
if (!rect.has_value()) {
|
||||
return std::nullopt;
|
||||
|
|
@ -188,10 +186,8 @@ static NSString *const kTaskName = @"objectDetector";
|
|||
regionOfInterest:(CGRect)roi
|
||||
error:(NSError **)error {
|
||||
std::optional<NormalizedRect> rect =
|
||||
[_visionTaskRunner normalizedRectFromRegionOfInterest:roi
|
||||
[_visionTaskRunner normalizedRectWithImageOrientation:image.orientation
|
||||
imageSize:CGSizeMake(image.width, image.height)
|
||||
imageOrientation:image.orientation
|
||||
ROIAllowed:NO
|
||||
error:error];
|
||||
if (!rect.has_value()) {
|
||||
return nil;
|
||||
|
|
|
|||
2
mediapipe/tasks/testdata/vision/BUILD
vendored
2
mediapipe/tasks/testdata/vision/BUILD
vendored
|
|
@ -54,6 +54,7 @@ mediapipe_files(srcs = [
|
|||
"hand_landmark_full.tflite",
|
||||
"hand_landmark_lite.tflite",
|
||||
"hand_landmarker.task",
|
||||
"gesture_recognizer.task",
|
||||
"left_hands.jpg",
|
||||
"left_hands_rotated.jpg",
|
||||
"mobilenet_v1_0.25_192_quantized_1_default_1.tflite",
|
||||
|
|
@ -104,7 +105,6 @@ exports_files(
|
|||
"expected_right_down_hand_landmarks.prototxt",
|
||||
"expected_right_up_hand_landmarks.prototxt",
|
||||
"face_geometry_expected_out.pbtxt",
|
||||
"gesture_recognizer.task",
|
||||
"portrait_expected_detection.pbtxt",
|
||||
"portrait_expected_face_geometry.pbtxt",
|
||||
"portrait_rotated_expected_detection.pbtxt",
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user