diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD index 26fa4d6cf..efe33718f 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD @@ -38,3 +38,20 @@ objc_library( "//mediapipe/tasks/ios/vision/core:MPPRunningMode", ], ) + +objc_library( + name = "MPPGestureRecognizer", + hdrs = ["sources/MPPGestureRecognizer.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + module_name = "MPPGestureRecognizer", + deps = [ + ":MPPGestureRecognizerOptions", + ":MPPGestureRecognizerResult", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/vision/core:MPPImage", + ], +) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h new file mode 100644 index 000000000..ed8ff30f9 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h @@ -0,0 +1,165 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h" +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Performs gesture recognition on images. + * + * This API expects a pre-trained TFLite hand gesture recognizer model or a custom one created using + * MediaPipe Solutions Model Maker. See + * https://developers.google.com/mediapipe/solutions/model_maker. + */ +NS_SWIFT_NAME(GestureRecognizer) +@interface MPPGestureRecognizer : NSObject + +/** + * Creates a new instance of `MPPGestureRecognizer` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `MPPGestureRecognizerOptions`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * @param error An optional error parameter populated when there is an error in initializing the + * gesture recognizer. + * + * @return A new instance of `MPPGestureRecognizer` with the given model path. `nil` if there is an + * error in initializing the gesture recognizer. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `MPPGestureRecognizer` from the given `MPPGestureRecognizerOptions`. + * + * @param options The options of type `MPPGestureRecognizerOptions` to use for configuring the + * `MPPGestureRecognizer`. + * @param error An optional error parameter populated when there is an error in initializing the + * gesture recognizer. + * + * @return A new instance of `MPPGestureRecognizer` with the given options. `nil` if there is an + * error in initializing the gesture recognizer. + */ +- (nullable instancetype)initWithOptions:(MPPGestureRecognizerOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs gesture recognition on the provided MPPImage using the whole image as region of + * interest. Rotation will be applied according to the `orientation` property of the provided + * `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with + * `MPPRunningModeImage`. + * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which gesture recognition is to be performed. + * @param error An optional error parameter populated when there is an error in performing gesture + * recognition on the input image. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ +- (nullable MPPGestureRecognizerResult *)recognizeImage:(MPPImage *)image + error:(NSError **)error + NS_SWIFT_NAME(recognize(image:)); + +/** + * Performs gesture recognition on the provided video frame of type `MPPImage` using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with + * `MPPRunningModeVideo`. + * + * It's required to provide the video frame's timestamp (in milliseconds). The input timestamps must + * be monotonically increasing. + * + * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which gesture recognition is to be performed. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing gesture + * recognition on the input video frame. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ +- (nullable MPPGestureRecognizerResult *)recognizeVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(recognize(videoFrame:timestampInMilliseconds:)); + +/** + * Sends live stream image data of type `MPPImage` to perform gesture recognition using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with + * `MPPRunningModeLiveStream`. + * + * The object which needs to be continuously notified of the available results of gesture + * recognition must confirm to `MPPGestureRecognizerLiveStreamDelegate` protocol and implement the + * `gestureRecognizer:didFinishRecognitionWithResult:timestampInMilliseconds:error:` + * delegate method. + * + * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent + * to the gesture recognizer. The input timestamps must be monotonically increasing. + * + * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color + * space is RGB with an Alpha channel. + * + * If this method is used for performing gesture recognition on live camera frames using + * `AVFoundation`, ensure that you request `AVCaptureVideoDataOutput` to output frames in + * `kCMPixelFormat_32RGBA` using its `videoSettings` property. + * + * @param image A live stream image data of type `MPPImage` on which gesture recognition is to be + * performed. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image is sent to the gesture recognizer. The input timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing gesture + * recognition on the input live stream image data. + * + * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + */ +- (BOOL)recognizeAsyncImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(recognizeAsync(image:timestampInMilliseconds:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h index 002c77892..b896e3756 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h @@ -21,22 +21,73 @@ NS_ASSUME_NONNULL_BEGIN +@class MPPGestureRecognizer; + +/** + * This protocol defines an interface for the delegates of `MPPGestureRecognizer` object to receive + * results of performing asynchronous gesture recognition on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). + * + * The delegate of `MPPGestureRecognizer` must adopt `MPPGestureRecognizerLiveStreamDelegate` + * protocol. The methods in this protocol are optional. + */ +NS_SWIFT_NAME(GestureRecognizerLiveStreamDelegate) +@protocol MPPGestureRecognizerLiveStreamDelegate + +@optional + +/** + * This method notifies a delegate that the results of asynchronous gesture recognition of + * an image submitted to the `MPPGestureRecognizer` is available. + * + * This method is called on a private serial dispatch queue created by the `MPPGestureRecognizer` + * for performing the asynchronous delegates calls. + * + * @param gestureRecognizer The gesture recognizer which performed the gesture recognition. + * This is useful to test equality when there are multiple instances of `MPPGestureRecognizer`. + * @param result The `MPPGestureRecognizerResult` object that contains a list of detections, each + * detection has a bounding box that is expressed in the unrotated input frame of reference + * coordinates system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the + * underlying image data. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image was sent to the gesture recognizer. + * @param error An optional error parameter populated when there is an error in performing gesture + * recognition on the input live stream image data. + * + */ +- (void)gestureRecognizer:(MPPGestureRecognizer *)gestureRecognizer + didFinishRecognitionWithResult:(nullable MPPGestureRecognizerResult *)result + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(nullable NSError *)error + NS_SWIFT_NAME(gestureRecognizer(_:didFinishGestureRecognition:timestampInMilliseconds:error:)); +@end + /** Options for setting up a `MPPGestureRecognizer`. */ NS_SWIFT_NAME(GestureRecognizerOptions) @interface MPPGestureRecognizerOptions : MPPTaskOptions +/** + * Running mode of the gesture recognizer task. Defaults to `MPPRunningModeImage`. + * `MPPGestureRecognizer` can be created with one of the following running modes: + * 1. `MPPRunningModeImage`: The mode for performing gesture recognition on single image inputs. + * 2. `MPPRunningModeVideo`: The mode for performing gesture recognition on the decoded frames of a + * video. + * 3. `MPPRunningModeLiveStream`: The mode for performing gesture recognition on a live stream of + * input data, such as from the camera. + */ @property(nonatomic) MPPRunningMode runningMode; /** - * The user-defined result callback for processing live stream data. The result callback should only - * be specified when the running mode is set to the live stream mode. - * TODO: Add parameter `MPPImage` in the callback. + * An object that confirms to `MPPGestureRecognizerLiveStreamDelegate` protocol. This object must + * implement `gestureRecognizer:didFinishRecognitionWithResult:timestampInMilliseconds:error:` to + * receive the results of performing asynchronous gesture recognition on images (i.e, when + * `runningMode` = `MPPRunningModeLiveStream`). */ -@property(nonatomic, copy) void (^completion) - (MPPGestureRecognizerResult *result, NSInteger timestampMs, NSError *error); +@property(nonatomic, weak, nullable) id + gestureRecognizerLiveStreamDelegate; /** Sets the maximum number of hands can be detected by the GestureRecognizer. */ -@property(nonatomic) NSInteger numHands; +@property(nonatomic) NSInteger numberOfHands NS_SWIFT_NAME(numHands); /** Sets minimum confidence score for the hand detection to be considered successful */ @property(nonatomic) float minHandDetectionConfidence; diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m index 720385f33..099ab33e1 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m @@ -19,7 +19,7 @@ - (instancetype)init { self = [super init]; if (self) { - _numHands = 1; + _numberOfHands = 1; _minHandDetectionConfidence = 0.5f; _minHandPresenceConfidence = 0.5f; _minTrackingConfidence = 0.5f; @@ -31,8 +31,9 @@ MPPGestureRecognizerOptions *gestureRecognizerOptions = [super copyWithZone:zone]; gestureRecognizerOptions.runningMode = self.runningMode; - gestureRecognizerOptions.completion = self.completion; - gestureRecognizerOptions.numHands = self.numHands; + gestureRecognizerOptions.gestureRecognizerLiveStreamDelegate = + self.gestureRecognizerLiveStreamDelegate; + gestureRecognizerOptions.numberOfHands = self.numberOfHands; gestureRecognizerOptions.minHandDetectionConfidence = self.minHandDetectionConfidence; gestureRecognizerOptions.minHandPresenceConfidence = self.minHandPresenceConfidence; gestureRecognizerOptions.minTrackingConfidence = self.minTrackingConfidence; diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h index 68a31c834..d56df2855 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h @@ -25,7 +25,7 @@ NS_SWIFT_NAME(GestureRecognizerResult) @interface MPPGestureRecognizerResult : MPPTaskResult /** Hand landmarks of detected hands. */ -@property(nonatomic, readonly) NSArray *> *landmarks; +@property(nonatomic, readonly) NSArray *> *landmarks; /** Hand landmarks in world coordniates of detected hands. */ @property(nonatomic, readonly) NSArray *> *worldLandmarks; @@ -54,11 +54,11 @@ NS_SWIFT_NAME(GestureRecognizerResult) * landmarks, handedness and gestures. * */ -- (instancetype)initWithLandmarks:(NSArray *> *)landmarks - worldLandmarks:(NSArray *> *)worldLandmarks - handedness:(NSArray *> *)handedness - gestures:(NSArray *> *)gestures - timestampInMilliseconds:(NSInteger)timestampInMilliseconds; +- (instancetype)initWithGestures:(NSArray *> *)gestures + handedness:(NSArray *> *)handedness + landmarks:(NSArray *> *)landmarks + worldLandmarks:(NSArray *> *)worldLandmarks + timestampInMilliseconds:(NSInteger)timestampInMilliseconds; @end diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m index 3ffb15392..b1fa1be14 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m @@ -16,11 +16,11 @@ @implementation MPPGestureRecognizerResult -- (instancetype)initWithLandmarks:(NSArray *> *)landmarks - worldLandmarks:(NSArray *> *)worldLandmarks - handedness:(NSArray *> *)handedness - gestures:(NSArray *> *)gestures - timestampInMilliseconds:(NSInteger)timestampInMilliseconds { +- (instancetype)initWithGestures:(NSArray *> *)gestures + handedness:(NSArray *> *)handedness + landmarks:(NSArray *> *)landmarks + worldLandmarks:(NSArray *> *)worldLandmarks + timestampInMilliseconds:(NSInteger)timestampInMilliseconds { self = [super initWithTimestampInMilliseconds:timestampInMilliseconds]; if (self) { _landmarks = landmarks; diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD new file mode 100644 index 000000000..d015761c4 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD @@ -0,0 +1,51 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPGestureRecognizerOptionsHelpers", + srcs = ["sources/MPPGestureRecognizerOptions+Helpers.mm"], + hdrs = ["sources/MPPGestureRecognizerOptions+Helpers.h"], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_classifier_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/components/processors/utils:MPPClassifierOptionsHelpers", + "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol", + "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", + "//mediapipe/tasks/ios/vision/gesture_recognizer:MPPGestureRecognizerOptions", + ], +) + +objc_library( + name = "MPPGestureRecognizerResultHelpers", + srcs = ["sources/MPPGestureRecognizerResult+Helpers.mm"], + hdrs = ["sources/MPPGestureRecognizerResult+Helpers.h"], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/tasks/ios/components/containers/utils:MPPCategoryHelpers", + "//mediapipe/tasks/ios/components/containers/utils:MPPLandmarkHelpers", + "//mediapipe/tasks/ios/vision/gesture_recognizer:MPPGestureRecognizerResult", + ], +) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h new file mode 100644 index 000000000..505bbd515 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h @@ -0,0 +1,32 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/calculator_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPGestureRecognizerOptions (Helpers) + +/** + * Populates the provided `CalculatorOptions` proto container with the current settings. + * + * @param optionsProto The `CalculatorOptions` proto object to copy the settings to. + */ +- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm new file mode 100644 index 000000000..d234d86a0 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm @@ -0,0 +1,92 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_classifier_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_recognizer_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/hand_detector/proto/hand_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarker_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options.pb.h" + +namespace { +using CalculatorOptionsProto = mediapipe::CalculatorOptions; +using GestureClassifierGraphOptionsProto = + ::mediapipe::tasks::vision::gesture_recognizer::proto::GestureClassifierGraphOptions; +using GestureRecognizerGraphOptionsProto = + ::mediapipe::tasks::vision::gesture_recognizer::proto::GestureRecognizerGraphOptions; +using HandGestureRecognizerGraphOptionsProto = + ::mediapipe::tasks::vision::gesture_recognizer::proto::HandGestureRecognizerGraphOptions; +using HandLandmarkerGraphOptionsProto = + ::mediapipe::tasks::vision::hand_landmarker::proto::HandLandmarkerGraphOptions; +using HandDetectorGraphOptionsProto = + ::mediapipe::tasks::vision::hand_detector::proto::HandDetectorGraphOptions; +using HandLandmarksDetectorGraphOptionsProto = + ::mediapipe::tasks::vision::hand_landmarker::proto::HandLandmarksDetectorGraphOptions; +using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; +} // namespace + +@implementation MPPGestureRecognizerOptions (Helpers) + +- (void)copyToProto:(CalculatorOptionsProto *)optionsProto { + GestureRecognizerGraphOptionsProto *gestureRecognizerGraphOptionsProto = + optionsProto->MutableExtension(GestureRecognizerGraphOptionsProto::ext); + gestureRecognizerGraphOptionsProto->Clear(); + + [self.baseOptions copyToProto:gestureRecognizerGraphOptionsProto->mutable_base_options() + withUseStreamMode:self.runningMode != MPPRunningModeImage]; + + HandLandmarkerGraphOptionsProto *handLandmarkerGraphOptionsProto = + gestureRecognizerGraphOptionsProto->mutable_hand_landmarker_graph_options(); + handLandmarkerGraphOptionsProto->Clear(); + handLandmarkerGraphOptionsProto->set_min_tracking_confidence(self.minTrackingConfidence); + + HandDetectorGraphOptionsProto *handDetectorGraphOptionsProto = + handLandmarkerGraphOptionsProto->mutable_hand_detector_graph_options(); + handDetectorGraphOptionsProto->Clear(); + handDetectorGraphOptionsProto->set_num_hands(self.numberOfHands); + handDetectorGraphOptionsProto->set_min_detection_confidence(self.minHandDetectionConfidence); + + HandLandmarksDetectorGraphOptionsProto *handLandmarksDetectorGraphOptionsProto = + handLandmarkerGraphOptionsProto->mutable_hand_landmarks_detector_graph_options(); + handLandmarksDetectorGraphOptionsProto->Clear(); + handLandmarksDetectorGraphOptionsProto->set_min_detection_confidence( + self.minHandPresenceConfidence); + + HandGestureRecognizerGraphOptionsProto *handGestureRecognizerGraphOptionsProto = + gestureRecognizerGraphOptionsProto->mutable_hand_gesture_recognizer_graph_options(); + + if (self.cannedGesturesClassifierOptions) { + GestureClassifierGraphOptionsProto *cannedGesturesClassifierOptionsProto = + handGestureRecognizerGraphOptionsProto->mutable_canned_gesture_classifier_graph_options(); + cannedGesturesClassifierOptionsProto->Clear(); + [self.cannedGesturesClassifierOptions + copyToProto:cannedGesturesClassifierOptionsProto->mutable_classifier_options()]; + } + + if (self.customGesturesClassifierOptions) { + GestureClassifierGraphOptionsProto *customGesturesClassifierOptionsProto = + handGestureRecognizerGraphOptionsProto->mutable_custom_gesture_classifier_graph_options(); + customGesturesClassifierOptionsProto->Clear(); + [self.customGesturesClassifierOptions + copyToProto:customGesturesClassifierOptionsProto->mutable_classifier_options()]; + } +} + +@end diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h new file mode 100644 index 000000000..649c11c8a --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h @@ -0,0 +1,46 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" + +#include "mediapipe/framework/packet.h" + +NS_ASSUME_NONNULL_BEGIN + +static const int kMicroSecondsPerMilliSecond = 1000; + +@interface MPPGestureRecognizerResult (Helpers) + +/** + * Creates an `MPPGestureRecognizerResult` from hand gestures, handedness, hand landmarks and world + * landmarks packets. + * + * @param handGesturesPacket a MediaPipe packet wrapping a`std::vector`. + * @param handednessPacket a MediaPipe packet wrapping a`std::vector`. + * @param handLandmarksPacket a MediaPipe packet wrapping + * a`std::vector`. + * @param handLandmarksPacket a MediaPipe packet wrapping a`std::vector`. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultWithHandGesturesPacket:(const mediapipe::Packet &)handGesturesPacket + handednessPacket:(const mediapipe::Packet &)handednessPacket + handLandmarksPacket:(const mediapipe::Packet &)handLandmarksPacket + worldLandmarksPacket:(const mediapipe::Packet &)worldLandmarksPacket; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm new file mode 100644 index 000000000..70773a940 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm @@ -0,0 +1,127 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h" + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h" + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/packet.h" + +namespace { +using ClassificationListProto = ::mediapipe::ClassificationList; +using LandmarkListProto = ::mediapipe::LandmarkList; +using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList; +using ::mediapipe::Packet; +} // namespace + +@implementation MPPGestureRecognizerResult (Helpers) + ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultWithHandGesturesPacket:(const Packet &)handGesturesPacket + handednessPacket:(const Packet &)handednessPacket + handLandmarksPacket:(const Packet &)handLandmarksPacket + worldLandmarksPacket:(const Packet &)worldLandmarksPacket { + NSInteger timestampInMilliseconds = + (NSInteger)(handGesturesPacket.Timestamp().Value() / kMicroSecondsPerMilliSecond); + + if (handGesturesPacket.IsEmpty()) { + return [[MPPGestureRecognizerResult alloc] initWithGestures:@[] + handedness:@[] + landmarks:@[] + worldLandmarks:@[] + timestampInMilliseconds:timestampInMilliseconds]; + } + + if (!handGesturesPacket.ValidateAsType>().ok() || + !handednessPacket.ValidateAsType>().ok() || + !handLandmarksPacket.ValidateAsType>().ok() || + !worldLandmarksPacket.ValidateAsType>().ok()) { + return nil; + } + + const std::vector &handGesturesClassificationListProtos = + handGesturesPacket.Get>(); + NSMutableArray *> *multiHandGestures = + [NSMutableArray arrayWithCapacity:(NSUInteger)handGesturesClassificationListProtos.size()]; + + for (const auto &classificationListProto : handGesturesClassificationListProtos) { + NSMutableArray *gestures = [NSMutableArray + arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; + for (const auto &classificationProto : classificationListProto.classification()) { + MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; + [gestures addObject:category]; + } + [multiHandGestures addObject:gestures]; + } + + const std::vector &handednessClassificationListProtos = + handednessPacket.Get>(); + NSMutableArray *> *multiHandHandedness = + [NSMutableArray arrayWithCapacity:(NSUInteger)handednessClassificationListProtos.size()]; + + for (const auto &classificationListProto : handednessClassificationListProtos) { + NSMutableArray *handedness = [NSMutableArray + arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; + for (const auto &classificationProto : classificationListProto.classification()) { + MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; + [handedness addObject:category]; + } + [multiHandHandedness addObject:handedness]; + } + + const std::vector &handLandmarkListProtos = + handLandmarksPacket.Get>(); + NSMutableArray *> *multiHandLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProtos.size()]; + + for (const auto &handLandmarkListProto : handLandmarkListProtos) { + NSMutableArray *handLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProto.landmark().size()]; + for (const auto &normalizedLandmarkProto : handLandmarkListProto.landmark()) { + MPPNormalizedLandmark *normalizedLandmark = + [MPPNormalizedLandmark normalizedLandmarkWithProto:normalizedLandmarkProto]; + [handLandmarks addObject:normalizedLandmark]; + } + [multiHandLandmarks addObject:handLandmarks]; + } + + const std::vector &worldLandmarkListProtos = + worldLandmarksPacket.Get>(); + NSMutableArray *> *multiHandWorldLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProtos.size()]; + + for (const auto &worldLandmarkListProto : worldLandmarkListProtos) { + NSMutableArray *worldLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProto.landmark().size()]; + for (const auto &landmarkProto : worldLandmarkListProto.landmark()) { + MPPLandmark *landmark = [MPPLandmark landmarkWithProto:landmarkProto]; + [worldLandmarks addObject:landmark]; + } + [multiHandWorldLandmarks addObject:worldLandmarks]; + } + + MPPGestureRecognizerResult *gestureRecognizerResult = + [[MPPGestureRecognizerResult alloc] initWithGestures:multiHandGestures + handedness:multiHandHandedness + landmarks:multiHandLandmarks + worldLandmarks:multiHandWorldLandmarks + timestampInMilliseconds:timestampInMilliseconds]; + + return gestureRecognizerResult; +} + +@end