From ae2901459dcfb9775c7e41468129d4a468a19abe Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 17 May 2023 21:50:11 +0530 Subject: [PATCH 01/10] Updated property types in MPPGestureRecognizerResult --- .../sources/MPPGestureRecognizerResult.h | 12 ++++++------ .../sources/MPPGestureRecognizerResult.m | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h index 68a31c834..d56df2855 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h @@ -25,7 +25,7 @@ NS_SWIFT_NAME(GestureRecognizerResult) @interface MPPGestureRecognizerResult : MPPTaskResult /** Hand landmarks of detected hands. */ -@property(nonatomic, readonly) NSArray *> *landmarks; +@property(nonatomic, readonly) NSArray *> *landmarks; /** Hand landmarks in world coordniates of detected hands. */ @property(nonatomic, readonly) NSArray *> *worldLandmarks; @@ -54,11 +54,11 @@ NS_SWIFT_NAME(GestureRecognizerResult) * landmarks, handedness and gestures. * */ -- (instancetype)initWithLandmarks:(NSArray *> *)landmarks - worldLandmarks:(NSArray *> *)worldLandmarks - handedness:(NSArray *> *)handedness - gestures:(NSArray *> *)gestures - timestampInMilliseconds:(NSInteger)timestampInMilliseconds; +- (instancetype)initWithGestures:(NSArray *> *)gestures + handedness:(NSArray *> *)handedness + landmarks:(NSArray *> *)landmarks + worldLandmarks:(NSArray *> *)worldLandmarks + timestampInMilliseconds:(NSInteger)timestampInMilliseconds; @end diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m index 3ffb15392..f9ea738de 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m @@ -16,10 +16,10 @@ @implementation MPPGestureRecognizerResult -- (instancetype)initWithLandmarks:(NSArray *> *)landmarks +- (instancetype)initWithGestures:(NSArray *> *)gestures + handedness:(NSArray *> *)handedness + landmarks:(NSArray *> *)landmarks worldLandmarks:(NSArray *> *)worldLandmarks - handedness:(NSArray *> *)handedness - gestures:(NSArray *> *)gestures timestampInMilliseconds:(NSInteger)timestampInMilliseconds { self = [super initWithTimestampInMilliseconds:timestampInMilliseconds]; if (self) { From a4c280310b51d51548466509ad8fabb86638699c Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 17 May 2023 21:50:53 +0530 Subject: [PATCH 02/10] Added delegates in iOS gesture recognizer options --- .../sources/MPPGestureRecognizerOptions.h | 63 +++++++++++++++++-- .../sources/MPPGestureRecognizerOptions.m | 6 +- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h index 002c77892..421073d45 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h @@ -21,22 +21,73 @@ NS_ASSUME_NONNULL_BEGIN +@class MPPGestureRecognizer; + +/** + * This protocol defines an interface for the delegates of `MPPGestureRecognizer` object to receive + * results of performing asynchronous gesture recognition on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). + * + * The delegate of `MPPGestureRecognizer` must adopt `MPPGestureRecognizerLiveStreamDelegate` + * protocol. The methods in this protocol are optional. + */ +NS_SWIFT_NAME(GestureRecognizerLiveStreamDelegate) +@protocol MPPGestureRecognizerLiveStreamDelegate + +@optional + +/** + * This method notifies a delegate that the results of asynchronous object detection of + * an image submitted to the `MPPGestureRecognizer` is available. + * + * This method is called on a private serial dispatch queue created by the `MPPGestureRecognizer` + * for performing the asynchronous delegates calls. + * + * @param gestureRecognizer The gesture recognizer which performed the gesture recognition. + * This is useful to test equality when there are multiple instances of `MPPGestureRecognizer`. + * @param result The `MPPGestureRecognizerResult` object that contains a list of detections, each + * detection has a bounding box that is expressed in the unrotated input frame of reference + * coordinates system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the + * underlying image data. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image was sent to the object detector. + * @param error An optional error parameter populated when there is an error in performing object + * detection on the input live stream image data. + * + */ +- (void)gestureRecognizer:(MPPGestureRecognizer *)gestureRecognizer + didFinishRecognitionWithResult:(nullable MPPGestureRecognizerResult *)result + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(nullable NSError *)error + NS_SWIFT_NAME(gestureRecognizer(_:didFinishGestureRecognition:timestampInMilliseconds:error:)); +@end + /** Options for setting up a `MPPGestureRecognizer`. */ NS_SWIFT_NAME(GestureRecognizerOptions) @interface MPPGestureRecognizerOptions : MPPTaskOptions +/** + * Running mode of the gesture recognizer task. Defaults to `MPPRunningModeImage`. + * `MPPGestureRecognizer` can be created with one of the following running modes: + * 1. `MPPRunningModeImage`: The mode for performing object detection on single image inputs. + * 2. `MPPRunningModeVideo`: The mode for performing object detection on the decoded frames of a + * video. + * 3. `MPPRunningModeLiveStream`: The mode for performing object detection on a live stream of + * input data, such as from the camera. + */ @property(nonatomic) MPPRunningMode runningMode; /** - * The user-defined result callback for processing live stream data. The result callback should only - * be specified when the running mode is set to the live stream mode. - * TODO: Add parameter `MPPImage` in the callback. + * An object that confirms to `MPPGestureRecognizerLiveStreamDelegate` protocol. This object must + * implement `gestureRecognizer:didFinishRecognitionWithResult:timestampInMilliseconds:error:` to + * receive the results of performing asynchronous gesture recognition on images (i.e, when + * `runningMode` = `MPPRunningModeLiveStream`). */ -@property(nonatomic, copy) void (^completion) - (MPPGestureRecognizerResult *result, NSInteger timestampMs, NSError *error); +@property(nonatomic, weak, nullable) id + gestureRecognizerLiveStreamDelegate; /** Sets the maximum number of hands can be detected by the GestureRecognizer. */ -@property(nonatomic) NSInteger numHands; +@property(nonatomic) NSInteger numberOfHands; /** Sets minimum confidence score for the hand detection to be considered successful */ @property(nonatomic) float minHandDetectionConfidence; diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m index 720385f33..7f874c8fd 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m @@ -19,7 +19,7 @@ - (instancetype)init { self = [super init]; if (self) { - _numHands = 1; + _numberOfHands = 1; _minHandDetectionConfidence = 0.5f; _minHandPresenceConfidence = 0.5f; _minTrackingConfidence = 0.5f; @@ -31,8 +31,8 @@ MPPGestureRecognizerOptions *gestureRecognizerOptions = [super copyWithZone:zone]; gestureRecognizerOptions.runningMode = self.runningMode; - gestureRecognizerOptions.completion = self.completion; - gestureRecognizerOptions.numHands = self.numHands; + gestureRecognizerOptions.gestureRecognizerLiveStreamDelegate = self.gestureRecognizerLiveStreamDelegate; + gestureRecognizerOptions.numberOfHands = self.numberOfHands; gestureRecognizerOptions.minHandDetectionConfidence = self.minHandDetectionConfidence; gestureRecognizerOptions.minHandPresenceConfidence = self.minHandPresenceConfidence; gestureRecognizerOptions.minTrackingConfidence = self.minTrackingConfidence; From ebd154550621dad536ba702e15e2ffd5965a670c Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 17 May 2023 21:51:40 +0530 Subject: [PATCH 03/10] Added MPPGestureRecognizerOptionsHelpers --- .../ios/vision/gesture_recognizer/utils/BUILD | 33 ++++++++ .../MPPGestureRecognizerOptions+Helpers.h | 32 +++++++ .../MPPGestureRecognizerOptions+Helpers.mm | 83 +++++++++++++++++++ 3 files changed, 148 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD new file mode 100644 index 000000000..f643c83f5 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD @@ -0,0 +1,33 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPGestureRecognizerOptionsHelpers", + srcs = ["sources/MPPGestureRecognizerOptions+Helpers.mm"], + hdrs = ["sources/MPPGestureRecognizerOptions+Helpers.h"], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/components/processors/utils:MPPClassifierOptionsHelpers", + "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol", + "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", + "//mediapipe/tasks/ios/vision/gesture_recognizer:MPPGestureRecognizerOptions", + ], +) + diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h new file mode 100644 index 000000000..505bbd515 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h @@ -0,0 +1,32 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/calculator_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPGestureRecognizerOptions (Helpers) + +/** + * Populates the provided `CalculatorOptions` proto container with the current settings. + * + * @param optionsProto The `CalculatorOptions` proto object to copy the settings to. + */ +- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm new file mode 100644 index 000000000..111dabab2 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm @@ -0,0 +1,83 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +#include "mediapipe/tasks/cc/vision/gesture_recognizer/proto/gesture_recognizer_graph_options.pb.h" + +namespace { +using CalculatorOptionsProto = mediapipe::CalculatorOptions; +using GestureClassifierGraphOptionsProto = + ::mediapipe::tasks::vision::gesture_recognizer::proto::GestureClassifierGraphOptions; +using GestureRecognizerGraphOptionsProto = + ::mediapipe::tasks::vision::gesture_recognizer::proto::GestureRecognizerGraphOptions; +using HandGestureRecognizerGraphOptionsProto = + ::mediapipe::tasks::vision::gesture_recognizer::proto::HandGestureRecognizerGraphOptions; +using HandLandmarkerGraphOptionsProto = + ::mediapipe::tasks::vision::hand_landmarker::proto::HandLandmarkerGraphOptions; +using HandDetectorGraphOptionsProto = + ::mediapipe::tasks::vision::hand_detector::proto::HandDetectorGraphOptions; +using HandLandmarksDetectorGraphOptionsProto = + ::mediapipe::tasks::vision::hand_landmarker::proto::HandLandmarksDetectorGraphOptions; +using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; +} // namespace + +@implementation MPPGestureRecognizerOptions (Helpers) + +- (void)copyToProto:(CalculatorOptionsProto *)optionsProto { + GestureRecognizerGraphOptionsProto *gestureRecognizerGraphOptionsProto = + optionsProto->MutableExtension(GestureRecognizerGraphOptionsProto::ext); + + [self.baseOptions copyToProto:gestureRecognizerGraphOptionsProto->mutable_base_options() + withUseStreamMode:self.runningMode != MPPRunningModeImage]; + + HandLandmarkerGraphOptionsProto *handLandmarkerGraphOptionsProto = + gestureRecognizerGraphOptionsProto->mutable_hand_landmarker_graph_options(); + handLandmarkerGraphOptionsProto->set_min_tracking_confidence(self.minTrackingConfidence); + + HandDetectorGraphOptionsProto *handDetectorGraphOptionsProto = + handLandmarkerGraphOptionsProto->mutable_hand_detector_graph_options(); + handDetectorGraphOptionsProto->Clear(); + handDetectorGraphOptionsProto->set_num_hands(self.numberOfHands); + handDetectorGraphOptionsProto->set_min_detection_confidence(self.minHandDetectionConfidence); + + HandLandmarksDetectorGraphOptionsProto *handLandmarksDetectorGraphOptionsProto = + handLandmarkerGraphOptionsProto->mutable_hand_landmarks_detector_graph_options(); + handLandmarksDetectorGraphOptionsProto->Clear(); + handLandmarksDetectorGraphOptionsProto->set_min_detection_confidence( + self.minHandPresenceConfidence); + + HandGestureRecognizerGraphOptionsProto *handGestureRecognizerGraphOptionsProto = + gestureRecognizerGraphOptionsProto->mutable_hand_gesture_recognizer_graph_options(); + + if (self.cannedGesturesClassifierOptions) { + GestureClassifierGraphOptionsProto *cannedGesturesClassifierOptionsProto = + handGestureRecognizerGraphOptionsProto->mutable_canned_gesture_classifier_graph_options(); + [self.cannedGesturesClassifierOptions + copyToProto:cannedGesturesClassifierOptionsProto->mutable_classifier_options()]; + } + + if (self.customGesturesClassifierOptions) { + GestureClassifierGraphOptionsProto *customGesturesClassifierOptionsProto = + handGestureRecognizerGraphOptionsProto->mutable_custom_gesture_classifier_graph_options(); + [self.customGesturesClassifierOptions + copyToProto:customGesturesClassifierOptionsProto->mutable_classifier_options()]; + } +} + +@end From 501b4bbc7b46d50139d269c00db0014f0f99b1de Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 17 May 2023 21:52:05 +0530 Subject: [PATCH 04/10] Added MPPGestureRecognizerResultHelpers --- .../ios/vision/gesture_recognizer/utils/BUILD | 13 ++ .../MPPGestureRecognizerResult+Helpers.h | 46 +++++++ .../MPPGestureRecognizerResult+Helpers.mm | 128 ++++++++++++++++++ 3 files changed, 187 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD index f643c83f5..8db5a63db 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/BUILD @@ -31,3 +31,16 @@ objc_library( ], ) +objc_library( + name = "MPPGestureRecognizerResultHelpers", + srcs = ["sources/MPPGestureRecognizerResult+Helpers.mm"], + hdrs = ["sources/MPPGestureRecognizerResult+Helpers.h"], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/tasks/ios/components/containers/utils:MPPCategoryHelpers", + "//mediapipe/tasks/ios/components/containers/utils:MPPLandmarkHelpers", + "//mediapipe/tasks/ios/vision/gesture_recognizer:MPPGestureRecognizerResult", + ], +) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h new file mode 100644 index 000000000..649c11c8a --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h @@ -0,0 +1,46 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" + +#include "mediapipe/framework/packet.h" + +NS_ASSUME_NONNULL_BEGIN + +static const int kMicroSecondsPerMilliSecond = 1000; + +@interface MPPGestureRecognizerResult (Helpers) + +/** + * Creates an `MPPGestureRecognizerResult` from hand gestures, handedness, hand landmarks and world + * landmarks packets. + * + * @param handGesturesPacket a MediaPipe packet wrapping a`std::vector`. + * @param handednessPacket a MediaPipe packet wrapping a`std::vector`. + * @param handLandmarksPacket a MediaPipe packet wrapping + * a`std::vector`. + * @param handLandmarksPacket a MediaPipe packet wrapping a`std::vector`. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultWithHandGesturesPacket:(const mediapipe::Packet &)handGesturesPacket + handednessPacket:(const mediapipe::Packet &)handednessPacket + handLandmarksPacket:(const mediapipe::Packet &)handLandmarksPacket + worldLandmarksPacket:(const mediapipe::Packet &)worldLandmarksPacket; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm new file mode 100644 index 000000000..d125ad26f --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm @@ -0,0 +1,128 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.h" + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h" + +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/packet.h" + +namespace { +using ClassificationListProto = ::mediapipe::ClassificationList; +using LandmarkListProto = ::mediapipe::LandmarkList; +using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList; +using ::mediapipe::Packet; +} // namespace + +@implementation MPPGestureRecognizerResult (Helpers) + ++ (MPPGestureRecognizerResult *) + gestureRecognizerResultWithHandGesturesPacket:(const Packet &)handGesturesPacket + handednessPacket:(const Packet &)handednessPacket + handLandmarksPacket:(const Packet &)handLandmarksPacket + worldLandmarksPacket:(const Packet &)worldLandmarksPacket { + NSInteger timestampInMilliseconds = + (NSInteger)(handGesturesPacket.Timestamp().Value() / kMicroSecondsPerMilliSecond); + + if (handGesturesPacket.IsEmpty()) { + return [[MPPGestureRecognizerResult alloc] initWithGestures:@[] + handedness:@[] + landmarks:@[] + worldLandmarks:@[] + timestampInMilliseconds:timestampInMilliseconds]; + } + + if (!handGesturesPacket.ValidateAsType>().ok() || + !handednessPacket.ValidateAsType>().ok() || + !handLandmarksPacket.ValidateAsType>().ok() || + !worldLandmarksPacket.ValidateAsType>().ok()) { + return nil; + } + + const std::vector &handGesturesClassificationListProtos = + handGesturesPacket.Get>(); + NSMutableArray *> *multiHandGestures = + [NSMutableArray arrayWithCapacity:(NSUInteger)handGesturesClassificationListProtos.size()]; + + for (const auto &classificationListProto : handGesturesClassificationListProtos) { + NSMutableArray *gestures = [NSMutableArray + arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; + for (const auto &classificationProto : classificationListProto.classification()) { + MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; + [gestures addObject:category]; + } + [multiHandGestures addObject:[gestures copy]]; + } + + const std::vector &handednessClassificationListProtos = + handednessPacket.Get>(); + NSMutableArray *> *multiHandHandedness = + [NSMutableArray arrayWithCapacity:(NSUInteger)handednessClassificationListProtos.size()]; + + for (const auto &classificationListProto : handednessClassificationListProtos) { + NSMutableArray *handedness = [NSMutableArray + arrayWithCapacity:(NSUInteger)classificationListProto.classification().size()]; + for (const auto &classificationProto : classificationListProto.classification()) { + MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; + [handedness addObject:category]; + } + [multiHandHandedness addObject:[handedness copy]]; + } + + const std::vector &handLandmarkListProtos = + handLandmarksPacket.Get>(); + NSMutableArray *> *multiHandLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProtos.size()]; + + for (const auto &handLandmarkListProto : handLandmarkListProtos) { + NSMutableArray *handLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)handLandmarkListProto.landmark().size()]; + for (const auto &normalizedLandmarkProto : handLandmarkListProto.landmark()) { + MPPNormalizedLandmark *normalizedLandmark = + [MPPNormalizedLandmark normalizedLandmarkWithProto:normalizedLandmarkProto]; + [handLandmarks addObject:normalizedLandmark]; + } + [multiHandLandmarks addObject:[handLandmarks copy]]; + } + + const std::vector &worldLandmarkListProtos = + worldLandmarksPacket.Get>(); + NSMutableArray *> *multiHandWorldLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProtos.size()]; + + for (const auto &worldLandmarkListProto : worldLandmarkListProtos) { + NSMutableArray *worldLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProto.landmark().size()]; + for (const auto &landmarkProto : worldLandmarkListProto.landmark()) { + MPPLandmark *landmark = + [MPPLandmark landmarkWithProto:landmarkProto]; + [worldLandmarks addObject:landmark]; + } + [multiHandWorldLandmarks addObject:[worldLandmarks copy]]; + } + + MPPGestureRecognizerResult *gestureRecognizerResult = + [[MPPGestureRecognizerResult alloc] initWithGestures:[multiHandGestures copy] + handedness:[multiHandHandedness copy] + landmarks:[multiHandLandmarks copy] + worldLandmarks:[multiHandWorldLandmarks copy] + timestampInMilliseconds:timestampInMilliseconds]; + + return gestureRecognizerResult; +} + +@end From dcb0414d4b6ef1b82d054487df8fc524220932dd Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 17 May 2023 21:53:25 +0530 Subject: [PATCH 05/10] Added MPPGestureRecognizer header --- .../tasks/ios/vision/gesture_recognizer/BUILD | 19 ++ .../sources/MPPGestureRecognizer.h | 274 ++++++++++++++++++ 2 files changed, 293 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD index 26fa4d6cf..a9a62d720 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD @@ -38,3 +38,22 @@ objc_library( "//mediapipe/tasks/ios/vision/core:MPPRunningMode", ], ) + +objc_library( + name = "MPPGestureRecognizer", + srcs = ["sources/MPPGestureRecognizer.mm"], + hdrs = ["sources/MPPGestureRecognizer.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + module_name = "MPPGestureRecognizer", + deps = [ + ":MPPGestureRecognizerOptions", + ":MPPGestureRecognizerResult", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/vision/core:MPPImage", + ], +) + diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h new file mode 100644 index 000000000..2e3ca937b --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h @@ -0,0 +1,274 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h" +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Performs gesture recognition on images. + * + * This API expects a pre-trained TFLite hand gesture recognizer model or a custom one created using + * MediaPipe Solutions Model Maker. See + * https://developers.google.com/mediapipe/solutions/model_maker. + */ +NS_SWIFT_NAME(GestureRecognizer) +@interface MPPGestureRecognizer : NSObject + +/** + * Creates a new instance of `MPPGestureRecognizer` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `MPPGestureRecognizerOptions`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * @param error An optional error parameter populated when there is an error in initializing the + * gesture recognizer. + * + * @return A new instance of `MPPGestureRecognizer` with the given model path. `nil` if there is an + * error in initializing the gesture recognizer. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `MPPGestureRecognizer` from the given `MPPGestureRecognizerOptions`. + * + * @param options The options of type `MPPGestureRecognizerOptions` to use for configuring the + * `MPPGestureRecognizer`. + * @param error An optional error parameter populated when there is an error in initializing the + * gesture recognizer. + * + * @return A new instance of `MPPGestureRecognizer` with the given options. `nil` if there is an + * error in initializing the gesture recognizer. + */ +- (nullable instancetype)initWithOptions:(MPPGestureRecognizerOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs gesture recognition on the provided MPPImage using the whole image as region of + * interest. Rotation will be applied according to the `orientation` property of the provided + * `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with + * `MPPRunningModeImage`. + * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which gesture recognition is to be performed. + * @param error An optional error parameter populated when there is an error in performing gesture + * recognition on the input image. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ +- (nullable MPPGestureRecognizerResult *)recognizeImage:(MPPImage *)image + error:(NSError **)error + NS_SWIFT_NAME(recognize(image:)); + +/** + * Performs gesture recognition on the provided `MPPImage` cropped to the specified region of + * interest. Rotation will be applied on the cropped image according to the `orientation` property + * of the provided `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with + * `MPPRunningModeImage`. + * + * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which gesture recognition is to be performed. + * @param roi A `CGRect` specifying the region of interest within the given `MPPImage`, on which + * gesture recognition should be performed. + * @param error An optional error parameter populated when there is an error in performing gesture + * recognition on the input image. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ +- (nullable MPPGestureRecognizerResult *)recognizeImage:(MPPImage *)image + regionOfInterest:(CGRect)roi + error:(NSError **)error + NS_SWIFT_NAME(recognize(image:regionOfInterest:)); + +/** + * Performs gesture recognition on the provided video frame of type `MPPImage` using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with + * `MPPRunningModeVideo`. + * + * It's required to provide the video frame's timestamp (in milliseconds). The input timestamps must + * be monotonically increasing. + * + * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which gesture recognition is to be performed. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing gesture + * recognition on the input video frame. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ +- (nullable MPPGestureRecognizerResult *)recognizeVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(recognize(videoFrame:timestampInMilliseconds:)); + +/** + * Performs gesture recognition on the provided video frame of type `MPPImage` cropped to the + * specified region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with + * `MPPRunningModeVideo`. + * + * It's required to provide the video frame's timestamp (in milliseconds). The input timestamps must + * be monotonically increasing. + * + * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image A live stream image data of type `MPPImage` on which gesture recognition is to be + * performed. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param roi A `CGRect` specifying the region of interest within the video frame of type + * `MPPImage`, on which gesture recognition should be performed. + * @param error An optional error parameter populated when there is an error in performing gesture + * recognition on the input video frame. + * + * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition + * results. + */ +- (nullable MPPGestureRecognizerResult *)recognizeVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + regionOfInterest:(CGRect)roi + error:(NSError **)error + NS_SWIFT_NAME(recognize(videoFrame:timestampInMilliseconds:regionOfInterest:)); + +/** + * Sends live stream image data of type `MPPImage` to perform gesture recognition using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with + * `MPPRunningModeLiveStream`. + * + * The object which needs to be continuously notified of the available results of gesture + * recognition must confirm to `MPPGestureRecognizerLiveStreamDelegate` protocol and implement the + * `gestureRecognizer:didFinishRecognitionWithResult:timestampInMilliseconds:error:` + * delegate method. + * + * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent + * to the gesture recognizer. The input timestamps must be monotonically increasing. + * + * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color + * space is RGB with an Alpha channel. + * + * If this method is used for performing gesture recognition on live camera frames using + * `AVFoundation`, ensure that you request `AVCaptureVideoDataOutput` to output frames in + * `kCMPixelFormat_32RGBA` using its `videoSettings` property. + * + * @param image A live stream image data of type `MPPImage` on which gesture recognition is to be + * performed. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image is sent to the gesture recognizer. The input timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing gesture + * recognition on the input live stream image data. + * + * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + */ +- (BOOL)recognizeAsyncImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(recognizeAsync(image:timestampInMilliseconds:)); + +/** + * Sends live stream image data of type `MPPImage` to perform gesture recognition, cropped to the + * specified region of interest.. Rotation will be applied according to the `orientation` property + * of the provided `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with + * `MPPRunningModeLiveStream`. + * + * The object which needs to be continuously notified of the available results of gesture + * recognition must confirm to `MPPGestureRecognizerLiveStreamDelegate` protocol and implement the + * `gestureRecognizer:didFinishRecognitionWithResult:timestampInMilliseconds:error:` delegate + * method. + * + * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent + * to the gesture recognizer. The input timestamps must be monotonically increasing. + * + * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color + * space is RGB with an Alpha channel. + * + * If this method is used for preforming gesture recognition on live camera frames using + * `AVFoundation`, ensure that you request `AVCaptureVideoDataOutput` to output frames in + * `kCMPixelFormat_32RGBA` using its `videoSettings` property. + * + * @param image A live stream image data of type `MPPImage` on which gesture recognition is to be + * performed. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image is sent to the gesture recognizer. The input timestamps must be monotonically increasing. + * @param roi A `CGRect` specifying the region of interest within the given live stream image data + * of type `MPPImage`, on which gesture recognition should be performed. + * @param error An optional error parameter populated when there is an error in performing gesture + * recognition on the input live stream image data. + * + * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + */ +- (BOOL)recognizeAsyncImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + regionOfInterest:(CGRect)roi + error:(NSError **)error + NS_SWIFT_NAME(recognizeAsync(image:timestampInMilliseconds:regionOfInterest:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END From 36b7514b19156a599a25b391c238582a7e25d248 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 17 May 2023 21:56:31 +0530 Subject: [PATCH 06/10] Removed srcs in MPPGestureRecognizer target --- mediapipe/tasks/ios/vision/gesture_recognizer/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD index a9a62d720..c44e89d67 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD @@ -41,7 +41,6 @@ objc_library( objc_library( name = "MPPGestureRecognizer", - srcs = ["sources/MPPGestureRecognizer.mm"], hdrs = ["sources/MPPGestureRecognizer.h"], copts = [ "-ObjC++", From d6d5a948451b39d7f1357568025dfbe975f26052 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 17 May 2023 21:59:26 +0530 Subject: [PATCH 07/10] Reverted copy of gesture recognizer result containers --- .../MPPGestureRecognizerResult+Helpers.mm | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm index d125ad26f..ecb0e7831 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerResult+Helpers.mm @@ -65,7 +65,7 @@ using ::mediapipe::Packet; MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; [gestures addObject:category]; } - [multiHandGestures addObject:[gestures copy]]; + [multiHandGestures addObject:gestures]; } const std::vector &handednessClassificationListProtos = @@ -80,7 +80,7 @@ using ::mediapipe::Packet; MPPCategory *category = [MPPCategory categoryWithProto:classificationProto]; [handedness addObject:category]; } - [multiHandHandedness addObject:[handedness copy]]; + [multiHandHandedness addObject:handedness]; } const std::vector &handLandmarkListProtos = @@ -96,7 +96,7 @@ using ::mediapipe::Packet; [MPPNormalizedLandmark normalizedLandmarkWithProto:normalizedLandmarkProto]; [handLandmarks addObject:normalizedLandmark]; } - [multiHandLandmarks addObject:[handLandmarks copy]]; + [multiHandLandmarks addObject:handLandmarks]; } const std::vector &worldLandmarkListProtos = @@ -112,14 +112,14 @@ using ::mediapipe::Packet; [MPPLandmark landmarkWithProto:landmarkProto]; [worldLandmarks addObject:landmark]; } - [multiHandWorldLandmarks addObject:[worldLandmarks copy]]; + [multiHandWorldLandmarks addObject:worldLandmarks]; } MPPGestureRecognizerResult *gestureRecognizerResult = - [[MPPGestureRecognizerResult alloc] initWithGestures:[multiHandGestures copy] - handedness:[multiHandHandedness copy] - landmarks:[multiHandLandmarks copy] - worldLandmarks:[multiHandWorldLandmarks copy] + [[MPPGestureRecognizerResult alloc] initWithGestures:multiHandGestures + handedness:multiHandHandedness + landmarks:multiHandLandmarks + worldLandmarks:multiHandWorldLandmarks timestampInMilliseconds:timestampInMilliseconds]; return gestureRecognizerResult; From dc7c018b3994e42143542a2da898096a9aa951dd Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 17 May 2023 22:03:10 +0530 Subject: [PATCH 08/10] Added clearing of all graph options protos in MPPGestureRecognizerOptions Helpers --- .../utils/sources/MPPGestureRecognizerOptions+Helpers.mm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm index 111dabab2..60af976f1 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm @@ -42,12 +42,14 @@ using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto - (void)copyToProto:(CalculatorOptionsProto *)optionsProto { GestureRecognizerGraphOptionsProto *gestureRecognizerGraphOptionsProto = optionsProto->MutableExtension(GestureRecognizerGraphOptionsProto::ext); + gestureRecognizerGraphOptionsProto->Clear(); [self.baseOptions copyToProto:gestureRecognizerGraphOptionsProto->mutable_base_options() withUseStreamMode:self.runningMode != MPPRunningModeImage]; HandLandmarkerGraphOptionsProto *handLandmarkerGraphOptionsProto = gestureRecognizerGraphOptionsProto->mutable_hand_landmarker_graph_options(); + handLandmarkerGraphOptionsProto->Clear(); handLandmarkerGraphOptionsProto->set_min_tracking_confidence(self.minTrackingConfidence); HandDetectorGraphOptionsProto *handDetectorGraphOptionsProto = @@ -68,6 +70,7 @@ using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto if (self.cannedGesturesClassifierOptions) { GestureClassifierGraphOptionsProto *cannedGesturesClassifierOptionsProto = handGestureRecognizerGraphOptionsProto->mutable_canned_gesture_classifier_graph_options(); + cannedGesturesClassifierOptionsProto->Clear(); [self.cannedGesturesClassifierOptions copyToProto:cannedGesturesClassifierOptionsProto->mutable_classifier_options()]; } @@ -75,6 +78,7 @@ using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto if (self.customGesturesClassifierOptions) { GestureClassifierGraphOptionsProto *customGesturesClassifierOptionsProto = handGestureRecognizerGraphOptionsProto->mutable_custom_gesture_classifier_graph_options(); + customGesturesClassifierOptionsProto->Clear(); [self.customGesturesClassifierOptions copyToProto:customGesturesClassifierOptionsProto->mutable_classifier_options()]; } From e905a9fe39637d709785ae76ec3afb98ce971e9f Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 18 May 2023 09:52:23 +0530 Subject: [PATCH 09/10] Removed roi methods from MPPGestureRecognizer --- .../sources/MPPGestureRecognizer.h | 109 ------------------ 1 file changed, 109 deletions(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h index 2e3ca937b..ed8ff30f9 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizer.h @@ -83,35 +83,6 @@ NS_SWIFT_NAME(GestureRecognizer) error:(NSError **)error NS_SWIFT_NAME(recognize(image:)); -/** - * Performs gesture recognition on the provided `MPPImage` cropped to the specified region of - * interest. Rotation will be applied on the cropped image according to the `orientation` property - * of the provided `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with - * `MPPRunningModeImage`. - * - * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: - * 1. kCVPixelFormatType_32BGRA - * 2. kCVPixelFormatType_32RGBA - * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. - * - * @param image The `MPPImage` on which gesture recognition is to be performed. - * @param roi A `CGRect` specifying the region of interest within the given `MPPImage`, on which - * gesture recognition should be performed. - * @param error An optional error parameter populated when there is an error in performing gesture - * recognition on the input image. - * - * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition - * results. - */ -- (nullable MPPGestureRecognizerResult *)recognizeImage:(MPPImage *)image - regionOfInterest:(CGRect)roi - error:(NSError **)error - NS_SWIFT_NAME(recognize(image:regionOfInterest:)); - /** * Performs gesture recognition on the provided video frame of type `MPPImage` using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of @@ -144,42 +115,6 @@ NS_SWIFT_NAME(GestureRecognizer) error:(NSError **)error NS_SWIFT_NAME(recognize(videoFrame:timestampInMilliseconds:)); -/** - * Performs gesture recognition on the provided video frame of type `MPPImage` cropped to the - * specified region of interest. Rotation will be applied according to the `orientation` property of - * the provided `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with - * `MPPRunningModeVideo`. - * - * It's required to provide the video frame's timestamp (in milliseconds). The input timestamps must - * be monotonically increasing. - * - * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: - * 1. kCVPixelFormatType_32BGRA - * 2. kCVPixelFormatType_32RGBA - * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. - * - * @param image A live stream image data of type `MPPImage` on which gesture recognition is to be - * performed. - * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input - * timestamps must be monotonically increasing. - * @param roi A `CGRect` specifying the region of interest within the video frame of type - * `MPPImage`, on which gesture recognition should be performed. - * @param error An optional error parameter populated when there is an error in performing gesture - * recognition on the input video frame. - * - * @return An `MPPGestureRecognizerResult` object that contains the hand gesture recognition - * results. - */ -- (nullable MPPGestureRecognizerResult *)recognizeVideoFrame:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - regionOfInterest:(CGRect)roi - error:(NSError **)error - NS_SWIFT_NAME(recognize(videoFrame:timestampInMilliseconds:regionOfInterest:)); - /** * Sends live stream image data of type `MPPImage` to perform gesture recognition using the whole * image as region of interest. Rotation will be applied according to the `orientation` property of @@ -221,50 +156,6 @@ NS_SWIFT_NAME(GestureRecognizer) error:(NSError **)error NS_SWIFT_NAME(recognizeAsync(image:timestampInMilliseconds:)); -/** - * Sends live stream image data of type `MPPImage` to perform gesture recognition, cropped to the - * specified region of interest.. Rotation will be applied according to the `orientation` property - * of the provided `MPPImage`. Only use this method when the `MPPGestureRecognizer` is created with - * `MPPRunningModeLiveStream`. - * - * The object which needs to be continuously notified of the available results of gesture - * recognition must confirm to `MPPGestureRecognizerLiveStreamDelegate` protocol and implement the - * `gestureRecognizer:didFinishRecognitionWithResult:timestampInMilliseconds:error:` delegate - * method. - * - * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent - * to the gesture recognizer. The input timestamps must be monotonically increasing. - * - * This method supports gesture recognition of RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: - * 1. kCVPixelFormatType_32BGRA - * 2. kCVPixelFormatType_32RGBA - * - * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color - * space is RGB with an Alpha channel. - * - * If this method is used for preforming gesture recognition on live camera frames using - * `AVFoundation`, ensure that you request `AVCaptureVideoDataOutput` to output frames in - * `kCMPixelFormat_32RGBA` using its `videoSettings` property. - * - * @param image A live stream image data of type `MPPImage` on which gesture recognition is to be - * performed. - * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input - * image is sent to the gesture recognizer. The input timestamps must be monotonically increasing. - * @param roi A `CGRect` specifying the region of interest within the given live stream image data - * of type `MPPImage`, on which gesture recognition should be performed. - * @param error An optional error parameter populated when there is an error in performing gesture - * recognition on the input live stream image data. - * - * @return `YES` if the image was sent to the task successfully, otherwise `NO`. - */ -- (BOOL)recognizeAsyncImage:(MPPImage *)image - timestampInMilliseconds:(NSInteger)timestampInMilliseconds - regionOfInterest:(CGRect)roi - error:(NSError **)error - NS_SWIFT_NAME(recognizeAsync(image:timestampInMilliseconds:regionOfInterest:)); - - (instancetype)init NS_UNAVAILABLE; + (instancetype)new NS_UNAVAILABLE; From bb5fcc2d6426cec83d1b6ab70bafe8c10b409198 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 18 May 2023 10:00:31 +0530 Subject: [PATCH 10/10] Fixed Typos --- .../sources/MPPGestureRecognizerOptions.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h index 421073d45..925d5610b 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h @@ -37,7 +37,7 @@ NS_SWIFT_NAME(GestureRecognizerLiveStreamDelegate) @optional /** - * This method notifies a delegate that the results of asynchronous object detection of + * This method notifies a delegate that the results of asynchronous gesture recognition of * an image submitted to the `MPPGestureRecognizer` is available. * * This method is called on a private serial dispatch queue created by the `MPPGestureRecognizer` @@ -50,9 +50,9 @@ NS_SWIFT_NAME(GestureRecognizerLiveStreamDelegate) * coordinates system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the * underlying image data. * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input - * image was sent to the object detector. - * @param error An optional error parameter populated when there is an error in performing object - * detection on the input live stream image data. + * image was sent to the gesture recognizer. + * @param error An optional error parameter populated when there is an error in performing gesture + * recognition on the input live stream image data. * */ - (void)gestureRecognizer:(MPPGestureRecognizer *)gestureRecognizer @@ -69,10 +69,10 @@ NS_SWIFT_NAME(GestureRecognizerOptions) /** * Running mode of the gesture recognizer task. Defaults to `MPPRunningModeImage`. * `MPPGestureRecognizer` can be created with one of the following running modes: - * 1. `MPPRunningModeImage`: The mode for performing object detection on single image inputs. - * 2. `MPPRunningModeVideo`: The mode for performing object detection on the decoded frames of a + * 1. `MPPRunningModeImage`: The mode for performing gesture recognition on single image inputs. + * 2. `MPPRunningModeVideo`: The mode for performing gesture recognition on the decoded frames of a * video. - * 3. `MPPRunningModeLiveStream`: The mode for performing object detection on a live stream of + * 3. `MPPRunningModeLiveStream`: The mode for performing gesture recognition on a live stream of * input data, such as from the camera. */ @property(nonatomic) MPPRunningMode runningMode;