Merge pull request #4467 from priankakariatyml:ios-hand-landmarker-containers

PiperOrigin-RevId: 536447484

mediapipe/tasks/ios/vision/hand_landmarker/BUILD

@@ -0,0 +1,39 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+objc_library(
+    name = "MPPHandLandmarkerResult",
+    srcs = ["sources/MPPHandLandmarkerResult.m"],
+    hdrs = ["sources/MPPHandLandmarkerResult.h"],
+    deps = [
+        "//mediapipe/tasks/ios/components/containers:MPPCategory",
+        "//mediapipe/tasks/ios/components/containers:MPPLandmark",
+        "//mediapipe/tasks/ios/core:MPPTaskResult",
+    ],
+)
+
+objc_library(
+    name = "MPPHandLandmarkerOptions",
+    srcs = ["sources/MPPHandLandmarkerOptions.m"],
+    hdrs = ["sources/MPPHandLandmarkerOptions.h"],
+    deps = [
+        ":MPPHandLandmarkerResult",
+        "//mediapipe/tasks/ios/core:MPPTaskOptions",
+        "//mediapipe/tasks/ios/vision/core:MPPRunningMode",
+    ],
+)

mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarkerOptions.h

@@ -0,0 +1,102 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+
+#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h"
+#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h"
+#import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarkerResult.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@class MPPHandLandmarker;
+
+/**
+ * This protocol defines an interface for the delegates of `MPPHandLandmarker` object to receive
+ * results of performing asynchronous hand landmark detection on images (i.e, when `runningMode` =
+ * `MPPRunningModeLiveStream`).
+ *
+ * The delegate of `MPPHandLandmarker` must adopt `MPPHandLandmarkerLiveStreamDelegate` protocol.
+ * The methods in this protocol are optional.
+ */
+NS_SWIFT_NAME(HandLandmarkerLiveStreamDelegate)
+@protocol MPPHandLandmarkerLiveStreamDelegate <NSObject>
+
+@optional
+
+/**
+ * This method notifies a delegate that the results of asynchronous hand landmark detection of an
+ * image submitted to the `MPPHandLandmarker` is available.
+ *
+ * This method is called on a private serial dispatch queue created by the `MPPHandLandmarker`
+ * for performing the asynchronous delegates calls.
+ *
+ * @param handLandmarker The hand landmarker which performed the hand landmarking.
+ * This is useful to test equality when there are multiple instances of `MPPHandLandmarker`.
+ * @param result The `MPPHandLandmarkerResult` object that contains a list of detections, each
+ * detection has a bounding box that is expressed in the unrotated input frame of reference
+ * coordinates system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the
+ * underlying image data.
+ * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input
+ * image was sent to the hand landmarker.
+ * @param error An optional error parameter populated when there is an error in performing hand
+ * landmark detection on the input live stream image data.
+ */
+- (void)handLandmarker:(MPPHandLandmarker *)handLandmarker
+    didFinishDetectionWithResult:(nullable MPPHandLandmarkerResult *)result
+         timestampInMilliseconds:(NSInteger)timestampInMilliseconds
+                           error:(nullable NSError *)error
+    NS_SWIFT_NAME(handLandmarker(_:didFinishDetection:timestampInMilliseconds:error:));
+@end
+
+/** Options for setting up a `MPPHandLandmarker`. */
+NS_SWIFT_NAME(HandLandmarkerOptions)
+@interface MPPHandLandmarkerOptions : MPPTaskOptions <NSCopying>
+
+/**
+ * Running mode of the hand landmarker task. Defaults to `MPPRunningModeImage`.
+ * `MPPHandLandmarker` can be created with one of the following running modes:
+ *  1. `MPPRunningModeImage`: The mode for performing hand landmark detection on single image
+ * inputs.
+ *  2. `MPPRunningModeVideo`: The mode for performing hand landmark detection on the decoded frames
+ * of a video.
+ *  3. `MPPRunningModeLiveStream`: The mode for performing hand landmark detection on a live stream
+ * of input data, such as from the camera.
+ */
+@property(nonatomic) MPPRunningMode runningMode;
+
+/**
+ * An object that confirms to `MPPHandLandmarkerLiveStreamDelegate` protocol. This object must
+ * implement `handLandmarker:didFinishDetectionWithResult:timestampInMilliseconds:error:` to
+ * receive the results of performing asynchronous hand landmark detection on images (i.e, when
+ * `runningMode` = `MPPRunningModeLiveStream`).
+ */
+@property(nonatomic, weak, nullable) id<MPPHandLandmarkerLiveStreamDelegate>
+    handLandmarkerLiveStreamDelegate;
+
+/** The maximum number of hands that can be detected by the `MPPHandLandmarker`. */
+@property(nonatomic) NSInteger numHands;
+
+/** The minimum confidence score for the hand detection to be considered successful. */
+@property(nonatomic) float minHandDetectionConfidence;
+
+/** The minimum confidence score of hand presence score in the hand landmark detection. */
+@property(nonatomic) float minHandPresenceConfidence;
+
+/** The minimum confidence score for the hand tracking to be considered successful. */
+@property(nonatomic) float minTrackingConfidence;
+
+@end
+
+NS_ASSUME_NONNULL_END

mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarkerOptions.m

@@ -0,0 +1,43 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarkerOptions.h"
+
+@implementation MPPHandLandmarkerOptions
+
+- (instancetype)init {
+  self = [super init];
+  if (self) {
+    _numHands = 1;
+    _minHandDetectionConfidence = 0.5f;
+    _minHandPresenceConfidence = 0.5f;
+    _minTrackingConfidence = 0.5f;
+  }
+  return self;
+}
+
+- (id)copyWithZone:(NSZone *)zone {
+  MPPHandLandmarkerOptions *handLandmarkerOptions = [super copyWithZone:zone];
+
+  handLandmarkerOptions.runningMode = self.runningMode;
+  handLandmarkerOptions.handLandmarkerLiveStreamDelegate = self.handLandmarkerLiveStreamDelegate;
+  handLandmarkerOptions.numHands = self.numHands;
+  handLandmarkerOptions.minHandDetectionConfidence = self.minHandDetectionConfidence;
+  handLandmarkerOptions.minHandPresenceConfidence = self.minHandPresenceConfidence;
+  handLandmarkerOptions.minTrackingConfidence = self.minTrackingConfidence;
+
+  return handLandmarkerOptions;
+}
+
+@end

mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarkerResult.h

@@ -0,0 +1,56 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+
+#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h"
+#import "mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h"
+#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+/** Represents the hand landmarker results generated by MPPHandLandmarker. */
+NS_SWIFT_NAME(HandLandmarkerResult)
+@interface MPPHandLandmarkerResult : MPPTaskResult
+
+/** Hand landmarks of detected hands. */
+@property(nonatomic, readonly) NSArray<NSArray<MPPNormalizedLandmark *> *> *landmarks;
+
+/** Hand landmarks in world coordniates of detected hands. */
+@property(nonatomic, readonly) NSArray<NSArray<MPPLandmark *> *> *worldLandmarks;
+
+/** Handedness of detected hands. */
+@property(nonatomic, readonly) NSArray<NSArray<MPPCategory *> *> *handedness;
+
+/**
+ * Initializes a new `MPPHandLandmarkerResult` with the given landmarks, world landmarks,
+ * handedness and timestamp (in milliseconds).
+ *
+ * @param landmarks The hand landmarks of detected hands.
+ * @param worldLandmarks The hand landmarks in world coordniates of detected hands.
+ * @param handedness The handedness of detected hands.
+ * @param timestampInMilliseconds The timestamp for this result.
+ *
+ * @return An instance of `MPPGHandLandmarkerResult` initialized with the given landmarks, world
+ * landmarks, handedness and timestamp (in milliseconds).
+ *
+ */
+- (instancetype)initWithLandmarks:(NSArray<NSArray<MPPNormalizedLandmark *> *> *)landmarks
+                   worldLandmarks:(NSArray<NSArray<MPPLandmark *> *> *)worldLandmarks
+                       handedness:(NSArray<NSArray<MPPCategory *> *> *)handedness
+          timestampInMilliseconds:(NSInteger)timestampInMilliseconds;
+
+@end
+
+NS_ASSUME_NONNULL_END

mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarkerResult.m

@@ -0,0 +1,32 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarkerResult.h"
+
+@implementation MPPHandLandmarkerResult
+
+- (instancetype)initWithLandmarks:(NSArray<NSArray<MPPNormalizedLandmark *> *> *)landmarks
+                   worldLandmarks:(NSArray<NSArray<MPPLandmark *> *> *)worldLandmarks
+                       handedness:(NSArray<NSArray<MPPCategory *> *> *)handedness
+          timestampInMilliseconds:(NSInteger)timestampInMilliseconds {
+  self = [super initWithTimestampInMilliseconds:timestampInMilliseconds];
+  if (self) {
+    _landmarks = landmarks;
+    _worldLandmarks = worldLandmarks;
+    _handedness = handedness;
+  }
+  return self;
+}
+
+@end

mediapipe/tasks/ios/vision/hand_landmarker/utils/BUILD

@@ -0,0 +1,33 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+objc_library(
+    name = "MPPHandLandmarkerOptionsHelpers",
+    srcs = ["sources/MPPHandLandmarkerOptions+Helpers.mm"],
+    hdrs = ["sources/MPPHandLandmarkerOptions+Helpers.h"],
+    deps = [
+        "//mediapipe/framework:calculator_options_cc_proto",
+        "//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_cc_proto",
+        "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_cc_proto",
+        "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_cc_proto",
+        "//mediapipe/tasks/ios/common/utils:NSStringHelpers",
+        "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol",
+        "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers",
+        "//mediapipe/tasks/ios/vision/hand_landmarker:MPPHandLandmarkerOptions",
+    ],
+)

mediapipe/tasks/ios/vision/hand_landmarker/utils/sources/MPPHandLandmarkerOptions+Helpers.h

@@ -0,0 +1,32 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator_options.pb.h"
+#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h"
+#import "mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarkerOptions.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface MPPHandLandmarkerOptions (Helpers) <MPPTaskOptionsProtocol>
+
+/**
+ * Populates the provided `CalculatorOptions` proto container with the current settings.
+ *
+ * @param optionsProto The `CalculatorOptions` proto object to copy the settings to.
+ */
+- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto;
+
+@end
+
+NS_ASSUME_NONNULL_END

mediapipe/tasks/ios/vision/hand_landmarker/utils/sources/MPPHandLandmarkerOptions+Helpers.mm

@@ -0,0 +1,58 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "mediapipe/tasks/ios/vision/hand_landmarker/utils/sources/MPPHandLandmarkerOptions+Helpers.h"
+
+#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h"
+#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h"
+
+#include "mediapipe/tasks/cc/vision/hand_detector/proto/hand_detector_graph_options.pb.h"
+#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarker_graph_options.pb.h"
+#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options.pb.h"
+
+namespace {
+using CalculatorOptionsProto = mediapipe::CalculatorOptions;
+using HandLandmarkerGraphOptionsProto =
+    ::mediapipe::tasks::vision::hand_landmarker::proto::HandLandmarkerGraphOptions;
+using HandDetectorGraphOptionsProto =
+    ::mediapipe::tasks::vision::hand_detector::proto::HandDetectorGraphOptions;
+using HandLandmarksDetectorGraphOptionsProto =
+    ::mediapipe::tasks::vision::hand_landmarker::proto::HandLandmarksDetectorGraphOptions;
+}  // namespace
+
+@implementation MPPHandLandmarkerOptions (Helpers)
+
+- (void)copyToProto:(CalculatorOptionsProto *)optionsProto {
+  HandLandmarkerGraphOptionsProto *handLandmarkerGraphOptionsProto =
+      optionsProto
+          ->MutableExtension(HandLandmarkerGraphOptionsProto::ext);
+              handLandmarkerGraphOptionsProto->Clear();
+
+  [self.baseOptions copyToProto:handLandmarkerGraphOptionsProto->mutable_base_options()
+              withUseStreamMode:self.runningMode != MPPRunningModeImage];
+
+  handLandmarkerGraphOptionsProto->set_min_tracking_confidence(self.minTrackingConfidence);
+
+  HandDetectorGraphOptionsProto *handDetectorGraphOptionsProto =
+      handLandmarkerGraphOptionsProto->mutable_hand_detector_graph_options();
+  handDetectorGraphOptionsProto->set_num_hands(self.numHands);
+  handDetectorGraphOptionsProto->set_min_detection_confidence(self.minHandDetectionConfidence);
+
+  HandLandmarksDetectorGraphOptionsProto *handLandmarksDetectorGraphOptionsProto =
+      handLandmarkerGraphOptionsProto->mutable_hand_landmarks_detector_graph_options();
+  handLandmarksDetectorGraphOptionsProto->set_min_detection_confidence(
+      self.minHandPresenceConfidence);
+}
+
+@end