Merge pull request #4850 from priankakariatyml:ios-pose-landmarker-containers

PiperOrigin-RevId: 574954344

mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h

@@ -27,7 +27,7 @@ NS_SWIFT_NAME(GestureRecognizerResult)
 /** Hand landmarks of detected hands. */
 @property(nonatomic, readonly) NSArray<NSArray<MPPNormalizedLandmark *> *> *landmarks;
 
-/** Hand landmarks in world coordniates of detected hands. */
+/** Hand landmarks in world coordinates of detected hands. */
 @property(nonatomic, readonly) NSArray<NSArray<MPPLandmark *> *> *worldLandmarks;
 
 /** Handedness of detected hands. */
@@ -45,7 +45,7 @@ NS_SWIFT_NAME(GestureRecognizerResult)
  * handedness, gestures and timestamp (in milliseconds).
  *
  * @param landmarks The hand landmarks of detected hands.
- * @param worldLandmarks The hand landmarks in world coordniates of detected hands.
+ * @param worldLandmarks The hand landmarks in world coordinates of detected hands.
  * @param handedness The handedness of detected hands.
  * @param handedness The recognized hand gestures of detected hands.
  * @param timestampInMilliseconds The timestamp for this result.

mediapipe/tasks/ios/vision/hand_landmarker/sources/MPPHandLandmarkerResult.h

@@ -27,7 +27,7 @@ NS_SWIFT_NAME(HandLandmarkerResult)
 /** Hand landmarks of detected hands. */
 @property(nonatomic, readonly) NSArray<NSArray<MPPNormalizedLandmark *> *> *landmarks;
 
-/** Hand landmarks in world coordniates of detected hands. */
+/** Hand landmarks in world coordinates of detected hands. */
 @property(nonatomic, readonly) NSArray<NSArray<MPPLandmark *> *> *worldLandmarks;
 
 /** Handedness of detected hands. */
@@ -38,7 +38,7 @@ NS_SWIFT_NAME(HandLandmarkerResult)
  * and timestamp (in milliseconds).
  *
  * @param landmarks The hand landmarks of detected hands.
- * @param worldLandmarks The hand landmarks in world coordniates of detected hands.
+ * @param worldLandmarks The hand landmarks in world coordinates of detected hands.
  * @param handedness The handedness of detected hands.
  * @param timestampInMilliseconds The timestamp for this result.
  *

mediapipe/tasks/ios/vision/pose_landmarker/BUILD

@@ -0,0 +1,39 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+objc_library(
+    name = "MPPPoseLandmarkerResult",
+    srcs = ["sources/MPPPoseLandmarkerResult.m"],
+    hdrs = ["sources/MPPPoseLandmarkerResult.h"],
+    deps = [
+        "//mediapipe/tasks/ios/components/containers:MPPLandmark",
+        "//mediapipe/tasks/ios/core:MPPTaskResult",
+        "//mediapipe/tasks/ios/vision/core:MPPMask",
+    ],
+)
+
+objc_library(
+    name = "MPPPoseLandmarkerOptions",
+    srcs = ["sources/MPPPoseLandmarkerOptions.m"],
+    hdrs = ["sources/MPPPoseLandmarkerOptions.h"],
+    deps = [
+        ":MPPPoseLandmarkerResult",
+        "//mediapipe/tasks/ios/core:MPPTaskOptions",
+        "//mediapipe/tasks/ios/vision/core:MPPRunningMode",
+    ],
+)

mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h

@@ -0,0 +1,107 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+
+#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h"
+#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h"
+#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@class MPPPoseLandmarker;
+
+/**
+ * This protocol defines an interface for the delegates of `PoseLandmarker` to receive
+ * results of performing asynchronous pose landmark detection on images (i.e, when `runningMode` =
+ * `.liveStream`).
+ *
+ * The delegate of `PoseLandmarker` must adopt `PoseLandmarkerLiveStreamDelegate` protocol.
+ * The methods in this protocol are optional.
+ */
+NS_SWIFT_NAME(PoseLandmarkerLiveStreamDelegate)
+@protocol MPPPoseLandmarkerLiveStreamDelegate <NSObject>
+
+/**
+ * This method notifies a delegate that the results of asynchronous pose landmark detection of an
+ * image submitted to the `PoseLandmarker` is available.
+ *
+ * This method is called on a private serial dispatch queue created by the `PoseLandmarker`
+ * for performing the asynchronous delegates calls.
+ *
+ * @param poseLandmarker The pose landmarker which performed the pose landmark detection.
+ * This is useful to test equality when there are multiple instances of `PoseLandmarker`.
+ * @param result The `PoseLandmarkerResult` object that contains a list of landmark.
+ * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input
+ * image was sent to the pose landmarker.
+ * @param error An optional error parameter populated when there is an error in performing pose
+ * landmark detection on the input live stream image data.
+ */
+- (void)poseLandmarker:(MPPPoseLandmarker *)poseLandmarker
+    didFinishDetectionWithResult:(nullable MPPPoseLandmarkerResult *)result
+         timestampInMilliseconds:(NSInteger)timestampInMilliseconds
+                           error:(nullable NSError *)error
+    NS_SWIFT_NAME(poseLandmarker(_:didFinishDetection:timestampInMilliseconds:error:));
+@end
+
+/** Options for setting up a `PoseLandmarker`. */
+NS_SWIFT_NAME(PoseLandmarkerOptions)
+@interface MPPPoseLandmarkerOptions : MPPTaskOptions <NSCopying>
+
+/**
+ * Running mode of the pose landmark dection task. Defaults to `.image`. `PoseLandmarker` can be
+ * created with one of the following running modes:
+ *  1. `.image`: The mode for performing pose landmark detection on single image inputs.
+ *  2. `.video`: The mode for performing pose landmark detection on the decoded frames of a video.
+ *  3. `.liveStream`: The mode for performing pose landmark detection on a live stream of input
+ * data, such as from the camera.
+ */
+@property(nonatomic) MPPRunningMode runningMode;
+
+/**
+ * An object that confirms to `PoseLandmarkerLiveStreamDelegate` protocol. This object must
+ * implement `poseLandmarker(_:didFinishDetectionWithResult:timestampInMilliseconds:error:)` to
+ * receive the results of performing asynchronous pose landmark detection on images (i.e, when
+ * `runningMode` = `.liveStream`).
+ */
+@property(nonatomic, weak, nullable) id<MPPPoseLandmarkerLiveStreamDelegate>
+    poseLandmarkerLiveStreamDelegate;
+
+/** The maximum number of poses that can be detected by the `PoseLandmarker`. Defaults to 1. */
+@property(nonatomic) NSInteger numPoses;
+
+/**
+ * The minimum confidence score for pose detection to be considered successful. Defaults to 0.5.
+ */
+@property(nonatomic) float minPoseDetectionConfidence;
+
+/**
+ * The minimum confidence score of pose presence score in the pose landmark detection. Defaults to
+ * 0.5.
+ */
+@property(nonatomic) float minPosePresenceConfidence;
+
+/**
+ * The minimum confidence score for pose tracking to be considered successful. Defaults to 0.5.
+ */
+@property(nonatomic) float minTrackingConfidence;
+
+/**
+ * Whether to output segmentation masks. Defaults to `false`.
+ */
+@property(nonatomic) BOOL shouldOutputSegmentationMasks;
+
+@end
+
+NS_ASSUME_NONNULL_END

mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.m

@@ -0,0 +1,45 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h"
+
+@implementation MPPPoseLandmarkerOptions
+
+- (instancetype)init {
+  self = [super init];
+  if (self) {
+    _numPoses = 1;
+    _minPoseDetectionConfidence = 0.5f;
+    _minPosePresenceConfidence = 0.5f;
+    _minTrackingConfidence = 0.5f;
+    _shouldOutputSegmentationMasks = NO;
+  }
+  return self;
+}
+
+- (id)copyWithZone:(NSZone *)zone {
+  MPPPoseLandmarkerOptions *poseLandmarkerOptions = [super copyWithZone:zone];
+
+  poseLandmarkerOptions.runningMode = self.runningMode;
+  poseLandmarkerOptions.numPoses = self.numPoses;
+  poseLandmarkerOptions.minPoseDetectionConfidence = self.minPoseDetectionConfidence;
+  poseLandmarkerOptions.minPosePresenceConfidence = self.minPosePresenceConfidence;
+  poseLandmarkerOptions.minTrackingConfidence = self.minTrackingConfidence;
+  poseLandmarkerOptions.shouldOutputSegmentationMasks = self.shouldOutputSegmentationMasks;
+  poseLandmarkerOptions.poseLandmarkerLiveStreamDelegate = self.poseLandmarkerLiveStreamDelegate;
+
+  return poseLandmarkerOptions;
+}
+
+@end

mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h

@@ -0,0 +1,60 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+#import "mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h"
+#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h"
+#import "mediapipe/tasks/ios/vision/core/sources/MPPMask.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+/** Represents the pose landmarks deection results generated by `PoseLandmarker`. */
+NS_SWIFT_NAME(PoseLandmarkerResult)
+@interface MPPPoseLandmarkerResult : MPPTaskResult
+
+/** Pose landmarks of detected poses. */
+@property(nonatomic, readonly) NSArray<NSArray<MPPNormalizedLandmark *> *> *landmarks;
+
+/** Pose landmarks in world coordinates of detected poses. */
+@property(nonatomic, readonly) NSArray<NSArray<MPPLandmark *> *> *worldLandmarks;
+
+/** Pose segmentation masks. */
+@property(nonatomic, readonly) NSArray<MPPMask *> *segmentationMasks;
+
+/**
+ * Initializes a new `PoseLandmarkerResult` with the given array of landmarks, world landmarks,
+ * segmentation masks of the detected poses and timestamp (in milliseconds).
+ *
+ * @param landmarks An array of `NormalizedLandmark` objects.
+ * @param worldLandmarks An array of `Landmark` objects.
+ * @param segmentationMasks An array of `Mask` objects.
+ * @param timestampInMilliseconds The timestamp (in milliseconds) for this result.
+ *
+ * @return An instance of `PoseLandmarkerResult` initialized with the given array of landmarks,
+ * world landmarks, segmentation masks of the detected poses and timestamp (in milliseconds).
+ */
+- (instancetype)initWithLandmarks:(NSArray<NSArray<MPPNormalizedLandmark *> *> *)landmarks
+                   worldLandmarks:(NSArray<NSArray<MPPLandmark *> *> *)worldLandmarks
+                segmentationMasks:(NSArray<MPPMask *> *)segmentationMasks
+          timestampInMilliseconds:(NSInteger)timestampInMilliseconds NS_DESIGNATED_INITIALIZER;
+
+- (instancetype)initWithTimestampInMilliseconds:(NSInteger)timestampInMilliseconds NS_UNAVAILABLE;
+
+- (instancetype)init NS_UNAVAILABLE;
+
++ (instancetype)new NS_UNAVAILABLE;
+
+@end
+
+NS_ASSUME_NONNULL_END

mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.m

@@ -0,0 +1,34 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+
+#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h"
+
+@implementation MPPPoseLandmarkerResult
+
+- (instancetype)initWithLandmarks:(NSArray<NSArray<MPPNormalizedLandmark *> *> *)landmarks
+                   worldLandmarks:(NSArray<NSArray<MPPLandmark *> *> *)worldLandmarks
+                segmentationMasks:(NSArray<MPPMask *> *)segmentationMasks
+          timestampInMilliseconds:(NSInteger)timestampInMilliseconds {
+  self = [super initWithTimestampInMilliseconds:timestampInMilliseconds];
+  if (self) {
+    _landmarks = [landmarks copy];
+    _worldLandmarks = [worldLandmarks copy];
+    _segmentationMasks = [segmentationMasks copy];
+  }
+  return self;
+}
+
+@end

mediapipe/tasks/ios/vision/pose_landmarker/utils/BUILD

@@ -0,0 +1,38 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+objc_library(
+    name = "MPPPoseLandmarkerOptionsHelpers",
+    srcs = ["sources/MPPPoseLandmarkerOptions+Helpers.mm"],
+    hdrs = ["sources/MPPPoseLandmarkerOptions+Helpers.h"],
+    copts = [
+        "-ObjC++",
+        "-std=c++17",
+        "-x objective-c++",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_options_cc_proto",
+        "//mediapipe/tasks/cc/vision/pose_detector/proto:pose_detector_graph_options_cc_proto",
+        "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarker_graph_options_cc_proto",
+        "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarks_detector_graph_options_cc_proto",
+        "//mediapipe/tasks/ios/common/utils:NSStringHelpers",
+        "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol",
+        "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers",
+        "//mediapipe/tasks/ios/vision/pose_landmarker:MPPPoseLandmarkerOptions",
+    ],
+)

mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.h

@@ -0,0 +1,36 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __cplusplus
+#error "This file requires Objective-C++."
+#endif  // __cplusplus
+
+#include "mediapipe/framework/calculator_options.pb.h"
+#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h"
+#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface MPPPoseLandmarkerOptions (Helpers) <MPPTaskOptionsProtocol>
+
+/**
+ * Populates the provided `CalculatorOptions` proto container with the current settings.
+ *
+ * @param optionsProto The `CalculatorOptions` proto object to copy the settings to.
+ */
+- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto;
+
+@end
+
+NS_ASSUME_NONNULL_END

mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.mm

@@ -0,0 +1,52 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.h"
+
+#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h"
+#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h"
+
+#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarker_graph_options.pb.h"
+#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.pb.h"
+#include "mediapipe/tasks/cc/vision/pose_detector/proto/pose_detector_graph_options.pb.h"
+
+using CalculatorOptionsProto = ::mediapipe::CalculatorOptions;
+using PoseDetectorGraphOptionsProto =
+    ::mediapipe::tasks::vision::pose_detector::proto::PoseDetectorGraphOptions;
+using PoseLandmarksDetectorGraphOptionsProto =
+    ::mediapipe::tasks::vision::pose_landmarker::proto::PoseLandmarksDetectorGraphOptions;
+using PoseLandmarkerGraphOptionsProto =
+    ::mediapipe::tasks::vision::pose_landmarker::proto::PoseLandmarkerGraphOptions;
+
+@implementation MPPPoseLandmarkerOptions (Helpers)
+
+- (void)copyToProto:(CalculatorOptionsProto *)optionsProto {
+  PoseLandmarkerGraphOptionsProto *poseLandmarkerGraphOptions =
+      optionsProto->MutableExtension(PoseLandmarkerGraphOptionsProto::ext);
+  poseLandmarkerGraphOptions->Clear();
+
+  [self.baseOptions copyToProto:poseLandmarkerGraphOptions->mutable_base_options()];
+  poseLandmarkerGraphOptions->set_min_tracking_confidence(self.minTrackingConfidence);
+
+  PoseLandmarksDetectorGraphOptionsProto *poseLandmarksDetectorGraphOptions =
+      poseLandmarkerGraphOptions->mutable_pose_landmarks_detector_graph_options();
+  poseLandmarksDetectorGraphOptions->set_min_detection_confidence(self.minPosePresenceConfidence);
+
+  PoseDetectorGraphOptionsProto *poseDetectorGraphOptions =
+      poseLandmarkerGraphOptions->mutable_pose_detector_graph_options();
+  poseDetectorGraphOptions->set_num_poses(self.numPoses);
+  poseDetectorGraphOptions->set_min_detection_confidence(self.minPoseDetectionConfidence);
+}
+
+@end

mediapipe/tasks/java/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerResult.java

@@ -100,7 +100,7 @@ public abstract class PoseLandmarkerResult implements TaskResult {
   /** Pose landmarks of detected poses. */
   public abstract List<List<NormalizedLandmark>> landmarks();
 
-  /** Pose landmarks in world coordniates of detected poses. */
+  /** Pose landmarks in world coordinates of detected poses. */
   public abstract List<List<Landmark>> worldLandmarks();
 
   /** Pose segmentation masks. */

mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer_result.d.ts

@@ -26,7 +26,7 @@ export declare interface GestureRecognizerResult {
   /** Hand landmarks of detected hands. */
   landmarks: NormalizedLandmark[][];
 
-  /** Hand landmarks in world coordniates of detected hands. */
+  /** Hand landmarks in world coordinates of detected hands. */
   worldLandmarks: Landmark[][];
 
   /** Handedness of detected hands. */