Merge pull request #4850 from priankakariatyml:ios-pose-landmarker-containers

PiperOrigin-RevId: 574954344
This commit is contained in:
Copybara-Service 2023-10-19 11:54:25 -07:00
commit ddf46a2a61
12 changed files with 417 additions and 6 deletions

View File

@ -27,7 +27,7 @@ NS_SWIFT_NAME(GestureRecognizerResult)
/** Hand landmarks of detected hands. */
@property(nonatomic, readonly) NSArray<NSArray<MPPNormalizedLandmark *> *> *landmarks;
/** Hand landmarks in world coordniates of detected hands. */
/** Hand landmarks in world coordinates of detected hands. */
@property(nonatomic, readonly) NSArray<NSArray<MPPLandmark *> *> *worldLandmarks;
/** Handedness of detected hands. */
@ -45,7 +45,7 @@ NS_SWIFT_NAME(GestureRecognizerResult)
* handedness, gestures and timestamp (in milliseconds).
*
* @param landmarks The hand landmarks of detected hands.
* @param worldLandmarks The hand landmarks in world coordniates of detected hands.
* @param worldLandmarks The hand landmarks in world coordinates of detected hands.
* @param handedness The handedness of detected hands.
* @param handedness The recognized hand gestures of detected hands.
* @param timestampInMilliseconds The timestamp for this result.

View File

@ -27,7 +27,7 @@ NS_SWIFT_NAME(HandLandmarkerResult)
/** Hand landmarks of detected hands. */
@property(nonatomic, readonly) NSArray<NSArray<MPPNormalizedLandmark *> *> *landmarks;
/** Hand landmarks in world coordniates of detected hands. */
/** Hand landmarks in world coordinates of detected hands. */
@property(nonatomic, readonly) NSArray<NSArray<MPPLandmark *> *> *worldLandmarks;
/** Handedness of detected hands. */
@ -38,7 +38,7 @@ NS_SWIFT_NAME(HandLandmarkerResult)
* and timestamp (in milliseconds).
*
* @param landmarks The hand landmarks of detected hands.
* @param worldLandmarks The hand landmarks in world coordniates of detected hands.
* @param worldLandmarks The hand landmarks in world coordinates of detected hands.
* @param handedness The handedness of detected hands.
* @param timestampInMilliseconds The timestamp for this result.
*

View File

@ -0,0 +1,39 @@
# Copyright 2023 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
objc_library(
name = "MPPPoseLandmarkerResult",
srcs = ["sources/MPPPoseLandmarkerResult.m"],
hdrs = ["sources/MPPPoseLandmarkerResult.h"],
deps = [
"//mediapipe/tasks/ios/components/containers:MPPLandmark",
"//mediapipe/tasks/ios/core:MPPTaskResult",
"//mediapipe/tasks/ios/vision/core:MPPMask",
],
)
objc_library(
name = "MPPPoseLandmarkerOptions",
srcs = ["sources/MPPPoseLandmarkerOptions.m"],
hdrs = ["sources/MPPPoseLandmarkerOptions.h"],
deps = [
":MPPPoseLandmarkerResult",
"//mediapipe/tasks/ios/core:MPPTaskOptions",
"//mediapipe/tasks/ios/vision/core:MPPRunningMode",
],
)

View File

@ -0,0 +1,107 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <Foundation/Foundation.h>
#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h"
#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h"
#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h"
NS_ASSUME_NONNULL_BEGIN
@class MPPPoseLandmarker;
/**
* This protocol defines an interface for the delegates of `PoseLandmarker` to receive
* results of performing asynchronous pose landmark detection on images (i.e, when `runningMode` =
* `.liveStream`).
*
* The delegate of `PoseLandmarker` must adopt `PoseLandmarkerLiveStreamDelegate` protocol.
* The methods in this protocol are optional.
*/
NS_SWIFT_NAME(PoseLandmarkerLiveStreamDelegate)
@protocol MPPPoseLandmarkerLiveStreamDelegate <NSObject>
/**
* This method notifies a delegate that the results of asynchronous pose landmark detection of an
* image submitted to the `PoseLandmarker` is available.
*
* This method is called on a private serial dispatch queue created by the `PoseLandmarker`
* for performing the asynchronous delegates calls.
*
* @param poseLandmarker The pose landmarker which performed the pose landmark detection.
* This is useful to test equality when there are multiple instances of `PoseLandmarker`.
* @param result The `PoseLandmarkerResult` object that contains a list of landmark.
* @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input
* image was sent to the pose landmarker.
* @param error An optional error parameter populated when there is an error in performing pose
* landmark detection on the input live stream image data.
*/
- (void)poseLandmarker:(MPPPoseLandmarker *)poseLandmarker
didFinishDetectionWithResult:(nullable MPPPoseLandmarkerResult *)result
timestampInMilliseconds:(NSInteger)timestampInMilliseconds
error:(nullable NSError *)error
NS_SWIFT_NAME(poseLandmarker(_:didFinishDetection:timestampInMilliseconds:error:));
@end
/** Options for setting up a `PoseLandmarker`. */
NS_SWIFT_NAME(PoseLandmarkerOptions)
@interface MPPPoseLandmarkerOptions : MPPTaskOptions <NSCopying>
/**
* Running mode of the pose landmark dection task. Defaults to `.image`. `PoseLandmarker` can be
* created with one of the following running modes:
* 1. `.image`: The mode for performing pose landmark detection on single image inputs.
* 2. `.video`: The mode for performing pose landmark detection on the decoded frames of a video.
* 3. `.liveStream`: The mode for performing pose landmark detection on a live stream of input
* data, such as from the camera.
*/
@property(nonatomic) MPPRunningMode runningMode;
/**
* An object that confirms to `PoseLandmarkerLiveStreamDelegate` protocol. This object must
* implement `poseLandmarker(_:didFinishDetectionWithResult:timestampInMilliseconds:error:)` to
* receive the results of performing asynchronous pose landmark detection on images (i.e, when
* `runningMode` = `.liveStream`).
*/
@property(nonatomic, weak, nullable) id<MPPPoseLandmarkerLiveStreamDelegate>
poseLandmarkerLiveStreamDelegate;
/** The maximum number of poses that can be detected by the `PoseLandmarker`. Defaults to 1. */
@property(nonatomic) NSInteger numPoses;
/**
* The minimum confidence score for pose detection to be considered successful. Defaults to 0.5.
*/
@property(nonatomic) float minPoseDetectionConfidence;
/**
* The minimum confidence score of pose presence score in the pose landmark detection. Defaults to
* 0.5.
*/
@property(nonatomic) float minPosePresenceConfidence;
/**
* The minimum confidence score for pose tracking to be considered successful. Defaults to 0.5.
*/
@property(nonatomic) float minTrackingConfidence;
/**
* Whether to output segmentation masks. Defaults to `false`.
*/
@property(nonatomic) BOOL shouldOutputSegmentationMasks;
@end
NS_ASSUME_NONNULL_END

View File

@ -0,0 +1,45 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h"
@implementation MPPPoseLandmarkerOptions
- (instancetype)init {
self = [super init];
if (self) {
_numPoses = 1;
_minPoseDetectionConfidence = 0.5f;
_minPosePresenceConfidence = 0.5f;
_minTrackingConfidence = 0.5f;
_shouldOutputSegmentationMasks = NO;
}
return self;
}
- (id)copyWithZone:(NSZone *)zone {
MPPPoseLandmarkerOptions *poseLandmarkerOptions = [super copyWithZone:zone];
poseLandmarkerOptions.runningMode = self.runningMode;
poseLandmarkerOptions.numPoses = self.numPoses;
poseLandmarkerOptions.minPoseDetectionConfidence = self.minPoseDetectionConfidence;
poseLandmarkerOptions.minPosePresenceConfidence = self.minPosePresenceConfidence;
poseLandmarkerOptions.minTrackingConfidence = self.minTrackingConfidence;
poseLandmarkerOptions.shouldOutputSegmentationMasks = self.shouldOutputSegmentationMasks;
poseLandmarkerOptions.poseLandmarkerLiveStreamDelegate = self.poseLandmarkerLiveStreamDelegate;
return poseLandmarkerOptions;
}
@end

View File

@ -0,0 +1,60 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <Foundation/Foundation.h>
#import "mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h"
#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h"
#import "mediapipe/tasks/ios/vision/core/sources/MPPMask.h"
NS_ASSUME_NONNULL_BEGIN
/** Represents the pose landmarks deection results generated by `PoseLandmarker`. */
NS_SWIFT_NAME(PoseLandmarkerResult)
@interface MPPPoseLandmarkerResult : MPPTaskResult
/** Pose landmarks of detected poses. */
@property(nonatomic, readonly) NSArray<NSArray<MPPNormalizedLandmark *> *> *landmarks;
/** Pose landmarks in world coordinates of detected poses. */
@property(nonatomic, readonly) NSArray<NSArray<MPPLandmark *> *> *worldLandmarks;
/** Pose segmentation masks. */
@property(nonatomic, readonly) NSArray<MPPMask *> *segmentationMasks;
/**
* Initializes a new `PoseLandmarkerResult` with the given array of landmarks, world landmarks,
* segmentation masks of the detected poses and timestamp (in milliseconds).
*
* @param landmarks An array of `NormalizedLandmark` objects.
* @param worldLandmarks An array of `Landmark` objects.
* @param segmentationMasks An array of `Mask` objects.
* @param timestampInMilliseconds The timestamp (in milliseconds) for this result.
*
* @return An instance of `PoseLandmarkerResult` initialized with the given array of landmarks,
* world landmarks, segmentation masks of the detected poses and timestamp (in milliseconds).
*/
- (instancetype)initWithLandmarks:(NSArray<NSArray<MPPNormalizedLandmark *> *> *)landmarks
worldLandmarks:(NSArray<NSArray<MPPLandmark *> *> *)worldLandmarks
segmentationMasks:(NSArray<MPPMask *> *)segmentationMasks
timestampInMilliseconds:(NSInteger)timestampInMilliseconds NS_DESIGNATED_INITIALIZER;
- (instancetype)initWithTimestampInMilliseconds:(NSInteger)timestampInMilliseconds NS_UNAVAILABLE;
- (instancetype)init NS_UNAVAILABLE;
+ (instancetype)new NS_UNAVAILABLE;
@end
NS_ASSUME_NONNULL_END

View File

@ -0,0 +1,34 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <Foundation/Foundation.h>
#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h"
@implementation MPPPoseLandmarkerResult
- (instancetype)initWithLandmarks:(NSArray<NSArray<MPPNormalizedLandmark *> *> *)landmarks
worldLandmarks:(NSArray<NSArray<MPPLandmark *> *> *)worldLandmarks
segmentationMasks:(NSArray<MPPMask *> *)segmentationMasks
timestampInMilliseconds:(NSInteger)timestampInMilliseconds {
self = [super initWithTimestampInMilliseconds:timestampInMilliseconds];
if (self) {
_landmarks = [landmarks copy];
_worldLandmarks = [worldLandmarks copy];
_segmentationMasks = [segmentationMasks copy];
}
return self;
}
@end

View File

@ -0,0 +1,38 @@
# Copyright 2023 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
objc_library(
name = "MPPPoseLandmarkerOptionsHelpers",
srcs = ["sources/MPPPoseLandmarkerOptions+Helpers.mm"],
hdrs = ["sources/MPPPoseLandmarkerOptions+Helpers.h"],
copts = [
"-ObjC++",
"-std=c++17",
"-x objective-c++",
],
deps = [
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/tasks/cc/vision/pose_detector/proto:pose_detector_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarker_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarks_detector_graph_options_cc_proto",
"//mediapipe/tasks/ios/common/utils:NSStringHelpers",
"//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol",
"//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers",
"//mediapipe/tasks/ios/vision/pose_landmarker:MPPPoseLandmarkerOptions",
],
)

View File

@ -0,0 +1,36 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __cplusplus
#error "This file requires Objective-C++."
#endif // __cplusplus
#include "mediapipe/framework/calculator_options.pb.h"
#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h"
#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h"
NS_ASSUME_NONNULL_BEGIN
@interface MPPPoseLandmarkerOptions (Helpers) <MPPTaskOptionsProtocol>
/**
* Populates the provided `CalculatorOptions` proto container with the current settings.
*
* @param optionsProto The `CalculatorOptions` proto object to copy the settings to.
*/
- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto;
@end
NS_ASSUME_NONNULL_END

View File

@ -0,0 +1,52 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.h"
#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h"
#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h"
#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarker_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/pose_detector/proto/pose_detector_graph_options.pb.h"
using CalculatorOptionsProto = ::mediapipe::CalculatorOptions;
using PoseDetectorGraphOptionsProto =
::mediapipe::tasks::vision::pose_detector::proto::PoseDetectorGraphOptions;
using PoseLandmarksDetectorGraphOptionsProto =
::mediapipe::tasks::vision::pose_landmarker::proto::PoseLandmarksDetectorGraphOptions;
using PoseLandmarkerGraphOptionsProto =
::mediapipe::tasks::vision::pose_landmarker::proto::PoseLandmarkerGraphOptions;
@implementation MPPPoseLandmarkerOptions (Helpers)
- (void)copyToProto:(CalculatorOptionsProto *)optionsProto {
PoseLandmarkerGraphOptionsProto *poseLandmarkerGraphOptions =
optionsProto->MutableExtension(PoseLandmarkerGraphOptionsProto::ext);
poseLandmarkerGraphOptions->Clear();
[self.baseOptions copyToProto:poseLandmarkerGraphOptions->mutable_base_options()];
poseLandmarkerGraphOptions->set_min_tracking_confidence(self.minTrackingConfidence);
PoseLandmarksDetectorGraphOptionsProto *poseLandmarksDetectorGraphOptions =
poseLandmarkerGraphOptions->mutable_pose_landmarks_detector_graph_options();
poseLandmarksDetectorGraphOptions->set_min_detection_confidence(self.minPosePresenceConfidence);
PoseDetectorGraphOptionsProto *poseDetectorGraphOptions =
poseLandmarkerGraphOptions->mutable_pose_detector_graph_options();
poseDetectorGraphOptions->set_num_poses(self.numPoses);
poseDetectorGraphOptions->set_min_detection_confidence(self.minPoseDetectionConfidence);
}
@end

View File

@ -100,7 +100,7 @@ public abstract class PoseLandmarkerResult implements TaskResult {
/** Pose landmarks of detected poses. */
public abstract List<List<NormalizedLandmark>> landmarks();
/** Pose landmarks in world coordniates of detected poses. */
/** Pose landmarks in world coordinates of detected poses. */
public abstract List<List<Landmark>> worldLandmarks();
/** Pose segmentation masks. */

View File

@ -26,7 +26,7 @@ export declare interface GestureRecognizerResult {
/** Hand landmarks of detected hands. */
landmarks: NormalizedLandmark[][];
/** Hand landmarks in world coordniates of detected hands. */
/** Hand landmarks in world coordinates of detected hands. */
worldLandmarks: Landmark[][];
/** Handedness of detected hands. */