From cfa261b34fd638bc50e2f596070210450f343976 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 7 Apr 2023 00:57:01 +0530 Subject: [PATCH 1/7] Added MPPLandmark --- .../tasks/ios/components/containers/BUILD | 6 + .../containers/sources/MPPLandmark.h | 126 ++++++++++++++++++ .../containers/sources/MPPLandmark.m | 105 +++++++++++++++ 3 files changed, 237 insertions(+) create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPLandmark.m diff --git a/mediapipe/tasks/ios/components/containers/BUILD b/mediapipe/tasks/ios/components/containers/BUILD index 3ed7669cb..8873847af 100644 --- a/mediapipe/tasks/ios/components/containers/BUILD +++ b/mediapipe/tasks/ios/components/containers/BUILD @@ -54,3 +54,9 @@ objc_library( "//third_party/apple_frameworks:UIKit", ], ) + +objc_library( + name = "MPPLandmark", + srcs = ["sources/MPPLandmark.m"], + hdrs = ["sources/MPPLandmark.h"], +) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h b/mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h new file mode 100644 index 000000000..bb1dbc2c8 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h @@ -0,0 +1,126 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + * Landmark represents a point in 3D space with x, y, z coordinates. The landmark coordinates are in + * meters. z represents the landmark depth, and the smaller the value the closer the world landmark + * is to the camera. + */ +NS_SWIFT_NAME(Landmark) +@interface MPPLandmark : NSObject + +/** The x coordinates of the landmark. */ +@property(nonatomic, readonly) float x; + +/** The y coordinates of the landmark. */ +@property(nonatomic, readonly) float y; + +/** The z coordinates of the landmark. */ +@property(nonatomic, readonly) float z; + +/** + * Landmark visibility. Should be `nil` if not supported. Float score of whether landmark is visible + * or occluded by other objects. Landmark considered as invisible also if it is not present on the + * screen (out of scene bounds). Depending on the model, visibility value is either a sigmoid or an + * argument of sigmoid. + */ +@property(nonatomic, readonly, nullable) NSNumber *visibility; + +/** + * Landmark presence. Should stay unset if not supported. Float score of whether landmark is present + * on the scene (located within scene bounds). Depending on the model, presence value is either a + * result of sigmoid or an argument of sigmoid function to get landmark presence probability. + */ +@property(nonatomic, readonly) NSNumber *presence; + +/** + * Initializes a new `MPPLandmark` object with the given x, y and z coordinates. + * + * @param x The x coordinates of the landmark. + * @param y The y coordinates of the landmark. + * @param z The z coordinates of the landmark. + * + * @return An instance of `MPPLandmark` initialized with the given x, y and z coordinates. + */ +- (instancetype)initWithX:(float)x + y:(float)y + z:(float)z + visibility:(nullable NSNumber *)visibility + presence:(nullable NSNumber *)presence NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +/** + * Normalized Landmark represents a point in 3D space with x, y, z coordinates. x and y are + * normalized to [0.0, 1.0] by the image width and height respectively. z represents the landmark + * depth, and the smaller the value the closer the landmark is to the camera. The magnitude of z + * uses roughly the same scale as x. + */ +NS_SWIFT_NAME(NormalizedLandmark) +@interface MPPNormalizedLandmark : NSObject + +/** The x coordinates of the landmark. */ +@property(nonatomic, readonly) float x; + +/** The y coordinates of the landmark. */ +@property(nonatomic, readonly) float y; + +/** The z coordinates of the landmark. */ +@property(nonatomic, readonly) float z; + +/** + * Landmark visibility. Should be `nil` if not supported. Float score of whether landmark is visible + * or occluded by other objects. Landmark considered as invisible also if it is not present on the + * screen (out of scene bounds). Depending on the model, visibility value is either a sigmoid or an + * argument of sigmoid. + */ +@property(nonatomic, readonly, nullable) NSNumber *visibility; + +/** + * Landmark presence. Should stay unset if not supported. Float score of whether landmark is present + * on the scene (located within scene bounds). Depending on the model, presence value is either a + * result of sigmoid or an argument of sigmoid function to get landmark presence probability. + */ +@property(nonatomic, readonly) NSNumber *presence; + +/** + * Initializes a new `MPPNormalizedLandmark` object with the given x, y and z coordinates. + * + * @param x The x coordinates of the landmark. + * @param y The y coordinates of the landmark. + * @param z The z coordinates of the landmark. + * + * @return An instance of `MPPNormalizedLandmark` initialized with the given x, y and z coordinates. + */ +- (instancetype)initWithX:(float)x + y:(float)y + z:(float)z + visibility:(nullable NSNumber *)visibility + presence:(nullable NSNumber *)presence NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPLandmark.m b/mediapipe/tasks/ios/components/containers/sources/MPPLandmark.m new file mode 100644 index 000000000..2617c7b9d --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPLandmark.m @@ -0,0 +1,105 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h" + +static const float kFloatDifferenceTolerance = 1e-6f; + +@implementation MPPLandmark + +- (instancetype)initWithX:(float)x + y:(float)y + z:(float)z + visibility:(NSNumber *)visibility + presence:(NSNumber *)presence { + self = [super init]; + if (self) { + _x = x; + _y = y; + _z = z; + _visibility = visibility; + _presence = presence; + } + return self; +} + +- (NSUInteger)hash { + return @(self.x).hash ^ @(self.y).hash ^ @(self.z).hash; +} + +- (BOOL)isEqual:(nullable id)object { + if (!object) { + return NO; + } + + if (self == object) { + return YES; + } + + if (![object isKindOfClass:[MPPLandmark class]]) { + return NO; + } + + MPPLandmark *otherLandmark = (MPPLandmark *)object; + + return fabsf(otherLandmark.x - self.x) < kFloatDifferenceTolerance && + fabsf(otherLandmark.y - self.y) < kFloatDifferenceTolerance && + fabsf(otherLandmark.z - self.z) < kFloatDifferenceTolerance; +} + +@end + +@implementation MPPNormalizedLandmark + +- (instancetype)initWithX:(float)x + y:(float)y + z:(float)z + visibility:(NSNumber *)visibility + presence:(NSNumber *)presence { + self = [super init]; + if (self) { + _x = x; + _y = y; + _z = z; + _visibility = visibility; + _presence = presence; + } + return self; +} + +- (NSUInteger)hash { + return @(self.x).hash ^ @(self.y).hash ^ @(self.z).hash; +} + +- (BOOL)isEqual:(nullable id)object { + if (!object) { + return NO; + } + + if (self == object) { + return YES; + } + + if (![object isKindOfClass:[MPPNormalizedLandmark class]]) { + return NO; + } + + MPPNormalizedLandmark *otherLandmark = (MPPNormalizedLandmark *)object; + + return fabsf(otherLandmark.x - self.x) < kFloatDifferenceTolerance && + fabsf(otherLandmark.y - self.y) < kFloatDifferenceTolerance && + fabsf(otherLandmark.z - self.z) < kFloatDifferenceTolerance; +} + +@end From 5272980c2e87a483c0fbb45a559bb492eaf95e59 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 7 Apr 2023 00:57:19 +0530 Subject: [PATCH 2/7] Added MPPLandmarkHelpers --- .../ios/components/containers/utils/BUILD | 11 ++++ .../utils/sources/MPPLandmark+Helpers.h | 33 ++++++++++++ .../utils/sources/MPPLandmark+Helpers.mm | 53 +++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h create mode 100644 mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.mm diff --git a/mediapipe/tasks/ios/components/containers/utils/BUILD b/mediapipe/tasks/ios/components/containers/utils/BUILD index 3f93c0f36..59431db13 100644 --- a/mediapipe/tasks/ios/components/containers/utils/BUILD +++ b/mediapipe/tasks/ios/components/containers/utils/BUILD @@ -73,3 +73,14 @@ objc_library( "//mediapipe/tasks/ios/components/containers:MPPDetection", ], ) + +objc_library( + name = "MPPLandmarkHelpers", + srcs = ["sources/MPPLandmark+Helpers.mm"], + hdrs = ["sources/MPPLandmark+Helpers.h"], + deps = [ + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/components/containers:MPPLandmark", + ], +) diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h b/mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h new file mode 100644 index 000000000..63612f509 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h @@ -0,0 +1,33 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/formats/landmark.pb.h" +#import "mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPLandmark (Helpers) + ++ (MPPLandmark *)landmarkWithProto:(const mediapipe::Landmark &)landmarkProto; + +@end + +@interface MPPNormalizedLandmark (Helpers) + ++ (MPPNormalizedLandmark *)normalizedLandmarkWithProto: + (const mediapipe::NormalizedLandmark &)normalizedLandmarProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.mm new file mode 100644 index 000000000..b94ff27ba --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.mm @@ -0,0 +1,53 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +static const NSInteger kDefaultCategoryIndex = -1; + +namespace { +using LandmarkProto = ::mediapipe::Landmark; +using NormalizedLandmarkProto = ::mediapipe::NormalizedLandmark; +} // namespace + +@implementation MPPLandmark (Helpers) + ++ (MPPLandmark *)landmarkWithProto:(const mediapipe::Landmark &)landmarkProto { + return [[MPPLandmark alloc] + initWithX:landmarkProto.x() + y:landmarkProto.y() + z:landmarkProto.z() + visibility:landmarkProto.has_visibility() ? @(landmarkProto.visibility()) : nil + presence:landmarkProto.has_presence() ? @(landmarkProto.presence()) : nil]; +} + +@end + +@implementation MPPNormalizedLandmark (Helpers) + ++ (MPPNormalizedLandmark *)normalizedLandmarkWithProto: + (const mediapipe::NormalizedLandmark &)normalizedLandmarkProto { + return [[MPPNormalizedLandmark alloc] + initWithX:normalizedLandmarkProto.x() + y:normalizedLandmarkProto.y() + z:normalizedLandmarkProto.z() + visibility:normalizedLandmarkProto.has_visibility() ? @(normalizedLandmarkProto.visibility()) + : nil + presence:normalizedLandmarkProto.has_presence() ? @(normalizedLandmarkProto.presence()) + : nil]; +} + +@end From 9ce300c7113463df3025204b57f79eaf72c1d0bc Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 7 Apr 2023 00:57:33 +0530 Subject: [PATCH 3/7] Added MPPClassifierOptions --- .../tasks/ios/components/processors/BUILD | 26 +++++++++ .../processors/sources/MPPClassifierOptions.h | 57 +++++++++++++++++++ .../processors/sources/MPPClassifierOptions.m | 40 +++++++++++++ 3 files changed, 123 insertions(+) create mode 100644 mediapipe/tasks/ios/components/processors/BUILD create mode 100644 mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h create mode 100644 mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m diff --git a/mediapipe/tasks/ios/components/processors/BUILD b/mediapipe/tasks/ios/components/processors/BUILD new file mode 100644 index 000000000..29bef771b --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/BUILD @@ -0,0 +1,26 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = [ + "//mediapipe/tasks:internal", + "//mediapipe/tasks:users", +]) + +licenses(["notice"]) + +objc_library( + name = "MPPClassifierOptions", + srcs = ["sources/MPPClassifierOptions.m"], + hdrs = ["sources/MPPClassifierOptions.h"], +) diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h new file mode 100644 index 000000000..3f1106ca7 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -0,0 +1,57 @@ +// Copyright 2023 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** Classifier options shared across MediaPipe iOS classification tasks. */ +NS_SWIFT_NAME(ClassifierOptions) +@interface MPPClassifierOptions : NSObject + +/** + * The locale to use for display names specified through the TFLite Model Metadata, if any. Defaults + * to English. + */ +@property(nonatomic, copy) NSString *displayNamesLocale; + +/** + * The maximum number of top-scored classification results to return. If < 0, all available results + * will be returned. If 0, an invalid argument error is returned. + */ +@property(nonatomic) NSInteger maxResults; + +/** + * Score threshold to override the one provided in the model metadata (if any). Results below this + * value are rejected. + */ +@property(nonatomic) float scoreThreshold; + +/** + * The allowlist of category names. If non-empty, detection results whose category name is not in + * this set will be filtered out. Duplicate or unknown category names are ignored. Mutually + * exclusive with categoryDenylist. + */ +@property(nonatomic, copy) NSArray *categoryAllowlist; + +/** + * The denylist of category names. If non-empty, detection results whose category name is in this + * set will be filtered out. Duplicate or unknown category names are ignored. Mutually exclusive + * with categoryAllowlist. + */ +@property(nonatomic, copy) NSArray *categoryDenylist; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m new file mode 100644 index 000000000..9b2207203 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m @@ -0,0 +1,40 @@ +// Copyright 2023 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" + +@implementation MPPClassifierOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _maxResults = -1; + _scoreThreshold = 0; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPClassifierOptions *classifierOptions = [[MPPClassifierOptions alloc] init]; + + classifierOptions.displayNamesLocale = self.displayNamesLocale; + classifierOptions.maxResults = self.maxResults; + classifierOptions.scoreThreshold = self.scoreThreshold; + classifierOptions.categoryAllowlist = self.categoryAllowlist; + classifierOptions.categoryDenylist = self.categoryDenylist; + + return classifierOptions; +} + +@end From 1992dfdf2cf4ec5d45fc2f70e2afcfe10350c9b9 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 7 Apr 2023 00:57:52 +0530 Subject: [PATCH 4/7] Added MPPClassifierOptions --- .../ios/components/processors/utils/BUILD | 31 +++++++++++++ .../sources/MPPClassifierOptions+Helpers.h | 27 ++++++++++++ .../sources/MPPClassifierOptions+Helpers.mm | 44 +++++++++++++++++++ 3 files changed, 102 insertions(+) create mode 100644 mediapipe/tasks/ios/components/processors/utils/BUILD create mode 100644 mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm diff --git a/mediapipe/tasks/ios/components/processors/utils/BUILD b/mediapipe/tasks/ios/components/processors/utils/BUILD new file mode 100644 index 000000000..0e4c3b09d --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/BUILD @@ -0,0 +1,31 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = [ + "//mediapipe/tasks:internal", + "//mediapipe/tasks:users", +]) + +licenses(["notice"]) + +objc_library( + name = "MPPClassifierOptionsHelpers", + srcs = ["sources/MPPClassifierOptions+Helpers.mm"], + hdrs = ["sources/MPPClassifierOptions+Helpers.h"], + deps = [ + "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/components/processors:MPPClassifierOptions", + ], +) diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h new file mode 100644 index 000000000..07def5f42 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h @@ -0,0 +1,27 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" +#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPClassifierOptions (Helpers) + +- (void)copyToProto: + (mediapipe::tasks::components::processors::proto::ClassifierOptions *)classifierOptionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm new file mode 100644 index 000000000..6f6ddbe29 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm @@ -0,0 +1,44 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +namespace { +using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; +} + +@implementation MPPClassifierOptions (Helpers) + +- (void)copyToProto:(ClassifierOptionsProto *)classifierOptionsProto { + classifierOptionsProto->Clear(); + + if (self.displayNamesLocale) { + classifierOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString); + } + + classifierOptionsProto->set_max_results((int)self.maxResults); + classifierOptionsProto->set_score_threshold(self.scoreThreshold); + + for (NSString *category in self.categoryAllowlist) { + classifierOptionsProto->add_category_allowlist(category.cppString); + } + + for (NSString *category in self.categoryDenylist) { + classifierOptionsProto->add_category_denylist(category.cppString); + } +} + +@end From 6f5b12d0568cf79a730e933d9fbc62f4aae4eaf3 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 7 Apr 2023 00:58:26 +0530 Subject: [PATCH 5/7] Added MPPGestureRecognizerResult --- .../tasks/ios/vision/gesture_recognizer/BUILD | 29 +++++++++ .../sources/MPPGestureRecognizerResult.h | 65 +++++++++++++++++++ .../sources/MPPGestureRecognizerResult.m | 34 ++++++++++ 3 files changed, 128 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/BUILD create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD new file mode 100644 index 000000000..c34b09654 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD @@ -0,0 +1,29 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPGestureRecognizerResult", + srcs = ["sources/MPPGestureRecognizerResult.m"], + hdrs = ["sources/MPPGestureRecognizerResult.h"], + deps = [ + "//mediapipe/tasks/ios/components/containers:MPPCategory", + "//mediapipe/tasks/ios/components/containers:MPPLandmark", + "//mediapipe/tasks/ios/core:MPPTaskResult", + ], +) + diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h new file mode 100644 index 000000000..697987607 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h @@ -0,0 +1,65 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h" +#import "mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Represents the gesture recognition results generated by MPPGestureRecognizer. */ +NS_SWIFT_NAME(GestureRecognizerResult) +@interface MPPGestureRecognizerResult : MPPTaskResult + +/** Hand landmarks of detected hands. */ +@property(nonatomic, readonly) NSArray *> *landmarks; + +/** Hand landmarks in world coordniates of detected hands. */ +@property(nonatomic, readonly) NSArray *> *worldLandmarks; + +/** Handedness of detected hands. */ +@property(nonatomic, readonly) NSArray *> *handedness; + +/** + * Recognized hand gestures of detected hands. Note that the index of the gesture is always -1, + * because the raw indices from multiple gesture classifiers cannot consolidate to a meaningful + * index. + */ +@property(nonatomic, readonly) NSArray *> *gestures; + +/** + * Initializes a new `MPPGestureRecognizerResult` with the given landmarks, world landmarks, + * handedness, gestures and timestamp (in milliseconds). + * + * @param landmarks The hand landmarks of detected hands. + * @param worldLandmarks The hand landmarks in world coordniates of detected hands. + * @param handedness The handedness of detected hands. + * @param handedness The recognized hand gestures of detected hands. + * @param timestampMs The timestamp for this result. + * + * @return An instance of `MPPGestureRecognizerResult` initialized with the given landmarks, world + * landmarks, handedness and gestures. + * + */ +- (instancetype)initWithLandmarks:(NSArray *> *)landmarks + worldLandmarks:(NSArray *> *)worldLandmarks + handedness:(NSArray *> *)handedness + gestures:(NSArray *> *)gestures + timestampMs:(NSInteger)timestampMs; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m new file mode 100644 index 000000000..a8e5e784d --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.m @@ -0,0 +1,34 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" + +@implementation MPPGestureRecognizerResult + +- (instancetype)initWithLandmarks:(NSArray *> *)landmarks + worldLandmarks:(NSArray *> *)worldLandmarks + handedness:(NSArray *> *)handedness + gestures:(NSArray *> *)gestures + timestampMs:(NSInteger)timestampMs { + self = [super initWithTimestampMs:timestampMs]; + if (self) { + _landmarks = landmarks; + _worldLandmarks = worldLandmarks; + _handedness = handedness; + _gestures = gestures; + } + return self; +} + +@end From 49c614280fbfdf2ef922d355f6fcf6bdfed4f757 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 7 Apr 2023 00:58:53 +0530 Subject: [PATCH 6/7] Added MPPGestureRecognizerOptions --- .../tasks/ios/vision/gesture_recognizer/BUILD | 11 +++ .../sources/MPPGestureRecognizerOptions.h | 70 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD index c34b09654..1c6f0eeeb 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/BUILD @@ -27,3 +27,14 @@ objc_library( ], ) +objc_library( + name = "MPPGestureRecognizerOptions", + srcs = ["sources/MPPGestureRecognizerOptions.m"], + hdrs = ["sources/MPPGestureRecognizerOptions.h"], + deps = [ + ":MPPGestureRecognizerResult", + "//mediapipe/tasks/ios/components/processors:MPPClassifierOptions", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/vision/core:MPPRunningMode", + ], +) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h new file mode 100644 index 000000000..5c7462201 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h @@ -0,0 +1,70 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Options for setting up a `MPPGestureRecognizer`. */ +NS_SWIFT_NAME(GestureRecognizerOptions) +@interface MPPGestureRecognizerOptions : MPPTaskOptions + +@property(nonatomic) MPPRunningMode runningMode; + +/** + * The user-defined result callback for processing live stream data. The result callback should only + * be specified when the running mode is set to the live stream mode. + * TODO: Add parameter `MPPImage` in the callback. + */ +@property(nonatomic, copy) void (^completion) + (MPPGestureRecognizerResult *result, NSInteger timestampMs, NSError *error); + +/** Sets the maximum number of hands can be detected by the GestureRecognizer. */ +@property(nonatomic) NSInteger numHands; + +/** Sets minimum confidence score for the hand detection to be considered successful */ +@property(nonatomic) float minHandDetectionConfidence; + +/** Sets minimum confidence score of hand presence score in the hand landmark detection. */ +@property(nonatomic) float minHandPresenceConfidence; + +/** Sets the minimum confidence score for the hand tracking to be considered successful. */ +@property(nonatomic) float minTrackingConfidence; + +/** + * Sets the optional `MPPClassifierOptions` controlling the canned gestures classifier, such as + * score threshold, allow list and deny list of gestures. The categories for canned gesture + * classifiers are: ["None", "Closed_Fist", "Open_Palm", "Pointing_Up", "Thumb_Down", "Thumb_Up", + * "Victory", "ILoveYou"]. + * + * TODO: Note this option is subject to change, after scoring merging calculator is implemented. + */ +@property(nonatomic, copy) MPPClassifierOptions *cannedGesturesClassifierOptions; + +/** + * Sets the optional {@link ClassifierOptions} controlling the custom gestures classifier, such as + * score threshold, allow list and deny list of gestures. + * + * TODO: Note this option is subject to change, after scoring merging calculator is implemented. + */ +@property(nonatomic, copy) MPPClassifierOptions *customGesturesClassifierOptions; + +@end + +NS_ASSUME_NONNULL_END From fe5ca090302edb7709813385838578edc75d4177 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 7 Apr 2023 01:01:39 +0530 Subject: [PATCH 7/7] Added MPPGestureRecognizerOptions.m --- .../sources/MPPGestureRecognizerOptions.m | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m new file mode 100644 index 000000000..720385f33 --- /dev/null +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m @@ -0,0 +1,45 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h" + +@implementation MPPGestureRecognizerOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _numHands = 1; + _minHandDetectionConfidence = 0.5f; + _minHandPresenceConfidence = 0.5f; + _minTrackingConfidence = 0.5f; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPGestureRecognizerOptions *gestureRecognizerOptions = [super copyWithZone:zone]; + + gestureRecognizerOptions.runningMode = self.runningMode; + gestureRecognizerOptions.completion = self.completion; + gestureRecognizerOptions.numHands = self.numHands; + gestureRecognizerOptions.minHandDetectionConfidence = self.minHandDetectionConfidence; + gestureRecognizerOptions.minHandPresenceConfidence = self.minHandPresenceConfidence; + gestureRecognizerOptions.minTrackingConfidence = self.minTrackingConfidence; + gestureRecognizerOptions.cannedGesturesClassifierOptions = self.cannedGesturesClassifierOptions; + gestureRecognizerOptions.customGesturesClassifierOptions = self.customGesturesClassifierOptions; + + return gestureRecognizerOptions; +} + +@end