From e997a19289d85071775751b453aa2e1b982f3891 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 21 Dec 2022 01:22:32 +0530 Subject: [PATCH 01/27] Added common utils and string helpers --- mediapipe/tasks/ios/common/utils/BUILD | 41 ++++++ .../ios/common/utils/sources/MPPCommonUtils.h | 78 ++++++++++ .../common/utils/sources/MPPCommonUtils.mm | 137 ++++++++++++++++++ .../common/utils/sources/NSString+Helpers.h | 28 ++++ .../common/utils/sources/NSString+Helpers.mm | 27 ++++ 5 files changed, 311 insertions(+) create mode 100644 mediapipe/tasks/ios/common/utils/BUILD create mode 100644 mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h create mode 100644 mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm create mode 100644 mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h create mode 100644 mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm diff --git a/mediapipe/tasks/ios/common/utils/BUILD b/mediapipe/tasks/ios/common/utils/BUILD new file mode 100644 index 000000000..f2ffda39e --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/BUILD @@ -0,0 +1,41 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPCommonUtils", + srcs = ["sources/MPPCommonUtils.mm"], + hdrs = ["sources/MPPCommonUtils.h"], + deps = [ + "//mediapipe/tasks/cc:common", + "//mediapipe/tasks/ios/common:MPPCommon", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:cord", + ], +) + +objc_library( + name = "NSStringHelpers", + srcs = ["sources/NSString+Helpers.mm"], + hdrs = ["sources/NSString+Helpers.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], +) + diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h new file mode 100644 index 000000000..8a90856c7 --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h @@ -0,0 +1,78 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#include "mediapipe/tasks/cc/common.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Error domain of Mediapipe Task related errors. */ +extern NSString *const MPPTasksErrorDomain; + +/** Helper utility for the all tasks which encapsulates common functionality. */ +@interface MPPCommonUtils : NSObject + +/** + * Creates and saves an NSError in the Mediapipe task library domain, with the given code and + * description. + * + * @param code Error code. + * @param description Error description. + * @param error Pointer to the memory location where the created error should be saved. If `nil`, + * no error will be saved. + */ ++ (void)createCustomError:(NSError **)error + withCode:(NSUInteger)code + description:(NSString *)description; + +/** + * Creates and saves an NSError with the given domain, code and description. + * + * @param error Pointer to the memory location where the created error should be saved. If `nil`, + * no error will be saved. + * @param domain Error domain. + * @param code Error code. + * @param description Error description. + */ ++ (void)createCustomError:(NSError **)error + withDomain:(NSString *)domain + code:(NSUInteger)code + description:(NSString *)description; + +/** + * Converts an absl status to an NSError. + * + * @param status absl status. + * @param error Pointer to the memory location where the created error should be saved. If `nil`, + * no error will be saved. + */ ++ (BOOL)checkCppError:(const absl::Status &)status toError:(NSError **)error; + +/** + * Allocates a block of memory with the specified size and returns a pointer to it. If memory + * cannot be allocated because of an invalid memSize, it saves an error. In other cases, it + * terminates program execution. + * + * @param memSize size of memory to be allocated + * @param error Pointer to the memory location where errors if any should be saved. If `nil`, no + * error will be saved. + * + * @return Pointer to the allocated block of memory on successfull allocation. nil in case as + * error is encountered because of invalid memSize. If failure is due to any other reason, method + * terminates program execution. + */ ++ (void *)mallocWithSize:(size_t)memSize error:(NSError **)error; +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm new file mode 100644 index 000000000..574f2ef9a --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm @@ -0,0 +1,137 @@ +// Copyright 2022 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" + +#include + +#include "absl/status/status.h" // from @com_google_absl +#include "absl/strings/cord.h" // from @com_google_absl + +#include "mediapipe/tasks/cc/common.h" + +/** Error domain of MediaPipe task library errors. */ +NSString *const MPPTasksErrorDomain = @"org.mediapipe.tasks"; + +@implementation MPPCommonUtils + ++ (void)createCustomError:(NSError **)error + withCode:(NSUInteger)code + description:(NSString *)description { + [MPPCommonUtils createCustomError:error + withDomain:MPPTasksErrorDomain + code:code + description:description]; +} + ++ (void)createCustomError:(NSError **)error + withDomain:(NSString *)domain + code:(NSUInteger)code + description:(NSString *)description { + if (error) { + *error = [NSError errorWithDomain:domain + code:code + userInfo:@{NSLocalizedDescriptionKey : description}]; + } +} + ++ (void *)mallocWithSize:(size_t)memSize error:(NSError **)error { + if (!memSize) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"memSize cannot be zero."]; + return NULL; + } + + void *allocedMemory = malloc(memSize); + if (!allocedMemory) { + exit(-1); + } + + return allocedMemory; +} + ++ (BOOL)checkCppError:(const absl::Status &)status toError:(NSError *_Nullable *)error { + if (status.ok()) { + return YES; + } + // Payload of absl::Status created by the Media Pipe task library stores an appropriate value of + // the enum MediaPipeTasksStatus. The integer value corresponding to the MediaPipeTasksStatus enum + // stored in the payload is extracted here to later map to the appropriate error code to be + // returned. In cases where the enum is not stored in (payload is NULL or the payload string + // cannot be converted to an integer), we set the error code value to be 1 + // (MPPTasksErrorCodeError of MPPTasksErrorCode used in the iOS library to signify + // any errors not falling into other categories.) Since payload is of type absl::Cord that can be + // type cast into an absl::optional, we use the std::stoi function to convert it into + // an integer code if possible. + NSUInteger genericErrorCode = MPPTasksErrorCodeError; + NSUInteger errorCode; + try { + // Try converting payload to integer if payload is not empty. Otherwise convert a string + // signifying generic error code MPPTasksErrorCodeError to integer. + errorCode = + (NSUInteger)std::stoi(static_cast>( + status.GetPayload(mediapipe::tasks::kMediaPipeTasksPayload)) + .value_or(std::to_string(genericErrorCode))); + } catch (std::invalid_argument &e) { + // If non empty payload string cannot be converted to an integer. Set error code to 1(kError). + errorCode = MPPTasksErrorCodeError; + } + + // If errorCode is outside the range of enum values possible or is + // MPPTasksErrorCodeError, we try to map the absl::Status::code() to assign + // appropriate MPPTasksErrorCode in default cases. Note: + // The mapping to absl::Status::code() is done to generate a more specific error code than + // MPPTasksErrorCodeError in cases when the payload can't be mapped to + // MPPTasksErrorCode. This can happen when absl::Status returned by TFLite library are in turn returned + // without modification by Mediapipe cc library methods. + if (errorCode > MPPTasksErrorCodeLast || errorCode <= MPPTasksErrorCodeFirst) { + switch (status.code()) { + case absl::StatusCode::kInternal: + errorCode = MPPTasksErrorCodeError; + break; + case absl::StatusCode::kInvalidArgument: + errorCode = MPPTasksErrorCodeInvalidArgumentError; + break; + case absl::StatusCode::kNotFound: + errorCode = MPPTasksErrorCodeError; + break; + default: + errorCode = MPPTasksErrorCodeError; + break; + } + } + + // Creates the NSEror with the appropriate error + // MPPTasksErrorCode and message. MPPTasksErrorCode has a one to one + // mapping with MediaPipeTasksStatus starting from the value 1(MPPTasksErrorCodeError) + // and hence will be correctly initialized if directly cast from the integer code derived from + // MediaPipeTasksStatus stored in its payload. MPPTasksErrorCode omits kOk = 0 of + // MediaPipeTasksStatusx. + // + // Stores a string including absl status code and message(if non empty) as the + // error message See + // https://github.com/abseil/abseil-cpp/blob/master/absl/status/status.h#L514 + // for explanation. absl::Status::message() can also be used but not always + // guaranteed to be non empty. + NSString *description = [NSString + stringWithCString:status.ToString(absl::StatusToStringMode::kWithNoExtraData).c_str() + encoding:NSUTF8StringEncoding]; + [MPPCommonUtils createCustomError:error withCode:errorCode description:description]; + return NO; +} + +@end diff --git a/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h new file mode 100644 index 000000000..aac7485da --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h @@ -0,0 +1,28 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#include + +NS_ASSUME_NONNULL_BEGIN + +@interface NSString (Helpers) + +@property(readonly) std::string cppString; + ++ (NSString *)stringWithCppString:(std::string)text; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm new file mode 100644 index 000000000..183ed4365 --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm @@ -0,0 +1,27 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +@implementation NSString (Helpers) + +- (std::string)cppString { + return std::string(self.UTF8String, [self lengthOfBytesUsingEncoding:NSUTF8StringEncoding]); +} + ++ (NSString *)stringWithCppString:(std::string)text { + return [NSString stringWithCString:text.c_str() encoding:[NSString defaultCStringEncoding]]; +} + +@end From 03bfbca53940d71b68c5c6e5c6a697abbf9fe5fe Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 21 Dec 2022 01:22:44 +0530 Subject: [PATCH 02/27] Added classifier options --- .../tasks/ios/components/processors/BUILD | 24 +++++++++++ .../processors/sources/MPPClassifierOptions.h | 42 +++++++++++++++++++ .../processors/sources/MPPClassifierOptions.m | 40 ++++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 mediapipe/tasks/ios/components/processors/BUILD create mode 100644 mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h create mode 100644 mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m diff --git a/mediapipe/tasks/ios/components/processors/BUILD b/mediapipe/tasks/ios/components/processors/BUILD new file mode 100644 index 000000000..6d1cfdf59 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/BUILD @@ -0,0 +1,24 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPClassifierOptions", + srcs = ["sources/MPPClassifierOptions.m"], + hdrs = ["sources/MPPClassifierOptions.h"], +) + diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h new file mode 100644 index 000000000..8c4981642 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -0,0 +1,42 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + * Holds settings for any single iOS Mediapipe classification task. + */ +NS_SWIFT_NAME(ClassifierOptions) +@interface MPPClassifierOptions : NSObject + +/** If set, all classes in this list will be filtered out from the results . */ +@property(nonatomic, copy) NSArray *labelDenyList; + +/** If set, all classes not in this list will be filtered out from the results . */ +@property(nonatomic, copy) NSArray *labelAllowList; + +/** Display names local for display names*/ +@property(nonatomic, copy) NSString *displayNamesLocale; + +/** Results with score threshold greater than this value are returned . */ +@property(nonatomic) float scoreThreshold; + +/** Limit to the number of classes that can be returned in results. */ +@property(nonatomic) NSInteger maxResults; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m new file mode 100644 index 000000000..52dce23e4 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m @@ -0,0 +1,40 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" + +@implementation MPPClassifierOptions + +- (instancetype)init { + self = [super init]; + if (self) { + self.maxResults = -1; + self.scoreThreshold = 0; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPClassifierOptions *classifierOptions = [[MPPClassifierOptions alloc] init]; + + classifierOptions.scoreThreshold = self.scoreThreshold; + classifierOptions.maxResults = self.maxResults; + classifierOptions.labelDenyList = self.labelDenyList; + classifierOptions.labelAllowList = self.labelAllowList; + classifierOptions.displayNamesLocale = self.displayNamesLocale; + + return classifierOptions; +} + +@end From c56ef735d7f8bf4fb7482c2b7dd01a61c3d0ffc4 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 21 Dec 2022 01:22:57 +0530 Subject: [PATCH 03/27] Added classifier options helpers --- .../ios/components/processors/utils/BUILD | 29 ++++++++++++++ .../sources/MPPClassifierOptions+Helpers.h | 25 ++++++++++++ .../sources/MPPClassifierOptions+Helpers.mm | 38 +++++++++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 mediapipe/tasks/ios/components/processors/utils/BUILD create mode 100644 mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm diff --git a/mediapipe/tasks/ios/components/processors/utils/BUILD b/mediapipe/tasks/ios/components/processors/utils/BUILD new file mode 100644 index 000000000..820c6bb56 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/BUILD @@ -0,0 +1,29 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPClassifierOptionsHelpers", + srcs = ["sources/MPPClassifierOptions+Helpers.mm"], + hdrs = ["sources/MPPClassifierOptions+Helpers.h"], + deps = [ + "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto", + "//mediapipe/tasks/ios/components/processors:MPPClassifierOptions", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + ] +) + diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h new file mode 100644 index 000000000..6644a6255 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h @@ -0,0 +1,25 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" +#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPClassifierOptions (Helpers) +- (void)copyToProto: + (mediapipe::tasks::components::processors::proto::ClassifierOptions *)classifierOptionsProto; +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm new file mode 100644 index 000000000..25e657599 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm @@ -0,0 +1,38 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" + +namespace { +using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; +} + +@implementation MPPClassifierOptions (Helpers) +- (void)copyToProto:(ClassifierOptionsProto *)classifierOptionsProto { + if (self.displayNamesLocale) { + classifierOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString); + } + classifierOptionsProto->set_max_results((int)self.maxResults); + classifierOptionsProto->set_score_threshold(self.scoreThreshold); + for (NSString *category in self.labelAllowList) { + classifierOptionsProto->add_category_allowlist(category.cppString); + } + + for (NSString *category in self.labelDenyList) { + classifierOptionsProto->add_category_denylist(category.cppString); + } +} + +@end From 6d02108bf5244f190dc07e035913694864138467 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 21 Dec 2022 01:23:29 +0530 Subject: [PATCH 04/27] Added task info --- .../tasks/ios/core/sources/MPPTaskInfo.h | 69 +++++++++ .../tasks/ios/core/sources/MPPTaskInfo.mm | 136 ++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 mediapipe/tasks/ios/core/sources/MPPTaskInfo.h create mode 100644 mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h new file mode 100644 index 000000000..fca660fae --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h @@ -0,0 +1,69 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#include "mediapipe/framework/calculator.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" + + +NS_ASSUME_NONNULL_BEGIN + +/** + * Holds all needed informaton to initialize a MediaPipe Task. + */ +@interface MPPTaskInfo : NSObject + +@property(nonatomic, copy, nonnull) NSString *taskGraphName; + +/** + * A task-specific options that is derived from MPPTaskOptions and confirms to + * MPPTaskOptionsProtocol. + */ +@property(nonatomic, copy) id taskOptions; + +/** + * List of task graph input stream info strings in the form TAG:name. + */ +@property(nonatomic, copy) NSArray *inputStreams; + +/** + * List of task graph output stream info in the form TAG:name. + */ +@property(nonatomic, copy) NSArray *outputStreams; + +/** + * If the task requires a flow limiter. + */ +@property(nonatomic) BOOL enableFlowLimiting; + ++ (instancetype)new NS_UNAVAILABLE; + +- (instancetype)initWithTaskGraphName:(NSString *)taskGraphName + inputStreams:(NSArray *)inputStreams + outputStreams:(NSArray *)outputStreams + taskOptions:(id)taskOptions + enableFlowLimiting:(BOOL)enableFlowLimiting + error:(NSError **)error; + +/** + * Creates a MediaPipe Task protobuf message from the MPPTaskInfo instance. + */ +- (mediapipe::CalculatorGraphConfig)generateGraphConfig; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm new file mode 100644 index 000000000..7d2fd6f28 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm @@ -0,0 +1,136 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +#include "mediapipe/calculators/core/flow_limiter_calculator.pb.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/calculator_options.pb.h" + +namespace { +using CalculatorGraphConfig = ::mediapipe::CalculatorGraphConfig; +using Node = ::mediapipe::CalculatorGraphConfig::Node; +using ::mediapipe::InputStreamInfo; +using ::mediapipe::CalculatorOptions; +using ::mediapipe::FlowLimiterCalculatorOptions; +} // namespace + +@implementation MPPTaskInfo + +- (instancetype)initWithTaskGraphName:(NSString *)taskGraphName + inputStreams:(NSArray *)inputStreams + outputStreams:(NSArray *)outputStreams + taskOptions:(id)taskOptions + enableFlowLimiting:(BOOL)enableFlowLimiting + error:(NSError **)error { + if (!taskGraphName || !inputStreams.count || !outputStreams.count) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"Task graph's name, input streams, and output streams should be non-empty."]; + } + + self = [super init]; + + if (self) { + _taskGraphName = taskGraphName; + _inputStreams = inputStreams; + _outputStreams = outputStreams; + _taskOptions = taskOptions; + _enableFlowLimiting = enableFlowLimiting; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] init]; + + taskInfo.taskGraphName = self.taskGraphName; + taskInfo.inputStreams = self.inputStreams; + taskInfo.outputStreams = self.outputStreams; + taskInfo.taskOptions = self.taskOptions; + taskInfo.enableFlowLimiting = self.enableFlowLimiting; + + return taskInfo; +} + +- (CalculatorGraphConfig)generateGraphConfig { + CalculatorGraphConfig graph_config; + + Node *task_subgraph_node = graph_config.add_node(); + task_subgraph_node->set_calculator(self.taskGraphName.cppString); + [self.taskOptions copyToProto:task_subgraph_node->mutable_options()]; + + for (NSString *outputStream in self.outputStreams) { + auto cpp_output_stream = std::string(outputStream.cppString); + task_subgraph_node->add_output_stream(cpp_output_stream); + graph_config.add_output_stream(cpp_output_stream); + } + + if (self.enableFlowLimiting) { + Node *flow_limit_calculator_node = graph_config.add_node(); + + flow_limit_calculator_node->set_calculator("FlowLimiterCalculator"); + + InputStreamInfo *input_stream_info = flow_limit_calculator_node->add_input_stream_info(); + input_stream_info->set_tag_index("FINISHED"); + input_stream_info->set_back_edge(true); + + FlowLimiterCalculatorOptions *flow_limit_calculator_options = + flow_limit_calculator_node->mutable_options()->MutableExtension( + FlowLimiterCalculatorOptions::ext); + flow_limit_calculator_options->set_max_in_flight(1); + flow_limit_calculator_options->set_max_in_queue(1); + + for (NSString *inputStream in self.inputStreams) { + graph_config.add_input_stream(inputStream.cppString); + + NSString *strippedInputStream = [MPPTaskInfo stripTagIndex:inputStream]; + flow_limit_calculator_node->add_input_stream(strippedInputStream.cppString); + + NSString *taskInputStream = [MPPTaskInfo addStreamNamePrefix:inputStream]; + task_subgraph_node->add_input_stream(taskInputStream.cppString); + + NSString *strippedTaskInputStream = [MPPTaskInfo stripTagIndex:taskInputStream]; + flow_limit_calculator_node->add_output_stream(strippedTaskInputStream.cppString); + } + + NSString *firstOutputStream = self.outputStreams[0]; + auto finished_output_stream = "FINISHED:" + firstOutputStream.cppString; + flow_limit_calculator_node->add_input_stream(finished_output_stream); + } else { + for (NSString *inputStream in self.inputStreams) { + auto cpp_input_stream = inputStream.cppString; + task_subgraph_node->add_input_stream(cpp_input_stream); + graph_config.add_input_stream(cpp_input_stream); + } + } + + return graph_config; +} + ++ (NSString *)stripTagIndex:(NSString *)tagIndexName { + return [tagIndexName componentsSeparatedByString:@":"][1]; +} + ++ (NSString *)addStreamNamePrefix:(NSString *)tagIndexName { + NSArray *splits = [tagIndexName componentsSeparatedByString:@":"]; + return [NSString stringWithFormat:@"%@:throttled_%@", splits[0], splits[1]]; +} + +@end From 64cf5e9b4e7208b3b974038d9e160e1509be1945 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 21 Dec 2022 01:23:41 +0530 Subject: [PATCH 05/27] Added iOS task options protocol --- .../ios/core/sources/MPPTaskOptionsProtocol.h | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h b/mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h new file mode 100644 index 000000000..c6f115451 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h @@ -0,0 +1,32 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#include "mediapipe/framework/calculator_options.pb.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Any mediapipe task options should confirm to this protocol. + */ +@protocol MPPTaskOptionsProtocol + +/** + * Copies the iOS Mediapipe task options to an object of mediapipe::CalculatorOptions proto. + */ +- (void)copyToProto:(mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END From e9fc3713f0bd0ab0fceb9ea07e78373fc8c50efd Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 21 Dec 2022 01:23:51 +0530 Subject: [PATCH 06/27] Added iOS task runner --- .../tasks/ios/core/sources/MPPTaskRunner.h | 47 ++++++++++++++++ .../tasks/ios/core/sources/MPPTaskRunner.mm | 56 +++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 mediapipe/tasks/ios/core/sources/MPPTaskRunner.h create mode 100644 mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h new file mode 100644 index 000000000..64e34b82e --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -0,0 +1,47 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/tasks/cc/core/task_runner.h" + + +NS_ASSUME_NONNULL_BEGIN + +/** + * This class is used to create and call appropriate methods on the C++ Task Runner. + */ +@interface MPPTaskRunner : NSObject +/** + * Initializes a new `MPPTaskRunner` with the mediapipe task graph config proto. + * + * @param graphConfig A mediapipe task graph config proto. + * + * @return An instance of `MPPTaskRunner` initialized to the given graph config proto. + */ +- (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig + error:(NSError **)error; + +- (absl::StatusOr)process:(const mediapipe::tasks::core::PacketMap&)packetMap error:(NSError **)error; + +- (void)close; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm new file mode 100644 index 000000000..404f6c582 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm @@ -0,0 +1,56 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" + +namespace { +using ::mediapipe::CalculatorGraphConfig; +using ::mediapipe::Packet; +using ::mediapipe::tasks::core::PacketMap; +using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; +} // namespace + +@interface MPPTaskRunner () { + // Cpp Task Runner + std::unique_ptr _cppTaskRunner; +} +@end + +@implementation MPPTaskRunner + +- (instancetype)initWithCalculatorGraphConfig:(CalculatorGraphConfig)graphConfig + error:(NSError **)error { + self = [super init]; + if (self) { + auto taskRunnerResult = TaskRunnerCpp::Create(std::move(graphConfig)); + + if (![MPPCommonUtils checkCppError:taskRunnerResult.status() toError:error]) { + return nil; + } + + _cppTaskRunner = std::move(taskRunnerResult.value()); + } + return self; +} + +- (absl::StatusOr)process:(const PacketMap&)packetMap { + return _cppTaskRunner->Process(packetMap); +} + +- (void)close { + _cppTaskRunner->Close(); +} + +@end From 4fedea60a93adb6ac9db50212b7f06f29758576e Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 21 Dec 2022 01:24:02 +0530 Subject: [PATCH 07/27] Added text packet creator --- .../ios/core/sources/MPPTextPacketCreator.h | 26 +++++++++++++++++ .../ios/core/sources/MPPTextPacketCreator.mm | 29 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h create mode 100644 mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm diff --git a/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h new file mode 100644 index 000000000..03f946dd0 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h @@ -0,0 +1,26 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#include "mediapipe/framework/packet.h" + +/* This class is an Objective-C wrapper around a MediaPipe graph object, and + * helps interface it with iOS technologies such as AVFoundation. + */ +@interface MPPTextPacketCreator : NSObject + ++ (mediapipe::Packet)createWithText:(NSString *)text; + +@end diff --git a/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm new file mode 100644 index 000000000..ca86e7a0b --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm @@ -0,0 +1,29 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +namespace { +using ::mediapipe::MakePacket; +using ::mediapipe::Packet; +} // namespace + +@implementation MPPTextPacketCreator + ++ (Packet)createWithText:(NSString *)text { + return MakePacket(text.cppString); +} + +@end From ff901a80a5398276b04e03e561c9acf0892d3aaa Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 21 Dec 2022 01:24:11 +0530 Subject: [PATCH 08/27] Added targets in core --- mediapipe/tasks/ios/core/BUILD | 53 ++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/mediapipe/tasks/ios/core/BUILD b/mediapipe/tasks/ios/core/BUILD index 7b648945e..adc37d901 100644 --- a/mediapipe/tasks/ios/core/BUILD +++ b/mediapipe/tasks/ios/core/BUILD @@ -36,3 +36,56 @@ objc_library( srcs = ["sources/MPPTaskResult.m"], hdrs = ["sources/MPPTaskResult.h"], ) + +objc_library( + name = "MPPTaskOptionsProtocol", + hdrs = ["sources/MPPTaskOptionsProtocol.h"], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + ], +) + +objc_library( + name = "MPPTaskInfo", + srcs = ["sources/MPPTaskInfo.mm"], + hdrs = ["sources/MPPTaskInfo.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/calculators/core:flow_limiter_calculator_cc_proto", + ":MPPTaskOptions", + ":MPPTaskOptionsProtocol", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/common:MPPCommon", + ], +) + +objc_library( + name = "MPPTextPacketCreator", + srcs = ["sources/MPPTextPacketCreator.mm"], + hdrs = ["sources/MPPTextPacketCreator.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + ], +) + +objc_library( + name = "MPPTaskRunner", + srcs = ["sources/MPPTaskRunner.mm"], + hdrs = ["sources/MPPTaskRunner.h"], + deps = [ + "//mediapipe/tasks/cc/core:task_runner", + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + ], +) From ae28948ca150fd3a801c2ef1387b460151083204 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 00:49:24 +0530 Subject: [PATCH 09/27] Marked designated initializers --- mediapipe/tasks/ios/core/sources/MPPTaskInfo.h | 2 +- mediapipe/tasks/ios/core/sources/MPPTaskRunner.h | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h index fca660fae..4c01787a8 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h @@ -55,7 +55,7 @@ NS_ASSUME_NONNULL_BEGIN outputStreams:(NSArray *)outputStreams taskOptions:(id)taskOptions enableFlowLimiting:(BOOL)enableFlowLimiting - error:(NSError **)error; + error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** * Creates a MediaPipe Task protobuf message from the MPPTaskInfo instance. diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 64e34b82e..e07cb344d 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -24,6 +24,7 @@ NS_ASSUME_NONNULL_BEGIN * This class is used to create and call appropriate methods on the C++ Task Runner. */ @interface MPPTaskRunner : NSObject + /** * Initializes a new `MPPTaskRunner` with the mediapipe task graph config proto. * @@ -32,7 +33,7 @@ NS_ASSUME_NONNULL_BEGIN * @return An instance of `MPPTaskRunner` initialized to the given graph config proto. */ - (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig - error:(NSError **)error; + error:(NSError **)error NS_DESIGNATED_INITIALIZER; - (absl::StatusOr)process:(const mediapipe::tasks::core::PacketMap&)packetMap error:(NSError **)error; From 481f4e960e009df7431f7281312e985507428c87 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 00:49:44 +0530 Subject: [PATCH 10/27] Updated comments --- mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h b/mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h index c6f115451..44fba4c0b 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h @@ -18,12 +18,12 @@ NS_ASSUME_NONNULL_BEGIN /** - * Any mediapipe task options should confirm to this protocol. + * Any MediaPipe task options should confirm to this protocol. */ @protocol MPPTaskOptionsProtocol /** - * Copies the iOS Mediapipe task options to an object of mediapipe::CalculatorOptions proto. + * Copies the iOS MediaPipe task options to an object of mediapipe::CalculatorOptions proto. */ - (void)copyToProto:(mediapipe::CalculatorOptions *)optionsProto; From 2943d1668e0e0c66dad9a0e6626dc1c82e38cd3d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 00:51:20 +0530 Subject: [PATCH 11/27] Updated comments --- mediapipe/tasks/ios/core/sources/MPPTaskRunner.h | 1 + 1 file changed, 1 insertion(+) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index e07cb344d..9dfef02e1 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -23,6 +23,7 @@ NS_ASSUME_NONNULL_BEGIN /** * This class is used to create and call appropriate methods on the C++ Task Runner. */ + @interface MPPTaskRunner : NSObject /** From 20f2e136c520b937d81a8241aec7a4ca869d3f70 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 00:59:22 +0530 Subject: [PATCH 12/27] Updated empty spaces --- .../processors/utils/sources/MPPClassifierOptions+Helpers.mm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm index 25e657599..db7fa6bfd 100644 --- a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm @@ -20,12 +20,16 @@ using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto } @implementation MPPClassifierOptions (Helpers) + - (void)copyToProto:(ClassifierOptionsProto *)classifierOptionsProto { if (self.displayNamesLocale) { classifierOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString); } + classifierOptionsProto->set_max_results((int)self.maxResults); + classifierOptionsProto->set_score_threshold(self.scoreThreshold); + for (NSString *category in self.labelAllowList) { classifierOptionsProto->add_category_allowlist(category.cppString); } From 1491b3f5a2da5ac3415edd8cd946e0e9b639887b Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 01:00:36 +0530 Subject: [PATCH 13/27] Updated comments --- mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h index 8a90856c7..d2e6067d5 100644 --- a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h @@ -24,7 +24,7 @@ extern NSString *const MPPTasksErrorDomain; @interface MPPCommonUtils : NSObject /** - * Creates and saves an NSError in the Mediapipe task library domain, with the given code and + * Creates and saves an NSError in the MediPipe task library domain, with the given code and * description. * * @param code Error code. From 1de369417572ebb09e26c72d8a6fa3e5f7685795 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 01:02:07 +0530 Subject: [PATCH 14/27] Updated comments --- mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h index d2e6067d5..1a44ee45a 100644 --- a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h @@ -51,9 +51,9 @@ extern NSString *const MPPTasksErrorDomain; description:(NSString *)description; /** - * Converts an absl status to an NSError. + * Converts an absl::Status to an NSError. * - * @param status absl status. + * @param status absl::Status. * @param error Pointer to the memory location where the created error should be saved. If `nil`, * no error will be saved. */ @@ -68,7 +68,7 @@ extern NSString *const MPPTasksErrorDomain; * @param error Pointer to the memory location where errors if any should be saved. If `nil`, no * error will be saved. * - * @return Pointer to the allocated block of memory on successfull allocation. nil in case as + * @return Pointer to the allocated block of memory on successfull allocation. `nil` in case as * error is encountered because of invalid memSize. If failure is due to any other reason, method * terminates program execution. */ From 99c11ff9743fab7799bfd7db9e2e63755fe4a123 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 01:03:39 +0530 Subject: [PATCH 15/27] Updated comments --- mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h index 1a44ee45a..407d87aba 100644 --- a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h @@ -61,7 +61,7 @@ extern NSString *const MPPTasksErrorDomain; /** * Allocates a block of memory with the specified size and returns a pointer to it. If memory - * cannot be allocated because of an invalid memSize, it saves an error. In other cases, it + * cannot be allocated because of an invalid `memSize`, it saves an error. In other cases, it * terminates program execution. * * @param memSize size of memory to be allocated @@ -69,7 +69,7 @@ extern NSString *const MPPTasksErrorDomain; * error will be saved. * * @return Pointer to the allocated block of memory on successfull allocation. `nil` in case as - * error is encountered because of invalid memSize. If failure is due to any other reason, method + * error is encountered because of invalid `memSize`. If failure is due to any other reason, method * terminates program execution. */ + (void *)mallocWithSize:(size_t)memSize error:(NSError **)error; From 7ae4b7e6394b5e75315fb46b4ee9c44b6e02ecc1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 01:05:01 +0530 Subject: [PATCH 16/27] Updated error domain --- mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm index 574f2ef9a..4d4880a87 100644 --- a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm @@ -24,7 +24,7 @@ #include "mediapipe/tasks/cc/common.h" /** Error domain of MediaPipe task library errors. */ -NSString *const MPPTasksErrorDomain = @"org.mediapipe.tasks"; +NSString *const MPPTasksErrorDomain = @"com.google.mediapipe.tasks"; @implementation MPPCommonUtils @@ -68,7 +68,7 @@ NSString *const MPPTasksErrorDomain = @"org.mediapipe.tasks"; if (status.ok()) { return YES; } - // Payload of absl::Status created by the Media Pipe task library stores an appropriate value of + // Payload of absl::Status created by the MediaPipe task library stores an appropriate value of // the enum MediaPipeTasksStatus. The integer value corresponding to the MediaPipeTasksStatus enum // stored in the payload is extracted here to later map to the appropriate error code to be // returned. In cases where the enum is not stored in (payload is NULL or the payload string From 54d36dfedad1d2e84f680fb69defb13b6eae45b9 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 01:05:50 +0530 Subject: [PATCH 17/27] Update MPPClassifierOptions.h --- .../ios/components/processors/sources/MPPClassifierOptions.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h index 8c4981642..b31dadb63 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -17,7 +17,7 @@ NS_ASSUME_NONNULL_BEGIN /** - * Holds settings for any single iOS Mediapipe classification task. + * Holds settings for any single iOS MediaPipe classification task. */ NS_SWIFT_NAME(ClassifierOptions) @interface MPPClassifierOptions : NSObject From 673b38dfe87c35504ac81f5b29935ab6b25beaa1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 01:08:13 +0530 Subject: [PATCH 18/27] Updated comments --- .../processors/sources/MPPClassifierOptions.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h index b31dadb63..d6b9a9582 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -22,16 +22,18 @@ NS_ASSUME_NONNULL_BEGIN NS_SWIFT_NAME(ClassifierOptions) @interface MPPClassifierOptions : NSObject -/** If set, all classes in this list will be filtered out from the results . */ +/** If set, all classes in this list will be filtered out from the results. */ @property(nonatomic, copy) NSArray *labelDenyList; -/** If set, all classes not in this list will be filtered out from the results . */ +/** If set, all classes not in this list will be filtered out from the results. */ @property(nonatomic, copy) NSArray *labelAllowList; -/** Display names local for display names*/ +/** The locale to use for display names specified through the TFLite Model + * Metadata, if any. Defaults to English. + */ @property(nonatomic, copy) NSString *displayNamesLocale; -/** Results with score threshold greater than this value are returned . */ +/** Results with score threshold greater than this value are returned. */ @property(nonatomic) float scoreThreshold; /** Limit to the number of classes that can be returned in results. */ From 66ee8d47c0d13c4f0a4f4ee91bde7bf570fbaa61 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 01:10:07 +0530 Subject: [PATCH 19/27] Resorted options --- .../processors/sources/MPPClassifierOptions.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h index d6b9a9582..0c22ed9de 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -22,22 +22,22 @@ NS_ASSUME_NONNULL_BEGIN NS_SWIFT_NAME(ClassifierOptions) @interface MPPClassifierOptions : NSObject -/** If set, all classes in this list will be filtered out from the results. */ -@property(nonatomic, copy) NSArray *labelDenyList; - -/** If set, all classes not in this list will be filtered out from the results. */ -@property(nonatomic, copy) NSArray *labelAllowList; - /** The locale to use for display names specified through the TFLite Model * Metadata, if any. Defaults to English. */ @property(nonatomic, copy) NSString *displayNamesLocale; +/** Limit to the number of classes that can be returned in results. */ +@property(nonatomic) NSInteger maxResults; + /** Results with score threshold greater than this value are returned. */ @property(nonatomic) float scoreThreshold; -/** Limit to the number of classes that can be returned in results. */ -@property(nonatomic) NSInteger maxResults; +/** If set, all classes not in this list will be filtered out from the results. */ +@property(nonatomic, copy) NSArray *labelAllowList; + +/** If set, all classes in this list will be filtered out from the results. */ +@property(nonatomic, copy) NSArray *labelDenyList; @end From e1dfcf03cf41f0f9519206ea4fa97f255161191f Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 01:12:34 +0530 Subject: [PATCH 20/27] Updated comments in MPPClassifierOptions.h --- .../ios/components/processors/sources/MPPClassifierOptions.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h index 0c22ed9de..371472cab 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -30,7 +30,9 @@ NS_SWIFT_NAME(ClassifierOptions) /** Limit to the number of classes that can be returned in results. */ @property(nonatomic) NSInteger maxResults; -/** Results with score threshold greater than this value are returned. */ +/** Score threshold to override the one provided in the model metadata (if any). + * Results below this value are rejected. + */ @property(nonatomic) float scoreThreshold; /** If set, all classes not in this list will be filtered out from the results. */ From c185dc9ad7ba33844fac9560f33c59fb2c9e4ad6 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 01:19:01 +0530 Subject: [PATCH 21/27] Renamed label to category in classifier options --- .../ios/components/processors/sources/MPPClassifierOptions.h | 4 ++-- .../ios/components/processors/sources/MPPClassifierOptions.m | 4 ++-- .../processors/utils/sources/MPPClassifierOptions+Helpers.mm | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h index 371472cab..0f0abe398 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -36,10 +36,10 @@ NS_SWIFT_NAME(ClassifierOptions) @property(nonatomic) float scoreThreshold; /** If set, all classes not in this list will be filtered out from the results. */ -@property(nonatomic, copy) NSArray *labelAllowList; +@property(nonatomic, copy) NSArray *categoryAllowList; /** If set, all classes in this list will be filtered out from the results. */ -@property(nonatomic, copy) NSArray *labelDenyList; +@property(nonatomic, copy) NSArray *categoryDenyList; @end diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m index 52dce23e4..1d9191802 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m @@ -30,8 +30,8 @@ classifierOptions.scoreThreshold = self.scoreThreshold; classifierOptions.maxResults = self.maxResults; - classifierOptions.labelDenyList = self.labelDenyList; - classifierOptions.labelAllowList = self.labelAllowList; + classifierOptions.categoryDenyList = self.categoryDenyList; + classifierOptions.categoryAllowList = self.categoryAllowList; classifierOptions.displayNamesLocale = self.displayNamesLocale; return classifierOptions; diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm index db7fa6bfd..3d8397efa 100644 --- a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm @@ -30,11 +30,11 @@ using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto classifierOptionsProto->set_score_threshold(self.scoreThreshold); - for (NSString *category in self.labelAllowList) { + for (NSString *category in self.categoryAllowList) { classifierOptionsProto->add_category_allowlist(category.cppString); } - for (NSString *category in self.labelDenyList) { + for (NSString *category in self.categoryDenyList) { classifierOptionsProto->add_category_denylist(category.cppString); } } From 20c3388ab68c11082de43aff825762686b3bc8e1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 01:59:38 +0530 Subject: [PATCH 22/27] Updated category allowlist and denylist names --- .../ios/components/processors/sources/MPPClassifierOptions.h | 4 ++-- .../ios/components/processors/sources/MPPClassifierOptions.m | 4 ++-- .../processors/utils/sources/MPPClassifierOptions+Helpers.mm | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h index 0f0abe398..e95de89e4 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -36,10 +36,10 @@ NS_SWIFT_NAME(ClassifierOptions) @property(nonatomic) float scoreThreshold; /** If set, all classes not in this list will be filtered out from the results. */ -@property(nonatomic, copy) NSArray *categoryAllowList; +@property(nonatomic, copy) NSArray *categoryAllowlist; /** If set, all classes in this list will be filtered out from the results. */ -@property(nonatomic, copy) NSArray *categoryDenyList; +@property(nonatomic, copy) NSArray *categoryDenylist; @end diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m index 1d9191802..accb6c7dd 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m @@ -30,8 +30,8 @@ classifierOptions.scoreThreshold = self.scoreThreshold; classifierOptions.maxResults = self.maxResults; - classifierOptions.categoryDenyList = self.categoryDenyList; - classifierOptions.categoryAllowList = self.categoryAllowList; + classifierOptions.categoryDenylist = self.categoryDenylist; + classifierOptions.categoryAllowlist = self.categoryAllowlist; classifierOptions.displayNamesLocale = self.displayNamesLocale; return classifierOptions; diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm index 3d8397efa..81fe57d13 100644 --- a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm @@ -30,11 +30,11 @@ using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto classifierOptionsProto->set_score_threshold(self.scoreThreshold); - for (NSString *category in self.categoryAllowList) { + for (NSString *category in self.categoryAllowlist) { classifierOptionsProto->add_category_allowlist(category.cppString); } - for (NSString *category in self.categoryDenyList) { + for (NSString *category in self.categoryDenylist) { classifierOptionsProto->add_category_denylist(category.cppString); } } From b4a7644428ac05af45dc737bd1002b6a8f6154cc Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 02:01:04 +0530 Subject: [PATCH 23/27] Updated comments --- .../ios/components/processors/sources/MPPClassifierOptions.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h index e95de89e4..348e94e96 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -27,7 +27,10 @@ NS_SWIFT_NAME(ClassifierOptions) */ @property(nonatomic, copy) NSString *displayNamesLocale; -/** Limit to the number of classes that can be returned in results. */ +/** The maximum number of top-scored classification results to return. If < 0, + * all available results will be returned. If 0, an invalid argument error is + * returned. + */ @property(nonatomic) NSInteger maxResults; /** Score threshold to override the one provided in the model metadata (if any). From e559613b9de8d73e0d4956688561174b58e2dcb9 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 02:02:48 +0530 Subject: [PATCH 24/27] Updated comments in MPPClassifierOptions.h --- .../processors/sources/MPPClassifierOptions.h | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h index 348e94e96..7bf5744f7 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -38,10 +38,16 @@ NS_SWIFT_NAME(ClassifierOptions) */ @property(nonatomic) float scoreThreshold; -/** If set, all classes not in this list will be filtered out from the results. */ +/** The allowlist of category names. If non-empty, detection results whose + * category name is not in this set will be filtered out. Duplicate or unknown + * category names are ignored. Mutually exclusive with categoryDenylist. + */ @property(nonatomic, copy) NSArray *categoryAllowlist; -/** If set, all classes in this list will be filtered out from the results. */ +/** The denylist of category names. If non-empty, detection results whose + * category name is in this set will be filtered out. Duplicate or unknown + * category names are ignored. Mutually exclusive with categoryAllowlist. + */ @property(nonatomic, copy) NSArray *categoryDenylist; @end From e47256ae55af3921d0878cf131c32625a2500082 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 12:10:23 +0530 Subject: [PATCH 25/27] Clearing proto before assigining new values in MPPClassifierOptions Helpers --- .../processors/utils/sources/MPPClassifierOptions+Helpers.mm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm index 81fe57d13..efe9572e1 100644 --- a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm @@ -22,6 +22,8 @@ using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto @implementation MPPClassifierOptions (Helpers) - (void)copyToProto:(ClassifierOptionsProto *)classifierOptionsProto { + classifierOptionsProto->Clear(); + if (self.displayNamesLocale) { classifierOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString); } From 613ed588908ac3bd39b48bf05e21c2fa52eeb9ad Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 12:16:33 +0530 Subject: [PATCH 26/27] Inverted condition check in MPPTaskInfo --- .../tasks/ios/core/sources/MPPTaskInfo.mm | 67 ++++++++++--------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm index 7d2fd6f28..be3c8cbf7 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm @@ -24,9 +24,9 @@ namespace { using CalculatorGraphConfig = ::mediapipe::CalculatorGraphConfig; using Node = ::mediapipe::CalculatorGraphConfig::Node; -using ::mediapipe::InputStreamInfo; using ::mediapipe::CalculatorOptions; using ::mediapipe::FlowLimiterCalculatorOptions; +using ::mediapipe::InputStreamInfo; } // namespace @implementation MPPTaskInfo @@ -82,45 +82,46 @@ using ::mediapipe::FlowLimiterCalculatorOptions; graph_config.add_output_stream(cpp_output_stream); } - if (self.enableFlowLimiting) { - Node *flow_limit_calculator_node = graph_config.add_node(); - - flow_limit_calculator_node->set_calculator("FlowLimiterCalculator"); - - InputStreamInfo *input_stream_info = flow_limit_calculator_node->add_input_stream_info(); - input_stream_info->set_tag_index("FINISHED"); - input_stream_info->set_back_edge(true); - - FlowLimiterCalculatorOptions *flow_limit_calculator_options = - flow_limit_calculator_node->mutable_options()->MutableExtension( - FlowLimiterCalculatorOptions::ext); - flow_limit_calculator_options->set_max_in_flight(1); - flow_limit_calculator_options->set_max_in_queue(1); - - for (NSString *inputStream in self.inputStreams) { - graph_config.add_input_stream(inputStream.cppString); - - NSString *strippedInputStream = [MPPTaskInfo stripTagIndex:inputStream]; - flow_limit_calculator_node->add_input_stream(strippedInputStream.cppString); - - NSString *taskInputStream = [MPPTaskInfo addStreamNamePrefix:inputStream]; - task_subgraph_node->add_input_stream(taskInputStream.cppString); - - NSString *strippedTaskInputStream = [MPPTaskInfo stripTagIndex:taskInputStream]; - flow_limit_calculator_node->add_output_stream(strippedTaskInputStream.cppString); - } - - NSString *firstOutputStream = self.outputStreams[0]; - auto finished_output_stream = "FINISHED:" + firstOutputStream.cppString; - flow_limit_calculator_node->add_input_stream(finished_output_stream); - } else { + if (!self.enableFlowLimiting) { for (NSString *inputStream in self.inputStreams) { auto cpp_input_stream = inputStream.cppString; task_subgraph_node->add_input_stream(cpp_input_stream); graph_config.add_input_stream(cpp_input_stream); } + return graph_config; } + Node *flow_limit_calculator_node = graph_config.add_node(); + + flow_limit_calculator_node->set_calculator("FlowLimiterCalculator"); + + InputStreamInfo *input_stream_info = flow_limit_calculator_node->add_input_stream_info(); + input_stream_info->set_tag_index("FINISHED"); + input_stream_info->set_back_edge(true); + + FlowLimiterCalculatorOptions *flow_limit_calculator_options = + flow_limit_calculator_node->mutable_options()->MutableExtension( + FlowLimiterCalculatorOptions::ext); + flow_limit_calculator_options->set_max_in_flight(1); + flow_limit_calculator_options->set_max_in_queue(1); + + for (NSString *inputStream in self.inputStreams) { + graph_config.add_input_stream(inputStream.cppString); + + NSString *strippedInputStream = [MPPTaskInfo stripTagIndex:inputStream]; + flow_limit_calculator_node->add_input_stream(strippedInputStream.cppString); + + NSString *taskInputStream = [MPPTaskInfo addStreamNamePrefix:inputStream]; + task_subgraph_node->add_input_stream(taskInputStream.cppString); + + NSString *strippedTaskInputStream = [MPPTaskInfo stripTagIndex:taskInputStream]; + flow_limit_calculator_node->add_output_stream(strippedTaskInputStream.cppString); + } + + NSString *firstOutputStream = self.outputStreams[0]; + auto finished_output_stream = "FINISHED:" + firstOutputStream.cppString; + flow_limit_calculator_node->add_input_stream(finished_output_stream); + return graph_config; } From 48eeae4d9d3582661f002ddc2424e3e6c8cdd512 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 22 Dec 2022 12:16:43 +0530 Subject: [PATCH 27/27] Formatted code --- mediapipe/tasks/ios/core/sources/MPPTaskInfo.h | 1 - mediapipe/tasks/ios/core/sources/MPPTaskRunner.h | 5 +++-- mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h index 4c01787a8..ae4c9eba1 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h @@ -17,7 +17,6 @@ #import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" - NS_ASSUME_NONNULL_BEGIN /** diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 9dfef02e1..6561e136d 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -17,7 +17,6 @@ #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/tasks/cc/core/task_runner.h" - NS_ASSUME_NONNULL_BEGIN /** @@ -36,7 +35,9 @@ NS_ASSUME_NONNULL_BEGIN - (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig error:(NSError **)error NS_DESIGNATED_INITIALIZER; -- (absl::StatusOr)process:(const mediapipe::tasks::core::PacketMap&)packetMap error:(NSError **)error; +- (absl::StatusOr) + process:(const mediapipe::tasks::core::PacketMap &)packetMap + error:(NSError **)error; - (void)close; diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm index 404f6c582..e08d0bc1b 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm @@ -45,7 +45,7 @@ using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; return self; } -- (absl::StatusOr)process:(const PacketMap&)packetMap { +- (absl::StatusOr)process:(const PacketMap &)packetMap { return _cppTaskRunner->Process(packetMap); }