diff --git a/mediapipe/tasks/ios/common/utils/BUILD b/mediapipe/tasks/ios/common/utils/BUILD new file mode 100644 index 000000000..a29c700da --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/BUILD @@ -0,0 +1,40 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPCommonUtils", + srcs = ["sources/MPPCommonUtils.mm"], + hdrs = ["sources/MPPCommonUtils.h"], + deps = [ + "//mediapipe/tasks/cc:common", + "//mediapipe/tasks/ios/common:MPPCommon", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:cord", + ], +) + +objc_library( + name = "NSStringHelpers", + srcs = ["sources/NSString+Helpers.mm"], + hdrs = ["sources/NSString+Helpers.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], +) diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h new file mode 100644 index 000000000..5404a074d --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h @@ -0,0 +1,80 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#include "mediapipe/tasks/cc/common.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Error domain of Mediapipe Task related errors. */ +extern NSString *const MPPTasksErrorDomain; + +/** Helper utility for the all tasks which encapsulates common functionality. */ +@interface MPPCommonUtils : NSObject + +/** + * Creates and saves an NSError in the MediPipe task library domain, with the given code and + * description. + * + * @param code Error code. + * @param description Error description. + * @param error Pointer to the memory location where the created error should be saved. If `nil`, + * no error will be saved. + */ ++ (void)createCustomError:(NSError **)error + withCode:(NSUInteger)code + description:(NSString *)description; + +/** + * Creates and saves an NSError with the given domain, code and description. + * + * @param error Pointer to the memory location where the created error should be saved. If `nil`, + * no error will be saved. + * @param domain Error domain. + * @param code Error code. + * @param description Error description. + */ ++ (void)createCustomError:(NSError **)error + withDomain:(NSString *)domain + code:(NSUInteger)code + description:(NSString *)description; + +/** + * Converts an absl::Status to an NSError. + * + * @param status absl::Status. + * @param error Pointer to the memory location where the created error should be saved. If `nil`, + * no error will be saved. + * @return YES when there is no error, NO otherwise. + */ ++ (BOOL)checkCppError:(const absl::Status &)status toError:(NSError **)error; + +/** + * Allocates a block of memory with the specified size and returns a pointer to it. If memory + * cannot be allocated because of an invalid `memSize`, it saves an error. In other cases, it + * terminates program execution. + * + * @param memSize size of memory to be allocated + * @param error Pointer to the memory location where errors if any should be saved. If `nil`, no + * error will be saved. + * + * @return Pointer to the allocated block of memory on successfull allocation. `nil` in case as + * error is encountered because of invalid `memSize`. If failure is due to any other reason, method + * terminates program execution. + */ ++ (void *)mallocWithSize:(size_t)memSize error:(NSError **)error; +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm new file mode 100644 index 000000000..8234ac6d3 --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm @@ -0,0 +1,136 @@ +// Copyright 2022 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" + +#include + +#include "absl/status/status.h" // from @com_google_absl +#include "absl/strings/cord.h" // from @com_google_absl +#include "mediapipe/tasks/cc/common.h" + +/** Error domain of MediaPipe task library errors. */ +NSString *const MPPTasksErrorDomain = @"com.google.mediapipe.tasks"; + +@implementation MPPCommonUtils + ++ (void)createCustomError:(NSError **)error + withCode:(NSUInteger)code + description:(NSString *)description { + [MPPCommonUtils createCustomError:error + withDomain:MPPTasksErrorDomain + code:code + description:description]; +} + ++ (void)createCustomError:(NSError **)error + withDomain:(NSString *)domain + code:(NSUInteger)code + description:(NSString *)description { + if (error) { + *error = [NSError errorWithDomain:domain + code:code + userInfo:@{NSLocalizedDescriptionKey : description}]; + } +} + ++ (void *)mallocWithSize:(size_t)memSize error:(NSError **)error { + if (!memSize) { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"memSize cannot be zero."]; + return NULL; + } + + void *allocedMemory = malloc(memSize); + if (!allocedMemory) { + exit(-1); + } + + return allocedMemory; +} + ++ (BOOL)checkCppError:(const absl::Status &)status toError:(NSError *_Nullable *)error { + if (status.ok()) { + return YES; + } + // Payload of absl::Status created by the MediaPipe task library stores an appropriate value of + // the enum MediaPipeTasksStatus. The integer value corresponding to the MediaPipeTasksStatus enum + // stored in the payload is extracted here to later map to the appropriate error code to be + // returned. In cases where the enum is not stored in (payload is NULL or the payload string + // cannot be converted to an integer), we set the error code value to be 1 + // (MPPTasksErrorCodeError of MPPTasksErrorCode used in the iOS library to signify + // any errors not falling into other categories.) Since payload is of type absl::Cord that can be + // type cast into an absl::optional, we use the std::stoi function to convert it into + // an integer code if possible. + NSUInteger genericErrorCode = MPPTasksErrorCodeError; + NSUInteger errorCode; + try { + // Try converting payload to integer if payload is not empty. Otherwise convert a string + // signifying generic error code MPPTasksErrorCodeError to integer. + errorCode = + (NSUInteger)std::stoi(static_cast>( + status.GetPayload(mediapipe::tasks::kMediaPipeTasksPayload)) + .value_or(std::to_string(genericErrorCode))); + } catch (std::invalid_argument &e) { + // If non empty payload string cannot be converted to an integer. Set error code to 1(kError). + errorCode = MPPTasksErrorCodeError; + } + + // If errorCode is outside the range of enum values possible or is + // MPPTasksErrorCodeError, we try to map the absl::Status::code() to assign + // appropriate MPPTasksErrorCode in default cases. Note: + // The mapping to absl::Status::code() is done to generate a more specific error code than + // MPPTasksErrorCodeError in cases when the payload can't be mapped to + // MPPTasksErrorCode. This can happen when absl::Status returned by TFLite library are in turn + // returned without modification by Mediapipe cc library methods. + if (errorCode > MPPTasksErrorCodeLast || errorCode <= MPPTasksErrorCodeFirst) { + switch (status.code()) { + case absl::StatusCode::kInternal: + errorCode = MPPTasksErrorCodeError; + break; + case absl::StatusCode::kInvalidArgument: + errorCode = MPPTasksErrorCodeInvalidArgumentError; + break; + case absl::StatusCode::kNotFound: + errorCode = MPPTasksErrorCodeError; + break; + default: + errorCode = MPPTasksErrorCodeError; + break; + } + } + + // Creates the NSEror with the appropriate error + // MPPTasksErrorCode and message. MPPTasksErrorCode has a one to one + // mapping with MediaPipeTasksStatus starting from the value 1(MPPTasksErrorCodeError) + // and hence will be correctly initialized if directly cast from the integer code derived from + // MediaPipeTasksStatus stored in its payload. MPPTasksErrorCode omits kOk = 0 of + // MediaPipeTasksStatusx. + // + // Stores a string including absl status code and message(if non empty) as the + // error message See + // https://github.com/abseil/abseil-cpp/blob/master/absl/status/status.h#L514 + // for explanation. absl::Status::message() can also be used but not always + // guaranteed to be non empty. + NSString *description = [NSString + stringWithCString:status.ToString(absl::StatusToStringMode::kWithNoExtraData).c_str() + encoding:NSUTF8StringEncoding]; + [MPPCommonUtils createCustomError:error withCode:errorCode description:description]; + return NO; +} + +@end diff --git a/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h new file mode 100644 index 000000000..66f9c5ccc --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h @@ -0,0 +1,29 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#include + +NS_ASSUME_NONNULL_BEGIN + +@interface NSString (Helpers) + +@property(readonly, nonatomic) std::string cppString; + ++ (NSString *)stringWithCppString:(std::string)text; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm new file mode 100644 index 000000000..183ed4365 --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm @@ -0,0 +1,27 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +@implementation NSString (Helpers) + +- (std::string)cppString { + return std::string(self.UTF8String, [self lengthOfBytesUsingEncoding:NSUTF8StringEncoding]); +} + ++ (NSString *)stringWithCppString:(std::string)text { + return [NSString stringWithCString:text.c_str() encoding:[NSString defaultCStringEncoding]]; +} + +@end diff --git a/mediapipe/tasks/ios/components/processors/BUILD b/mediapipe/tasks/ios/components/processors/BUILD new file mode 100644 index 000000000..165145076 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/BUILD @@ -0,0 +1,23 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPClassifierOptions", + srcs = ["sources/MPPClassifierOptions.m"], + hdrs = ["sources/MPPClassifierOptions.h"], +) diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h new file mode 100644 index 000000000..13dca4030 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -0,0 +1,60 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + * Holds settings for any single iOS MediaPipe classification task. + */ +NS_SWIFT_NAME(ClassifierOptions) +@interface MPPClassifierOptions : NSObject + +/** + * The locale to use for display names specified through the TFLite Model + * Metadata, if any. Defaults to English. + */ +@property(nonatomic, copy) NSString *displayNamesLocale; + +/** + * The maximum number of top-scored classification results to return. If < 0, + * all available results will be returned. If 0, an invalid argument error is + * returned. + */ +@property(nonatomic) NSInteger maxResults; + +/** + * Score threshold to override the one provided in the model metadata (if any). + * Results below this value are rejected. + */ +@property(nonatomic) float scoreThreshold; + +/** + * The allowlist of category names. If non-empty, detection results whose + * category name is not in this set will be filtered out. Duplicate or unknown + * category names are ignored. Mutually exclusive with categoryDenylist. + */ +@property(nonatomic, copy) NSArray *categoryAllowlist; + +/** + * The denylist of category names. If non-empty, detection results whose + * category name is in this set will be filtered out. Duplicate or unknown + * category names are ignored. Mutually exclusive with categoryAllowlist. + */ +@property(nonatomic, copy) NSArray *categoryDenylist; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m new file mode 100644 index 000000000..01f498184 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m @@ -0,0 +1,40 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" + +@implementation MPPClassifierOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _maxResults = -1; + _scoreThreshold = 0; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPClassifierOptions *classifierOptions = [[MPPClassifierOptions alloc] init]; + + classifierOptions.scoreThreshold = self.scoreThreshold; + classifierOptions.maxResults = self.maxResults; + classifierOptions.categoryDenylist = self.categoryDenylist; + classifierOptions.categoryAllowlist = self.categoryAllowlist; + classifierOptions.displayNamesLocale = self.displayNamesLocale; + + return classifierOptions; +} + +@end diff --git a/mediapipe/tasks/ios/components/processors/utils/BUILD b/mediapipe/tasks/ios/components/processors/utils/BUILD new file mode 100644 index 000000000..5344c5fdf --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/BUILD @@ -0,0 +1,28 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPClassifierOptionsHelpers", + srcs = ["sources/MPPClassifierOptions+Helpers.mm"], + hdrs = ["sources/MPPClassifierOptions+Helpers.h"], + deps = [ + "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/components/processors:MPPClassifierOptions", + ], +) diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h new file mode 100644 index 000000000..e156020df --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h @@ -0,0 +1,26 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" + +#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPClassifierOptions (Helpers) +- (void)copyToProto: + (mediapipe::tasks::components::processors::proto::ClassifierOptions *)classifierOptionsProto; +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm new file mode 100644 index 000000000..24b54fd6a --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm @@ -0,0 +1,43 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" + +namespace { +using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; +} + +@implementation MPPClassifierOptions (Helpers) + +- (void)copyToProto:(ClassifierOptionsProto *)classifierOptionsProto { + classifierOptionsProto->Clear(); + + if (self.displayNamesLocale) { + classifierOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString); + } + + classifierOptionsProto->set_max_results((int)self.maxResults); + classifierOptionsProto->set_score_threshold(self.scoreThreshold); + + for (NSString *category in self.categoryAllowlist) { + classifierOptionsProto->add_category_allowlist(category.cppString); + } + + for (NSString *category in self.categoryDenylist) { + classifierOptionsProto->add_category_denylist(category.cppString); + } +} + +@end diff --git a/mediapipe/tasks/ios/core/BUILD b/mediapipe/tasks/ios/core/BUILD index 7b648945e..434d20085 100644 --- a/mediapipe/tasks/ios/core/BUILD +++ b/mediapipe/tasks/ios/core/BUILD @@ -36,3 +36,60 @@ objc_library( srcs = ["sources/MPPTaskResult.m"], hdrs = ["sources/MPPTaskResult.h"], ) + +objc_library( + name = "MPPTaskOptionsProtocol", + hdrs = ["sources/MPPTaskOptionsProtocol.h"], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + ], +) + +objc_library( + name = "MPPTaskInfo", + srcs = ["sources/MPPTaskInfo.mm"], + hdrs = ["sources/MPPTaskInfo.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + ":MPPTaskOptions", + ":MPPTaskOptionsProtocol", + "//mediapipe/calculators/core:flow_limiter_calculator_cc_proto", + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + ], +) + +objc_library( + name = "MPPTextPacketCreator", + srcs = ["sources/MPPTextPacketCreator.mm"], + hdrs = ["sources/MPPTextPacketCreator.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + ], +) + +objc_library( + name = "MPPTaskRunner", + srcs = ["sources/MPPTaskRunner.mm"], + hdrs = ["sources/MPPTaskRunner.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/tasks/cc/core:task_runner", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + ], +) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h new file mode 100644 index 000000000..b94e704d1 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h @@ -0,0 +1,70 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#include "mediapipe/framework/calculator.pb.h" + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Holds all needed informaton to initialize a MediaPipe Task. + */ +@interface MPPTaskInfo : NSObject + +@property(nonatomic, copy, nonnull) NSString *taskGraphName; + +/** + * A task-specific options that is derived from MPPTaskOptions and confirms to + * MPPTaskOptionsProtocol. + */ +@property(nonatomic, copy) id taskOptions; + +/** + * List of task graph input stream info strings in the form TAG:name. + */ +@property(nonatomic, copy) NSArray *inputStreams; + +/** + * List of task graph output stream info in the form TAG:name. + */ +@property(nonatomic, copy) NSArray *outputStreams; + +/** + * If the task requires a flow limiter. + */ +@property(nonatomic) BOOL enableFlowLimiting; + ++ (instancetype)new NS_UNAVAILABLE; + +- (instancetype)initWithTaskGraphName:(NSString *)taskGraphName + inputStreams:(NSArray *)inputStreams + outputStreams:(NSArray *)outputStreams + taskOptions:(id)taskOptions + enableFlowLimiting:(BOOL)enableFlowLimiting + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Creates a MediaPipe Task protobuf message from the MPPTaskInfo instance. + */ +- (::mediapipe::CalculatorGraphConfig)generateGraphConfig; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm new file mode 100644 index 000000000..5f2290497 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm @@ -0,0 +1,136 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +#include "mediapipe/calculators/core/flow_limiter_calculator.pb.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/calculator_options.pb.h" + +namespace { +using CalculatorGraphConfig = ::mediapipe::CalculatorGraphConfig; +using Node = ::mediapipe::CalculatorGraphConfig::Node; +using ::mediapipe::FlowLimiterCalculatorOptions; +using ::mediapipe::InputStreamInfo; +} // namespace + +@implementation MPPTaskInfo + +- (instancetype)initWithTaskGraphName:(NSString *)taskGraphName + inputStreams:(NSArray *)inputStreams + outputStreams:(NSArray *)outputStreams + taskOptions:(id)taskOptions + enableFlowLimiting:(BOOL)enableFlowLimiting + error:(NSError **)error { + if (!taskGraphName || !inputStreams.count || !outputStreams.count) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"Task graph's name, input streams, and output streams should be non-empty."]; + } + + self = [super init]; + + if (self) { + _taskGraphName = taskGraphName; + _inputStreams = inputStreams; + _outputStreams = outputStreams; + _taskOptions = taskOptions; + _enableFlowLimiting = enableFlowLimiting; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] init]; + + taskInfo.taskGraphName = self.taskGraphName; + taskInfo.inputStreams = self.inputStreams; + taskInfo.outputStreams = self.outputStreams; + taskInfo.taskOptions = self.taskOptions; + taskInfo.enableFlowLimiting = self.enableFlowLimiting; + + return taskInfo; +} + +- (CalculatorGraphConfig)generateGraphConfig { + CalculatorGraphConfig graph_config; + + Node *task_subgraph_node = graph_config.add_node(); + task_subgraph_node->set_calculator(self.taskGraphName.cppString); + [self.taskOptions copyToProto:task_subgraph_node->mutable_options()]; + + for (NSString *outputStream in self.outputStreams) { + auto cpp_output_stream = std::string(outputStream.cppString); + task_subgraph_node->add_output_stream(cpp_output_stream); + graph_config.add_output_stream(cpp_output_stream); + } + + if (!self.enableFlowLimiting) { + for (NSString *inputStream in self.inputStreams) { + auto cpp_input_stream = inputStream.cppString; + task_subgraph_node->add_input_stream(cpp_input_stream); + graph_config.add_input_stream(cpp_input_stream); + } + return graph_config; + } + + Node *flow_limit_calculator_node = graph_config.add_node(); + + flow_limit_calculator_node->set_calculator("FlowLimiterCalculator"); + + InputStreamInfo *input_stream_info = flow_limit_calculator_node->add_input_stream_info(); + input_stream_info->set_tag_index("FINISHED"); + input_stream_info->set_back_edge(true); + + FlowLimiterCalculatorOptions *flow_limit_calculator_options = + flow_limit_calculator_node->mutable_options()->MutableExtension( + FlowLimiterCalculatorOptions::ext); + flow_limit_calculator_options->set_max_in_flight(1); + flow_limit_calculator_options->set_max_in_queue(1); + + for (NSString *inputStream in self.inputStreams) { + graph_config.add_input_stream(inputStream.cppString); + + NSString *strippedInputStream = [MPPTaskInfo stripTagIndex:inputStream]; + flow_limit_calculator_node->add_input_stream(strippedInputStream.cppString); + + NSString *taskInputStream = [MPPTaskInfo addStreamNamePrefix:inputStream]; + task_subgraph_node->add_input_stream(taskInputStream.cppString); + + NSString *strippedTaskInputStream = [MPPTaskInfo stripTagIndex:taskInputStream]; + flow_limit_calculator_node->add_output_stream(strippedTaskInputStream.cppString); + } + + NSString *firstOutputStream = self.outputStreams[0]; + auto finished_output_stream = "FINISHED:" + firstOutputStream.cppString; + flow_limit_calculator_node->add_input_stream(finished_output_stream); + + return graph_config; +} + ++ (NSString *)stripTagIndex:(NSString *)tagIndexName { + return [tagIndexName componentsSeparatedByString:@":"][1]; +} + ++ (NSString *)addStreamNamePrefix:(NSString *)tagIndexName { + NSArray *splits = [tagIndexName componentsSeparatedByString:@":"]; + return [NSString stringWithFormat:@"%@:throttled_%@", splits[0], splits[1]]; +} + +@end diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h b/mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h new file mode 100644 index 000000000..c03165c1d --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h @@ -0,0 +1,33 @@ +// Copyright 2022 The MediaPipe Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#include "mediapipe/framework/calculator_options.pb.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Any MediaPipe task options should confirm to this protocol. + */ +@protocol MPPTaskOptionsProtocol + +/** + * Copies the iOS MediaPipe task options to an object of mediapipe::CalculatorOptions proto. + */ +- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h new file mode 100644 index 000000000..2b9f2ecdb --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -0,0 +1,48 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/tasks/cc/core/task_runner.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * This class is used to create and call appropriate methods on the C++ Task Runner. + */ +@interface MPPTaskRunner : NSObject + +/** + * Initializes a new `MPPTaskRunner` with the mediapipe task graph config proto. + * + * @param graphConfig A mediapipe task graph config proto. + * + * @return An instance of `MPPTaskRunner` initialized to the given graph config proto. + */ +- (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +- (absl::StatusOr)process: + (const mediapipe::tasks::core::PacketMap &)packetMap; + +- (absl::Status)close; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm new file mode 100644 index 000000000..c5c307fd5 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm @@ -0,0 +1,55 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" + +namespace { +using ::mediapipe::CalculatorGraphConfig; +using ::mediapipe::tasks::core::PacketMap; +using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; +} // namespace + +@interface MPPTaskRunner () { + // Cpp Task Runner + std::unique_ptr _cppTaskRunner; +} +@end + +@implementation MPPTaskRunner + +- (instancetype)initWithCalculatorGraphConfig:(CalculatorGraphConfig)graphConfig + error:(NSError **)error { + self = [super init]; + if (self) { + auto taskRunnerResult = TaskRunnerCpp::Create(std::move(graphConfig)); + + if (![MPPCommonUtils checkCppError:taskRunnerResult.status() toError:error]) { + return nil; + } + + _cppTaskRunner = std::move(taskRunnerResult.value()); + } + return self; +} + +- (absl::StatusOr)process:(const PacketMap &)packetMap { + return _cppTaskRunner->Process(packetMap); +} + +- (absl::Status)close { + return _cppTaskRunner->Close(); +} + +@end diff --git a/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h new file mode 100644 index 000000000..03f946dd0 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h @@ -0,0 +1,26 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#include "mediapipe/framework/packet.h" + +/* This class is an Objective-C wrapper around a MediaPipe graph object, and + * helps interface it with iOS technologies such as AVFoundation. + */ +@interface MPPTextPacketCreator : NSObject + ++ (mediapipe::Packet)createWithText:(NSString *)text; + +@end diff --git a/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm new file mode 100644 index 000000000..ca86e7a0b --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm @@ -0,0 +1,29 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +namespace { +using ::mediapipe::MakePacket; +using ::mediapipe::Packet; +} // namespace + +@implementation MPPTextPacketCreator + ++ (Packet)createWithText:(NSString *)text { + return MakePacket(text.cppString); +} + +@end