From d7b0f660e66cb1500e06431ba2bc48ed391e68ad Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 1 Dec 2022 02:44:52 +0530 Subject: [PATCH] Added Files for Text Classifier iOS Task --- mediapipe/tasks/ios/common/BUILD | 26 +++ .../tasks/ios/common/sources/MPPCommon.h | 179 ++++++++++++++++++ mediapipe/tasks/ios/common/utils/BUILD | 38 ++++ .../ios/common/utils/sources/MPPCommonUtils.h | 78 ++++++++ .../common/utils/sources/MPPCommonUtils.mm | 129 +++++++++++++ .../common/utils/sources/NSString+Helpers.h | 27 +++ .../common/utils/sources/NSString+Helpers.mm | 23 +++ .../tasks/ios/components/containers/BUILD | 32 ++++ .../containers/sources/MPPCategory.h | 62 ++++++ .../containers/sources/MPPCategory.m | 33 ++++ .../sources/MPPClassificationResult.h | 94 +++++++++ .../sources/MPPClassificationResult.m | 50 +++++ .../tasks/ios/components/processors/BUILD | 24 +++ .../processors/sources/MPPClassifierOptions.h | 42 ++++ .../processors/sources/MPPClassifierOptions.m | 40 ++++ .../ios/components/processors/utils/BUILD | 29 +++ .../sources/MPPClassifierOptions+Helpers.h | 25 +++ .../sources/MPPClassifierOptions+Helpers.mm | 38 ++++ mediapipe/tasks/ios/core/BUILD | 67 +++++++ .../tasks/ios/core/sources/MPPBaseOptions.h | 51 +++++ .../tasks/ios/core/sources/MPPBaseOptions.m | 36 ++++ .../tasks/ios/core/sources/MPPExternalFile.h | 28 +++ .../tasks/ios/core/sources/MPPExternalFile.m | 27 +++ .../tasks/ios/core/sources/MPPTaskInfo.h | 64 +++++++ .../tasks/ios/core/sources/MPPTaskInfo.mm | 135 +++++++++++++ .../tasks/ios/core/sources/MPPTaskOptions.h | 58 ++++++ .../tasks/ios/core/sources/MPPTaskOptions.m | 36 ++++ mediapipe/tasks/ios/core/utils/BUILD | 27 +++ .../utils/sources/MPPBaseOptions+Helpers.h | 26 +++ .../utils/sources/MPPBaseOptions+Helpers.mm | 40 ++++ mediapipe/tasks/ios/text/core/BUILD | 33 ++++ .../text/core/sources/MPPBaseTextTaskApi.h | 44 +++++ .../text/core/sources/MPPBaseTextTaskApi.mm | 52 +++++ .../tasks/ios/text/text_classifier/BUILD | 50 +++++ .../sources/MPPTextClassifier.h | 61 ++++++ .../sources/MPPTextClassifier.mm | 65 +++++++ .../sources/MPPTextClassifierOptions.h | 51 +++++ .../sources/MPPTextClassifierOptions.mm | 27 +++ .../ios/text/text_classifier/utils/BUILD | 30 +++ .../MPPTextClassifierOptions+Helpers.h | 26 +++ .../MPPTextClassifierOptions+Helpers.mm | 36 ++++ 41 files changed, 2039 insertions(+) create mode 100644 mediapipe/tasks/ios/common/BUILD create mode 100644 mediapipe/tasks/ios/common/sources/MPPCommon.h create mode 100644 mediapipe/tasks/ios/common/utils/BUILD create mode 100644 mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h create mode 100644 mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm create mode 100644 mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h create mode 100644 mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm create mode 100644 mediapipe/tasks/ios/components/containers/BUILD create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPCategory.h create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPCategory.m create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m create mode 100644 mediapipe/tasks/ios/components/processors/BUILD create mode 100644 mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h create mode 100644 mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m create mode 100644 mediapipe/tasks/ios/components/processors/utils/BUILD create mode 100644 mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm create mode 100644 mediapipe/tasks/ios/core/BUILD create mode 100644 mediapipe/tasks/ios/core/sources/MPPBaseOptions.h create mode 100644 mediapipe/tasks/ios/core/sources/MPPBaseOptions.m create mode 100644 mediapipe/tasks/ios/core/sources/MPPExternalFile.h create mode 100644 mediapipe/tasks/ios/core/sources/MPPExternalFile.m create mode 100644 mediapipe/tasks/ios/core/sources/MPPTaskInfo.h create mode 100644 mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm create mode 100644 mediapipe/tasks/ios/core/sources/MPPTaskOptions.h create mode 100644 mediapipe/tasks/ios/core/sources/MPPTaskOptions.m create mode 100644 mediapipe/tasks/ios/core/utils/BUILD create mode 100644 mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm create mode 100644 mediapipe/tasks/ios/text/core/BUILD create mode 100644 mediapipe/tasks/ios/text/core/sources/MPPBaseTextTaskApi.h create mode 100644 mediapipe/tasks/ios/text/core/sources/MPPBaseTextTaskApi.mm create mode 100644 mediapipe/tasks/ios/text/text_classifier/BUILD create mode 100644 mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h create mode 100644 mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm create mode 100644 mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h create mode 100644 mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.mm create mode 100644 mediapipe/tasks/ios/text/text_classifier/utils/BUILD create mode 100644 mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm diff --git a/mediapipe/tasks/ios/common/BUILD b/mediapipe/tasks/ios/common/BUILD new file mode 100644 index 000000000..0d00c423f --- /dev/null +++ b/mediapipe/tasks/ios/common/BUILD @@ -0,0 +1,26 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPCommon", + hdrs = [ + "sources/MPPCommon.h", + ], + module_name = "MPPCommon", +) + diff --git a/mediapipe/tasks/ios/common/sources/MPPCommon.h b/mediapipe/tasks/ios/common/sources/MPPCommon.h new file mode 100644 index 000000000..1f450370e --- /dev/null +++ b/mediapipe/tasks/ios/common/sources/MPPCommon.h @@ -0,0 +1,179 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + * @enum TFLSupportErrorCode + * This enum specifies error codes for TensorFlow Lite Task Library. + * It maintains a 1:1 mapping to TfLiteSupportErrorCode of C libray. + */ +typedef NS_ENUM(NSUInteger, MPPTasksErrorCode) { + + // Generic error codes. + + // Unspecified error. + MPPTasksErrorCodeError = 1, + // Invalid argument specified. + MPPTasksErrorCodeInvalidArgumentError = 2, + // Invalid FlatBuffer file or buffer specified. + MPPTasksErrorCodeInvalidFlatBufferError = 3, + // Model contains a builtin op that isn't supported by the OpResolver or + // delegates. + MPPTasksErrorCodeUnsupportedBuiltinOp = 4, + // Model contains a custom op that isn't supported by the OpResolver or + // delegates. + MPPTasksErrorCodeUnsupportedCustomOp = 5, + + // File I/O error codes. + + // No such file. + MPPTasksErrorCodeFileNotFoundError = 100, + // Permission issue. + MPPTasksErrorCodeFilePermissionDeniedError, + // I/O error when reading file. + MPPTasksErrorCodeFileReadError, + // I/O error when mmap-ing file. + MPPTasksErrorCodeFileMmapError, + // ZIP I/O error when unpacMPPTasksErrorCodeing the zip file. + MPPTasksErrorCodeFileZipError, + + // TensorFlow Lite metadata error codes. + + // Unexpected schema version (aMPPTasksErrorCodea file_identifier) in the Metadata FlatBuffer. + MPPTasksErrorCodeMetadataInvalidSchemaVersionError = 200, + // No such associated file within metadata, or file has not been pacMPPTasksErrorCodeed. + MPPTasksErrorCodeMetadataAssociatedFileNotFoundError, + // ZIP I/O error when unpacMPPTasksErrorCodeing an associated file. + MPPTasksErrorCodeMetadataAssociatedFileZipError, + // Inconsistency error between the metadata and actual TF Lite model. + // E.g.: number of labels and output tensor values differ. + MPPTasksErrorCodeMetadataInconsistencyError, + // Invalid process units specified. + // E.g.: multiple ProcessUnits with the same type for a given tensor. + MPPTasksErrorCodeMetadataInvalidProcessUnitsError, + // Inconsistency error with the number of labels. + // E.g.: label files for different locales have a different number of labels. + MPPTasksErrorCodeMetadataNumLabelsMismatchError, + // Score calibration parameters parsing error. + // E.g.: too many parameters provided in the corresponding associated file. + MPPTasksErrorCodeMetadataMalformedScoreCalibrationError, + // Unexpected number of subgraphs for the current task. + // E.g.: image classification expects a single subgraph. + MPPTasksErrorCodeMetadataInvalidNumSubgraphsError, + // A given tensor requires NormalizationOptions but none were found. + // E.g.: float input tensor requires normalization to preprocess input images. + MPPTasksErrorCodeMetadataMissingNormalizationOptionsError, + // Invalid ContentProperties specified. + // E.g. expected ImageProperties, got BoundingBoxProperties. + MPPTasksErrorCodeMetadataInvalidContentPropertiesError, + // Metadata is mandatory but was not found. + // E.g. current task requires TFLite Model Metadata but none was found. + MPPTasksErrorCodeMetadataNotFoundError, + // Associated TENSOR_AXIS_LABELS or TENSOR_VALUE_LABELS file is mandatory but + // none was found or it was empty. + // E.g. current task requires labels but none were found. + MPPTasksErrorCodeMetadataMissingLabelsError, + // The ProcessingUnit for tokenizer is not correctly configured. + // E.g BertTokenizer doesn't have a valid vocab file associated. + MPPTasksErrorCodeMetadataInvalidTokenizerError, + + // Input tensor(s) error codes. + + // Unexpected number of input tensors for the current task. + // E.g. current task expects a single input tensor. + MPPTasksErrorCodeInvalidNumInputTensorsError = 300, + // Unexpected input tensor dimensions for the current task. + // E.g.: only 4D input tensors supported. + MPPTasksErrorCodeInvalidInputTensorDimensionsError, + // Unexpected input tensor type for the current task. + // E.g.: current task expects a uint8 pixel image as input. + MPPTasksErrorCodeInvalidInputTensorTypeError, + // Unexpected input tensor bytes size. + // E.g.: size in bytes does not correspond to the expected number of pixels. + MPPTasksErrorCodeInvalidInputTensorSizeError, + // No correct input tensor found for the model. + // E.g.: input tensor name is not part of the text model's input tensors. + MPPTasksErrorCodeInputTensorNotFoundError, + + // Output tensor(s) error codes. + + // Unexpected output tensor dimensions for the current task. + // E.g.: only a batch size of 1 is supported. + MPPTasksErrorCodeInvalidOutputTensorDimensionsError = 400, + // Unexpected input tensor type for the current task. + // E.g.: multi-head model with different output tensor types. + MPPTasksErrorCodeInvalidOutputTensorTypeError, + // No correct output tensor found for the model. + // E.g.: output tensor name is not part of the text model's output tensors. + MPPTasksErrorCodeOutputTensorNotFoundError, + // Unexpected number of output tensors for the current task. + // E.g.: current task expects a single output tensor. + MPPTasksErrorCodeInvalidNumOutputTensorsError, + + // Image processing error codes. + + // Unspecified image processing failures. + MPPTasksErrorCodeImageProcessingError = 500, + // Unexpected input or output buffer metadata. + // E.g.: rotate RGBA buffer to Grayscale buffer by 90 degrees. + MPPTasksErrorCodeImageProcessingInvalidArgumentError, + // Image processing operation failures. + // E.g. libyuv rotation failed for an unknown reason. + MPPTasksErrorCodeImageProcessingBackendError, + + // Task runner error codes. + MPPTasksErrorCodeRunnerError = 600, + // Task runner is not initialized. + MPPTasksErrorCodeRunnerInitializationError, + // Task runner is not started successfully. + MPPTasksErrorCodeRunnerFailsToStartError, + // Task runner is not started. + MPPTasksErrorCodeRunnerNotStartedError, + // Task runner API is called in the wrong processing mode. + MPPTasksErrorCodeRunnerApiCalledInWrongModeError, + // Task runner receives/produces invalid MediaPipe packet timestamp. + MPPTasksErrorCodeRunnerInvalidTimestampError, + // Task runner receives unexpected MediaPipe graph input packet. + // E.g. The packet type doesn't match the graph input stream's data type. + MPPTasksErrorCodeRunnerUnexpectedInputError, + // Task runner produces unexpected MediaPipe graph output packet. + // E.g. The number of output packets is not equal to the number of graph + // output streams. + MPPTasksErrorCodeRunnerUnexpectedOutputError, + // Task runner is not closed successfully. + MPPTasksErrorCodeRunnerFailsToCloseError, + // Task runner's model resources cache service is unavailable or the + // targeting model resources bundle is not found. + MPPTasksErrorCodeRunnerModelResourcesCacheServiceError, + + // Task graph error codes. + MPPTasksErrorCodeGraphError = 700, + // Task graph is not implemented. + MPPTasksErrorCodeTaskGraphNotImplementedError, + // Task graph config is invalid. + MPPTasksErrorCodeInvalidTaskGraphConfigError, + + MPPTasksErrorCodeFirst = MPPTasksErrorCodeError, + + /** + * The last error code in TFLSupportErrorCode (for internal use only). + */ + MPPTasksErrorCodeLast = MPPTasksErrorCodeInvalidTaskGraphConfigError, + +} NS_SWIFT_NAME(TasksErrorCode); + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/common/utils/BUILD b/mediapipe/tasks/ios/common/utils/BUILD new file mode 100644 index 000000000..9c8c75586 --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/BUILD @@ -0,0 +1,38 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPCommonUtils", + srcs = ["sources/MPPCommonUtils.mm"], + hdrs = ["sources/MPPCommonUtils.h"], + deps = [ + "//mediapipe/tasks/cc:common", + "//mediapipe/tasks/ios/common:MPPCommon", + ], +) + +objc_library( + name = "NSStringHelpers", + srcs = ["sources/NSString+Helpers.mm"], + hdrs = ["sources/NSString+Helpers.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], +) + diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h new file mode 100644 index 000000000..6b4f40bc6 --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h @@ -0,0 +1,78 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#import +#include "mediapipe/tasks/cc/common.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Error domain of TensorFlow Lite Support related errors. */ +extern NSString *const MPPTasksErrorDomain; + +/** Helper utility for the all tasks which encapsulates common functionality. */ +@interface MPPCommonUtils : NSObject + +/** + * Creates and saves an NSError in the Tensorflow Lite Task Library domain, with the given code and + * description. + * + * @param code Error code. + * @param description Error description. + * @param error Pointer to the memory location where the created error should be saved. If `nil`, + * no error will be saved. + */ ++ (void)createCustomError:(NSError **)error + withCode:(NSUInteger)code + description:(NSString *)description; + +/** + * Creates and saves an NSError with the given domain, code and description. + * + * @param error Pointer to the memory location where the created error should be saved. If `nil`, + * no error will be saved. + * @param domain Error domain. + * @param code Error code. + * @param description Error description. + */ ++ (void)createCustomError:(NSError **)error + withDomain:(NSString *)domain + code:(NSUInteger)code + description:(NSString *)description; + +/** + * Converts an absl status to an NSError. + * + * @param status absl status. + * @param error Pointer to the memory location where the created error should be saved. If `nil`, + * no error will be saved. + */ ++ (BOOL)checkCppError:(const absl::Status &)status toError:(NSError **)error; + +/** + * Allocates a block of memory with the specified size and returns a pointer to it. If memory + * cannot be allocated because of an invalid memSize, it saves an error. In other cases, it + * terminates program execution. + * + * @param memSize size of memory to be allocated + * @param error Pointer to the memory location where errors if any should be saved. If `nil`, no + * error will be saved. + * + * @return Pointer to the allocated block of memory on successfull allocation. nil in case as + * error is encountered because of invalid memSize. If failure is due to any other reason, method + * terminates program execution. + */ ++ (void *)mallocWithSize:(size_t)memSize error:(NSError **)error; +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm new file mode 100644 index 000000000..5d8cc6887 --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm @@ -0,0 +1,129 @@ +// Copyright 2022 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" + +/** Error domain of TensorFlow Lite Support related errors. */ +NSString *const TFLSupportTaskErrorDomain = @"org.tensorflow.lite.tasks"; + +@implementation TFLCommonUtils + ++ (void)createCustomError:(NSError **)error + withCode:(NSUInteger)code + description:(NSString *)description { + [TFLCommonUtils createCustomError:error + withDomain:TFLSupportTaskErrorDomain + code:code + description:description]; +} + ++ (void)createCustomError:(NSError **)error + withDomain:(NSString *)domain + code:(NSUInteger)code + description:(NSString *)description { + if (error) { + *error = [NSError errorWithDomain:domain + code:code + userInfo:@{NSLocalizedDescriptionKey : description}]; + } +} + ++ (void *)mallocWithSize:(size_t)memSize error:(NSError **)error { + if (!memSize) { + [TFLCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"memSize cannot be zero."]; + return NULL; + } + + void *allocedMemory = malloc(memSize); + if (!allocedMemory) { + exit(-1); + } + + return allocedMemory; +} + ++ (BOOL)checkCppError:(const absl::Status &)status toError:(NSError *_Nullable *)error { + if (status.ok()) { + return YES; + } + // Payload of absl::Status created by the tflite task library stores an appropriate value of the + // enum TfLiteSupportStatus. The integer value corresponding to the TfLiteSupportStatus enum + // stored in the payload is extracted here to later map to the appropriate error code to be + // returned. In cases where the enum is not stored in (payload is NULL or the payload string + // cannot be converted to an integer), we set the error code value to be 1 + // (TFLSupportErrorCodeUnspecifiedError of TFLSupportErrorCode used in the iOS library to signify + // any errors not falling into other categories.) Since payload is of type absl::Cord that can be + // type cast into an absl::optional, we use the std::stoi function to convert it into + // an integer code if possible. + NSUInteger genericErrorCode = MPPTasksErrorCodeError; + NSUInteger errorCode; + try { + // Try converting payload to integer if payload is not empty. Otherwise convert a string + // signifying generic error code TFLSupportErrorCodeUnspecifiedError to integer. + errorCode = + (NSUInteger)std::stoi(static_cast>( + status.GetPayload(mediapipe::tasks::kMediaPipeTasksPayload)) + .value_or(std::to_string(genericErrorCode))); + } catch (std::invalid_argument &e) { + // If non empty payload string cannot be converted to an integer. Set error code to 1(kError). + errorCode = MPPTasksErrorCodeError; + } + + // If errorCode is outside the range of enum values possible or is + // TFLSupportErrorCodeUnspecifiedError, we try to map the absl::Status::code() to assign + // appropriate TFLSupportErrorCode or TFLSupportErrorCodeUnspecifiedError in default cases. Note: + // The mapping to absl::Status::code() is done to generate a more specific error code than + // TFLSupportErrorCodeUnspecifiedError in cases when the payload can't be mapped to + // TfLiteSupportStatus. This can happen when absl::Status returned by TfLite are in turn returned + // without moodification by TfLite Support Methods. + if (errorCode > MPPTasksErrorCodeLast || errorCode <= MPPTasksErrorCodeFirst) { + switch (status.code()) { + case absl::StatusCode::kInternal: + errorCode = MPPTasksErrorCodeError; + break; + case absl::StatusCode::kInvalidArgument: + errorCode = MPPTasksErrorCodeInvalidArgumentError; + break; + case absl::StatusCode::kNotFound: + errorCode = MPPTasksErrorCodeError; + break; + default: + errorCode = MPPTasksErrorCodeError; + break; + } + } + + // Creates the NSEror with the appropriate error + // TFLSupportErrorCode and message. TFLSupportErrorCode has a one to one + // mapping with TfLiteSupportStatus starting from the value 1(TFLSupportErrorCodeUnspecifiedError) + // and hence will be correctly initialized if directly cast from the integer code derived from + // TfLiteSupportStatus stored in its payload. TFLSupportErrorCode omits kOk = 0 of + // TfLiteSupportStatus. + // + // Stores a string including absl status code and message(if non empty) as the + // error message See + // https://github.com/abseil/abseil-cpp/blob/master/absl/status/status.h#L514 + // for explanation. absl::Status::message() can also be used but not always + // guaranteed to be non empty. + NSString *description = [NSString + stringWithCString:status.ToString(absl::StatusToStringMode::kWithNoExtraData).c_str() + encoding:NSUTF8StringEncoding]; + [TFLCommonUtils createCustomError:error withCode:errorCode description:description]; + return NO; +} + +@end diff --git a/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h new file mode 100644 index 000000000..31828a367 --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h @@ -0,0 +1,27 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +#import +#include + +NS_ASSUME_NONNULL_BEGIN + +@interface NSString (Helpers) + +@property(readonly) std::string cppString; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm new file mode 100644 index 000000000..540903c6c --- /dev/null +++ b/mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.mm @@ -0,0 +1,23 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +@implementation NSString (Helpers) + +- (std::string)cppString { + return std::string(self.UTF8String, [self lengthOfBytesUsingEncoding:NSUTF8StringEncoding]); +} + +@end diff --git a/mediapipe/tasks/ios/components/containers/BUILD b/mediapipe/tasks/ios/components/containers/BUILD new file mode 100644 index 000000000..ce80571e9 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/BUILD @@ -0,0 +1,32 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPCategory", + srcs = ["sources/MPPCategory.m"], + hdrs = ["sources/MPPCategory.h"], +) + +objc_library( + name = "MPPClassificationResult", + srcs = ["sources/MPPClassificationResult.m"], + hdrs = ["sources/MPPClassificationResult.h"], + deps = [ + ":MPPCategory", + ], +) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h new file mode 100644 index 000000000..e2f8a0729 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h @@ -0,0 +1,62 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import + +NS_ASSUME_NONNULL_BEGIN + +/** Encapsulates information about a class in the classification results. */ +NS_SWIFT_NAME(ClassificationCategory) +@interface TFLCategory : NSObject + +/** Index of the class in the corresponding label map, usually packed in the TFLite Model + * Metadata. */ +@property(nonatomic, readonly) NSInteger index; + +/** Confidence score for this class . */ +@property(nonatomic, readonly) float score; + +/** Class name of the class. */ +@property(nonatomic, readonly, nullable) NSString *label; + +/** Display name of the class. */ +@property(nonatomic, readonly, nullable) NSString *displayName; + +/** + * Initializes a new `TFLCategory` with the given index, score, label and display name. + * + * @param index Index of the class in the corresponding label map, usually packed in the TFLite + * Model Metadata. + * + * @param score Confidence score for this class. + * + * @param label Class name of the class. + * + * @param displayName Display name of the class. + * + * @return An instance of `TFLCategory` initialized with the given index, score, label and display + * name. + */ +- (instancetype)initWithIndex:(NSInteger)index + score:(float)score + label:(nullable NSString *)label + displayName:(nullable NSString *)displayName; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.m b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.m new file mode 100644 index 000000000..30efa239a --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.m @@ -0,0 +1,33 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/components/containers/sources/TFLCategory.h" + +@implementation TFLCategory + +- (instancetype)initWithIndex:(NSInteger)index + score:(float)score + label:(nullable NSString *)label + displayName:(nullable NSString *)displayName { + self = [super init]; + if (self) { + _index = index; + _score = score; + _label = label; + _displayName = displayName; + } + return self; +} + +@end diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h new file mode 100644 index 000000000..120780a7b --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h @@ -0,0 +1,94 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#import +#import "mediapipe/tasks/ios/task/components/containers/sources/MPPCategory.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Encapsulates list of predicted classes (aka labels) for a given image classifier head. */ +NS_SWIFT_NAME(Classifications) +@interface MPPClassifications : NSObject + +/** + * The index of the classifier head these classes refer to. This is useful for multi-head + * models. + */ +@property(nonatomic, readonly) NSInteger headIndex; + +/** The name of the classifier head, which is the corresponding tensor metadata + * name. + */ +@property(nonatomic, readonly) NSString *headName; + +/** The array of predicted classes, usually sorted by descending scores (e.g.from high to low + * probability). */ +@property(nonatomic, readonly) NSArray *categories; + +/** + * Initializes a new `MPPClassifications` with the given head index and array of categories. + * head name is initialized to `nil`. + * + * @param headIndex The index of the image classifier head these classes refer to. + * @param categories An array of `MPPCategory` objects encapsulating a list of + * predictions usually sorted by descending scores (e.g. from high to low probability). + * + * @return An instance of `MPPClassifications` initialized with the given head index and + * array of categories. + */ +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + categories:(NSArray *)categories; + +/** + * Initializes a new `MPPClassifications` with the given head index, head name and array of + * categories. + * + * @param headIndex The index of the classifier head these classes refer to. + * @param headName The name of the classifier head, which is the corresponding tensor metadata + * name. + * @param categories An array of `MPPCategory` objects encapsulating a list of + * predictions usually sorted by descending scores (e.g. from high to low probability). + * + * @return An object of `MPPClassifications` initialized with the given head index, head name and + * array of categories. + */ +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + headName:(nullable NSString *)headName + categories:(NSArray *)categories; + +@end + +/** Encapsulates results of any classification task. */ +NS_SWIFT_NAME(ClassificationResult) +@interface MPPClassificationResult : NSObject + +/** Array of MPPClassifications objects containing classifier predictions per image classifier + * head. + */ +@property(nonatomic, readonly) NSArray *classifications; + +/** + * Initializes a new `MPPClassificationResult` with the given array of classifications. + * + * @param classifications An Aaray of `MPPClassifications` objects containing classifier + * predictions per classifier head. + * + * @return An instance of MPPClassificationResult initialized with the given array of + * classifications. + */ +- (instancetype)initWithClassifications:(NSArray *)classifications; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m new file mode 100644 index 000000000..266b401e0 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m @@ -0,0 +1,50 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#import "mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h" + +@implementation MPPClassifications + +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + headName:(nullable NSString *)headName + categories:(NSArray *)categories { + self = [super init]; + if (self) { + _headIndex = headIndex; + _headName = headName; + _categories = categories; + } + return self; +} + +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + categories:(NSArray *)categories { + return [self initWithHeadIndex:headIndex headName:nil categories:categories]; +} + +@end + +@implementation MPPClassificationResult { + NSArray *_classifications; +} + +- (instancetype)initWithClassifications:(NSArray *)classifications { + self = [super init]; + if (self) { + _classifications = classifications; + } + return self; +} + +@end diff --git a/mediapipe/tasks/ios/components/processors/BUILD b/mediapipe/tasks/ios/components/processors/BUILD new file mode 100644 index 000000000..6d1cfdf59 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/BUILD @@ -0,0 +1,24 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPClassifierOptions", + srcs = ["sources/MPPClassifierOptions.m"], + hdrs = ["sources/MPPClassifierOptions.h"], +) + diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h new file mode 100644 index 000000000..7538a625d --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h @@ -0,0 +1,42 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + * Holds settings for any single iOS Mediapipe classification task. + */ +NS_SWIFT_NAME(ClassifierOptions) +@interface MPPClassifierOptions : NSObject + +/** If set, all classes in this list will be filtered out from the results . */ +@property(nonatomic, copy) NSArray *labelDenyList; + +/** If set, all classes not in this list will be filtered out from the results . */ +@property(nonatomic, copy) NSArray *labelAllowList; + +/** Display names local for display names*/ +@property(nonatomic, copy) NSString *displayNamesLocale; + +/** Results with score threshold greater than this value are returned . */ +@property(nonatomic) float scoreThreshold; + +/** Limit to the number of classes that can be returned in results. */ +@property(nonatomic) NSInteger maxResults; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m new file mode 100644 index 000000000..5a5baf79f --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m @@ -0,0 +1,40 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" + +@implementation MPPClassifierOptions + +- (instancetype)init { + self = [super init]; + if (self) { + self.maxResults = -1; + self.scoreThreshold = 0; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPClassifierOptions *classifierOptions = [[MPPClassifierOptions alloc] init]; + + classifierOptions.scoreThreshold = self.scoreThreshold; + classifierOptions.maxResults = self.maxResults; + classifierOptions.labelDenyList = self.labelDenyList; + classifierOptions.labelAllowList = self.labelAllowList; + classifierOptions.displayNamesLocale = self.displayNamesLocale; + + return classifierOptions; +} + +@end diff --git a/mediapipe/tasks/ios/components/processors/utils/BUILD b/mediapipe/tasks/ios/components/processors/utils/BUILD new file mode 100644 index 000000000..820c6bb56 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/BUILD @@ -0,0 +1,29 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPClassifierOptionsHelpers", + srcs = ["sources/MPPClassifierOptions+Helpers.mm"], + hdrs = ["sources/MPPClassifierOptions+Helpers.h"], + deps = [ + "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto", + "//mediapipe/tasks/ios/components/processors:MPPClassifierOptions", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + ] +) + diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h new file mode 100644 index 000000000..4defb3ee7 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h @@ -0,0 +1,25 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" +#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPClassifierOptions (Helpers) +- (void)copyToProto: + (mediapipe::tasks::components::processors::proto::ClassifierOptions *)classifierOptionsProto; +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm new file mode 100644 index 000000000..efb220147 --- /dev/null +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm @@ -0,0 +1,38 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" + +namespace { +using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; +} + +@implementation MPPClassifierOptions (Helpers) +- (void)copyToProto:(ClassifierOptionsProto *)classifierOptionsProto { + if (self.displayNamesLocale) { + classifierOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString); + } + classifierOptionsProto->set_max_results((int)self.maxResults); + classifierOptionsProto->set_score_threshold(self.scoreThreshold); + for (NSString *category in self.labelAllowList) { + classifierOptionsProto->add_category_allowlist(category.cppString); + } + + for (NSString *category in self.labelDenyList) { + classifierOptionsProto->add_category_denylist(category.cppString); + } +} + +@end diff --git a/mediapipe/tasks/ios/core/BUILD b/mediapipe/tasks/ios/core/BUILD new file mode 100644 index 000000000..91257b4de --- /dev/null +++ b/mediapipe/tasks/ios/core/BUILD @@ -0,0 +1,67 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPExternalFile", + srcs = ["sources/MPPExternalFile.m"], + hdrs = ["sources/MPPExternalFile.h"], +) + +objc_library( + name = "MPPBaseOptions", + srcs = ["sources/MPPBaseOptions.m"], + hdrs = ["sources/MPPBaseOptions.h"], + deps = [ + ":MPPExternalFile", + + ], +) + +objc_library( + name = "MPPTaskOptions", + srcs = ["sources/MPPTaskOptions.m"], + hdrs = ["sources/MPPTaskOptions.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + ":MPPBaseOptions", + ], +) + +objc_library( + name = "MPPTaskInfo", + srcs = ["sources/MPPTaskInfo.mm"], + hdrs = ["sources/MPPTaskInfo.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/calculators/core:flow_limiter_calculator_cc_proto", + ":MPPTaskOptions", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/common:MPPCommon", + ], +) + diff --git a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h new file mode 100644 index 000000000..686e50add --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h @@ -0,0 +1,51 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import +#import "mediapipe/tasks/ios/core/sources/MPPExternalFile.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * MediaPipe Tasks delegate. + */ +typedef NS_ENUM(NSUInteger, MPPDelegate) { + + /** CPU. */ + MPPDelegateCPU, + + /** GPU. */ + MPPDelegateGPU +} NS_SWIFT_NAME(Delegate); + +/** + * Holds the base options that is used for creation of any type of task. It has fields with + * important information acceleration configuration, tflite model source etc. + */ +NS_SWIFT_NAME(BaseOptions) +@interface MPPBaseOptions : NSObject + +/** + * The external model file, as a single standalone TFLite file. It could be packed with TFLite Model + * Metadata[1] and associated files if exist. Fail to provide the necessary metadata and associated + * files might result in errors. + */ +@property(nonatomic, copy) MPPExternalFile *modelAssetFile; + +/** + * device delegate to run the MediaPipe pipeline. If the delegate is not set, the default + * delegate CPU is used. + */ +@property(nonatomic) MPPDelegate delegate; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m new file mode 100644 index 000000000..4c25b80e8 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m @@ -0,0 +1,36 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/core/sources/MPPBaseOptions.h" + +@implementation MPPBaseOptions + +- (instancetype)init { + self = [super init]; + if (self) { + self.modelAssetFile = [[MPPExternalFile alloc] init]; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPBaseOptions *baseOptions = [[MPPBaseOptions alloc] init]; + + baseOptions.modelAssetFile = self.modelAssetFile; + baseOptions.delegate = self.delegate; + + return baseOptions; +} + +@end diff --git a/mediapipe/tasks/ios/core/sources/MPPExternalFile.h b/mediapipe/tasks/ios/core/sources/MPPExternalFile.h new file mode 100644 index 000000000..a97802002 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPExternalFile.h @@ -0,0 +1,28 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + * Holds information about an external file. + */ +NS_SWIFT_NAME(ExternalFile) +@interface MPPExternalFile : NSObject + +/** Path to the file in bundle. */ +@property(nonatomic, copy) NSString *filePath; +/// Add provision for other sources in future. + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPExternalFile.m b/mediapipe/tasks/ios/core/sources/MPPExternalFile.m new file mode 100644 index 000000000..70d85657c --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPExternalFile.m @@ -0,0 +1,27 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/core/sources/MPPExternalFile.h" + +@implementation MPPExternalFile + +- (id)copyWithZone:(NSZone *)zone { + MPPExternalFile *externalFile = [[MPPExternalFile alloc] init]; + + externalFile.filePath = self.filePath; + + return externalFile; +} + +@end diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h new file mode 100644 index 000000000..620184518 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.h @@ -0,0 +1,64 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import +#include "mediapipe/framework/calculator.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Holds all needed informaton to initialize a MediaPipe Task. + */ +@interface MPPTaskInfo : NSObject + +@property(nonatomic, copy, nonnull) NSString *taskGraphName; + +/** + * A task-specific options that is derived from MPPTaskOptions and confirms to + * MPPTaskOptionsProtocol. + */ +@property(nonatomic, copy) id taskOptions; + +/** + * List of task graph input stream info strings in the form TAG:name. + */ +@property(nonatomic, copy) NSArray *inputStreams; + +/** + * List of task graph output stream info in the form TAG:name. + */ +@property(nonatomic, copy) NSArray *outputStreams; + +/** + * If the task requires a flow limiter. + */ +@property(nonatomic) BOOL enableFlowLimiting; + ++ (instancetype)new NS_UNAVAILABLE; + +- (instancetype)initWithTaskGraphName:(NSString *)taskGraphName + inputStreams:(NSArray *)inputStreams + outputStreams:(NSArray *)outputStreams + taskOptions:(id)taskOptions + enableFlowLimiting:(BOOL)enableFlowLimiting + error:(NSError **)error; + +/** + * Creates a MediaPipe Task protobuf message from the MPPTaskInfo instance. + */ +- (mediapipe::CalculatorGraphConfig)generateGraphConfig; + +- (instancetype)init NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm new file mode 100644 index 000000000..7e42d6eae --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm @@ -0,0 +1,135 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +#include "mediapipe/calculators/core/flow_limiter_calculator.pb.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/calculator_options.pb.h" + +namespace { +using CalculatorGraphConfig = ::mediapipe::CalculatorGraphConfig; +using Node = ::mediapipe::CalculatorGraphConfig::Node; +using InputStreamInfo = ::mediapipe::InputStreamInfo; +using CalculatorOptions = ::mediapipe::CalculatorOptions; +using FlowLimiterCalculatorOptions = ::mediapipe::FlowLimiterCalculatorOptions; +} // namespace + +@implementation MPPTaskInfo + +- (instancetype)initWithTaskGraphName:(NSString *)taskGraphName + inputStreams:(NSArray *)inputStreams + outputStreams:(NSArray *)outputStreams + taskOptions:(id)taskOptions + enableFlowLimiting:(BOOL)enableFlowLimiting + error:(NSError **)error { + self = [super init]; + if (!taskGraphName || !inputStreams.count || !outputStreams.count) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description: + @"Task graph's name, input streams, and output streams should be non-empty."]; + } + + if (self) { + _taskGraphName = taskGraphName; + _inputStreams = inputStreams; + _outputStreams = outputStreams; + _taskOptions = taskOptions; + _enableFlowLimiting = enableFlowLimiting; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] init]; + + taskInfo.taskGraphName = self.taskGraphName; + taskInfo.inputStreams = self.inputStreams; + taskInfo.outputStreams = self.outputStreams; + taskInfo.taskOptions = self.taskOptions; + taskInfo.enableFlowLimiting = self.enableFlowLimiting; + + return taskInfo; +} + +- (CalculatorGraphConfig)generateGraphConfig { + CalculatorGraphConfig graph_config; + + Node *task_subgraph_node = graph_config.add_node(); + task_subgraph_node->set_calculator(self.taskGraphName.cppString); + [self.taskOptions copyToProto:task_subgraph_node->mutable_options()]; + + for (NSString *outputStream in self.outputStreams) { + auto cpp_output_stream = std::string(outputStream.cppString); + task_subgraph_node->add_output_stream(cpp_output_stream); + graph_config.add_output_stream(cpp_output_stream); + } + + if (self.enableFlowLimiting) { + Node *flow_limit_calculator_node = graph_config.add_node(); + + flow_limit_calculator_node->set_calculator("FlowLimiterCalculator"); + + InputStreamInfo *input_stream_info = flow_limit_calculator_node->add_input_stream_info(); + input_stream_info->set_tag_index("FINISHED"); + input_stream_info->set_back_edge(true); + + FlowLimiterCalculatorOptions *flow_limit_calculator_options = + flow_limit_calculator_node->mutable_options()->MutableExtension( + FlowLimiterCalculatorOptions::ext); + flow_limit_calculator_options->set_max_in_flight(1); + flow_limit_calculator_options->set_max_in_queue(1); + + for (NSString *inputStream in self.inputStreams) { + graph_config.add_input_stream(inputStream.cppString); + + NSString *strippedInputStream = [MPPTaskInfo stripTagIndex:inputStream]; + flow_limit_calculator_node->add_input_stream(strippedInputStream.cppString); + + NSString *taskInputStream = [MPPTaskInfo addStreamNamePrefix:inputStream]; + task_subgraph_node->add_input_stream(taskInputStream.cppString); + + NSString *strippedTaskInputStream = [MPPTaskInfo stripTagIndex:taskInputStream]; + flow_limit_calculator_node->add_output_stream(strippedTaskInputStream.cppString); + } + + NSString *firstOutputStream = self.outputStreams[0]; + auto finished_output_stream = "FINISHED:" + firstOutputStream.cppString; + flow_limit_calculator_node->add_input_stream(finished_output_stream); + } else { + for (NSString *inputStream in self.inputStreams) { + auto cpp_input_stream = inputStream.cppString; + task_subgraph_node->add_input_stream(cpp_input_stream); + graph_config.add_input_stream(cpp_input_stream); + } + } + + return graph_config; +} + ++ (NSString *)stripTagIndex:(NSString *)tagIndexName { + return [tagIndexName componentsSeparatedByString:@":"][1]; +} + ++ (NSString *)addStreamNamePrefix:(NSString *)tagIndexName { + NSArray *splits = [tagIndexName componentsSeparatedByString:@":"]; + return [NSString stringWithFormat:@"%@:throttled_%@", splits[0], splits[1]]; +} + +@end diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskOptions.h b/mediapipe/tasks/ios/core/sources/MPPTaskOptions.h new file mode 100644 index 000000000..e40e92657 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskOptions.h @@ -0,0 +1,58 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import +#include "mediapipe/framework/calculator_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPBaseOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * MediaPipe Tasks options base class. Any MediaPipe task-specific options class should extend + * this class. + */ +NS_SWIFT_NAME(TaskOptions) +@interface MPPTaskOptions : NSObject +/** + * Base options for configuring the Mediapipe task. + */ +@property(nonatomic, copy) MPPBaseOptions *baseOptions; + +/** + * Initializes a new `MPPTaskOptions` with the absolute path to the model file + * stored locally on the device, set to the given the model path. + * + * @discussion The external model file must be a single standalone TFLite file. It could be packed + * with TFLite Model Metadata[1] and associated files if they exist. Failure to provide the + * necessary metadata and associated files might result in errors. Check the [documentation] + * (https://www.tensorflow.org/lite/convert/metadata) for each task about the specific requirement. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * + * @return An instance of `MPPTaskOptions` initialized to the given model path. + */ +- (instancetype)initWithModelPath:(NSString *)modelPath; + +@end + +/** + * Any mediapipe task options should confirm to this protocol. + */ +@protocol MPPTaskOptionsProtocol + +/** + * Copies the iOS Mediapipe task options to an object of mediapipe::CalculatorOptions proto. + */ +- (void)copyToProto:(mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskOptions.m b/mediapipe/tasks/ios/core/sources/MPPTaskOptions.m new file mode 100644 index 000000000..ec1adbaf1 --- /dev/null +++ b/mediapipe/tasks/ios/core/sources/MPPTaskOptions.m @@ -0,0 +1,36 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/core/sources/MPPBaseOptions.h" + +@implementation MPPTaskOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _baseOptions = [[MPPBaseOptions alloc] init]; + } + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath { + self = [self init]; + if (self) { + _baseOptions.modelAssetFile.filePath = modelPath; + } + return self; +} + +@end diff --git a/mediapipe/tasks/ios/core/utils/BUILD b/mediapipe/tasks/ios/core/utils/BUILD new file mode 100644 index 000000000..d9fbaf375 --- /dev/null +++ b/mediapipe/tasks/ios/core/utils/BUILD @@ -0,0 +1,27 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPBaseOptionsHelpers", + srcs = ["sources/MPPBaseOptions+Helpers.mm"], + hdrs = ["sources/MPPBaseOptions+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/core:MPPBaseOptions", + "//mediapipe/tasks/cc/core/proto:base_options_cc_proto", + ], +) diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h new file mode 100644 index 000000000..f57844a8f --- /dev/null +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h @@ -0,0 +1,26 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#include "mediapipe/tasks/cc/core/proto/base_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPBaseOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPBaseOptions (Helpers) + +- (void)copyToProto:(mediapipe::tasks::core::proto::BaseOptions *)baseOptionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm new file mode 100644 index 000000000..f20f8602a --- /dev/null +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm @@ -0,0 +1,40 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +namespace { +using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions; +} + +@implementation MPPBaseOptions (Helpers) + +- (void)copyToProto:(BaseOptionsProto *)baseOptionsProto { + if (self.modelAssetFile.filePath) { + baseOptionsProto->mutable_model_asset()->set_file_name(self.modelAssetFile.filePath.UTF8String); + } + + switch (self.delegate) { + case MPPDelegateCPU: { + baseOptionsProto->mutable_acceleration()->mutable_tflite(); + break; + } + case MPPDelegateGPU: + break; + default: + break; + } +} + +@end diff --git a/mediapipe/tasks/ios/text/core/BUILD b/mediapipe/tasks/ios/text/core/BUILD new file mode 100644 index 000000000..abb8edc71 --- /dev/null +++ b/mediapipe/tasks/ios/text/core/BUILD @@ -0,0 +1,33 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPBaseTextTaskApi", + srcs = ["sources/MPPBaseTextTaskApi.mm"], + hdrs = ["sources/MPPBaseTextTaskApi.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/tasks/cc/core:task_runner", + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + ], +) + diff --git a/mediapipe/tasks/ios/text/core/sources/MPPBaseTextTaskApi.h b/mediapipe/tasks/ios/text/core/sources/MPPBaseTextTaskApi.h new file mode 100644 index 000000000..3d2fd4f43 --- /dev/null +++ b/mediapipe/tasks/ios/text/core/sources/MPPBaseTextTaskApi.h @@ -0,0 +1,44 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import + +#include "mediapipe/framework/calculator.pb.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * The base class of the user-facing iOS mediapipe text task api classes. + */ +NS_SWIFT_NAME(BaseTextTaskApi) +@interface MPPBaseTextTaskApi : NSObject + +/** + * Initializes a new `MPPBaseTextTaskApi` with the mediapipe text task graph config proto. + * + * @param graphConfig A mediapipe text task graph config proto. + * + * @return An instance of `MPPBaseTextTaskApi` initialized to the given graph config proto. + */ +- (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig + error:(NSError **)error; +- (void)close; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/core/sources/MPPBaseTextTaskApi.mm b/mediapipe/tasks/ios/text/core/sources/MPPBaseTextTaskApi.mm new file mode 100644 index 000000000..9d7142fe5 --- /dev/null +++ b/mediapipe/tasks/ios/text/core/sources/MPPBaseTextTaskApi.mm @@ -0,0 +1,52 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/text/core/sources/MPPBaseTextTaskApi.h" + +#include "mediapipe/tasks/cc/core/task_runner.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" + +namespace { +using ::mediapipe::CalculatorGraphConfig; +using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; +} // namespace + +@interface MPPBaseTextTaskApi () { + /** TextSearcher backed by C++ API */ + std::unique_ptr _taskRunner; +} +@end + +@implementation MPPBaseTextTaskApi + +- (instancetype)initWithCalculatorGraphConfig:(CalculatorGraphConfig)graphConfig + error:(NSError **)error { + self = [super init]; + if (self) { + auto taskRunnerResult = TaskRunnerCpp::Create(std::move(graphConfig)); + + if (![MPPCommonUtils checkCppError:taskRunnerResult.status() toError:error]) { + return nil; + } + + _taskRunner = std::move(taskRunnerResult.value()); + } + return self; +} + +- (void)close { + _taskRunner->Close(); +} + +@end diff --git a/mediapipe/tasks/ios/text/text_classifier/BUILD b/mediapipe/tasks/ios/text/text_classifier/BUILD new file mode 100644 index 000000000..9ed25a852 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/BUILD @@ -0,0 +1,50 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPTextClassifier", + srcs = ["sources/MPPTextClassifier.mm"], + hdrs = ["sources/MPPTextClassifier.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/text/core:MPPBaseTextTaskApi", + "//mediapipe/tasks/ios/text/text_classifier/utils:MPPTextClassifierOptionsHelpers", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + ":MPPTextClassifierOptions", + ], +) + +objc_library( + name = "MPPTextClassifierOptions", + srcs = ["sources/MPPTextClassifierOptions.mm"], + hdrs = ["sources/MPPTextClassifierOptions.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/components/processors:MPPClassifierOptions", + ], +) + diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h new file mode 100644 index 000000000..d6b2f770f --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h @@ -0,0 +1,61 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/text/core/sources/MPPBaseTextTaskApi.h" +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * A Mediapipe iOS Text Classifier. + */ +NS_SWIFT_NAME(TextClassifier) +@interface MPPTextClassifier : MPPBaseTextTaskApi + +/** + * Creates a new instance of `MPPTextClassifier` from an absolute path to a TensorFlow Lite model + * file stored locally on the device. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * + * @param error An optional error parameter populated when there is an error in initializing + * the text classifier. + * + * @return A new instance of `MPPTextClassifier` with the given model path. `nil` if there is an + * error in initializing the text classifier. + */ +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `MPPTextClassifier` from the given text classifier options. + * + * @param options The options to use for configuring the `MPPTextClassifier`. + * @param error An optional error parameter populated when there is an error in initializing + * the text classifier. + * + * @return A new instance of `MPPTextClassifier` with the given options. `nil` if there is an error + * in initializing the text classifier. + */ +- (instancetype)initWithOptions:(MPPTextClassifierOptions *)options error:(NSError **)error; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm new file mode 100644 index 000000000..56bc4930f --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm @@ -0,0 +1,65 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h" + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h" + +NSString *kClassificationsStreamName = @"classifications_out"; +NSString *kClassificationsTag = @"classifications"; +NSString *kTextInStreamName = @"text_in"; +NSString *kTextTag = @"TEXT"; +NSString *kTaskGraphName = @"mediapipe.tasks.text.text_classifier.TextClassifierGraph"; + +@implementation MPPTextClassifierOptions + +- (instancetype)initWithModelPath:(NSString *)modelPath { + self = [super initWithModelPath:modelPath]; + if (self) { + _classifierOptions = [[MPPClassifierOptions alloc] init]; + } + return self; +} + +@end + +@implementation MPPTextClassifier + +- (instancetype)initWithOptions:(MPPTextClassifierOptions *)options error:(NSError **)error { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] + initWithTaskGraphName:kTaskGraphName + inputStreams:@[ [NSString stringWithFormat:@"@:@", kTextTag, kTextInStreamName] ] + outputStreams:@[ [NSString stringWithFormat:@"@:@", kClassificationsTag, + kClassificationsStreamName] ] + taskOptions:options + enableFlowLimiting:NO + error:error]; + + if (!taskInfo) { + return nil; + } + + return [super initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] error:error]; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPTextClassifierOptions *options = + [[MPPTextClassifierOptions alloc] initWithModelPath:modelPath]; + + return [self initWithOptions:options error:error]; +} + +@end diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h new file mode 100644 index 000000000..47c44dd0d --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h @@ -0,0 +1,51 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import + +#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Options to configure MPPTextClassifierOptions. + */ +NS_SWIFT_NAME(TextClassifierOptions) +@interface MPPTextClassifierOptions : MPPTaskOptions + +/** + * Options controlling the behavior of the embedding model specified in the + * base options. + */ +@property(nonatomic, copy) MPPClassifierOptions *classifierOptions; + +/** + * Initializes a new `MPPTextClassifierOptions` with the absolute path to the model file + * stored locally on the device, set to the given the model path. + * + * @discussion The external model file must be a single standalone TFLite file. It could be packed + * with TFLite Model Metadata[1] and associated files if they exist. Failure to provide the + * necessary metadata and associated files might result in errors. Check the [documentation] + * (https://www.tensorflow.org/lite/convert/metadata) for each task about the specific requirement. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * + * @return An instance of `MPPTextClassifierOptions` initialized to the given model path. + */ +- (instancetype)initWithModelPath:(NSString *)modelPath; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.mm b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.mm new file mode 100644 index 000000000..8cab693cd --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.mm @@ -0,0 +1,27 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h" + +@implementation MPPTextClassifierOptions + +- (instancetype)initWithModelPath:(NSString *)modelPath { + self = [super initWithModelPath:modelPath]; + if (self) { + _classifierOptions = [[MPPClassifierOptions alloc] init]; + } + return self; +} + +@end \ No newline at end of file diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/BUILD b/mediapipe/tasks/ios/text/text_classifier/utils/BUILD new file mode 100644 index 000000000..453a30f54 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/BUILD @@ -0,0 +1,30 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPTextClassifierOptionsHelpers", + srcs = ["sources/MPPTextClassifierOptions+Helpers.mm"], + hdrs = ["sources/MPPTextClassifierOptions+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/text/text_classifier:MPPTextClassifierOptions", + "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", + "//mediapipe/tasks/ios/components/processors/utils:MPPClassifierOptionsHelpers", + "//mediapipe/tasks/cc/text/text_classifier/proto:text_classifier_graph_options_cc_proto", + "//mediapipe/framework:calculator_options_cc_proto", + ], +) diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h new file mode 100644 index 000000000..8df471d05 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h @@ -0,0 +1,26 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#include "mediapipe/framework/calculator_options.pb.h" +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPTextClassifierOptions (Helpers) + +- (void)copyToProto:(mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm new file mode 100644 index 000000000..3576cb8d2 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm @@ -0,0 +1,36 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ +#include "mediapipe/tasks/cc/text/text_classifier/proto/text_classifier_graph_options.pb.h" +#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" +#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h" + +namespace { +using CalculatorOptionsProto = ::mediapipe::CalculatorOptions; +using TextClassifierGraphOptionsProto = + ::mediapipe::tasks::text::text_classifier::proto::TextClassifierGraphOptions; + +} // namespace + +@implementation MPPTextClassifierOptions (Helpers) + +- (void)copyToProto:(CalculatorOptionsProto *)optionsProto { + TextClassifierGraphOptionsProto *graph_options = + optionsProto->MutableExtension(TextClassifierGraphOptionsProto::ext); + [self.baseOptions copyToProto:graph_options->mutable_base_options()]; + [self.classifierOptions copyToProto:graph_options->mutable_classifier_options()]; +} + +@end