diff --git a/mediapipe/tasks/ios/components/processors/utils/BUILD b/mediapipe/tasks/ios/components/containers/BUILD similarity index 60% rename from mediapipe/tasks/ios/components/processors/utils/BUILD rename to mediapipe/tasks/ios/components/containers/BUILD index 5344c5fdf..9d82fc55a 100644 --- a/mediapipe/tasks/ios/components/processors/utils/BUILD +++ b/mediapipe/tasks/ios/components/containers/BUILD @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,12 +17,16 @@ package(default_visibility = ["//mediapipe/tasks:internal"]) licenses(["notice"]) objc_library( - name = "MPPClassifierOptionsHelpers", - srcs = ["sources/MPPClassifierOptions+Helpers.mm"], - hdrs = ["sources/MPPClassifierOptions+Helpers.h"], + name = "MPPCategory", + srcs = ["sources/MPPCategory.m"], + hdrs = ["sources/MPPCategory.h"], +) + +objc_library( + name = "MPPClassificationResult", + srcs = ["sources/MPPClassificationResult.m"], + hdrs = ["sources/MPPClassificationResult.h"], deps = [ - "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto", - "//mediapipe/tasks/ios/common/utils:NSStringHelpers", - "//mediapipe/tasks/ios/components/processors:MPPClassifierOptions", + ":MPPCategory", ], ) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h new file mode 100644 index 000000000..648725d95 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h @@ -0,0 +1,68 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + * Category is a util class that contains a label, its display name, a float value as score, and the + * index of the label in the corresponding label file. Typically it's used as the result of + * classification tasks. + **/ +NS_SWIFT_NAME(ClassificationCategory) +@interface MPPCategory : NSObject + +/** + * The index of the label in the corresponding label file. Set to -1 if the index is + * not set. + **/ +@property(nonatomic, readonly) NSInteger index; + +/** Confidence score for this class . **/ +@property(nonatomic, readonly) float score; + +/** The label of this category object. **/ +@property(nonatomic, readonly, nullable) NSString *categoryName; + +/** + * The display name of the label, which may be translated for different locales. For example, a + * label, "apple", may be translated into Spanish for display purpose, so that the display name is + * "manzana". + **/ +@property(nonatomic, readonly, nullable) NSString *displayName; + +/** + * Initializes a new `MPPCategory` with the given index, score, category name and display name. + * + * @param index The index of the label in the corresponding label file. + * @param score The probability score of this label category. + * @param categoryName The label of this category object. + * @param displayName The display name of the label. + * + * @return An instance of `MPPCategory` initialized with the given index, score, category name and + * display name. + **/ +- (instancetype)initWithIndex:(NSInteger)index + score:(float)score + categoryName:(nullable NSString *)categoryName + displayName:(nullable NSString *)displayName NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.m b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.m new file mode 100644 index 000000000..824fae65e --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.m @@ -0,0 +1,33 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h" + +@implementation MPPCategory + +- (instancetype)initWithIndex:(NSInteger)index + score:(float)score + categoryName:(nullable NSString *)categoryName + displayName:(nullable NSString *)displayName { + self = [super init]; + if (self) { + _index = index; + _score = score; + _categoryName = categoryName; + _displayName = displayName; + } + return self; +} + +@end diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h new file mode 100644 index 000000000..9c8b9bd2e --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h @@ -0,0 +1,116 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Represents the list of classification for a given classifier head. Typically used as a result + * for classification tasks. + **/ +NS_SWIFT_NAME(Classifications) +@interface MPPClassifications : NSObject + +/** + * The index of the classifier head these entries refer to. This is useful for multi-head models. + **/ +@property(nonatomic, readonly) NSInteger headIndex; + +/** The optional name of the classifier head, which is the corresponding tensor metadata name. **/ +@property(nonatomic, readonly, nullable) NSString *headName; + +/** An array of `MPPCategory` objects containing the predicted categories. **/ +@property(nonatomic, readonly) NSArray *categories; + +/** + * Initializes a new `MPPClassifications` object with the given head index and array of categories. + * Head name is initialized to `nil`. + * + * @param headIndex The index of the classifier head. + * @param categories An array of `MPPCategory` objects containing the predicted categories. + * + * @return An instance of `MPPClassifications` initialized with the given head index and + * array of categories. + **/ +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + categories:(NSArray *)categories; + +/** + * Initializes a new `MPPClassifications` with the given head index, head name and array of + * categories. + * + * @param headIndex The index of the classifier head. + * @param headName The name of the classifier head, which is the corresponding tensor metadata + * name. + * @param categories An array of `MPPCategory` objects containing the predicted categories. + * + * @return An object of `MPPClassifications` initialized with the given head index, head name and + * array of categories. + **/ +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + headName:(nullable NSString *)headName + categories:(NSArray *)categories NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +/** + * Represents the classification results of a model. Typically used as a result for classification + * tasks. + **/ +NS_SWIFT_NAME(ClassificationResult) +@interface MPPClassificationResult : NSObject + +/** + * An Array of `MPPClassifications` objects containing the predicted categories for each head of + * the model. + **/ +@property(nonatomic, readonly) NSArray *classifications; + +/** + * The optional timestamp (in milliseconds) of the start of the chunk of data corresponding to + * these results. If it is set to the value -1, it signifies the absence of a timestamp. This is + * only used for classification on time series (e.g. audio classification). In these use cases, the + * amount of data to process might exceed the maximum size that the model can process: to solve + * this, the input data is split into multiple chunks starting at different timestamps. + **/ +@property(nonatomic, readonly) NSInteger timestampMs; + +/** + * Initializes a new `MPPClassificationResult` with the given array of classifications and time + * stamp (in milliseconds). + * + * @param classifications An Array of `MPPClassifications` objects containing the predicted + * categories for each head of the model. + * @param timestampMs The timestamp (in milliseconds) of the start of the chunk of data + * corresponding to these results. + * + * @return An instance of `MPPClassificationResult` initialized with the given array of + * classifications and timestampMs. + **/ +- (instancetype)initWithClassifications:(NSArray *)classifications + timestampMs:(NSInteger)timestampMs NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m new file mode 100644 index 000000000..6d42d22ca --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m @@ -0,0 +1,51 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h" + +@implementation MPPClassifications + +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + headName:(nullable NSString *)headName + categories:(NSArray *)categories { + self = [super init]; + if (self) { + _headIndex = headIndex; + _headName = headName; + _categories = categories; + } + return self; +} + +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + categories:(NSArray *)categories { + return [self initWithHeadIndex:headIndex headName:nil categories:categories]; +} + +@end + +@implementation MPPClassificationResult + +- (instancetype)initWithClassifications:(NSArray *)classifications + timestampMs:(NSInteger)timestampMs { + self = [super init]; + if (self) { + _classifications = classifications; + _timestampMs = timestampMs; + } + + return self; +} + +@end diff --git a/mediapipe/tasks/ios/components/containers/utils/BUILD b/mediapipe/tasks/ios/components/containers/utils/BUILD new file mode 100644 index 000000000..e4c76ac4b --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/BUILD @@ -0,0 +1,40 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPCategoryHelpers", + srcs = ["sources/MPPCategory+Helpers.mm"], + hdrs = ["sources/MPPCategory+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/components/containers:MPPCategory", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + ], +) + +objc_library( + name = "MPPClassificationResultHelpers", + srcs = ["sources/MPPClassificationResult+Helpers.mm"], + hdrs = ["sources/MPPClassificationResult+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/components/containers:MPPClassificationResult", + ":MPPCategoryHelpers", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto", + ], +) diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h similarity index 62% rename from mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h rename to mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h index e156020df..7580cfeeb 100644 --- a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h @@ -1,4 +1,4 @@ -// Copyright 2022 The MediaPipe Authors. +// Copyright 2023 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" - -#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" +#include "mediapipe/framework/formats/classification.pb.h" +#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h" NS_ASSUME_NONNULL_BEGIN -@interface MPPClassifierOptions (Helpers) -- (void)copyToProto: - (mediapipe::tasks::components::processors::proto::ClassifierOptions *)classifierOptionsProto; +@interface MPPCategory (Helpers) + ++ (MPPCategory *)categoryWithProto:(const mediapipe::Classification &)classificationProto; + @end NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm new file mode 100644 index 000000000..1c6c951d0 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm @@ -0,0 +1,43 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +namespace { +using ClassificationProto = ::mediapipe::Classification; +} + +@implementation MPPCategory (Helpers) + ++ (MPPCategory *)categoryWithProto:(const ClassificationProto &)clasificationProto { + NSString *categoryName; + NSString *displayName; + + if (clasificationProto.has_label()) { + categoryName = [NSString stringWithCppString:clasificationProto.label()]; + } + + if (clasificationProto.has_display_name()) { + displayName = [NSString stringWithCppString:clasificationProto.display_name()]; + } + + return [[MPPCategory alloc] initWithIndex:clasificationProto.index() + score:clasificationProto.score() + categoryName:categoryName + displayName:displayName]; +} + +@end diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h new file mode 100644 index 000000000..fde436feb --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h @@ -0,0 +1,35 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" +#import "mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPClassifications (Helpers) + ++ (MPPClassifications *)classificationsWithProto: + (const mediapipe::tasks::components::containers::proto::Classifications &)classificationsProto; + +@end + +@interface MPPClassificationResult (Helpers) + ++ (MPPClassificationResult *)classificationResultWithProto: + (const mediapipe::tasks::components::containers::proto::ClassificationResult &) + classificationResultProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm new file mode 100644 index 000000000..78bc0b6a3 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm @@ -0,0 +1,64 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h" + +namespace { +using ClassificationsProto = ::mediapipe::tasks::components::containers::proto::Classifications; +using ClassificationResultProto = + ::mediapipe::tasks::components::containers::proto::ClassificationResult; +} // namespace + +@implementation MPPClassifications (Helpers) + ++ (MPPClassifications *)classificationsWithProto: + (const ClassificationsProto &)classificationsProto { + NSMutableArray *categories = [NSMutableArray arrayWithCapacity:(NSUInteger)classificationsProto.classification_list().classification_size()]; + for (const auto &classification : classificationsProto.classification_list().classification()) { + [categories addObject:[MPPCategory categoryWithProto:classification]]; + } + + NSString *headName; + if (classificationsProto.has_head_name()) { + headName = [NSString stringWithCppString:classificationsProto.head_name()]; + } + + return [[MPPClassifications alloc] initWithHeadIndex:(NSInteger)classificationsProto.head_index() + headName:headName + categories:categories]; +} + +@end + +@implementation MPPClassificationResult (Helpers) + ++ (MPPClassificationResult *)classificationResultWithProto: + (const ClassificationResultProto &)classificationResultProto { + NSMutableArray *classifications = [NSMutableArray arrayWithCapacity:(NSUInteger)classificationResultProto.classifications_size()]; + for (const auto &classificationsProto : classificationResultProto.classifications()) { + [classifications addObject:[MPPClassifications classificationsWithProto:classificationsProto]]; + } + + NSInteger timestampMs; + if (classificationResultProto.has_timestamp_ms()) { + timestampMs = (NSInteger)classificationResultProto.timestamp_ms(); + } + + return [[MPPClassificationResult alloc] initWithClassifications:classifications timestampMs:timestampMs];; +} + +@end diff --git a/mediapipe/tasks/ios/core/BUILD b/mediapipe/tasks/ios/core/BUILD index 434d20085..e4797c34d 100644 --- a/mediapipe/tasks/ios/core/BUILD +++ b/mediapipe/tasks/ios/core/BUILD @@ -90,6 +90,8 @@ objc_library( deps = [ "//mediapipe/framework:calculator_cc_proto", "//mediapipe/tasks/cc/core:task_runner", + "//mediapipe/tasks/cc/core:mediapipe_builtin_op_resolver", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "@org_tensorflow//tensorflow/lite/core/api:op_resolver", ], ) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm index 80ff594a2..ae6ed2a70 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" + #import "mediapipe/tasks/ios/common/sources/MPPCommon.h" #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskOptions.m b/mediapipe/tasks/ios/core/sources/MPPTaskOptions.m index fe74517c3..ad11bbc6e 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskOptions.m +++ b/mediapipe/tasks/ios/core/sources/MPPTaskOptions.m @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" + #import "mediapipe/tasks/ios/core/sources/MPPBaseOptions.h" @implementation MPPTaskOptions diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 2b9f2ecdb..a1b1dfad4 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -20,23 +20,65 @@ NS_ASSUME_NONNULL_BEGIN /** - * This class is used to create and call appropriate methods on the C++ Task Runner. - */ + * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, + * execute and terminate any MediaPipe task. + * + * An instance of the newly created C++ task runner will be stored until this class is destroyed. + * When methods are called for processing (performing inference), closing etc., on this class, + * internally the appropriate methods will be called on the C++ task runner instance to execute the + * appropriate actions. For each type of task, a subclass of this class must be defined to add any + * additional functionality. For eg:, vision tasks must create an `MPPVisionTaskRunner` and provide + * additional functionality. An instance of `MPPVisionTaskRunner` can in turn be used by the each + * vision task for creation and execution of the task. Please see the documentation for the C++ Task + * Runner for more details on how the taks runner operates. + **/ @interface MPPTaskRunner : NSObject /** - * Initializes a new `MPPTaskRunner` with the mediapipe task graph config proto. + * Initializes a new `MPPTaskRunner` with the MediaPipe calculator configuration proto and an + * optional C++ packets callback. + * + * You can pass `nullptr` for `packetsCallback` in case the mode of operation requested by the user + * is synchronous. + * + * If the task is operating in asynchronous mode, any iOS MediaPipe task that uses the + * `MPPTaskRunner` must define a C++ callback function to obtain the results of inference + * asynchronously and deliver the results to the user. To accomplish this, the callback function + * should in turn invoke the block provided by the user in the task options supplied to create the + * task. Please see the documentation of the C++ Task Runner for more information on the synchronous + * and asynchronous modes of operation. * * @param graphConfig A mediapipe task graph config proto. + * @param packetsCallback An optional C++ callback function that takes a list of output packets as + * the input argument. If provided, the callback must in turn call the block provided by the user in + * the appropriate task options. * - * @return An instance of `MPPTaskRunner` initialized to the given graph config proto. - */ + * @return An instance of `MPPTaskRunner` initialized to the given graph config proto and optional + * packetsCallback. + **/ - (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig + packetsCallback: + (mediapipe::tasks::core::PacketsCallback)packetsCallback error:(NSError **)error NS_DESIGNATED_INITIALIZER; +/** + * A synchronous method for processing batch data or offline streaming data. This method is designed + * for processing either batch data such as unrelated images and texts or offline streaming data + * such as the decoded frames from a video file or audio file. The call blocks the current + * thread until a failure status or a successful result is returned. If the input packets have no + * timestamp, an internal timestamp will be assigend per invocation. Otherwise, when the timestamp + * is set in the input packets, the caller must ensure that the input packet timestamps are greater + * than the timestamps of the previous invocation. This method is thread-unsafe and it is the + * caller's responsibility to synchronize access to this method across multiple threads and to + * ensure that the input packet timestamps are in order. + **/ - (absl::StatusOr)process: (const mediapipe::tasks::core::PacketMap &)packetMap; +/** + * Shuts down the C++ task runner. After the runner is closed, any calls that send input data to the + * runner are illegal and will receive errors. + **/ - (absl::Status)close; - (instancetype)init NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm index c5c307fd5..a77f206b2 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm @@ -13,11 +13,17 @@ // limitations under the License. #import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" + #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" +#include "tensorflow/lite/core/api/op_resolver.h" + namespace { using ::mediapipe::CalculatorGraphConfig; +using ::mediapipe::tasks::core::MediaPipeBuiltinOpResolver; using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; } // namespace @@ -30,15 +36,17 @@ using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; @implementation MPPTaskRunner - (instancetype)initWithCalculatorGraphConfig:(CalculatorGraphConfig)graphConfig + packetsCallback:(PacketsCallback)packetsCallback error:(NSError **)error { self = [super init]; if (self) { - auto taskRunnerResult = TaskRunnerCpp::Create(std::move(graphConfig)); + auto taskRunnerResult = TaskRunnerCpp::Create(std::move(graphConfig), + absl::make_unique(), + std::move(packetsCallback)); if (![MPPCommonUtils checkCppError:taskRunnerResult.status() toError:error]) { return nil; } - _cppTaskRunner = std::move(taskRunnerResult.value()); } return self; diff --git a/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm index ca86e7a0b..fb59b363d 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h" + #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" namespace { diff --git a/mediapipe/tasks/ios/core/utils/BUILD b/mediapipe/tasks/ios/core/utils/BUILD new file mode 100644 index 000000000..1cfc75e6a --- /dev/null +++ b/mediapipe/tasks/ios/core/utils/BUILD @@ -0,0 +1,27 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPBaseOptionsHelpers", + srcs = ["sources/MPPBaseOptions+Helpers.mm"], + hdrs = ["sources/MPPBaseOptions+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/core:MPPBaseOptions", + "//mediapipe/tasks/cc/core/proto:base_options_cc_proto", + ], +) diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h new file mode 100644 index 000000000..d52df2ae4 --- /dev/null +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h @@ -0,0 +1,26 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/tasks/cc/core/proto/base_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPBaseOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPBaseOptions (Helpers) + +- (void)copyToProto:(mediapipe::tasks::core::proto::BaseOptions *)baseOptionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm new file mode 100644 index 000000000..3fd8fbda3 --- /dev/null +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm @@ -0,0 +1,44 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +namespace { +using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions; +} + +@implementation MPPBaseOptions (Helpers) + +- (void)copyToProto:(BaseOptionsProto *)baseOptionsProto { + baseOptionsProto->Clear(); + + if (self.modelAssetPath) { + baseOptionsProto->mutable_model_asset()->set_file_name(self.modelAssetPath.UTF8String); + } + + switch (self.delegate) { + case MPPDelegateCPU: { + baseOptionsProto->mutable_acceleration()->mutable_tflite(); + break; + } + case MPPDelegateGPU: { + // TODO: Provide an implementation for GPU Delegate. + [NSException raise:@"Invalid value for delegate" format:@"GPU Delegate is not implemented."]; + } + default: + break; + } +} + +@end diff --git a/mediapipe/tasks/ios/components/processors/BUILD b/mediapipe/tasks/ios/text/core/BUILD similarity index 67% rename from mediapipe/tasks/ios/components/processors/BUILD rename to mediapipe/tasks/ios/text/core/BUILD index 165145076..bf88f5734 100644 --- a/mediapipe/tasks/ios/components/processors/BUILD +++ b/mediapipe/tasks/ios/text/core/BUILD @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +17,15 @@ package(default_visibility = ["//mediapipe/tasks:internal"]) licenses(["notice"]) objc_library( - name = "MPPClassifierOptions", - srcs = ["sources/MPPClassifierOptions.m"], - hdrs = ["sources/MPPClassifierOptions.h"], + name = "MPPTextTaskRunner", + srcs = ["sources/MPPTextTaskRunner.mm"], + hdrs = ["sources/MPPTextTaskRunner.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskRunner", + ], ) + diff --git a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h new file mode 100644 index 000000000..e3df3de9d --- /dev/null +++ b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h @@ -0,0 +1,43 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, + * execute and terminate any MediaPipe text task. + **/ +@interface MPPTextTaskRunner : MPPTaskRunner + +/** + * Initializes a new `MPPTextTaskRunner` with the MediaPipe calculator config proto. + * + * @param graphConfig A MediaPipe calculator config proto. + * + * @return An instance of `MPPTextTaskRunner` initialized to the given MediaPipe calculator config + * proto. + **/ +- (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.mm b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.mm new file mode 100644 index 000000000..956448c17 --- /dev/null +++ b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.mm @@ -0,0 +1,29 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h" + +namespace { +using ::mediapipe::CalculatorGraphConfig; +} // namespace + +@implementation MPPTextTaskRunner + +- (instancetype)initWithCalculatorGraphConfig:(CalculatorGraphConfig)graphConfig + error:(NSError **)error { + self = [super initWithCalculatorGraphConfig:graphConfig packetsCallback:nullptr error:error]; + return self; +} + +@end diff --git a/mediapipe/tasks/ios/text/text_classifier/BUILD b/mediapipe/tasks/ios/text/text_classifier/BUILD new file mode 100644 index 000000000..e5242f50d --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/BUILD @@ -0,0 +1,58 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPTextClassifierOptions", + srcs = ["sources/MPPTextClassifierOptions.m"], + hdrs = ["sources/MPPTextClassifierOptions.h"], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskOptions", + ], +) + +objc_library( + name = "MPPTextClassifierResult", + srcs = ["sources/MPPTextClassifierResult.m"], + hdrs = ["sources/MPPTextClassifierResult.h"], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskResult", + "//mediapipe/tasks/ios/components/containers:MPPClassificationResult", + ], +) + +objc_library( + name = "MPPTextClassifier", + srcs = ["sources/MPPTextClassifier.mm"], + hdrs = ["sources/MPPTextClassifier.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/tasks/cc/text/text_classifier:text_classifier_graph", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/text/core:MPPTextTaskRunner", + "//mediapipe/tasks/ios/core:MPPTextPacketCreator", + "//mediapipe/tasks/ios/text/text_classifier/utils:MPPTextClassifierOptionsHelpers", + "//mediapipe/tasks/ios/text/text_classifier/utils:MPPTextClassifierResultHelpers", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + ":MPPTextClassifierOptions", + ], +) diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h new file mode 100644 index 000000000..73ed5ff7a --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h @@ -0,0 +1,103 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h" +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Performs classification on text. + * + * This API expects a TFLite model with (optional) [TFLite Model + * Metadata](https://www.tensorflow.org/lite/convert/metadata")that contains the mandatory + * (described below) input tensors, output tensor, and the optional (but recommended) label + * items as AssociatedFiles with type TENSOR_AXIS_LABELS per output classification tensor. + * + * Metadata is required for models with int32 input tensors because it contains the input + * process unit for the model's Tokenizer. No metadata is required for models with string + * input tensors. + * + * Input tensors + * - Three input tensors `kTfLiteInt32` of shape `[batch_size xbert_max_seq_len]` + * representing the input ids, mask ids, and segment ids. This input signature requires + * a Bert Tokenizer process unit in the model metadata. + * - Or one input tensor `kTfLiteInt32` of shape `[batch_size xmax_seq_len]` representing + * the input ids. This input signature requires a Regex Tokenizer process unit in the + * model metadata. + * - Or one input tensor (`kTfLiteString`) that is shapeless or has shape `[1]` containing + * the input string. + * + * At least one output tensor (`kTfLiteFloat32/kBool`) with: + * - `N` classes and shape `[1 x N]` + * - optional (but recommended) label map(s) as AssociatedFiles with type + * TENSOR_AXIS_LABELS, + * containing one label per line. The first such AssociatedFile (if any) is used to fill + * the `categoryName` field of the results. The `displayName` field is filled from the + * AssociatedFile (if any) whose locale matches the `displayNamesLocale` field of the + * `MPPTextClassifierOptions` used at creation time ("en" by default, i.e. English). If + * none of these are available, only the `index` field of the results will be filled. + **/ +NS_SWIFT_NAME(TextClassifier) +@interface MPPTextClassifier : NSObject + +/** + * Creates a new instance of `MPPTextClassifier` from an absolute path to a TensorFlow Lite + * model file stored locally on the device and the default `MPPTextClassifierOptions`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the + * device. + * @param error An optional error parameter populated when there is an error in initializing + * the text classifier. + * + * @return A new instance of `MPPTextClassifier` with the given model path. `nil` if there is an + * error in initializing the text classifier. + **/ +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `MPPTextClassifier` from the given `MPPTextClassifierOptions`. + * + * @param options The options of type `MPPTextClassifierOptions` to use for configuring the + * `MPPTextClassifier`. + * @param error An optional error parameter populated when there is an error in initializing + * the text classifier. + * + * @return A new instance of `MPPTextClassifier` with the given options. `nil` if there is an + * error in initializing the text classifier. + **/ +- (instancetype)initWithOptions:(MPPTextClassifierOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs classification on the input text. + * + * @param text The `NSString` on which classification is to be performed. + * @param error An optional error parameter populated when there is an error in performing + * classification on the input text. + * + * @return A `MPPTextClassifierResult` object that contains a list of text classifications. + **/ +- (nullable MPPTextClassifierResult *)classifyText:(NSString *)text error:(NSError **)error; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm new file mode 100644 index 000000000..aed05ec37 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm @@ -0,0 +1,98 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h" + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h" +#import "mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h" +#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h" +#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h" + +#include "absl/status/statusor.h" +#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" + +namespace { +using ::mediapipe::Packet; +using ::mediapipe::tasks::components::containers::proto::ClassificationResult; +using ::mediapipe::tasks::core::PacketMap; +} // namespace + +static NSString *const kClassificationsStreamName = @"classifications_out"; +static NSString *const kClassificationsTag = @"CLASSIFICATIONS"; +static NSString *const kTextInStreamName = @"text_in"; +static NSString *const kTextTag = @"TEXT"; +static NSString *const kTaskGraphName = @"mediapipe.tasks.text.text_classifier.TextClassifierGraph"; + +@interface MPPTextClassifier () { + /** iOS Text Task Runner */ + MPPTextTaskRunner *_textTaskRunner; +} +@end + +@implementation MPPTextClassifier + +- (instancetype)initWithOptions:(MPPTextClassifierOptions *)options error:(NSError **)error { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] + initWithTaskGraphName:kTaskGraphName + inputStreams:@[ [NSString stringWithFormat:@"%@:%@", kTextTag, kTextInStreamName] ] + outputStreams:@[ [NSString stringWithFormat:@"%@:%@", kClassificationsTag, + kClassificationsStreamName] ] + taskOptions:options + enableFlowLimiting:NO + error:error]; + + if (!taskInfo) { + return nil; + } + + _textTaskRunner = + [[MPPTextTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] + error:error]; + + if (!_textTaskRunner) { + return nil; + } + + self = [super init]; + + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPTextClassifierOptions *options = [[MPPTextClassifierOptions alloc] init]; + + options.baseOptions.modelAssetPath = modelPath; + + return [self initWithOptions:options error:error]; +} + +- (nullable MPPTextClassifierResult *)classifyText:(NSString *)text error:(NSError **)error { + Packet packet = [MPPTextPacketCreator createWithText:text]; + + std::map packetMap = {{kTextInStreamName.cppString, packet}}; + absl::StatusOr statusOrOutputPacketMap = [_textTaskRunner process:packetMap]; + + if (![MPPCommonUtils checkCppError:statusOrOutputPacketMap.status() toError:error]) { + return nil; + } + + return [MPPTextClassifierResult + textClassifierResultWithClassificationsPacket:statusOrOutputPacketMap.value() + [kClassificationsStreamName.cppString]]; +} + +@end diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h similarity index 87% rename from mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h rename to mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h index 13dca4030..d43d801d4 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h @@ -1,4 +1,4 @@ -// Copyright 2022 The MediaPipe Authors. +// Copyright 2023 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,13 +14,15 @@ #import +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" + NS_ASSUME_NONNULL_BEGIN /** - * Holds settings for any single iOS MediaPipe classification task. + * Options for setting up a `MPPTextClassifierOptions`. */ -NS_SWIFT_NAME(ClassifierOptions) -@interface MPPClassifierOptions : NSObject +NS_SWIFT_NAME(TextClassifierOptions) +@interface MPPTextClassifierOptions : MPPTaskOptions /** * The locale to use for display names specified through the TFLite Model diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m similarity index 55% rename from mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m rename to mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m index 01f498184..2d5c17cda 100644 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m @@ -1,4 +1,4 @@ -// Copyright 2022 The MediaPipe Authors. +// Copyright 2023 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h" -@implementation MPPClassifierOptions +@implementation MPPTextClassifierOptions - (instancetype)init { self = [super init]; @@ -26,15 +26,15 @@ } - (id)copyWithZone:(NSZone *)zone { - MPPClassifierOptions *classifierOptions = [[MPPClassifierOptions alloc] init]; + MPPTextClassifierOptions *textClassifierOptions = [super copyWithZone:zone]; - classifierOptions.scoreThreshold = self.scoreThreshold; - classifierOptions.maxResults = self.maxResults; - classifierOptions.categoryDenylist = self.categoryDenylist; - classifierOptions.categoryAllowlist = self.categoryAllowlist; - classifierOptions.displayNamesLocale = self.displayNamesLocale; + textClassifierOptions.scoreThreshold = self.scoreThreshold; + textClassifierOptions.maxResults = self.maxResults; + textClassifierOptions.categoryDenylist = self.categoryDenylist; + textClassifierOptions.categoryAllowlist = self.categoryAllowlist; + textClassifierOptions.displayNamesLocale = self.displayNamesLocale; - return classifierOptions; + return textClassifierOptions; } @end diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h new file mode 100644 index 000000000..c6b2f3205 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h @@ -0,0 +1,44 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Represents the classification results generated by `MPPTextClassifier`. **/ +NS_SWIFT_NAME(TextClassifierResult) +@interface MPPTextClassifierResult : MPPTaskResult + +/** The `MPPClassificationResult` instance containing one set of results per classifier head. **/ +@property(nonatomic, readonly) MPPClassificationResult *classificationResult; + +/** + * Initializes a new `MPPTextClassifierResult` with the given `MPPClassificationResult` and + * timestamp (in milliseconds). + * + * @param classificationResult The `MPPClassificationResult` instance containing one set of results + * per classifier head. + * @param timestampMs The timestamp for this result. + * + * @return An instance of `MPPTextClassifierResult` initialized with the given + * `MPPClassificationResult` and timestamp (in milliseconds). + **/ +- (instancetype)initWithClassificationResult:(MPPClassificationResult *)classificationResult + timestampMs:(NSInteger)timestampMs; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.m b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.m new file mode 100644 index 000000000..4d5c1104a --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.m @@ -0,0 +1,28 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h" + +@implementation MPPTextClassifierResult + +- (instancetype)initWithClassificationResult:(MPPClassificationResult *)classificationResult + timestampMs:(NSInteger)timestampMs { + self = [super initWithTimestampMs:timestampMs]; + if (self) { + _classificationResult = classificationResult; + } + return self; +} + +@end diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/BUILD b/mediapipe/tasks/ios/text/text_classifier/utils/BUILD new file mode 100644 index 000000000..299050b32 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/BUILD @@ -0,0 +1,41 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPTextClassifierOptionsHelpers", + srcs = ["sources/MPPTextClassifierOptions+Helpers.mm"], + hdrs = ["sources/MPPTextClassifierOptions+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/text/text_classifier:MPPTextClassifierOptions", + "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", + "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol", + "//mediapipe/tasks/cc/text/text_classifier/proto:text_classifier_graph_options_cc_proto", + ], +) + +objc_library( + name = "MPPTextClassifierResultHelpers", + srcs = ["sources/MPPTextClassifierResult+Helpers.mm"], + hdrs = ["sources/MPPTextClassifierResult+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/text/text_classifier:MPPTextClassifierResult", + "//mediapipe/tasks/ios/components/containers/utils:MPPClassificationResultHelpers", + "//mediapipe/framework:packet", + ], +) diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h new file mode 100644 index 000000000..1e52e5c87 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h @@ -0,0 +1,26 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPTextClassifierOptions (Helpers) + +- (void)copyToProto:(mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm similarity index 58% rename from mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm rename to mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm index 24b54fd6a..c370f11ef 100644 --- a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm @@ -1,4 +1,4 @@ -// Copyright 2022 The MediaPipe Authors. +// Copyright 2023 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,16 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. +#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h" + #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" -#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +#include "mediapipe/tasks/cc/text/text_classifier/proto/text_classifier_graph_options.pb.h" namespace { +using CalculatorOptionsProto = ::mediapipe::CalculatorOptions; +using TextClassifierGraphOptionsProto = + ::mediapipe::tasks::text::text_classifier::proto::TextClassifierGraphOptions; using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; -} +} // namespace -@implementation MPPClassifierOptions (Helpers) +@implementation MPPTextClassifierOptions (Helpers) -- (void)copyToProto:(ClassifierOptionsProto *)classifierOptionsProto { +- (void)copyToProto:(CalculatorOptionsProto *)optionsProto { + TextClassifierGraphOptionsProto *graphOptions = + optionsProto->MutableExtension(TextClassifierGraphOptionsProto::ext); + [self.baseOptions copyToProto:graphOptions->mutable_base_options()]; + + ClassifierOptionsProto *classifierOptionsProto = graphOptions->mutable_classifier_options(); classifierOptionsProto->Clear(); if (self.displayNamesLocale) { @@ -38,6 +50,7 @@ using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto for (NSString *category in self.categoryDenylist) { classifierOptionsProto->add_category_denylist(category.cppString); } + } @end diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h new file mode 100644 index 000000000..f1b728b0a --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h @@ -0,0 +1,28 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h" + +#include "mediapipe/framework/packet.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPTextClassifierResult (Helpers) + ++ (MPPTextClassifierResult *)textClassifierResultWithClassificationsPacket: + (const mediapipe::Packet &)packet; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.mm b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.mm new file mode 100644 index 000000000..62e0d8cb1 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.mm @@ -0,0 +1,43 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h" + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" + +#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" + +static const int kMicroSecondsPerMilliSecond = 1000; + +namespace { +using ClassificationResultProto = + ::mediapipe::tasks::components::containers::proto::ClassificationResult; +using ::mediapipe::Packet; +} // namespace + +#define int kMicroSecondsPerMilliSecond = 1000; + +@implementation MPPTextClassifierResult (Helpers) + ++ (MPPTextClassifierResult *)textClassifierResultWithClassificationsPacket:(const Packet &)packet { + MPPClassificationResult *classificationResult = [MPPClassificationResult + classificationResultWithProto:packet.Get()]; + + return [[MPPTextClassifierResult alloc] + initWithClassificationResult:classificationResult + timestampMs:(NSInteger)(packet.Timestamp().Value() / + kMicroSecondsPerMilliSecond)]; +} + +@end