From 1ea0e0a6fd8adf3d6e73bca0310c570f4904629b Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 18:27:56 +0530 Subject: [PATCH 01/45] Added MPPCategory and MPPClassificationResult --- .../tasks/ios/components/containers/BUILD | 32 +++++ .../containers/sources/MPPCategory.h | 65 ++++++++++ .../containers/sources/MPPCategory.m | 33 +++++ .../sources/MPPClassificationResult.h | 114 ++++++++++++++++++ .../sources/MPPClassificationResult.m | 57 +++++++++ 5 files changed, 301 insertions(+) create mode 100644 mediapipe/tasks/ios/components/containers/BUILD create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPCategory.h create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPCategory.m create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m diff --git a/mediapipe/tasks/ios/components/containers/BUILD b/mediapipe/tasks/ios/components/containers/BUILD new file mode 100644 index 000000000..9d82fc55a --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/BUILD @@ -0,0 +1,32 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPCategory", + srcs = ["sources/MPPCategory.m"], + hdrs = ["sources/MPPCategory.h"], +) + +objc_library( + name = "MPPClassificationResult", + srcs = ["sources/MPPClassificationResult.m"], + hdrs = ["sources/MPPClassificationResult.h"], + deps = [ + ":MPPCategory", + ], +) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h new file mode 100644 index 000000000..035cde09d --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h @@ -0,0 +1,65 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +NS_ASSUME_NONNULL_BEGIN + +/** Category is a util class, contains a label, its display name, a float value as score, and the + * index of the label in the corresponding label file. Typically it's used as the result of + * classification tasks. */ +NS_SWIFT_NAME(ClassificationCategory) +@interface MPPCategory : NSObject + +/** The index of the label in the corresponding label file. It takes the value -1 if the index is + * not set. */ +@property(nonatomic, readonly) NSInteger index; + +/** Confidence score for this class . */ +@property(nonatomic, readonly) float score; + +/** The label of this category object. */ +@property(nonatomic, readonly, nullable) NSString *categoryName; + +/** The display name of the label, which may be translated for different locales. For example, a + * label, "apple", may be translated into Spanish for display purpose, so that the display name is + * "manzana". */ +@property(nonatomic, readonly, nullable) NSString *displayName; + +/** + * Initializes a new `MPPCategory` with the given index, score, category name and display name. + * + * @param index The index of the label in the corresponding label file. + * + * @param score The probability score of this label category. + * + * @param categoryName The label of this category object.. + * + * @param displayName The display name of the label. + * + * @return An instance of `MPPCategory` initialized with the given index, score, category name and + * display name. + */ +- (instancetype)initWithIndex:(NSInteger)index + score:(float)score + categoryName:(nullable NSString *)categoryName + displayName:(nullable NSString *)displayName; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.m b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.m new file mode 100644 index 000000000..824fae65e --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.m @@ -0,0 +1,33 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h" + +@implementation MPPCategory + +- (instancetype)initWithIndex:(NSInteger)index + score:(float)score + categoryName:(nullable NSString *)categoryName + displayName:(nullable NSString *)displayName { + self = [super init]; + if (self) { + _index = index; + _score = score; + _categoryName = categoryName; + _displayName = displayName; + } + return self; +} + +@end diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h new file mode 100644 index 000000000..732d1f899 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h @@ -0,0 +1,114 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Represents the list of classification for a given classifier head. Typically used as a result + * for classification tasks. */ +NS_SWIFT_NAME(Classifications) +@interface MPPClassifications : NSObject + +/** The index of the classifier head these entries refer to. This is useful for multi-head models. + */ +@property(nonatomic, readonly) NSInteger headIndex; + +/** The optional name of the classifier head, which is the corresponding tensor metadata name. */ +@property(nonatomic, readonly, nullable) NSString *headName; + +/** An array of `MPPCategory` objects containing the predicted categories. */ +@property(nonatomic, readonly) NSArray *categories; + +/** + * Initializes a new `MPPClassifications` object with the given head index and array of categories. + * Head name is initialized to `nil`. + * + * @param headIndex The index of the classifier head. + * @param categories An array of `MPPCategory` objects containing the predicted categories. + * + * @return An instance of `MPPClassifications` initialized with the given head index and + * array of categories. + */ +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + categories:(NSArray *)categories; + +/** + * Initializes a new `MPPClassifications` with the given head index, head name and array of + * categories. + * + * @param headIndex The index of the classifier head. + * @param headName The name of the classifier head, which is the corresponding tensor metadata + * name. + * @param categories An array of `MPPCategory` objects containing the predicted categories. + * + * @return An object of `MPPClassifications` initialized with the given head index, head name and + * array of categories. + */ +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + headName:(nullable NSString *)headName + categories:(NSArray *)categories; + +@end + +/** + * Represents the classification results of a model. Typically used as a result for classification + * tasks. + */ +NS_SWIFT_NAME(ClassificationResult) +@interface MPPClassificationResult : NSObject + +/** An Array of `MPPClassifications` objects containing the predicted categories for each head of + * the model. */ +@property(nonatomic, readonly) NSArray *classifications; + +/** The optional timestamp (in milliseconds) of the start of the chunk of data corresponding to + * these results. If it is set to the value -1, it signifies the absence of a time stamp. This is + * only used for classification on time series (e.g. audio classification). In these use cases, the + * amount of data to process might exceed the maximum size that the model can process: to solve + * this, the input data is split into multiple chunks starting at different timestamps. */ +@property(nonatomic, readonly) NSInteger timestampMs; + +/** + * Initializes a new `MPPClassificationResult` with the given array of classifications. This method + * must be used when no time stamp needs to be specified. It sets the property `timestampMs` to -1. + * + * @param classifications An Aaray of `MPPClassifications` objects containing classifier + * predictions per classifier head. + * + * @return An instance of MPPClassificationResult initialized with the given array of + * classifications. + */ +- (instancetype)initWithClassifications:(NSArray *)classifications; + +/** + * Initializes a new `MPPClassificationResult` with the given array of classifications and time + * stamp (in milliseconds). + * + * @param classifications An Array of `MPPClassifications` objects containing the predicted + * categories for each head of the model. + * + * @param timeStampMs The timestamp (in milliseconds) of the start of the chunk of data + * corresponding to these results. + * + * @return An instance of `MPPClassificationResult` initialized with the given array of + * classifications and timestampMs. + */ +- (instancetype)initWithClassifications:(NSArray *)classifications + timestampMs:(NSInteger)timestampMs; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m new file mode 100644 index 000000000..6cf75234e --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m @@ -0,0 +1,57 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h" + +@implementation MPPClassifications + +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + headName:(nullable NSString *)headName + categories:(NSArray *)categories { + self = [super init]; + if (self) { + _headIndex = headIndex; + _headName = headName; + _categories = categories; + } + return self; +} + +- (instancetype)initWithHeadIndex:(NSInteger)headIndex + categories:(NSArray *)categories { + return [self initWithHeadIndex:headIndex headName:nil categories:categories]; +} + +@end + +@implementation MPPClassificationResult + +- (instancetype)initWithClassifications:(NSArray *)classifications + timestampMs:(NSInteger)timestampMs { + self = [super init]; + if (self) { + _classifications = classifications; + _timestampMs = timestampMs; + } + + return self; +} + +- (instancetype)initWithClassifications:(NSArray *)classifications { + return [self initWithClassifications:classifications timestampMs:-1]; + + return self; +} + +@end From a608e0d8c08488b36be6258324f808bb763c3d87 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 18:28:28 +0530 Subject: [PATCH 02/45] Added utils for iOS classification result classes --- .../ios/components/containers/utils/BUILD | 40 +++++++++++ .../utils/sources/MPPCategory+Helpers.h | 26 +++++++ .../utils/sources/MPPCategory+Helpers.mm | 42 ++++++++++++ .../sources/MPPClassificationResult+Helpers.h | 35 ++++++++++ .../MPPClassificationResult+Helpers.mm | 68 +++++++++++++++++++ 5 files changed, 211 insertions(+) create mode 100644 mediapipe/tasks/ios/components/containers/utils/BUILD create mode 100644 mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h create mode 100644 mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm create mode 100644 mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h create mode 100644 mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm diff --git a/mediapipe/tasks/ios/components/containers/utils/BUILD b/mediapipe/tasks/ios/components/containers/utils/BUILD new file mode 100644 index 000000000..e4c76ac4b --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/BUILD @@ -0,0 +1,40 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPCategoryHelpers", + srcs = ["sources/MPPCategory+Helpers.mm"], + hdrs = ["sources/MPPCategory+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/components/containers:MPPCategory", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + ], +) + +objc_library( + name = "MPPClassificationResultHelpers", + srcs = ["sources/MPPClassificationResult+Helpers.mm"], + hdrs = ["sources/MPPClassificationResult+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/components/containers:MPPClassificationResult", + ":MPPCategoryHelpers", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto", + ], +) diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h new file mode 100644 index 000000000..7580cfeeb --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h @@ -0,0 +1,26 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/formats/classification.pb.h" +#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPCategory (Helpers) + ++ (MPPCategory *)categoryWithProto:(const mediapipe::Classification &)classificationProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm new file mode 100644 index 000000000..f729d9720 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm @@ -0,0 +1,42 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h" + +namespace { +using ClassificationProto = ::mediapipe::Classification; +} + +@implementation MPPCategory (Helpers) + ++ (MPPCategory *)categoryWithProto:(const ClassificationProto &)clasificationProto { + NSString *categoryName; + NSString *displayName; + + if (clasificationProto.has_label()) { + categoryName = [NSString stringWithCppString:clasificationProto.label()]; + } + + if (clasificationProto.has_display_name()) { + displayName = [NSString stringWithCppString:clasificationProto.display_name()]; + } + + return [[MPPCategory alloc] initWithIndex:clasificationProto.index() + score:clasificationProto.score() + categoryName:categoryName + displayName:displayName]; +} + +@end diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h new file mode 100644 index 000000000..fde436feb --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h @@ -0,0 +1,35 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" +#import "mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPClassifications (Helpers) + ++ (MPPClassifications *)classificationsWithProto: + (const mediapipe::tasks::components::containers::proto::Classifications &)classificationsProto; + +@end + +@interface MPPClassificationResult (Helpers) + ++ (MPPClassificationResult *)classificationResultWithProto: + (const mediapipe::tasks::components::containers::proto::ClassificationResult &) + classificationResultProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm new file mode 100644 index 000000000..9ad284790 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm @@ -0,0 +1,68 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" + +namespace { +using ClassificationsProto = ::mediapipe::tasks::components::containers::proto::Classifications; +using ClassificationResultProto = + ::mediapipe::tasks::components::containers::proto::ClassificationResult; +} // namespace + +@implementation MPPClassifications (Helpers) + ++ (MPPClassifications *)classificationsWithProto: + (const ClassificationsProto &)classificationsProto { + NSMutableArray *categories = [[NSMutableArray alloc] init]; + for (const auto &classification : classificationsProto.classification_list().classification()) { + [categories addObject:[MPPCategory categoryWithProto:classification]]; + } + + NSString *headName; + + if (classificationsProto.has_head_name()) { + headName = [NSString stringWithCppString:classificationsProto.head_name()]; + } + + return [[MPPClassifications alloc] initWithHeadIndex:(NSInteger)classificationsProto.head_index() + headName:headName + categories:categories]; +} + +@end + +@implementation MPPClassificationResult (Helpers) + ++ (MPPClassificationResult *)classificationResultWithProto: + (const ClassificationResultProto &)classificationResultProto { + NSMutableArray *classifications = [[NSMutableArray alloc] init]; + for (const auto &classifications_proto : classificationResultProto.classifications()) { + [classifications addObject:[MPPClassifications classificationsWithProto:classifications_proto]]; + } + + MPPClassificationResult *classificationResult; + + if (classificationResultProto.has_timestamp_ms()) { + classificationResult = [[MPPClassificationResult alloc] initWithClassifications:classifications timestampMs:(NSInteger)classificationResultProto.timestamp_ms()]; + } + else { + classificationResult = [[MPPClassificationResult alloc] initWithClassifications:classifications]; + } + + return classificationResult; +} + +@end From dfe7c83ad52748baf55383ab596223ce4c47df02 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 18:29:01 +0530 Subject: [PATCH 03/45] Updated type and name of timestamp in MPPTaskResult --- mediapipe/tasks/ios/core/sources/MPPTaskResult.h | 4 ++-- mediapipe/tasks/ios/core/sources/MPPTaskResult.m | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskResult.h b/mediapipe/tasks/ios/core/sources/MPPTaskResult.h index d15d4f258..4ee7b2fc6 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskResult.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskResult.h @@ -26,11 +26,11 @@ NS_SWIFT_NAME(TaskResult) /** * Timestamp that is associated with the task result object. */ -@property(nonatomic, assign, readonly) long timestamp; +@property(nonatomic, assign, readonly) NSInteger timestampMs; - (instancetype)init NS_UNAVAILABLE; -- (instancetype)initWithTimestamp:(long)timestamp NS_DESIGNATED_INITIALIZER; +- (instancetype)initWithTimestampMs:(NSInteger)timestampMs NS_DESIGNATED_INITIALIZER; @end diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskResult.m b/mediapipe/tasks/ios/core/sources/MPPTaskResult.m index 7088eb246..6c08014ff 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskResult.m +++ b/mediapipe/tasks/ios/core/sources/MPPTaskResult.m @@ -16,16 +16,16 @@ @implementation MPPTaskResult -- (instancetype)initWithTimestamp:(long)timestamp { +- (instancetype)initWithTimestampMs:(NSInteger)timestampMs { self = [super init]; if (self) { - _timestamp = timestamp; + _timestampMs = timestampMs; } return self; } - (id)copyWithZone:(NSZone *)zone { - return [[MPPTaskResult alloc] initWithTimestamp:self.timestamp]; + return [[MPPTaskResult alloc] initWithTimestampMs:self.timestampMs]; } @end From 8ba81de20be221edc5aec535350648fb8e6ac4d1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 18:31:56 +0530 Subject: [PATCH 04/45] Added packets callback provision and built in op resolver in MPPTaskRunner --- mediapipe/tasks/ios/core/BUILD | 2 + .../tasks/ios/core/sources/MPPTaskRunner.h | 46 +++++++++++++++++-- .../tasks/ios/core/sources/MPPTaskRunner.mm | 10 +++- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/ios/core/BUILD b/mediapipe/tasks/ios/core/BUILD index 434d20085..e4797c34d 100644 --- a/mediapipe/tasks/ios/core/BUILD +++ b/mediapipe/tasks/ios/core/BUILD @@ -90,6 +90,8 @@ objc_library( deps = [ "//mediapipe/framework:calculator_cc_proto", "//mediapipe/tasks/cc/core:task_runner", + "//mediapipe/tasks/cc/core:mediapipe_builtin_op_resolver", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "@org_tensorflow//tensorflow/lite/core/api:op_resolver", ], ) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 2b9f2ecdb..97255234f 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -20,23 +20,63 @@ NS_ASSUME_NONNULL_BEGIN /** - * This class is used to create and call appropriate methods on the C++ Task Runner. + * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, + * execute and terminate any Mediapipe task. + * + * An instance of the newly created C++ task runner will + * be stored until this class is destroyed. When methods are called for processing (performing + * inference), closing etc., on this class, internally the appropriate methods will be called on the + * C++ task runner instance to execute the appropriate actions. For each type of task, a subclass of + * this class must be defined to add any additional functionality. For eg:, vision tasks must create + * an `MPPVisionTaskRunner` and provide additional functionality. An instance of + * `MPPVisionTaskRunner` can in turn be used by the each vision task for creation and execution of + * the task. Please see the documentation for the C++ Task Runner for more details on how the taks + * runner operates. */ @interface MPPTaskRunner : NSObject /** - * Initializes a new `MPPTaskRunner` with the mediapipe task graph config proto. + * Initializes a new `MPPTaskRunner` with the mediapipe task graph config proto and an optional C++ + * packets callback. + * + * You can pass `nullptr` for `packetsCallback` in case the mode of operation + * requested by the user is synchronous. + * + * If the task is operating in asynchronous mode, any iOS Mediapipe task that uses the `MPPTaskRunner` + * must define a C++ callback function to obtain the results of inference asynchronously and deliver + * the results to the user. To accomplish this, callback function will in turn invoke the block + * provided by the user in the task options supplied to create the task. + * Please see the documentation of the C++ Task Runner for more information on the synchronous and + * asynchronous modes of operation. * * @param graphConfig A mediapipe task graph config proto. * - * @return An instance of `MPPTaskRunner` initialized to the given graph config proto. + * @param packetsCallback An optional C++ callback function that takes a list of output packets as + * the input argument. If provided, the callback must in turn call the block provided by the user in + * the appropriate task options. + * + * @return An instance of `MPPTaskRunner` initialized to the given graph config proto and optional + * packetsCallback. */ - (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig + packetsCallback: + (mediapipe::tasks::core::PacketsCallback)packetsCallback error:(NSError **)error NS_DESIGNATED_INITIALIZER; +/** A synchronous method for processing batch data or offline streaming data. This method is +designed for processing either batch data such as unrelated images and texts or offline streaming +data such as the decoded frames from a video file and an audio file. The call blocks the current +thread until a failure status or a successful result is returned. If the input packets have no +timestamp, an internal timestamp will be assigend per invocation. Otherwise, when the timestamp is +set in the input packets, the caller must ensure that the input packet timestamps are greater than +the timestamps of the previous invocation. This method is thread-unsafe and it is the caller's +responsibility to synchronize access to this method across multiple threads and to ensure that the +input packet timestamps are in order.*/ - (absl::StatusOr)process: (const mediapipe::tasks::core::PacketMap &)packetMap; +/** Shuts down the C++ task runner. After the runner is closed, any calls that send input data to + * the runner are illegal and will receive errors. */ - (absl::Status)close; - (instancetype)init NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm index c5c307fd5..fd3f780fa 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm @@ -13,11 +13,15 @@ // limitations under the License. #import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" +#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#include "tensorflow/lite/core/api/op_resolver.h" namespace { using ::mediapipe::CalculatorGraphConfig; +using ::mediapipe::tasks::core::MediaPipeBuiltinOpResolver; using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; } // namespace @@ -30,15 +34,17 @@ using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; @implementation MPPTaskRunner - (instancetype)initWithCalculatorGraphConfig:(CalculatorGraphConfig)graphConfig + packetsCallback:(PacketsCallback)packetsCallback error:(NSError **)error { self = [super init]; if (self) { - auto taskRunnerResult = TaskRunnerCpp::Create(std::move(graphConfig)); + auto taskRunnerResult = TaskRunnerCpp::Create(std::move(graphConfig), + absl::make_unique(), + std::move(packetsCallback)); if (![MPPCommonUtils checkCppError:taskRunnerResult.status() toError:error]) { return nil; } - _cppTaskRunner = std::move(taskRunnerResult.value()); } return self; From e6657dc8c12c70c47add82f155330058d2cc2498 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 18:33:02 +0530 Subject: [PATCH 05/45] Added MPPBaseOptions helpers --- mediapipe/tasks/ios/core/utils/BUILD | 27 +++++++++++++ .../utils/sources/MPPBaseOptions+Helpers.h | 26 ++++++++++++ .../utils/sources/MPPBaseOptions+Helpers.mm | 40 +++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 mediapipe/tasks/ios/core/utils/BUILD create mode 100644 mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm diff --git a/mediapipe/tasks/ios/core/utils/BUILD b/mediapipe/tasks/ios/core/utils/BUILD new file mode 100644 index 000000000..d9fbaf375 --- /dev/null +++ b/mediapipe/tasks/ios/core/utils/BUILD @@ -0,0 +1,27 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPBaseOptionsHelpers", + srcs = ["sources/MPPBaseOptions+Helpers.mm"], + hdrs = ["sources/MPPBaseOptions+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/core:MPPBaseOptions", + "//mediapipe/tasks/cc/core/proto:base_options_cc_proto", + ], +) diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h new file mode 100644 index 000000000..d52df2ae4 --- /dev/null +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h @@ -0,0 +1,26 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/tasks/cc/core/proto/base_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPBaseOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPBaseOptions (Helpers) + +- (void)copyToProto:(mediapipe::tasks::core::proto::BaseOptions *)baseOptionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm new file mode 100644 index 000000000..66694a94e --- /dev/null +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm @@ -0,0 +1,40 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +namespace { +using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions; +} + +@implementation MPPBaseOptions (Helpers) + +- (void)copyToProto:(BaseOptionsProto *)baseOptionsProto { + if (self.modelAssetPath) { + baseOptionsProto->mutable_model_asset()->set_file_name(self.modelAssetPath.UTF8String); + } + + switch (self.delegate) { + case MPPDelegateCPU: { + baseOptionsProto->mutable_acceleration()->mutable_tflite(); + break; + } + case MPPDelegateGPU: + break; + default: + break; + } +} + +@end From 0eacc333752c5b0ebe698748bac86a9fc91bf8a2 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 19:18:50 +0530 Subject: [PATCH 06/45] Added MPPTextTaskRunner --- mediapipe/tasks/ios/text/core/BUILD | 31 ++++++++++++++++ .../ios/text/core/sources/MPPTextTaskRunner.h | 37 +++++++++++++++++++ .../text/core/sources/MPPTextTaskRunner.mm | 29 +++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 mediapipe/tasks/ios/text/core/BUILD create mode 100644 mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h create mode 100644 mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.mm diff --git a/mediapipe/tasks/ios/text/core/BUILD b/mediapipe/tasks/ios/text/core/BUILD new file mode 100644 index 000000000..6d558b22b --- /dev/null +++ b/mediapipe/tasks/ios/text/core/BUILD @@ -0,0 +1,31 @@ +# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPTextTaskRunner", + srcs = ["sources/MPPTextTaskRunner.mm"], + hdrs = ["sources/MPPTextTaskRunner.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskRunner", + ], +) + diff --git a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h new file mode 100644 index 000000000..dd5d96ce6 --- /dev/null +++ b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h @@ -0,0 +1,37 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, execute and terminate any Mediapipe text task. + */ +@interface MPPTextTaskRunner : MPPTaskRunner + +/** + * Initializes a new `MPPTextTaskRunner` with the mediapipe task graph config proto. + * + * @param graphConfig A mediapipe task graph config proto. + * + * @return An instance of `MPPTextTaskRunner` initialized to the given graph config proto. + */ +- (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.mm b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.mm new file mode 100644 index 000000000..956448c17 --- /dev/null +++ b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.mm @@ -0,0 +1,29 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h" + +namespace { +using ::mediapipe::CalculatorGraphConfig; +} // namespace + +@implementation MPPTextTaskRunner + +- (instancetype)initWithCalculatorGraphConfig:(CalculatorGraphConfig)graphConfig + error:(NSError **)error { + self = [super initWithCalculatorGraphConfig:graphConfig packetsCallback:nullptr error:error]; + return self; +} + +@end From febc4fd28320a527bcad42862e667b450cfa3209 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 19:25:13 +0530 Subject: [PATCH 07/45] Updated comments in BaseOptions Helpers --- .../tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm index 66694a94e..bd863b20b 100644 --- a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm @@ -21,6 +21,8 @@ using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions; @implementation MPPBaseOptions (Helpers) - (void)copyToProto:(BaseOptionsProto *)baseOptionsProto { + baseOptionsProto->Clear(); + if (self.modelAssetPath) { baseOptionsProto->mutable_model_asset()->set_file_name(self.modelAssetPath.UTF8String); } @@ -31,6 +33,7 @@ using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions; break; } case MPPDelegateGPU: + // TODO: Provide an implementation for GPU Delegate. break; default: break; From 5484d0934145bb00695221798401a6220edb0b86 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 19:32:35 +0530 Subject: [PATCH 08/45] Added exception for GPUDelegate --- .../tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm index bd863b20b..3fd8fbda3 100644 --- a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm @@ -32,9 +32,10 @@ using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions; baseOptionsProto->mutable_acceleration()->mutable_tflite(); break; } - case MPPDelegateGPU: + case MPPDelegateGPU: { // TODO: Provide an implementation for GPU Delegate. - break; + [NSException raise:@"Invalid value for delegate" format:@"GPU Delegate is not implemented."]; + } default: break; } From 607b646953adc852a50ff1712e867b67a6f84f61 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 19:36:54 +0530 Subject: [PATCH 09/45] Added iOS text classifier options and result --- .../tasks/ios/text/text_classifier/BUILD | 37 +++++++++++++++ .../sources/MPPTextClassifierOptions.h | 35 +++++++++++++++ .../sources/MPPTextClassifierOptions.m | 27 +++++++++++ .../sources/MPPTextClassifierResult.h | 45 +++++++++++++++++++ .../sources/MPPTextClassifierResult.m | 28 ++++++++++++ 5 files changed, 172 insertions(+) create mode 100644 mediapipe/tasks/ios/text/text_classifier/BUILD create mode 100644 mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h create mode 100644 mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m create mode 100644 mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h create mode 100644 mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.m diff --git a/mediapipe/tasks/ios/text/text_classifier/BUILD b/mediapipe/tasks/ios/text/text_classifier/BUILD new file mode 100644 index 000000000..7d3f4b05d --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/BUILD @@ -0,0 +1,37 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPTextClassifierOptions", + srcs = ["sources/MPPTextClassifierOptions.m"], + hdrs = ["sources/MPPTextClassifierOptions.h"], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/components/processors:MPPClassifierOptions", + ], +) + +objc_library( + name = "MPPTextClassifierResult", + srcs = ["sources/MPPTextClassifierResult.m"], + hdrs = ["sources/MPPTextClassifierResult.h"], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskResult", + "//mediapipe/tasks/ios/components/containers:MPPClassificationResult", + ], +) diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h new file mode 100644 index 000000000..25189578b --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h @@ -0,0 +1,35 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Options for setting up a `MPPTextClassifierOptions`. + */ +NS_SWIFT_NAME(TextClassifierOptions) +@interface MPPTextClassifierOptions : MPPTaskOptions + +/** + * Options for configuring the classifier behavior, such as score threshold, number of results, etc. + */ +@property(nonatomic, copy) MPPClassifierOptions *classifierOptions; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m new file mode 100644 index 000000000..8d4ffd36f --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m @@ -0,0 +1,27 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h" + +@implementation MPPTextClassifierOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _classifierOptions = [[MPPClassifierOptions alloc] init]; + } + return self; +} + +@end \ No newline at end of file diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h new file mode 100644 index 000000000..6926757e4 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h @@ -0,0 +1,45 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Represents the classification results generated by `MPPTextClassifier`. */ +NS_SWIFT_NAME(TextClassifierResult) +@interface MPPTextClassifierResult : MPPTaskResult + +/** The `MPPClassificationResult` instance containing one set of results per classifier head. */ +@property(nonatomic, readonly) MPPClassificationResult *classificationResult; + +/** + * Initializes a new `MPPTextClassifierResult` with the given `MPPClassificationResult` and time + * stamp (in milliseconds). + * + * @param classificationResult The `MPPClassificationResult` instance containing one set of results + * per classifier head. + * + * @param timeStampMs The time stamp for this result. + * + * @return An instance of `MPPTextClassifierResult` initialized with the given + * `MPPClassificationResult` and time stamp (in milliseconds). + */ +- (instancetype)initWithClassificationResult:(MPPClassificationResult *)classificationResult + timestampMs:(NSInteger)timestampMs; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.m b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.m new file mode 100644 index 000000000..4d5c1104a --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.m @@ -0,0 +1,28 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h" + +@implementation MPPTextClassifierResult + +- (instancetype)initWithClassificationResult:(MPPClassificationResult *)classificationResult + timestampMs:(NSInteger)timestampMs { + self = [super initWithTimestampMs:timestampMs]; + if (self) { + _classificationResult = classificationResult; + } + return self; +} + +@end From 73940714afed49d858dd9a5371b24f7ec3d2fac7 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 19:37:32 +0530 Subject: [PATCH 10/45] Added iOS text classifier options and result helpers --- .../ios/text/text_classifier/utils/BUILD | 41 ++++++++++++++++++ .../MPPTextClassifierOptions+Helpers.h | 26 ++++++++++++ .../MPPTextClassifierOptions+Helpers.mm | 36 ++++++++++++++++ .../sources/MPPTextClassifierResult+Helpers.h | 28 +++++++++++++ .../MPPTextClassifierResult+Helpers.mm | 42 +++++++++++++++++++ 5 files changed, 173 insertions(+) create mode 100644 mediapipe/tasks/ios/text/text_classifier/utils/BUILD create mode 100644 mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm create mode 100644 mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h create mode 100644 mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.mm diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/BUILD b/mediapipe/tasks/ios/text/text_classifier/utils/BUILD new file mode 100644 index 000000000..abc1fc23b --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/BUILD @@ -0,0 +1,41 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPTextClassifierOptionsHelpers", + srcs = ["sources/MPPTextClassifierOptions+Helpers.mm"], + hdrs = ["sources/MPPTextClassifierOptions+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/text/text_classifier:MPPTextClassifierOptions", + "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", + "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol", + "//mediapipe/tasks/ios/components/processors/utils:MPPClassifierOptionsHelpers", + "//mediapipe/tasks/cc/text/text_classifier/proto:text_classifier_graph_options_cc_proto", + ], +) + +objc_library( + name = "MPPTextClassifierResultHelpers", + srcs = ["sources/MPPTextClassifierResult+Helpers.mm"], + hdrs = ["sources/MPPTextClassifierResult+Helpers.h"], + deps = [ + "//mediapipe/tasks/ios/text/text_classifier:MPPTextClassifierResult", + "//mediapipe/tasks/ios/components/containers/utils:MPPClassificationResultHelpers", + "//mediapipe/framework:packet", + ], +) diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h new file mode 100644 index 000000000..1e52e5c87 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h @@ -0,0 +1,26 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPTextClassifierOptions (Helpers) + +- (void)copyToProto:(mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm new file mode 100644 index 000000000..728000b44 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm @@ -0,0 +1,36 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/tasks/cc/text/text_classifier/proto/text_classifier_graph_options.pb.h" +#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" +#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h" + +namespace { +using CalculatorOptionsProto = ::mediapipe::CalculatorOptions; +using TextClassifierGraphOptionsProto = + ::mediapipe::tasks::text::text_classifier::proto::TextClassifierGraphOptions; + +} // namespace + +@implementation MPPTextClassifierOptions (Helpers) + +- (void)copyToProto:(CalculatorOptionsProto *)optionsProto { + TextClassifierGraphOptionsProto *graph_options = + optionsProto->MutableExtension(TextClassifierGraphOptionsProto::ext); + [self.baseOptions copyToProto:graph_options->mutable_base_options()]; + [self.classifierOptions copyToProto:graph_options->mutable_classifier_options()]; +} + +@end diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h new file mode 100644 index 000000000..f1b728b0a --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h @@ -0,0 +1,28 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h" + +#include "mediapipe/framework/packet.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPTextClassifierResult (Helpers) + ++ (MPPTextClassifierResult *)textClassifierResultWithClassificationsPacket: + (const mediapipe::Packet &)packet; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.mm b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.mm new file mode 100644 index 000000000..f5d6aa1d3 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.mm @@ -0,0 +1,42 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" +#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h" + +#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" + +static const int kMicroSecondsPerMilliSecond = 1000; + +namespace { +using ClassificationResultProto = + ::mediapipe::tasks::components::containers::proto::ClassificationResult; +using ::mediapipe::Packet; +} // namespace + +#define int kMicroSecondsPerMilliSecond = 1000; + +@implementation MPPTextClassifierResult (Helpers) + ++ (MPPTextClassifierResult *)textClassifierResultWithClassificationsPacket:(const Packet &)packet { + MPPClassificationResult *classificationResult = [MPPClassificationResult + classificationResultWithProto:packet.Get()]; + + return [[MPPTextClassifierResult alloc] + initWithClassificationResult:classificationResult + timestampMs:(NSInteger)(packet.Timestamp().Value() / + kMicroSecondsPerMilliSecond)]; +} + +@end From 381cde355b7a4de3f25e520e53f566a47b5e63d0 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 19:41:36 +0530 Subject: [PATCH 11/45] Added iOS MPPTextClassifier --- .../tasks/ios/text/text_classifier/BUILD | 22 ++++ .../sources/MPPTextClassifier.h | 102 ++++++++++++++++++ .../sources/MPPTextClassifier.mm | 98 +++++++++++++++++ 3 files changed, 222 insertions(+) create mode 100644 mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h create mode 100644 mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm diff --git a/mediapipe/tasks/ios/text/text_classifier/BUILD b/mediapipe/tasks/ios/text/text_classifier/BUILD index 7d3f4b05d..d58c85be9 100644 --- a/mediapipe/tasks/ios/text/text_classifier/BUILD +++ b/mediapipe/tasks/ios/text/text_classifier/BUILD @@ -35,3 +35,25 @@ objc_library( "//mediapipe/tasks/ios/components/containers:MPPClassificationResult", ], ) + +objc_library( + name = "MPPTextClassifier", + srcs = ["sources/MPPTextClassifier.mm"], + hdrs = ["sources/MPPTextClassifier.h"], + copts = [ + "-ObjC++", + "-std=c++17", + ], + deps = [ + "//mediapipe/tasks/cc/text/text_classifier:text_classifier_graph", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/text/core:MPPTextTaskRunner", + "//mediapipe/tasks/ios/core:MPPTextPacketCreator", + "//mediapipe/tasks/ios/text/text_classifier/utils:MPPTextClassifierOptionsHelpers", + "//mediapipe/tasks/ios/text/text_classifier/utils:MPPTextClassifierResultHelpers", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + ":MPPTextClassifierOptions", + ], +) diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h new file mode 100644 index 000000000..ee6f25100 --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h @@ -0,0 +1,102 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h" +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * This API expects a TFLite model with (optional) [TFLite Model + * Metadata](https://www.tensorflow.org/lite/convert/metadata")that contains the mandatory + * (described below) input tensors, output tensor, and the optional (but recommended) label items as + * AssociatedFiles with type TENSOR_AXIS_LABELS per output classification tensor. + * + * Metadata is required for models with int32 input tensors because it contains the input process + * unit for the model's Tokenizer. No metadata is required for models with string input tensors. + * + * Input tensors + * - Three input tensors `kTfLiteInt32` of shape `[batch_size xbert_max_seq_len]` + * representing the input ids, mask ids, and segment ids. This input signature requires a + * Bert Tokenizer process unit in the model metadata. + * - Or one input tensor `kTfLiteInt32` of shape `[batch_size xmax_seq_len]` representing + * the input ids. This input signature requires a Regex Tokenizer process unit in the + * model metadata. + * - Or one input tensor ({@code kTfLiteString}) that is shapeless or has shape `[1]` containing + * the input string. + * + * At least one output tensor `(kTfLiteFloat32}/kBool)` with: + * - `N` classes and shape `[1 x N]` + * - optional (but recommended) label map(s) as AssociatedFile-s with type TENSOR_AXIS_LABELS, + * containing one label per line. The first such AssociatedFile (if any) is used to fill the + * `class_name` field of the results. The `display_name` field is filled from the AssociatedFile + * (if any) whose locale matches the `display_names_locale` field of the + * `MPPTextClassifierOptions` used at creation time ("en" by default, i.e. English). If none of + * these are available, only the `index` field of the results will be filled. + * + * @brief Performs classification on text. + */ +NS_SWIFT_NAME(TextClassifier) +@interface MPPTextClassifier : NSObject + +/** + * Creates a new instance of `MPPTextClassifier` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `MPPTextClassifierOptions`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * + * @param error An optional error parameter populated when there is an error in initializing + * the text classifier. + * + * @return A new instance of `MPPTextClassifier` with the given model path. `nil` if there is an + * error in initializing the text classifier. + */ +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `MPPTextClassifier` from the given `MPPTextClassifierOptions`. + * + * @param options The options of type `MPPTextClassifierOptions` to use for configuring the + * `MPPTextClassifier`. + * + * @param error An optional error parameter populated when there is an error in initializing + * the text classifier. + * + * @return A new instance of `MPPTextClassifier` with the given options. `nil` if there is an error + * in initializing the text classifier. + */ +- (instancetype)initWithOptions:(MPPTextClassifierOptions *)options error:(NSError **)error; + +/** + * Performs classification on the input text. + * + * @param text The `NSString` on which classification is to be performed. + * + * @param error An optional error parameter populated when there is an error in performing + * classification on the input text. + * + * @return A `MPPTextClassifierResult` object that contains a list of text classifications. + */ +- (nullable MPPTextClassifierResult *)classifyWithText:(NSString *)text error:(NSError **)error; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm new file mode 100644 index 000000000..f87d0681a --- /dev/null +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm @@ -0,0 +1,98 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h" +#import "mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h" +#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h" +#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h" + +#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" + +#include "absl/status/statusor.h" + +namespace { +using ::mediapipe::Packet; +using ::mediapipe::tasks::components::containers::proto::ClassificationResult; +using ::mediapipe::tasks::core::PacketMap; +} // namespace + +static NSString *const kClassificationsStreamName = @"classifications_out"; +static NSString *const kClassificationsTag = @"CLASSIFICATIONS"; +static NSString *const kTextInStreamName = @"text_in"; +static NSString *const kTextTag = @"TEXT"; +static NSString *const kTaskGraphName = @"mediapipe.tasks.text.text_classifier.TextClassifierGraph"; + +@interface MPPTextClassifier () { + /** iOS Text Task Runner */ + MPPTextTaskRunner *_textTaskRunner; +} +@end + +@implementation MPPTextClassifier + +- (instancetype)initWithOptions:(MPPTextClassifierOptions *)options error:(NSError **)error { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] + initWithTaskGraphName:kTaskGraphName + inputStreams:@[ [NSString stringWithFormat:@"%@:%@", kTextTag, kTextInStreamName] ] + outputStreams:@[ [NSString stringWithFormat:@"%@:%@", kClassificationsTag, + kClassificationsStreamName] ] + taskOptions:options + enableFlowLimiting:NO + error:error]; + + if (!taskInfo) { + return nil; + } + + _textTaskRunner = + [[MPPTextTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] + error:error]; + + if (!_textTaskRunner) { + return nil; + } + + self = [super init]; + + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPTextClassifierOptions *options = [[MPPTextClassifierOptions alloc] init]; + + options.baseOptions.modelAssetPath = modelPath; + + return [self initWithOptions:options error:error]; +} + +- (nullable MPPTextClassifierResult *)classifyWithText:(NSString *)text error:(NSError **)error { + Packet packet = [MPPTextPacketCreator createWithText:text]; + + std::map packet_map = {{kTextInStreamName.cppString, packet}}; + absl::StatusOr status_or_output_packet_map = [_textTaskRunner process:packet_map]; + + if (![MPPCommonUtils checkCppError:status_or_output_packet_map.status() toError:error]) { + return nil; + } + + return [MPPTextClassifierResult + textClassifierResultWithClassificationsPacket:status_or_output_packet_map.value() + [kClassificationsStreamName.cppString]]; +} + +@end From 17ddb624fd6ca9ada6ff1d2d65a6fadff5dd4faf Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 20:01:20 +0530 Subject: [PATCH 12/45] Added new line --- .../ios/text/text_classifier/sources/MPPTextClassifierOptions.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m index 8d4ffd36f..c12fbbf1c 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m @@ -24,4 +24,4 @@ return self; } -@end \ No newline at end of file +@end From 2121cc89a442841ef341899ce53c3639b7cfe2c8 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:18:48 +0530 Subject: [PATCH 13/45] Updated comments in MPPCategory --- .../containers/sources/MPPCategory.h | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h index 035cde09d..28e89f249 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h @@ -16,41 +16,47 @@ NS_ASSUME_NONNULL_BEGIN -/** Category is a util class, contains a label, its display name, a float value as score, and the +/** + * Category is a util class that contains a label, its display name, a float value as score, and the * index of the label in the corresponding label file. Typically it's used as the result of - * classification tasks. */ + * classification tasks. + **/ NS_SWIFT_NAME(ClassificationCategory) @interface MPPCategory : NSObject -/** The index of the label in the corresponding label file. It takes the value -1 if the index is - * not set. */ +/** + * The index of the label in the corresponding label file. It takes the value -1 if the index is + * not set. + **/ @property(nonatomic, readonly) NSInteger index; -/** Confidence score for this class . */ +/** Confidence score for this class . **/ @property(nonatomic, readonly) float score; -/** The label of this category object. */ +/** The label of this category object. **/ @property(nonatomic, readonly, nullable) NSString *categoryName; -/** The display name of the label, which may be translated for different locales. For example, a - * label, "apple", may be translated into Spanish for display purpose, so that the display name is - * "manzana". */ +/** + * The display name of the label, which may be translated for different locales. For example, a + * label, "apple", may be translated into Spanish for display purpose, so that the display name is + * "manzana". + **/ @property(nonatomic, readonly, nullable) NSString *displayName; /** - * Initializes a new `MPPCategory` with the given index, score, category name and display name. - * - * @param index The index of the label in the corresponding label file. - * - * @param score The probability score of this label category. - * - * @param categoryName The label of this category object.. - * - * @param displayName The display name of the label. - * - * @return An instance of `MPPCategory` initialized with the given index, score, category name and - * display name. - */ + * Initializes a new `MPPCategory` with the given index, score, category name and display name. + * + * @param index The index of the label in the corresponding label file. + * + * @param score The probability score of this label category. + * + * @param categoryName The label of this category object. + * + * @param displayName The display name of the label. + * + * @return An instance of `MPPCategory` initialized with the given index, score, category name and + * display name. + **/ - (instancetype)initWithIndex:(NSInteger)index score:(float)score categoryName:(nullable NSString *)categoryName From 9da4d15ef4c9dc800d6074dc91824424097033cd Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:20:03 +0530 Subject: [PATCH 14/45] Updated comments in MPPClassificationResult --- .../sources/MPPClassificationResult.h | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h index 732d1f899..65efc7dbb 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h @@ -18,18 +18,20 @@ NS_ASSUME_NONNULL_BEGIN /** Represents the list of classification for a given classifier head. Typically used as a result - * for classification tasks. */ + * for classification tasks. + **/ NS_SWIFT_NAME(Classifications) @interface MPPClassifications : NSObject -/** The index of the classifier head these entries refer to. This is useful for multi-head models. - */ +/** + * The index of the classifier head these entries refer to. This is useful for multi-head models. + **/ @property(nonatomic, readonly) NSInteger headIndex; -/** The optional name of the classifier head, which is the corresponding tensor metadata name. */ +/** The optional name of the classifier head, which is the corresponding tensor metadata name. **/ @property(nonatomic, readonly, nullable) NSString *headName; -/** An array of `MPPCategory` objects containing the predicted categories. */ +/** An array of `MPPCategory` objects containing the predicted categories. **/ @property(nonatomic, readonly) NSArray *categories; /** @@ -41,7 +43,7 @@ NS_SWIFT_NAME(Classifications) * * @return An instance of `MPPClassifications` initialized with the given head index and * array of categories. - */ + **/ - (instancetype)initWithHeadIndex:(NSInteger)headIndex categories:(NSArray *)categories; @@ -56,7 +58,7 @@ NS_SWIFT_NAME(Classifications) * * @return An object of `MPPClassifications` initialized with the given head index, head name and * array of categories. - */ + **/ - (instancetype)initWithHeadIndex:(NSInteger)headIndex headName:(nullable NSString *)headName categories:(NSArray *)categories; @@ -66,19 +68,21 @@ NS_SWIFT_NAME(Classifications) /** * Represents the classification results of a model. Typically used as a result for classification * tasks. - */ + **/ NS_SWIFT_NAME(ClassificationResult) @interface MPPClassificationResult : NSObject /** An Array of `MPPClassifications` objects containing the predicted categories for each head of - * the model. */ + * the model. + **/ @property(nonatomic, readonly) NSArray *classifications; /** The optional timestamp (in milliseconds) of the start of the chunk of data corresponding to * these results. If it is set to the value -1, it signifies the absence of a time stamp. This is * only used for classification on time series (e.g. audio classification). In these use cases, the * amount of data to process might exceed the maximum size that the model can process: to solve - * this, the input data is split into multiple chunks starting at different timestamps. */ + * this, the input data is split into multiple chunks starting at different timestamps. + **/ @property(nonatomic, readonly) NSInteger timestampMs; /** @@ -90,7 +94,7 @@ NS_SWIFT_NAME(ClassificationResult) * * @return An instance of MPPClassificationResult initialized with the given array of * classifications. - */ + **/ - (instancetype)initWithClassifications:(NSArray *)classifications; /** @@ -105,7 +109,7 @@ NS_SWIFT_NAME(ClassificationResult) * * @return An instance of `MPPClassificationResult` initialized with the given array of * classifications and timestampMs. - */ + **/ - (instancetype)initWithClassifications:(NSArray *)classifications timestampMs:(NSInteger)timestampMs; From 3cdb54e29d4291ce42b5130ca2b95e5ef8716da1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:25:49 +0530 Subject: [PATCH 15/45] Update MPPCategory.h --- .../tasks/ios/components/containers/sources/MPPCategory.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h index 28e89f249..31725983a 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h @@ -17,10 +17,10 @@ NS_ASSUME_NONNULL_BEGIN /** - * Category is a util class that contains a label, its display name, a float value as score, and the - * index of the label in the corresponding label file. Typically it's used as the result of - * classification tasks. - **/ + * Category is a util class that contains a label, its display name, a float value as score, and the + * index of the label in the corresponding label file. Typically it's used as the result of + * classification tasks. + **/ NS_SWIFT_NAME(ClassificationCategory) @interface MPPCategory : NSObject From fa9ec901869a5c47b3a4271f7e8ca6d302462496 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:32:27 +0530 Subject: [PATCH 16/45] Removed extra space between param comments --- .../tasks/ios/components/containers/sources/MPPCategory.h | 3 --- .../components/containers/sources/MPPClassificationResult.h | 1 - mediapipe/tasks/ios/core/sources/MPPTaskRunner.h | 1 - .../tasks/ios/text/text_classifier/sources/MPPTextClassifier.h | 3 --- .../ios/text/text_classifier/sources/MPPTextClassifierResult.h | 1 - 5 files changed, 9 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h index 31725983a..bb5526dc4 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h @@ -47,11 +47,8 @@ NS_SWIFT_NAME(ClassificationCategory) * Initializes a new `MPPCategory` with the given index, score, category name and display name. * * @param index The index of the label in the corresponding label file. - * * @param score The probability score of this label category. - * * @param categoryName The label of this category object. - * * @param displayName The display name of the label. * * @return An instance of `MPPCategory` initialized with the given index, score, category name and diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h index 65efc7dbb..1b2f79ecc 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h @@ -103,7 +103,6 @@ NS_SWIFT_NAME(ClassificationResult) * * @param classifications An Array of `MPPClassifications` objects containing the predicted * categories for each head of the model. - * * @param timeStampMs The timestamp (in milliseconds) of the start of the chunk of data * corresponding to these results. * diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 97255234f..31a867988 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -50,7 +50,6 @@ NS_ASSUME_NONNULL_BEGIN * asynchronous modes of operation. * * @param graphConfig A mediapipe task graph config proto. - * * @param packetsCallback An optional C++ callback function that takes a list of output packets as * the input argument. If provided, the callback must in turn call the block provided by the user in * the appropriate task options. diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h index ee6f25100..813311fc2 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h @@ -58,7 +58,6 @@ NS_SWIFT_NAME(TextClassifier) * file stored locally on the device and the default `MPPTextClassifierOptions`. * * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. - * * @param error An optional error parameter populated when there is an error in initializing * the text classifier. * @@ -72,7 +71,6 @@ NS_SWIFT_NAME(TextClassifier) * * @param options The options of type `MPPTextClassifierOptions` to use for configuring the * `MPPTextClassifier`. - * * @param error An optional error parameter populated when there is an error in initializing * the text classifier. * @@ -85,7 +83,6 @@ NS_SWIFT_NAME(TextClassifier) * Performs classification on the input text. * * @param text The `NSString` on which classification is to be performed. - * * @param error An optional error parameter populated when there is an error in performing * classification on the input text. * diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h index 6926757e4..15522fca1 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h @@ -31,7 +31,6 @@ NS_SWIFT_NAME(TextClassifierResult) * * @param classificationResult The `MPPClassificationResult` instance containing one set of results * per classifier head. - * * @param timeStampMs The time stamp for this result. * * @return An instance of `MPPTextClassifierResult` initialized with the given From 7ec12371ba96038b1dd9a1840e983aa0c763a649 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:35:41 +0530 Subject: [PATCH 17/45] Marked MPPCategory initializer as designated initializer --- mediapipe/tasks/ios/components/containers/sources/MPPCategory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h index bb5526dc4..7a760a52b 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h @@ -57,7 +57,7 @@ NS_SWIFT_NAME(ClassificationCategory) - (instancetype)initWithIndex:(NSInteger)index score:(float)score categoryName:(nullable NSString *)categoryName - displayName:(nullable NSString *)displayName; + displayName:(nullable NSString *)displayName NS_DESIGNATED_INITIALIZER; - (instancetype)init NS_UNAVAILABLE; From 0e8909497b80021c8d97c00d6cc7d5119225bb80 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:46:14 +0530 Subject: [PATCH 18/45] Added designated initializer in MPPClassifications --- .../components/containers/sources/MPPClassificationResult.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h index 1b2f79ecc..baded7c95 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h @@ -61,7 +61,11 @@ NS_SWIFT_NAME(Classifications) **/ - (instancetype)initWithHeadIndex:(NSInteger)headIndex headName:(nullable NSString *)headName - categories:(NSArray *)categories; + categories:(NSArray *)categories NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; @end From 387805f724aa83e0af85da065b06bfe8cc397291 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:46:37 +0530 Subject: [PATCH 19/45] Updated comments --- .../components/containers/sources/MPPClassificationResult.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h index baded7c95..46a7e2aba 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h @@ -82,7 +82,7 @@ NS_SWIFT_NAME(ClassificationResult) @property(nonatomic, readonly) NSArray *classifications; /** The optional timestamp (in milliseconds) of the start of the chunk of data corresponding to - * these results. If it is set to the value -1, it signifies the absence of a time stamp. This is + * these results. If it is set to the value -1, it signifies the absence of a timestamp. This is * only used for classification on time series (e.g. audio classification). In these use cases, the * amount of data to process might exceed the maximum size that the model can process: to solve * this, the input data is split into multiple chunks starting at different timestamps. @@ -107,7 +107,7 @@ NS_SWIFT_NAME(ClassificationResult) * * @param classifications An Array of `MPPClassifications` objects containing the predicted * categories for each head of the model. - * @param timeStampMs The timestamp (in milliseconds) of the start of the chunk of data + * @param timestampMs The timestamp (in milliseconds) of the start of the chunk of data * corresponding to these results. * * @return An instance of `MPPClassificationResult` initialized with the given array of From d804506799c3c50a71272ec8b41a2252e7c16126 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:47:10 +0530 Subject: [PATCH 20/45] Added designated initializer in MPPClassificationResult --- .../components/containers/sources/MPPClassificationResult.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h index 46a7e2aba..17fea4e53 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h @@ -114,7 +114,11 @@ NS_SWIFT_NAME(ClassificationResult) * classifications and timestampMs. **/ - (instancetype)initWithClassifications:(NSArray *)classifications - timestampMs:(NSInteger)timestampMs; + timestampMs:(NSInteger)timestampMs NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; @end From 8570136f80109f73003279067c1944801115a826 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:47:33 +0530 Subject: [PATCH 21/45] Removed convenience initializer in MPPClassificationResult --- .../containers/sources/MPPClassificationResult.h | 12 ------------ .../containers/sources/MPPClassificationResult.m | 6 ------ .../utils/sources/MPPClassificationResult+Helpers.mm | 10 ++++------ 3 files changed, 4 insertions(+), 24 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h index 17fea4e53..c38f42552 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h @@ -89,18 +89,6 @@ NS_SWIFT_NAME(ClassificationResult) **/ @property(nonatomic, readonly) NSInteger timestampMs; -/** - * Initializes a new `MPPClassificationResult` with the given array of classifications. This method - * must be used when no time stamp needs to be specified. It sets the property `timestampMs` to -1. - * - * @param classifications An Aaray of `MPPClassifications` objects containing classifier - * predictions per classifier head. - * - * @return An instance of MPPClassificationResult initialized with the given array of - * classifications. - **/ -- (instancetype)initWithClassifications:(NSArray *)classifications; - /** * Initializes a new `MPPClassificationResult` with the given array of classifications and time * stamp (in milliseconds). diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m index 6cf75234e..6d42d22ca 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.m @@ -48,10 +48,4 @@ return self; } -- (instancetype)initWithClassifications:(NSArray *)classifications { - return [self initWithClassifications:classifications timestampMs:-1]; - - return self; -} - @end diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm index 9ad284790..7cad3e155 100644 --- a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm @@ -54,15 +54,13 @@ using ClassificationResultProto = } MPPClassificationResult *classificationResult; + NSInteger timeStampMs; if (classificationResultProto.has_timestamp_ms()) { - classificationResult = [[MPPClassificationResult alloc] initWithClassifications:classifications timestampMs:(NSInteger)classificationResultProto.timestamp_ms()]; + timeStampMs = (NSInteger)classificationResultProto.timestamp_ms(); } - else { - classificationResult = [[MPPClassificationResult alloc] initWithClassifications:classifications]; - } - - return classificationResult; + + return [[MPPClassificationResult alloc] initWithClassifications:classifications timestampMs:timeStampMs];; } @end From bc15882cdc9d66d117b165529e696d225b8677be Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:51:07 +0530 Subject: [PATCH 22/45] Updated incorrect usage of timestamp --- .../utils/sources/MPPClassificationResult+Helpers.mm | 6 +++--- .../text/text_classifier/sources/MPPTextClassifierResult.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm index 7cad3e155..a5a1a3918 100644 --- a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm @@ -54,13 +54,13 @@ using ClassificationResultProto = } MPPClassificationResult *classificationResult; - NSInteger timeStampMs; + NSInteger timestampMs; if (classificationResultProto.has_timestamp_ms()) { - timeStampMs = (NSInteger)classificationResultProto.timestamp_ms(); + timestampMs = (NSInteger)classificationResultProto.timestamp_ms(); } - return [[MPPClassificationResult alloc] initWithClassifications:classifications timestampMs:timeStampMs];; + return [[MPPClassificationResult alloc] initWithClassifications:classifications timestampMs:timestampMs];; } @end diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h index 15522fca1..d44084a55 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h @@ -31,10 +31,10 @@ NS_SWIFT_NAME(TextClassifierResult) * * @param classificationResult The `MPPClassificationResult` instance containing one set of results * per classifier head. - * @param timeStampMs The time stamp for this result. + * @param timestampMs The timestamp for this result. * * @return An instance of `MPPTextClassifierResult` initialized with the given - * `MPPClassificationResult` and time stamp (in milliseconds). + * `MPPClassificationResult` and timestamp (in milliseconds). */ - (instancetype)initWithClassificationResult:(MPPClassificationResult *)classificationResult timestampMs:(NSInteger)timestampMs; From c70fa8095f035a25b6be786ad22990627d9d5bc0 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:52:18 +0530 Subject: [PATCH 23/45] Updated comments --- .../text_classifier/sources/MPPTextClassifierResult.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h index d44084a55..c6b2f3205 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierResult.h @@ -18,16 +18,16 @@ NS_ASSUME_NONNULL_BEGIN -/** Represents the classification results generated by `MPPTextClassifier`. */ +/** Represents the classification results generated by `MPPTextClassifier`. **/ NS_SWIFT_NAME(TextClassifierResult) @interface MPPTextClassifierResult : MPPTaskResult -/** The `MPPClassificationResult` instance containing one set of results per classifier head. */ +/** The `MPPClassificationResult` instance containing one set of results per classifier head. **/ @property(nonatomic, readonly) MPPClassificationResult *classificationResult; /** - * Initializes a new `MPPTextClassifierResult` with the given `MPPClassificationResult` and time - * stamp (in milliseconds). + * Initializes a new `MPPTextClassifierResult` with the given `MPPClassificationResult` and + * timestamp (in milliseconds). * * @param classificationResult The `MPPClassificationResult` instance containing one set of results * per classifier head. @@ -35,7 +35,7 @@ NS_SWIFT_NAME(TextClassifierResult) * * @return An instance of `MPPTextClassifierResult` initialized with the given * `MPPClassificationResult` and timestamp (in milliseconds). - */ + **/ - (instancetype)initWithClassificationResult:(MPPClassificationResult *)classificationResult timestampMs:(NSInteger)timestampMs; From 05a9b40fa7dc30fbf83e2cb706957e690baeafd2 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 13:54:27 +0530 Subject: [PATCH 24/45] Updated C++ variables in MPPTextClassifier to camel case --- .../ios/text/text_classifier/sources/MPPTextClassifier.mm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm index f87d0681a..8a1116285 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm @@ -83,15 +83,15 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.text.text_classifier.T - (nullable MPPTextClassifierResult *)classifyWithText:(NSString *)text error:(NSError **)error { Packet packet = [MPPTextPacketCreator createWithText:text]; - std::map packet_map = {{kTextInStreamName.cppString, packet}}; - absl::StatusOr status_or_output_packet_map = [_textTaskRunner process:packet_map]; + std::map packetMap = {{kTextInStreamName.cppString, packet}}; + absl::StatusOr statusOrOutputPacketMap = [_textTaskRunner process:packetMap]; - if (![MPPCommonUtils checkCppError:status_or_output_packet_map.status() toError:error]) { + if (![MPPCommonUtils checkCppError:statusOrOutputPacketMap.status() toError:error]) { return nil; } return [MPPTextClassifierResult - textClassifierResultWithClassificationsPacket:status_or_output_packet_map.value() + textClassifierResultWithClassificationsPacket:statusOrOutputPacketMap.value() [kClassificationsStreamName.cppString]]; } From 5e24b8ea697e62149e4be7fae720c8adfdd1e9f4 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:01:09 +0530 Subject: [PATCH 25/45] Updated C++ types to camel case in MPPTaskInfo --- .../tasks/ios/core/sources/MPPTaskInfo.mm | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm index 5f2290497..80ff594a2 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm @@ -69,59 +69,59 @@ using ::mediapipe::InputStreamInfo; } - (CalculatorGraphConfig)generateGraphConfig { - CalculatorGraphConfig graph_config; + CalculatorGraphConfig graphConfig; - Node *task_subgraph_node = graph_config.add_node(); - task_subgraph_node->set_calculator(self.taskGraphName.cppString); - [self.taskOptions copyToProto:task_subgraph_node->mutable_options()]; + Node *taskSubgraphNode = graphConfig.add_node(); + taskSubgraphNode->set_calculator(self.taskGraphName.cppString); + [self.taskOptions copyToProto:taskSubgraphNode->mutable_options()]; for (NSString *outputStream in self.outputStreams) { - auto cpp_output_stream = std::string(outputStream.cppString); - task_subgraph_node->add_output_stream(cpp_output_stream); - graph_config.add_output_stream(cpp_output_stream); + auto cppOutputStream = std::string(outputStream.cppString); + taskSubgraphNode->add_output_stream(cppOutputStream); + graphConfig.add_output_stream(cppOutputStream); } if (!self.enableFlowLimiting) { for (NSString *inputStream in self.inputStreams) { - auto cpp_input_stream = inputStream.cppString; - task_subgraph_node->add_input_stream(cpp_input_stream); - graph_config.add_input_stream(cpp_input_stream); + auto cppInputStream = inputStream.cppString; + taskSubgraphNode->add_input_stream(cppInputStream); + graphConfig.add_input_stream(cppInputStream); } - return graph_config; + return graphConfig; } - Node *flow_limit_calculator_node = graph_config.add_node(); + Node *flowLimitCalculatorNode = graphConfig.add_node(); - flow_limit_calculator_node->set_calculator("FlowLimiterCalculator"); + flowLimitCalculatorNode->set_calculator("FlowLimiterCalculator"); - InputStreamInfo *input_stream_info = flow_limit_calculator_node->add_input_stream_info(); - input_stream_info->set_tag_index("FINISHED"); - input_stream_info->set_back_edge(true); + InputStreamInfo *inputStreamInfo = flowLimitCalculatorNode->add_input_stream_info(); + inputStreamInfo->set_tag_index("FINISHED"); + inputStreamInfo->set_back_edge(true); - FlowLimiterCalculatorOptions *flow_limit_calculator_options = - flow_limit_calculator_node->mutable_options()->MutableExtension( + FlowLimiterCalculatorOptions *flowLimitCalculatorOptions = + flowLimitCalculatorNode->mutable_options()->MutableExtension( FlowLimiterCalculatorOptions::ext); - flow_limit_calculator_options->set_max_in_flight(1); - flow_limit_calculator_options->set_max_in_queue(1); + flowLimitCalculatorOptions->set_max_in_flight(1); + flowLimitCalculatorOptions->set_max_in_queue(1); for (NSString *inputStream in self.inputStreams) { - graph_config.add_input_stream(inputStream.cppString); + graphConfig.add_input_stream(inputStream.cppString); NSString *strippedInputStream = [MPPTaskInfo stripTagIndex:inputStream]; - flow_limit_calculator_node->add_input_stream(strippedInputStream.cppString); + flowLimitCalculatorNode->add_input_stream(strippedInputStream.cppString); NSString *taskInputStream = [MPPTaskInfo addStreamNamePrefix:inputStream]; - task_subgraph_node->add_input_stream(taskInputStream.cppString); + taskSubgraphNode->add_input_stream(taskInputStream.cppString); NSString *strippedTaskInputStream = [MPPTaskInfo stripTagIndex:taskInputStream]; - flow_limit_calculator_node->add_output_stream(strippedTaskInputStream.cppString); + flowLimitCalculatorNode->add_output_stream(strippedTaskInputStream.cppString); } NSString *firstOutputStream = self.outputStreams[0]; - auto finished_output_stream = "FINISHED:" + firstOutputStream.cppString; - flow_limit_calculator_node->add_input_stream(finished_output_stream); + auto finishedOutputStream = "FINISHED:" + firstOutputStream.cppString; + flowLimitCalculatorNode->add_input_stream(finishedOutputStream); - return graph_config; + return graphConfig; } + (NSString *)stripTagIndex:(NSString *)tagIndexName { From b7a9165e5a7bccb0a9883c9ef295675947bacd7b Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:02:03 +0530 Subject: [PATCH 26/45] Updated C++ types to camel case in MPPTextClassifierOptions+Helpers.mm --- .../utils/sources/MPPTextClassifierOptions+Helpers.mm | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm index 728000b44..79e4a38cc 100644 --- a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm @@ -27,10 +27,10 @@ using TextClassifierGraphOptionsProto = @implementation MPPTextClassifierOptions (Helpers) - (void)copyToProto:(CalculatorOptionsProto *)optionsProto { - TextClassifierGraphOptionsProto *graph_options = + TextClassifierGraphOptionsProto *graphOptions = optionsProto->MutableExtension(TextClassifierGraphOptionsProto::ext); - [self.baseOptions copyToProto:graph_options->mutable_base_options()]; - [self.classifierOptions copyToProto:graph_options->mutable_classifier_options()]; + [self.baseOptions copyToProto:graphOptions->mutable_base_options()]; + [self.classifierOptions copyToProto:graphOptions->mutable_classifier_options()]; } @end From e119363aa3edccb36c964d275a5567b91c651057 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:12:49 +0530 Subject: [PATCH 27/45] Updated incorrect spelling of MediaPipe in iOS files --- mediapipe/tasks/ios/common/sources/MPPCommon.h | 2 +- mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h | 2 +- mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm | 2 +- mediapipe/tasks/ios/core/sources/MPPTaskOptions.h | 2 +- mediapipe/tasks/ios/core/sources/MPPTaskRunner.h | 4 ++-- mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mediapipe/tasks/ios/common/sources/MPPCommon.h b/mediapipe/tasks/ios/common/sources/MPPCommon.h index 7ce791d12..09a61e20d 100644 --- a/mediapipe/tasks/ios/common/sources/MPPCommon.h +++ b/mediapipe/tasks/ios/common/sources/MPPCommon.h @@ -18,7 +18,7 @@ NS_ASSUME_NONNULL_BEGIN /** * @enum MPPTasksErrorCode - * This enum specifies error codes for Mediapipe Task Library. + * This enum specifies error codes for MediaPipe Task Library. * It maintains a 1:1 mapping to MediaPipeTasksStatus of the C ++libray. */ typedef NS_ENUM(NSUInteger, MPPTasksErrorCode) { diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h index 5404a074d..69c28b916 100644 --- a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h @@ -18,7 +18,7 @@ NS_ASSUME_NONNULL_BEGIN -/** Error domain of Mediapipe Task related errors. */ +/** Error domain of MediaPipe Task related errors. */ extern NSString *const MPPTasksErrorDomain; /** Helper utility for the all tasks which encapsulates common functionality. */ diff --git a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm index 8234ac6d3..1a37f8465 100644 --- a/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm +++ b/mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.mm @@ -96,7 +96,7 @@ NSString *const MPPTasksErrorDomain = @"com.google.mediapipe.tasks"; // The mapping to absl::Status::code() is done to generate a more specific error code than // MPPTasksErrorCodeError in cases when the payload can't be mapped to // MPPTasksErrorCode. This can happen when absl::Status returned by TFLite library are in turn - // returned without modification by Mediapipe cc library methods. + // returned without modification by MediaPipe cc library methods. if (errorCode > MPPTasksErrorCodeLast || errorCode <= MPPTasksErrorCodeFirst) { switch (status.code()) { case absl::StatusCode::kInternal: diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskOptions.h b/mediapipe/tasks/ios/core/sources/MPPTaskOptions.h index ee2f7d032..e10678348 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskOptions.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskOptions.h @@ -25,7 +25,7 @@ NS_SWIFT_NAME(TaskOptions) @interface MPPTaskOptions : NSObject /** - * Base options for configuring the Mediapipe task. + * Base options for configuring the MediaPipe task. */ @property(nonatomic, copy) MPPBaseOptions *baseOptions; diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 31a867988..05e7823c3 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -21,7 +21,7 @@ NS_ASSUME_NONNULL_BEGIN /** * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, - * execute and terminate any Mediapipe task. + * execute and terminate any MediaPipe task. * * An instance of the newly created C++ task runner will * be stored until this class is destroyed. When methods are called for processing (performing @@ -42,7 +42,7 @@ NS_ASSUME_NONNULL_BEGIN * You can pass `nullptr` for `packetsCallback` in case the mode of operation * requested by the user is synchronous. * - * If the task is operating in asynchronous mode, any iOS Mediapipe task that uses the `MPPTaskRunner` + * If the task is operating in asynchronous mode, any iOS MediaPipe task that uses the `MPPTaskRunner` * must define a C++ callback function to obtain the results of inference asynchronously and deliver * the results to the user. To accomplish this, callback function will in turn invoke the block * provided by the user in the task options supplied to create the task. diff --git a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h index dd5d96ce6..f5dcca1a1 100644 --- a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h +++ b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h @@ -18,7 +18,7 @@ NS_ASSUME_NONNULL_BEGIN /** - * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, execute and terminate any Mediapipe text task. + * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, execute and terminate any MediaPipe text task. */ @interface MPPTextTaskRunner : MPPTaskRunner From 7ff7d7f5dfb168ab591584dacc5023c9c75071cb Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:14:09 +0530 Subject: [PATCH 28/45] Updated C++ types to camel case in MPPClassificationResult+helpers.mm --- .../utils/sources/MPPClassificationResult+Helpers.mm | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm index a5a1a3918..6510b0ab1 100644 --- a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm @@ -32,7 +32,6 @@ using ClassificationResultProto = } NSString *headName; - if (classificationsProto.has_head_name()) { headName = [NSString stringWithCppString:classificationsProto.head_name()]; } @@ -49,13 +48,11 @@ using ClassificationResultProto = + (MPPClassificationResult *)classificationResultWithProto: (const ClassificationResultProto &)classificationResultProto { NSMutableArray *classifications = [[NSMutableArray alloc] init]; - for (const auto &classifications_proto : classificationResultProto.classifications()) { - [classifications addObject:[MPPClassifications classificationsWithProto:classifications_proto]]; + for (const auto &classificationsProto : classificationResultProto.classifications()) { + [classifications addObject:[MPPClassifications classificationsWithProto:classificationsProto]]; } - MPPClassificationResult *classificationResult; NSInteger timestampMs; - if (classificationResultProto.has_timestamp_ms()) { timestampMs = (NSInteger)classificationResultProto.timestamp_ms(); } From 2406c77636453ca5080338596798e1eb68085d52 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:18:37 +0530 Subject: [PATCH 29/45] Updated comments --- mediapipe/tasks/ios/core/sources/MPPTaskRunner.h | 2 +- mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 05e7823c3..5abdfbe71 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -36,7 +36,7 @@ NS_ASSUME_NONNULL_BEGIN @interface MPPTaskRunner : NSObject /** - * Initializes a new `MPPTaskRunner` with the mediapipe task graph config proto and an optional C++ + * Initializes a new `MPPTaskRunner` with the MediaPipe calculator configuration proto and an optional C++ * packets callback. * * You can pass `nullptr` for `packetsCallback` in case the mode of operation diff --git a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h index f5dcca1a1..4662e3aa2 100644 --- a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h +++ b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h @@ -23,9 +23,9 @@ NS_ASSUME_NONNULL_BEGIN @interface MPPTextTaskRunner : MPPTaskRunner /** - * Initializes a new `MPPTextTaskRunner` with the mediapipe task graph config proto. + * Initializes a new `MPPTextTaskRunner` with the MediaPipe task graph config proto. * - * @param graphConfig A mediapipe task graph config proto. + * @param graphConfig A MediaPipe task graph config proto. * * @return An instance of `MPPTextTaskRunner` initialized to the given graph config proto. */ From 6409c12cb211c85a98d1db065e737e2ea6c86c8c Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:22:58 +0530 Subject: [PATCH 30/45] Updated comments --- .../tasks/ios/core/sources/MPPTaskRunner.h | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 5abdfbe71..c1d36bf77 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -32,12 +32,12 @@ NS_ASSUME_NONNULL_BEGIN * `MPPVisionTaskRunner` can in turn be used by the each vision task for creation and execution of * the task. Please see the documentation for the C++ Task Runner for more details on how the taks * runner operates. - */ + **/ @interface MPPTaskRunner : NSObject /** - * Initializes a new `MPPTaskRunner` with the MediaPipe calculator configuration proto and an optional C++ - * packets callback. + * Initializes a new `MPPTaskRunner` with the MediaPipe calculator configuration proto and an optional + * C++ packets callback. * * You can pass `nullptr` for `packetsCallback` in case the mode of operation * requested by the user is synchronous. @@ -56,26 +56,29 @@ NS_ASSUME_NONNULL_BEGIN * * @return An instance of `MPPTaskRunner` initialized to the given graph config proto and optional * packetsCallback. - */ + **/ - (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig packetsCallback: (mediapipe::tasks::core::PacketsCallback)packetsCallback error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** A synchronous method for processing batch data or offline streaming data. This method is -designed for processing either batch data such as unrelated images and texts or offline streaming -data such as the decoded frames from a video file and an audio file. The call blocks the current -thread until a failure status or a successful result is returned. If the input packets have no -timestamp, an internal timestamp will be assigend per invocation. Otherwise, when the timestamp is -set in the input packets, the caller must ensure that the input packet timestamps are greater than -the timestamps of the previous invocation. This method is thread-unsafe and it is the caller's -responsibility to synchronize access to this method across multiple threads and to ensure that the -input packet timestamps are in order.*/ + * designed for processing either batch data such as unrelated images and texts or offline streaming + * data such as the decoded frames from a video file and an audio file. The call blocks the current + * thread until a failure status or a successful result is returned. If the input packets have no + * timestamp, an internal timestamp will be assigend per invocation. Otherwise, when the timestamp is + * set in the input packets, the caller must ensure that the input packet timestamps are greater than + * the timestamps of the previous invocation. This method is thread-unsafe and it is the caller's + * responsibility to synchronize access to this method across multiple threads and to ensure that the + * input packet timestamps are in order. + **/ - (absl::StatusOr)process: (const mediapipe::tasks::core::PacketMap &)packetMap; -/** Shuts down the C++ task runner. After the runner is closed, any calls that send input data to - * the runner are illegal and will receive errors. */ +/** + * Shuts down the C++ task runner. After the runner is closed, any calls that send input data to + * the runner are illegal and will receive errors. + **/ - (absl::Status)close; - (instancetype)init NS_UNAVAILABLE; From 5f41d4591c2a3e0e32e77b38d6ebf11dedefcbe7 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:24:37 +0530 Subject: [PATCH 31/45] Updated comments --- .../tasks/ios/core/sources/MPPTaskRunner.h | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index c1d36bf77..8625cf606 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -23,15 +23,14 @@ NS_ASSUME_NONNULL_BEGIN * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, * execute and terminate any MediaPipe task. * - * An instance of the newly created C++ task runner will - * be stored until this class is destroyed. When methods are called for processing (performing - * inference), closing etc., on this class, internally the appropriate methods will be called on the - * C++ task runner instance to execute the appropriate actions. For each type of task, a subclass of - * this class must be defined to add any additional functionality. For eg:, vision tasks must create - * an `MPPVisionTaskRunner` and provide additional functionality. An instance of - * `MPPVisionTaskRunner` can in turn be used by the each vision task for creation and execution of - * the task. Please see the documentation for the C++ Task Runner for more details on how the taks - * runner operates. + * An instance of the newly created C++ task runner will be stored until this class is destroyed. + * When methods are called for processing (performing inference), closing etc., on this class, + * internally the appropriate methods will be called on the C++ task runner instance to execute + * the appropriate actions. For each type of task, a subclass of this class must be defined to add + * any additional functionality. For eg:, vision tasks must create an `MPPVisionTaskRunner` and + * provide additional functionality. An instance of `MPPVisionTaskRunner` can in turn be used by + * the each vision task for creation and execution of the task. Please see the documentation for + * the C++ Task Runner for more details on how the taks runner operates. **/ @interface MPPTaskRunner : NSObject From 95f5ea7203038a591ee0a28cbb2076114eac109b Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:30:29 +0530 Subject: [PATCH 32/45] Reformatted using the clang formatter. --- .../tasks/ios/core/sources/MPPTaskRunner.h | 67 ++++++++++--------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 8625cf606..56e8e9df9 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -21,31 +21,31 @@ NS_ASSUME_NONNULL_BEGIN /** * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, - * execute and terminate any MediaPipe task. - * - * An instance of the newly created C++ task runner will be stored until this class is destroyed. - * When methods are called for processing (performing inference), closing etc., on this class, - * internally the appropriate methods will be called on the C++ task runner instance to execute - * the appropriate actions. For each type of task, a subclass of this class must be defined to add - * any additional functionality. For eg:, vision tasks must create an `MPPVisionTaskRunner` and - * provide additional functionality. An instance of `MPPVisionTaskRunner` can in turn be used by - * the each vision task for creation and execution of the task. Please see the documentation for - * the C++ Task Runner for more details on how the taks runner operates. + * execute and terminate any MediaPipe task. + * + * An instance of the newly created C++ task runner will be stored until this class is destroyed. + * When methods are called for processing (performing inference), closing etc., on this class, + * internally the appropriate methods will be called on the C++ task runner instance to execute the + * appropriate actions. For each type of task, a subclass of this class must be defined to add any + * additional functionality. For eg:, vision tasks must create an `MPPVisionTaskRunner` and provide + * additional functionality. An instance of `MPPVisionTaskRunner` can in turn be used by the each + * vision task for creation and execution of the task. Please see the documentation for the C++ Task + * Runner for more details on how the taks runner operates. **/ @interface MPPTaskRunner : NSObject /** - * Initializes a new `MPPTaskRunner` with the MediaPipe calculator configuration proto and an optional - * C++ packets callback. - * - * You can pass `nullptr` for `packetsCallback` in case the mode of operation - * requested by the user is synchronous. - * - * If the task is operating in asynchronous mode, any iOS MediaPipe task that uses the `MPPTaskRunner` - * must define a C++ callback function to obtain the results of inference asynchronously and deliver - * the results to the user. To accomplish this, callback function will in turn invoke the block - * provided by the user in the task options supplied to create the task. - * Please see the documentation of the C++ Task Runner for more information on the synchronous and + * Initializes a new `MPPTaskRunner` with the MediaPipe calculator configuration proto and an + * optional C++ packets callback. + * + * You can pass `nullptr` for `packetsCallback` in case the mode of operation requested by the user + * is synchronous. + * + * If the task is operating in asynchronous mode, any iOS MediaPipe task that uses the + * `MPPTaskRunner` must define a C++ callback function to obtain the results of inference + * asynchronously and deliver the results to the user. To accomplish this, callback function will in + * turn invoke the block provided by the user in the task options supplied to create the task. + * Please see the documentation of the C++ Task Runner for more information on the synchronous and * asynchronous modes of operation. * * @param graphConfig A mediapipe task graph config proto. @@ -61,22 +61,23 @@ NS_ASSUME_NONNULL_BEGIN (mediapipe::tasks::core::PacketsCallback)packetsCallback error:(NSError **)error NS_DESIGNATED_INITIALIZER; -/** A synchronous method for processing batch data or offline streaming data. This method is - * designed for processing either batch data such as unrelated images and texts or offline streaming - * data such as the decoded frames from a video file and an audio file. The call blocks the current - * thread until a failure status or a successful result is returned. If the input packets have no - * timestamp, an internal timestamp will be assigend per invocation. Otherwise, when the timestamp is - * set in the input packets, the caller must ensure that the input packet timestamps are greater than - * the timestamps of the previous invocation. This method is thread-unsafe and it is the caller's - * responsibility to synchronize access to this method across multiple threads and to ensure that the - * input packet timestamps are in order. +/** + * A synchronous method for processing batch data or offline streaming data. This method is designed + * for processing either batch data such as unrelated images and texts or offline streaming data + * such as the decoded frames from a video file and an audio file. The call blocks the current + * thread until a failure status or a successful result is returned. If the input packets have no + * timestamp, an internal timestamp will be assigend per invocation. Otherwise, when the timestamp + * is set in the input packets, the caller must ensure that the input packet timestamps are greater + * than the timestamps of the previous invocation. This method is thread-unsafe and it is the + * caller's responsibility to synchronize access to this method across multiple threads and to + * ensure that the input packet timestamps are in order. **/ - (absl::StatusOr)process: (const mediapipe::tasks::core::PacketMap &)packetMap; -/** - * Shuts down the C++ task runner. After the runner is closed, any calls that send input data to - * the runner are illegal and will receive errors. +/** + * Shuts down the C++ task runner. After the runner is closed, any calls that send input data to the + * runner are illegal and will receive errors. **/ - (absl::Status)close; From cd329bafc4f99d9b2347d9a2f3476034c6fb5fc8 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:32:49 +0530 Subject: [PATCH 33/45] Updated comments --- mediapipe/tasks/ios/core/sources/MPPTaskRunner.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 56e8e9df9..2c241ac03 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -43,10 +43,10 @@ NS_ASSUME_NONNULL_BEGIN * * If the task is operating in asynchronous mode, any iOS MediaPipe task that uses the * `MPPTaskRunner` must define a C++ callback function to obtain the results of inference - * asynchronously and deliver the results to the user. To accomplish this, callback function will in - * turn invoke the block provided by the user in the task options supplied to create the task. - * Please see the documentation of the C++ Task Runner for more information on the synchronous and - * asynchronous modes of operation. + * asynchronously and deliver the results to the user. To accomplish this, the callback function + * should in turn invoke the block provided by the user in the task options supplied to create the + * task. Please see the documentation of the C++ Task Runner for more information on the synchronous + * and asynchronous modes of operation. * * @param graphConfig A mediapipe task graph config proto. * @param packetsCallback An optional C++ callback function that takes a list of output packets as From 813500699241a7b9c4000b115bd77c8cc6d3c5a5 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:34:27 +0530 Subject: [PATCH 34/45] Updated comments in MPPTaskRunner --- mediapipe/tasks/ios/core/sources/MPPTaskRunner.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 2c241ac03..a1b1dfad4 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -64,7 +64,7 @@ NS_ASSUME_NONNULL_BEGIN /** * A synchronous method for processing batch data or offline streaming data. This method is designed * for processing either batch data such as unrelated images and texts or offline streaming data - * such as the decoded frames from a video file and an audio file. The call blocks the current + * such as the decoded frames from a video file or audio file. The call blocks the current * thread until a failure status or a successful result is returned. If the input packets have no * timestamp, an internal timestamp will be assigend per invocation. Otherwise, when the timestamp * is set in the input packets, the caller must ensure that the input packet timestamps are greater From 75b186d05a080975def3d721493a9dfca9f5ccb2 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:34:58 +0530 Subject: [PATCH 35/45] Fixed year in build file --- mediapipe/tasks/ios/core/utils/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/core/utils/BUILD b/mediapipe/tasks/ios/core/utils/BUILD index d9fbaf375..1cfc75e6a 100644 --- a/mediapipe/tasks/ios/core/utils/BUILD +++ b/mediapipe/tasks/ios/core/utils/BUILD @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 84f4d79fd6f453c240eb485d149f221d2b247b86 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:52:24 +0530 Subject: [PATCH 36/45] Updated ordering of imports in iOS files --- .../containers/utils/sources/MPPCategory+Helpers.mm | 3 ++- .../utils/sources/MPPClassificationResult+Helpers.mm | 3 ++- .../utils/sources/MPPClassifierOptions+Helpers.mm | 3 ++- mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm | 1 + mediapipe/tasks/ios/core/sources/MPPTaskOptions.m | 1 + mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm | 4 +++- mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm | 1 + .../ios/text/text_classifier/sources/MPPTextClassifier.mm | 4 ++-- .../utils/sources/MPPTextClassifierOptions+Helpers.mm | 6 ++++-- .../utils/sources/MPPTextClassifierResult+Helpers.mm | 3 ++- 10 files changed, 20 insertions(+), 9 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm index f729d9720..1c6c951d0 100644 --- a/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.mm @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + namespace { using ClassificationProto = ::mediapipe::Classification; } diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm index 6510b0ab1..02ad390cd 100644 --- a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" + #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h" -#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" namespace { using ClassificationsProto = ::mediapipe::tasks::components::containers::proto::Classifications; diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm index 24b54fd6a..94b3270c8 100644 --- a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm +++ b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + namespace { using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; } diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm index 80ff594a2..ae6ed2a70 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskInfo.mm @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" + #import "mediapipe/tasks/ios/common/sources/MPPCommon.h" #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskOptions.m b/mediapipe/tasks/ios/core/sources/MPPTaskOptions.m index fe74517c3..ad11bbc6e 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskOptions.m +++ b/mediapipe/tasks/ios/core/sources/MPPTaskOptions.m @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" + #import "mediapipe/tasks/ios/core/sources/MPPBaseOptions.h" @implementation MPPTaskOptions diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm index fd3f780fa..a77f206b2 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm @@ -13,8 +13,10 @@ // limitations under the License. #import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" -#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" + #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" + +#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" #include "tensorflow/lite/core/api/op_resolver.h" namespace { diff --git a/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm index ca86e7a0b..fb59b363d 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.mm @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h" + #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" namespace { diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm index 8a1116285..afe9b9a40 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h" + #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" @@ -21,9 +22,8 @@ #import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h" #import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h" -#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" - #include "absl/status/statusor.h" +#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" namespace { using ::mediapipe::Packet; diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm index 79e4a38cc..5a43f2d2f 100644 --- a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "mediapipe/tasks/cc/text/text_classifier/proto/text_classifier_graph_options.pb.h" +#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h" + #import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" #import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" -#import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h" + +#include "mediapipe/tasks/cc/text/text_classifier/proto/text_classifier_graph_options.pb.h" namespace { using CalculatorOptionsProto = ::mediapipe::CalculatorOptions; diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.mm b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.mm index f5d6aa1d3..62e0d8cb1 100644 --- a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.mm +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.mm @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" #import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierResult+Helpers.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" + #include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" static const int kMicroSecondsPerMilliSecond = 1000; From f6bfaa45866924d32a89ca450c984d1b960502d3 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:53:00 +0530 Subject: [PATCH 37/45] Updated year in comments --- mediapipe/tasks/ios/text/core/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/text/core/BUILD b/mediapipe/tasks/ios/text/core/BUILD index 6d558b22b..bf88f5734 100644 --- a/mediapipe/tasks/ios/text/core/BUILD +++ b/mediapipe/tasks/ios/text/core/BUILD @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. All Rights Reserved. +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 61a918d449301e0b0a4e3a8c908690e98b363a5a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:55:28 +0530 Subject: [PATCH 38/45] Updated comments in MPPTextTaskRunner --- .../ios/text/core/sources/MPPTextTaskRunner.h | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h index 4662e3aa2..a0db39958 100644 --- a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h +++ b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h @@ -18,17 +18,19 @@ NS_ASSUME_NONNULL_BEGIN /** - * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, execute and terminate any MediaPipe text task. - */ + * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, + * execute and terminate any MediaPipe text task. + **/ @interface MPPTextTaskRunner : MPPTaskRunner /** - * Initializes a new `MPPTextTaskRunner` with the MediaPipe task graph config proto. + * Initializes a new `MPPTextTaskRunner` with the MediaPipe calculator config proto. * - * @param graphConfig A MediaPipe task graph config proto. + * @param graphConfig A MediaPipe calculator config proto. * - * @return An instance of `MPPTextTaskRunner` initialized to the given graph config proto. - */ + * @return An instance of `MPPTextTaskRunner` initialized to the given MediaPipe calculator config + * proto. + **/ - (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig error:(NSError **)error NS_DESIGNATED_INITIALIZER; From 7d0678c1107112bf04ba5dbd2927b8c7b9e6ecae Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 14:56:42 +0530 Subject: [PATCH 39/45] Marked init and new as unavailable --- mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h index a0db39958..e3df3de9d 100644 --- a/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h +++ b/mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h @@ -34,6 +34,10 @@ NS_ASSUME_NONNULL_BEGIN - (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig error:(NSError **)error NS_DESIGNATED_INITIALIZER; +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + @end NS_ASSUME_NONNULL_END From ad7fd76fb14f5c1a11f6c3d080225a1348f41b77 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 15:11:34 +0530 Subject: [PATCH 40/45] Updated MPPTextClassifier header formatting --- .../sources/MPPTextClassifier.h | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h index 813311fc2..fe1264655 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h @@ -21,49 +21,52 @@ NS_ASSUME_NONNULL_BEGIN /** + * @brief Performs classification on text. + * * This API expects a TFLite model with (optional) [TFLite Model * Metadata](https://www.tensorflow.org/lite/convert/metadata")that contains the mandatory - * (described below) input tensors, output tensor, and the optional (but recommended) label items as - * AssociatedFiles with type TENSOR_AXIS_LABELS per output classification tensor. + * (described below) input tensors, output tensor, and the optional (but recommended) label + * items as AssociatedFiles with type TENSOR_AXIS_LABELS per output classification tensor. * - * Metadata is required for models with int32 input tensors because it contains the input process - * unit for the model's Tokenizer. No metadata is required for models with string input tensors. + * Metadata is required for models with int32 input tensors because it contains the input + * process unit for the model's Tokenizer. No metadata is required for models with string + * input tensors. * * Input tensors * - Three input tensors `kTfLiteInt32` of shape `[batch_size xbert_max_seq_len]` - * representing the input ids, mask ids, and segment ids. This input signature requires a - * Bert Tokenizer process unit in the model metadata. + * representing the input ids, mask ids, and segment ids. This input signature requires + * a Bert Tokenizer process unit in the model metadata. * - Or one input tensor `kTfLiteInt32` of shape `[batch_size xmax_seq_len]` representing * the input ids. This input signature requires a Regex Tokenizer process unit in the * model metadata. - * - Or one input tensor ({@code kTfLiteString}) that is shapeless or has shape `[1]` containing - * the input string. + * - Or one input tensor (`kTfLiteString`) that is shapeless or has shape `[1]` containing + * the input string. * - * At least one output tensor `(kTfLiteFloat32}/kBool)` with: + * At least one output tensor (`kTfLiteFloat32/kBool`) with: * - `N` classes and shape `[1 x N]` - * - optional (but recommended) label map(s) as AssociatedFile-s with type TENSOR_AXIS_LABELS, - * containing one label per line. The first such AssociatedFile (if any) is used to fill the - * `class_name` field of the results. The `display_name` field is filled from the AssociatedFile - * (if any) whose locale matches the `display_names_locale` field of the - * `MPPTextClassifierOptions` used at creation time ("en" by default, i.e. English). If none of - * these are available, only the `index` field of the results will be filled. - * - * @brief Performs classification on text. - */ + * - optional (but recommended) label map(s) as AssociatedFiles with type + * TENSOR_AXIS_LABELS, + * containing one label per line. The first such AssociatedFile (if any) is used to fill + * the `categoryName` field of the results. The `displayName` field is filled from the + * AssociatedFile (if any) whose locale matches the `displayNamesLocale` field of the + * `MPPTextClassifierOptions` used at creation time ("en" by default, i.e. English). If + * none of these are available, only the `index` field of the results will be filled. + **/ NS_SWIFT_NAME(TextClassifier) @interface MPPTextClassifier : NSObject /** - * Creates a new instance of `MPPTextClassifier` from an absolute path to a TensorFlow Lite model - * file stored locally on the device and the default `MPPTextClassifierOptions`. + * Creates a new instance of `MPPTextClassifier` from an absolute path to a TensorFlow Lite + * model file stored locally on the device and the default `MPPTextClassifierOptions`. * - * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the + * device. * @param error An optional error parameter populated when there is an error in initializing * the text classifier. * * @return A new instance of `MPPTextClassifier` with the given model path. `nil` if there is an * error in initializing the text classifier. - */ + **/ - (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; /** @@ -74,10 +77,11 @@ NS_SWIFT_NAME(TextClassifier) * @param error An optional error parameter populated when there is an error in initializing * the text classifier. * - * @return A new instance of `MPPTextClassifier` with the given options. `nil` if there is an error - * in initializing the text classifier. - */ -- (instancetype)initWithOptions:(MPPTextClassifierOptions *)options error:(NSError **)error; + * @return A new instance of `MPPTextClassifier` with the given options. `nil` if there is an + * error in initializing the text classifier. + **/ +- (instancetype)initWithOptions:(MPPTextClassifierOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; /** * Performs classification on the input text. @@ -87,7 +91,7 @@ NS_SWIFT_NAME(TextClassifier) * classification on the input text. * * @return A `MPPTextClassifierResult` object that contains a list of text classifications. - */ + **/ - (nullable MPPTextClassifierResult *)classifyWithText:(NSString *)text error:(NSError **)error; - (instancetype)init NS_UNAVAILABLE; From 1edfe3737a29600f0ce42bdf228df5898a034ec9 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 15:21:49 +0530 Subject: [PATCH 41/45] Updated method name in MPPTextClassifier --- .../tasks/ios/text/text_classifier/sources/MPPTextClassifier.h | 2 +- .../tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h index fe1264655..73ed5ff7a 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.h @@ -92,7 +92,7 @@ NS_SWIFT_NAME(TextClassifier) * * @return A `MPPTextClassifierResult` object that contains a list of text classifications. **/ -- (nullable MPPTextClassifierResult *)classifyWithText:(NSString *)text error:(NSError **)error; +- (nullable MPPTextClassifierResult *)classifyText:(NSString *)text error:(NSError **)error; - (instancetype)init NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm index afe9b9a40..aed05ec37 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifier.mm @@ -80,7 +80,7 @@ static NSString *const kTaskGraphName = @"mediapipe.tasks.text.text_classifier.T return [self initWithOptions:options error:error]; } -- (nullable MPPTextClassifierResult *)classifyWithText:(NSString *)text error:(NSError **)error { +- (nullable MPPTextClassifierResult *)classifyText:(NSString *)text error:(NSError **)error { Packet packet = [MPPTextPacketCreator createWithText:text]; std::map packetMap = {{kTextInStreamName.cppString, packet}}; From 770db81d2c382b86509778486790d1371676a91f Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 15:33:59 +0530 Subject: [PATCH 42/45] Allocated nsmutablearray with capacity --- .../utils/sources/MPPClassificationResult+Helpers.mm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm index 02ad390cd..78bc0b6a3 100644 --- a/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.mm @@ -27,7 +27,7 @@ using ClassificationResultProto = + (MPPClassifications *)classificationsWithProto: (const ClassificationsProto &)classificationsProto { - NSMutableArray *categories = [[NSMutableArray alloc] init]; + NSMutableArray *categories = [NSMutableArray arrayWithCapacity:(NSUInteger)classificationsProto.classification_list().classification_size()]; for (const auto &classification : classificationsProto.classification_list().classification()) { [categories addObject:[MPPCategory categoryWithProto:classification]]; } @@ -48,7 +48,7 @@ using ClassificationResultProto = + (MPPClassificationResult *)classificationResultWithProto: (const ClassificationResultProto &)classificationResultProto { - NSMutableArray *classifications = [[NSMutableArray alloc] init]; + NSMutableArray *classifications = [NSMutableArray arrayWithCapacity:(NSUInteger)classificationResultProto.classifications_size()]; for (const auto &classificationsProto : classificationResultProto.classifications()) { [classifications addObject:[MPPClassifications classificationsWithProto:classificationsProto]]; } From 447b8256091381e721266cdc9b043be4407f6ba1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 15:50:55 +0530 Subject: [PATCH 43/45] Removed classifier options and added it's properties in line to MPPTextClassifierOptions --- .../tasks/ios/components/processors/BUILD | 23 ------- .../processors/sources/MPPClassifierOptions.h | 60 ------------------- .../processors/sources/MPPClassifierOptions.m | 40 ------------- .../ios/components/processors/utils/BUILD | 28 --------- .../sources/MPPClassifierOptions+Helpers.h | 26 -------- .../sources/MPPClassifierOptions+Helpers.mm | 44 -------------- .../tasks/ios/text/text_classifier/BUILD | 1 - .../sources/MPPTextClassifierOptions.h | 35 +++++++++-- .../sources/MPPTextClassifierOptions.m | 15 ++++- .../ios/text/text_classifier/utils/BUILD | 2 +- .../MPPTextClassifierOptions+Helpers.mm | 24 +++++++- 11 files changed, 67 insertions(+), 231 deletions(-) delete mode 100644 mediapipe/tasks/ios/components/processors/BUILD delete mode 100644 mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h delete mode 100644 mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m delete mode 100644 mediapipe/tasks/ios/components/processors/utils/BUILD delete mode 100644 mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h delete mode 100644 mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm diff --git a/mediapipe/tasks/ios/components/processors/BUILD b/mediapipe/tasks/ios/components/processors/BUILD deleted file mode 100644 index 165145076..000000000 --- a/mediapipe/tasks/ios/components/processors/BUILD +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2022 The MediaPipe Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -package(default_visibility = ["//mediapipe/tasks:internal"]) - -licenses(["notice"]) - -objc_library( - name = "MPPClassifierOptions", - srcs = ["sources/MPPClassifierOptions.m"], - hdrs = ["sources/MPPClassifierOptions.h"], -) diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h deleted file mode 100644 index 13dca4030..000000000 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright 2022 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import - -NS_ASSUME_NONNULL_BEGIN - -/** - * Holds settings for any single iOS MediaPipe classification task. - */ -NS_SWIFT_NAME(ClassifierOptions) -@interface MPPClassifierOptions : NSObject - -/** - * The locale to use for display names specified through the TFLite Model - * Metadata, if any. Defaults to English. - */ -@property(nonatomic, copy) NSString *displayNamesLocale; - -/** - * The maximum number of top-scored classification results to return. If < 0, - * all available results will be returned. If 0, an invalid argument error is - * returned. - */ -@property(nonatomic) NSInteger maxResults; - -/** - * Score threshold to override the one provided in the model metadata (if any). - * Results below this value are rejected. - */ -@property(nonatomic) float scoreThreshold; - -/** - * The allowlist of category names. If non-empty, detection results whose - * category name is not in this set will be filtered out. Duplicate or unknown - * category names are ignored. Mutually exclusive with categoryDenylist. - */ -@property(nonatomic, copy) NSArray *categoryAllowlist; - -/** - * The denylist of category names. If non-empty, detection results whose - * category name is in this set will be filtered out. Duplicate or unknown - * category names are ignored. Mutually exclusive with categoryAllowlist. - */ -@property(nonatomic, copy) NSArray *categoryDenylist; - -@end - -NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m b/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m deleted file mode 100644 index 01f498184..000000000 --- a/mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.m +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2022 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" - -@implementation MPPClassifierOptions - -- (instancetype)init { - self = [super init]; - if (self) { - _maxResults = -1; - _scoreThreshold = 0; - } - return self; -} - -- (id)copyWithZone:(NSZone *)zone { - MPPClassifierOptions *classifierOptions = [[MPPClassifierOptions alloc] init]; - - classifierOptions.scoreThreshold = self.scoreThreshold; - classifierOptions.maxResults = self.maxResults; - classifierOptions.categoryDenylist = self.categoryDenylist; - classifierOptions.categoryAllowlist = self.categoryAllowlist; - classifierOptions.displayNamesLocale = self.displayNamesLocale; - - return classifierOptions; -} - -@end diff --git a/mediapipe/tasks/ios/components/processors/utils/BUILD b/mediapipe/tasks/ios/components/processors/utils/BUILD deleted file mode 100644 index 5344c5fdf..000000000 --- a/mediapipe/tasks/ios/components/processors/utils/BUILD +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2022 The MediaPipe Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -package(default_visibility = ["//mediapipe/tasks:internal"]) - -licenses(["notice"]) - -objc_library( - name = "MPPClassifierOptionsHelpers", - srcs = ["sources/MPPClassifierOptions+Helpers.mm"], - hdrs = ["sources/MPPClassifierOptions+Helpers.h"], - deps = [ - "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto", - "//mediapipe/tasks/ios/common/utils:NSStringHelpers", - "//mediapipe/tasks/ios/components/processors:MPPClassifierOptions", - ], -) diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h deleted file mode 100644 index e156020df..000000000 --- a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2022 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" - -#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" - -NS_ASSUME_NONNULL_BEGIN - -@interface MPPClassifierOptions (Helpers) -- (void)copyToProto: - (mediapipe::tasks::components::processors::proto::ClassifierOptions *)classifierOptionsProto; -@end - -NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm deleted file mode 100644 index 94b3270c8..000000000 --- a/mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.mm +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright 2022 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" - -#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" - -namespace { -using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; -} - -@implementation MPPClassifierOptions (Helpers) - -- (void)copyToProto:(ClassifierOptionsProto *)classifierOptionsProto { - classifierOptionsProto->Clear(); - - if (self.displayNamesLocale) { - classifierOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString); - } - - classifierOptionsProto->set_max_results((int)self.maxResults); - classifierOptionsProto->set_score_threshold(self.scoreThreshold); - - for (NSString *category in self.categoryAllowlist) { - classifierOptionsProto->add_category_allowlist(category.cppString); - } - - for (NSString *category in self.categoryDenylist) { - classifierOptionsProto->add_category_denylist(category.cppString); - } -} - -@end diff --git a/mediapipe/tasks/ios/text/text_classifier/BUILD b/mediapipe/tasks/ios/text/text_classifier/BUILD index d58c85be9..e5242f50d 100644 --- a/mediapipe/tasks/ios/text/text_classifier/BUILD +++ b/mediapipe/tasks/ios/text/text_classifier/BUILD @@ -22,7 +22,6 @@ objc_library( hdrs = ["sources/MPPTextClassifierOptions.h"], deps = [ "//mediapipe/tasks/ios/core:MPPTaskOptions", - "//mediapipe/tasks/ios/components/processors:MPPClassifierOptions", ], ) diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h index 25189578b..d43d801d4 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.h @@ -14,7 +14,6 @@ #import -#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" NS_ASSUME_NONNULL_BEGIN @@ -23,12 +22,40 @@ NS_ASSUME_NONNULL_BEGIN * Options for setting up a `MPPTextClassifierOptions`. */ NS_SWIFT_NAME(TextClassifierOptions) -@interface MPPTextClassifierOptions : MPPTaskOptions +@interface MPPTextClassifierOptions : MPPTaskOptions /** - * Options for configuring the classifier behavior, such as score threshold, number of results, etc. + * The locale to use for display names specified through the TFLite Model + * Metadata, if any. Defaults to English. */ -@property(nonatomic, copy) MPPClassifierOptions *classifierOptions; +@property(nonatomic, copy) NSString *displayNamesLocale; + +/** + * The maximum number of top-scored classification results to return. If < 0, + * all available results will be returned. If 0, an invalid argument error is + * returned. + */ +@property(nonatomic) NSInteger maxResults; + +/** + * Score threshold to override the one provided in the model metadata (if any). + * Results below this value are rejected. + */ +@property(nonatomic) float scoreThreshold; + +/** + * The allowlist of category names. If non-empty, detection results whose + * category name is not in this set will be filtered out. Duplicate or unknown + * category names are ignored. Mutually exclusive with categoryDenylist. + */ +@property(nonatomic, copy) NSArray *categoryAllowlist; + +/** + * The denylist of category names. If non-empty, detection results whose + * category name is in this set will be filtered out. Duplicate or unknown + * category names are ignored. Mutually exclusive with categoryAllowlist. + */ +@property(nonatomic, copy) NSArray *categoryDenylist; @end diff --git a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m index c12fbbf1c..2d5c17cda 100644 --- a/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m +++ b/mediapipe/tasks/ios/text/text_classifier/sources/MPPTextClassifierOptions.m @@ -19,9 +19,22 @@ - (instancetype)init { self = [super init]; if (self) { - _classifierOptions = [[MPPClassifierOptions alloc] init]; + _maxResults = -1; + _scoreThreshold = 0; } return self; } +- (id)copyWithZone:(NSZone *)zone { + MPPTextClassifierOptions *textClassifierOptions = [super copyWithZone:zone]; + + textClassifierOptions.scoreThreshold = self.scoreThreshold; + textClassifierOptions.maxResults = self.maxResults; + textClassifierOptions.categoryDenylist = self.categoryDenylist; + textClassifierOptions.categoryAllowlist = self.categoryAllowlist; + textClassifierOptions.displayNamesLocale = self.displayNamesLocale; + + return textClassifierOptions; +} + @end diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/BUILD b/mediapipe/tasks/ios/text/text_classifier/utils/BUILD index abc1fc23b..299050b32 100644 --- a/mediapipe/tasks/ios/text/text_classifier/utils/BUILD +++ b/mediapipe/tasks/ios/text/text_classifier/utils/BUILD @@ -21,10 +21,10 @@ objc_library( srcs = ["sources/MPPTextClassifierOptions+Helpers.mm"], hdrs = ["sources/MPPTextClassifierOptions+Helpers.h"], deps = [ + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", "//mediapipe/tasks/ios/text/text_classifier:MPPTextClassifierOptions", "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol", - "//mediapipe/tasks/ios/components/processors/utils:MPPClassifierOptionsHelpers", "//mediapipe/tasks/cc/text/text_classifier/proto:text_classifier_graph_options_cc_proto", ], ) diff --git a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm index 5a43f2d2f..c370f11ef 100644 --- a/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm +++ b/mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.mm @@ -14,7 +14,7 @@ #import "mediapipe/tasks/ios/text/text_classifier/utils/sources/MPPTextClassifierOptions+Helpers.h" -#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" #include "mediapipe/tasks/cc/text/text_classifier/proto/text_classifier_graph_options.pb.h" @@ -23,7 +23,7 @@ namespace { using CalculatorOptionsProto = ::mediapipe::CalculatorOptions; using TextClassifierGraphOptionsProto = ::mediapipe::tasks::text::text_classifier::proto::TextClassifierGraphOptions; - +using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; } // namespace @implementation MPPTextClassifierOptions (Helpers) @@ -32,7 +32,25 @@ using TextClassifierGraphOptionsProto = TextClassifierGraphOptionsProto *graphOptions = optionsProto->MutableExtension(TextClassifierGraphOptionsProto::ext); [self.baseOptions copyToProto:graphOptions->mutable_base_options()]; - [self.classifierOptions copyToProto:graphOptions->mutable_classifier_options()]; + + ClassifierOptionsProto *classifierOptionsProto = graphOptions->mutable_classifier_options(); + classifierOptionsProto->Clear(); + + if (self.displayNamesLocale) { + classifierOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString); + } + + classifierOptionsProto->set_max_results((int)self.maxResults); + classifierOptionsProto->set_score_threshold(self.scoreThreshold); + + for (NSString *category in self.categoryAllowlist) { + classifierOptionsProto->add_category_allowlist(category.cppString); + } + + for (NSString *category in self.categoryDenylist) { + classifierOptionsProto->add_category_denylist(category.cppString); + } + } @end From 3d324ecf29ac3bc2a68a0ced8a350c6c31b15187 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 15:59:00 +0530 Subject: [PATCH 44/45] Updated comments --- mediapipe/tasks/ios/components/containers/sources/MPPCategory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h index 7a760a52b..648725d95 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPCategory.h @@ -25,7 +25,7 @@ NS_SWIFT_NAME(ClassificationCategory) @interface MPPCategory : NSObject /** - * The index of the label in the corresponding label file. It takes the value -1 if the index is + * The index of the label in the corresponding label file. Set to -1 if the index is * not set. **/ @property(nonatomic, readonly) NSInteger index; From 9d50565ef2d18ecea6cd813d1b0601672fdcd65a Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 6 Jan 2023 16:00:37 +0530 Subject: [PATCH 45/45] Updated comments in MPPClassificationResult.h --- .../containers/sources/MPPClassificationResult.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h index c38f42552..9c8b9bd2e 100644 --- a/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h +++ b/mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h @@ -17,7 +17,8 @@ NS_ASSUME_NONNULL_BEGIN -/** Represents the list of classification for a given classifier head. Typically used as a result +/** + * Represents the list of classification for a given classifier head. Typically used as a result * for classification tasks. **/ NS_SWIFT_NAME(Classifications) @@ -76,12 +77,14 @@ NS_SWIFT_NAME(Classifications) NS_SWIFT_NAME(ClassificationResult) @interface MPPClassificationResult : NSObject -/** An Array of `MPPClassifications` objects containing the predicted categories for each head of +/** + * An Array of `MPPClassifications` objects containing the predicted categories for each head of * the model. **/ @property(nonatomic, readonly) NSArray *classifications; -/** The optional timestamp (in milliseconds) of the start of the chunk of data corresponding to +/** + * The optional timestamp (in milliseconds) of the start of the chunk of data corresponding to * these results. If it is set to the value -1, it signifies the absence of a timestamp. This is * only used for classification on time series (e.g. audio classification). In these use cases, the * amount of data to process might exceed the maximum size that the model can process: to solve