Merge pull request #3973 from priankakariatyml:ios-task-classifier-result

PiperOrigin-RevId: 501005242
This commit is contained in:
Copybara-Service 2023-01-10 08:47:45 -08:00
commit b747fc481b
14 changed files with 473 additions and 180 deletions

View File

@ -1,4 +1,4 @@
# Copyright 2022 The MediaPipe Authors. All Rights Reserved. # Copyright 2023 The MediaPipe Authors. All Rights Reserved.
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
@ -17,12 +17,18 @@ package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"]) licenses(["notice"])
objc_library( objc_library(
name = "MPPClassifierOptionsHelpers", name = "MPPCategory",
srcs = ["sources/MPPClassifierOptions+Helpers.mm"], srcs = ["sources/MPPCategory.m"],
hdrs = ["sources/MPPClassifierOptions+Helpers.h"], hdrs = ["sources/MPPCategory.h"],
deps = ["//third_party/apple_frameworks:Foundation"],
)
objc_library(
name = "MPPClassificationResult",
srcs = ["sources/MPPClassificationResult.m"],
hdrs = ["sources/MPPClassificationResult.h"],
deps = [ deps = [
"//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto", ":MPPCategory",
"//mediapipe/tasks/ios/common/utils:NSStringHelpers", "//third_party/apple_frameworks:Foundation",
"//mediapipe/tasks/ios/components/processors:MPPClassifierOptions",
], ],
) )

View File

@ -0,0 +1,68 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <Foundation/Foundation.h>
NS_ASSUME_NONNULL_BEGIN
/**
* Category is a util class that contains a label, its display name, a float value as score, and the
* index of the label in the corresponding label file. Typically it's used as the result of
* classification tasks.
*/
NS_SWIFT_NAME(ClassificationCategory)
@interface MPPCategory : NSObject
/**
* The index of the label in the corresponding label file. Set to -1 if the index is
* not set.
*/
@property(nonatomic, readonly) NSInteger index;
/** Confidence score for this class. */
@property(nonatomic, readonly) float score;
/** The label of this category object. */
@property(nonatomic, readonly, nullable) NSString *categoryName;
/**
* The display name of the label, which may be translated for different locales. For example, a
* label, "apple", may be translated into Spanish for display purpose, so that the display name is
* "manzana".
*/
@property(nonatomic, readonly, nullable) NSString *displayName;
/**
* Initializes a new `MPPCategory` with the given index, score, category name and display name.
*
* @param index The index of the label in the corresponding label file.
* @param score The probability score of this label category.
* @param categoryName The label of this category object.
* @param displayName The display name of the label.
*
* @return An instance of `MPPCategory` initialized with the given index, score, category name and
* display name.
*/
- (instancetype)initWithIndex:(NSInteger)index
score:(float)score
categoryName:(nullable NSString *)categoryName
displayName:(nullable NSString *)displayName NS_DESIGNATED_INITIALIZER;
- (instancetype)init NS_UNAVAILABLE;
+ (instancetype)new NS_UNAVAILABLE;
@end
NS_ASSUME_NONNULL_END

View File

@ -0,0 +1,33 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h"
@implementation MPPCategory
- (instancetype)initWithIndex:(NSInteger)index
score:(float)score
categoryName:(nullable NSString *)categoryName
displayName:(nullable NSString *)displayName {
self = [super init];
if (self) {
_index = index;
_score = score;
_categoryName = categoryName;
_displayName = displayName;
}
return self;
}
@end

View File

@ -0,0 +1,116 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <Foundation/Foundation.h>
#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h"
NS_ASSUME_NONNULL_BEGIN
/**
* Represents the list of classification for a given classifier head. Typically used as a result
* for classification tasks.
*/
NS_SWIFT_NAME(Classifications)
@interface MPPClassifications : NSObject
/**
* The index of the classifier head these entries refer to. This is useful for multi-head models.
*/
@property(nonatomic, readonly) NSInteger headIndex;
/** The optional name of the classifier head, which is the corresponding tensor metadata name. */
@property(nonatomic, readonly, nullable) NSString *headName;
/** An array of `MPPCategory` objects containing the predicted categories. */
@property(nonatomic, readonly) NSArray<MPPCategory *> *categories;
/**
* Initializes a new `MPPClassifications` object with the given head index and array of categories.
* Head name is initialized to `nil`.
*
* @param headIndex The index of the classifier head.
* @param categories An array of `MPPCategory` objects containing the predicted categories.
*
* @return An instance of `MPPClassifications` initialized with the given head index and
* array of categories.
*/
- (instancetype)initWithHeadIndex:(NSInteger)headIndex
categories:(NSArray<MPPCategory *> *)categories;
/**
* Initializes a new `MPPClassifications` with the given head index, head name and array of
* categories.
*
* @param headIndex The index of the classifier head.
* @param headName The name of the classifier head, which is the corresponding tensor metadata
* name.
* @param categories An array of `MPPCategory` objects containing the predicted categories.
*
* @return An object of `MPPClassifications` initialized with the given head index, head name and
* array of categories.
*/
- (instancetype)initWithHeadIndex:(NSInteger)headIndex
headName:(nullable NSString *)headName
categories:(NSArray<MPPCategory *> *)categories NS_DESIGNATED_INITIALIZER;
- (instancetype)init NS_UNAVAILABLE;
+ (instancetype)new NS_UNAVAILABLE;
@end
/**
* Represents the classification results of a model. Typically used as a result for classification
* tasks.
*/
NS_SWIFT_NAME(ClassificationResult)
@interface MPPClassificationResult : NSObject
/**
* An Array of `MPPClassifications` objects containing the predicted categories for each head of
* the model.
*/
@property(nonatomic, readonly) NSArray<MPPClassifications *> *classifications;
/**
* The optional timestamp (in milliseconds) of the start of the chunk of data corresponding to
* these results. If it is set to the value -1, it signifies the absence of a timestamp. This is
* only used for classification on time series (e.g. audio classification). In these use cases, the
* amount of data to process might exceed the maximum size that the model can process: to solve
* this, the input data is split into multiple chunks starting at different timestamps.
*/
@property(nonatomic, readonly) NSInteger timestampMs;
/**
* Initializes a new `MPPClassificationResult` with the given array of classifications and time
* stamp (in milliseconds).
*
* @param classifications An Array of `MPPClassifications` objects containing the predicted
* categories for each head of the model.
* @param timestampMs The timestamp (in milliseconds) of the start of the chunk of data
* corresponding to these results.
*
* @return An instance of `MPPClassificationResult` initialized with the given array of
* classifications and timestampMs.
*/
- (instancetype)initWithClassifications:(NSArray<MPPClassifications *> *)classifications
timestampMs:(NSInteger)timestampMs NS_DESIGNATED_INITIALIZER;
- (instancetype)init NS_UNAVAILABLE;
+ (instancetype)new NS_UNAVAILABLE;
@end
NS_ASSUME_NONNULL_END

View File

@ -0,0 +1,51 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h"
@implementation MPPClassifications
- (instancetype)initWithHeadIndex:(NSInteger)headIndex
headName:(nullable NSString *)headName
categories:(NSArray<MPPCategory *> *)categories {
self = [super init];
if (self) {
_headIndex = headIndex;
_headName = headName;
_categories = categories;
}
return self;
}
- (instancetype)initWithHeadIndex:(NSInteger)headIndex
categories:(NSArray<MPPCategory *> *)categories {
return [self initWithHeadIndex:headIndex headName:nil categories:categories];
}
@end
@implementation MPPClassificationResult
- (instancetype)initWithClassifications:(NSArray<MPPClassifications *> *)classifications
timestampMs:(NSInteger)timestampMs {
self = [super init];
if (self) {
_classifications = classifications;
_timestampMs = timestampMs;
}
return self;
}
@end

View File

@ -0,0 +1,40 @@
# Copyright 2023 The MediaPipe Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
objc_library(
name = "MPPCategoryHelpers",
srcs = ["sources/MPPCategory+Helpers.mm"],
hdrs = ["sources/MPPCategory+Helpers.h"],
deps = [
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/tasks/ios/common/utils:NSStringHelpers",
"//mediapipe/tasks/ios/components/containers:MPPCategory",
],
)
objc_library(
name = "MPPClassificationResultHelpers",
srcs = ["sources/MPPClassificationResult+Helpers.mm"],
hdrs = ["sources/MPPClassificationResult+Helpers.h"],
deps = [
":MPPCategoryHelpers",
"//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto",
"//mediapipe/tasks/ios/common/utils:NSStringHelpers",
"//mediapipe/tasks/ios/components/containers:MPPClassificationResult",
],
)

View File

@ -1,4 +1,4 @@
// Copyright 2022 The MediaPipe Authors. // Copyright 2023 The MediaPipe Authors.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License. // you may not use this file except in compliance with the License.
@ -12,15 +12,15 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" #include "mediapipe/framework/formats/classification.pb.h"
#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h"
#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h"
NS_ASSUME_NONNULL_BEGIN NS_ASSUME_NONNULL_BEGIN
@interface MPPClassifierOptions (Helpers) @interface MPPCategory (Helpers)
- (void)copyToProto:
(mediapipe::tasks::components::processors::proto::ClassifierOptions *)classifierOptionsProto; + (MPPCategory *)categoryWithProto:(const ::mediapipe::Classification &)classificationProto;
@end @end
NS_ASSUME_NONNULL_END NS_ASSUME_NONNULL_END

View File

@ -0,0 +1,42 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h"
#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h"
namespace {
using ClassificationProto = ::mediapipe::Classification;
}
@implementation MPPCategory (Helpers)
+ (MPPCategory *)categoryWithProto:(const ClassificationProto &)clasificationProto {
NSString *categoryName;
NSString *displayName;
if (clasificationProto.has_label()) {
categoryName = [NSString stringWithCppString:clasificationProto.label()];
}
if (clasificationProto.has_display_name()) {
displayName = [NSString stringWithCppString:clasificationProto.display_name()];
}
return [[MPPCategory alloc] initWithIndex:clasificationProto.index()
score:clasificationProto.score()
categoryName:categoryName
displayName:displayName];
}
@end

View File

@ -0,0 +1,35 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h"
#import "mediapipe/tasks/ios/components/containers/sources/MPPClassificationResult.h"
NS_ASSUME_NONNULL_BEGIN
@interface MPPClassifications (Helpers)
+ (MPPClassifications *)classificationsWithProto:
(const mediapipe::tasks::components::containers::proto::Classifications &)classificationsProto;
@end
@interface MPPClassificationResult (Helpers)
+ (MPPClassificationResult *)classificationResultWithProto:
(const mediapipe::tasks::components::containers::proto::ClassificationResult &)
classificationResultProto;
@end
NS_ASSUME_NONNULL_END

View File

@ -0,0 +1,68 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h"
#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPCategory+Helpers.h"
#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h"
namespace {
using ClassificationsProto = ::mediapipe::tasks::components::containers::proto::Classifications;
using ClassificationResultProto =
::mediapipe::tasks::components::containers::proto::ClassificationResult;
} // namespace
@implementation MPPClassifications (Helpers)
+ (MPPClassifications *)classificationsWithProto:
(const ClassificationsProto &)classificationsProto {
NSMutableArray<MPPCategory *> *categories =
[NSMutableArray arrayWithCapacity:(NSUInteger)classificationsProto.classification_list()
.classification_size()];
for (const auto &classification : classificationsProto.classification_list().classification()) {
[categories addObject:[MPPCategory categoryWithProto:classification]];
}
NSString *headName;
if (classificationsProto.has_head_name()) {
headName = [NSString stringWithCppString:classificationsProto.head_name()];
}
return [[MPPClassifications alloc] initWithHeadIndex:(NSInteger)classificationsProto.head_index()
headName:headName
categories:categories];
}
@end
@implementation MPPClassificationResult (Helpers)
+ (MPPClassificationResult *)classificationResultWithProto:
(const ClassificationResultProto &)classificationResultProto {
NSMutableArray *classifications = [NSMutableArray
arrayWithCapacity:(NSUInteger)classificationResultProto.classifications_size()];
for (const auto &classificationsProto : classificationResultProto.classifications()) {
[classifications addObject:[MPPClassifications classificationsWithProto:classificationsProto]];
}
NSInteger timestampMs = 0;
if (classificationResultProto.has_timestamp_ms()) {
timestampMs = (NSInteger)classificationResultProto.timestamp_ms();
}
return [[MPPClassificationResult alloc] initWithClassifications:classifications
timestampMs:timestampMs];
;
}
@end

View File

@ -1,23 +0,0 @@
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
objc_library(
name = "MPPClassifierOptions",
srcs = ["sources/MPPClassifierOptions.m"],
hdrs = ["sources/MPPClassifierOptions.h"],
)

View File

@ -1,60 +0,0 @@
// Copyright 2022 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import <Foundation/Foundation.h>
NS_ASSUME_NONNULL_BEGIN
/**
* Holds settings for any single iOS MediaPipe classification task.
*/
NS_SWIFT_NAME(ClassifierOptions)
@interface MPPClassifierOptions : NSObject <NSCopying>
/**
* The locale to use for display names specified through the TFLite Model
* Metadata, if any. Defaults to English.
*/
@property(nonatomic, copy) NSString *displayNamesLocale;
/**
* The maximum number of top-scored classification results to return. If < 0,
* all available results will be returned. If 0, an invalid argument error is
* returned.
*/
@property(nonatomic) NSInteger maxResults;
/**
* Score threshold to override the one provided in the model metadata (if any).
* Results below this value are rejected.
*/
@property(nonatomic) float scoreThreshold;
/**
* The allowlist of category names. If non-empty, detection results whose
* category name is not in this set will be filtered out. Duplicate or unknown
* category names are ignored. Mutually exclusive with categoryDenylist.
*/
@property(nonatomic, copy) NSArray<NSString *> *categoryAllowlist;
/**
* The denylist of category names. If non-empty, detection results whose
* category name is in this set will be filtered out. Duplicate or unknown
* category names are ignored. Mutually exclusive with categoryAllowlist.
*/
@property(nonatomic, copy) NSArray<NSString *> *categoryDenylist;
@end
NS_ASSUME_NONNULL_END

View File

@ -1,40 +0,0 @@
// Copyright 2022 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "mediapipe/tasks/ios/components/processors/sources/MPPClassifierOptions.h"
@implementation MPPClassifierOptions
- (instancetype)init {
self = [super init];
if (self) {
_maxResults = -1;
_scoreThreshold = 0;
}
return self;
}
- (id)copyWithZone:(NSZone *)zone {
MPPClassifierOptions *classifierOptions = [[MPPClassifierOptions alloc] init];
classifierOptions.scoreThreshold = self.scoreThreshold;
classifierOptions.maxResults = self.maxResults;
classifierOptions.categoryDenylist = self.categoryDenylist;
classifierOptions.categoryAllowlist = self.categoryAllowlist;
classifierOptions.displayNamesLocale = self.displayNamesLocale;
return classifierOptions;
}
@end

View File

@ -1,43 +0,0 @@
// Copyright 2022 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h"
#import "mediapipe/tasks/ios/components/processors/utils/sources/MPPClassifierOptions+Helpers.h"
namespace {
using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions;
}
@implementation MPPClassifierOptions (Helpers)
- (void)copyToProto:(ClassifierOptionsProto *)classifierOptionsProto {
classifierOptionsProto->Clear();
if (self.displayNamesLocale) {
classifierOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString);
}
classifierOptionsProto->set_max_results((int)self.maxResults);
classifierOptionsProto->set_score_threshold(self.scoreThreshold);
for (NSString *category in self.categoryAllowlist) {
classifierOptionsProto->add_category_allowlist(category.cppString);
}
for (NSString *category in self.categoryDenylist) {
classifierOptionsProto->add_category_denylist(category.cppString);
}
}
@end