From 7fe365489dfd2f6726a9db734b244d74e778e4fb Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 23 Jun 2023 20:09:05 +0530 Subject: [PATCH 1/3] Added iOS Image Segmenter Result --- .../tasks/ios/vision/image_segmenter/BUILD | 28 ++++++++ .../sources/MPPImageSegmenterResult.h | 65 +++++++++++++++++++ .../sources/MPPImageSegmenterResult.m | 32 +++++++++ 3 files changed, 125 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/image_segmenter/BUILD create mode 100644 mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h create mode 100644 mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.m diff --git a/mediapipe/tasks/ios/vision/image_segmenter/BUILD b/mediapipe/tasks/ios/vision/image_segmenter/BUILD new file mode 100644 index 000000000..c0700a8d9 --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/BUILD @@ -0,0 +1,28 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPImageSegmenterResult", + srcs = ["sources/MPPImageSegmenterResult.m"], + hdrs = ["sources/MPPImageSegmenterResult.h"], + deps = [ + "//mediapipe/tasks/ios/vision/core:MPPMask", + "//mediapipe/tasks/ios/core:MPPTaskResult", + ], +) + diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h new file mode 100644 index 000000000..c0c299f77 --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h @@ -0,0 +1,65 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPMask.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Represents the segmentation results generated by `MPPImageSegmenter`. */ +NS_SWIFT_NAME(ImageSegmenterResult) +@interface MPPImageSegmenterResult : MPPTaskResult + +/** An optional array of `MPPMask` objects. Each `MPPMask` in the array holds a 32 bit float array + * of size `image width` * `image height` which represents the confidence mask for each category. + * Each element of the float array represents the confidence with which the model predicted that the + * corresponding pixel belongs to the category that the mask represents, usually in the range [0, + * 1]. */ +@property(nonatomic, readonly, nullable) NSArray *confidenceMasks; + +/** An optional `MPPMask` that holds a`UInt8` array of size `image width` * `image height`. Each + * element of this array represents the class to which the pixel in the original image was predicted + * to belong to. */ +@property(nonatomic, readonly, nullable) MPPMask *categoryMask; + +/** + * The quality scores of the result masks, in the range of [0, 1]. Defaults + * to `1` if the model doesn't output quality scores. Each element corresponds to the score of + * the category in the model outputs. + */ +@property(nonatomic, readonly, nullable) NSArray *qualityScores; + +/** + * Initializes a new `MPPImageSegmenterResult` with the given array of confidence masks, category + * mask, quality scores and timestamp (in milliseconds). + * + * @param confidenceMasks An optional array of `MPPMask` objects. Each `MPPMask` in the array must + * be of type `MPPMaskDataTypeFloat32`. + * @param categoryMask An optional `MPMask` object of type `MPPMaskDataTypeUInt8`. + * @param qualityScores The quality scores of the result masks of type NSArray *. Each + * `NSNumber` in the array holds a `float`. + * @param timestampInMilliseconds The timestamp (in milliseconds) for this result. + * + * @return An instance of `MPPImageSegmenterResult` initialized with the given array of confidence + * masks, category mask, quality scores and timestamp (in milliseconds). + */ +- (instancetype)initWithConfidenceMasks:(nullable NSArray *)confidenceMasks + categoryMasks:(nullable MPPMask *)categoryMask + qualityScores:(nullable NSArray *)qualityScores + timestampInMilliseconds:(NSInteger)timestampInMilliseconds; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.m b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.m new file mode 100644 index 000000000..2b11fc160 --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.m @@ -0,0 +1,32 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h" + +@implementation MPPImageSegmenterResult + +- (instancetype)initWithConfidenceMasks:(NSArray *)confidenceMasks + categoryMask:(MPPMask *)categoryMask + qualityScores:(NSArray *)qualityScores + timestampInMilliseconds:(NSInteger)timestampInMilliseconds { + self = [super initWithTimestampInMilliseconds:timestampInMilliseconds]; + if (self) { + _confidenceMasks = confidenceMasks; + _categoryMask = categoryMask; + _qualityScores = qualityScores; + } + return self; +} + +@end From 7623c5a9410068deb4410dc7aad7f130b0eaabe6 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 23 Jun 2023 20:09:18 +0530 Subject: [PATCH 2/3] Added iOS Image Segmenter Options --- .../tasks/ios/vision/image_segmenter/BUILD | 10 ++ .../sources/MPPImageSegmenterOptions.h | 99 +++++++++++++++++++ .../sources/MPPImageSegmenterOptions.m | 40 ++++++++ 3 files changed, 149 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h create mode 100644 mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m diff --git a/mediapipe/tasks/ios/vision/image_segmenter/BUILD b/mediapipe/tasks/ios/vision/image_segmenter/BUILD index c0700a8d9..eb6411852 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/BUILD +++ b/mediapipe/tasks/ios/vision/image_segmenter/BUILD @@ -26,3 +26,13 @@ objc_library( ], ) +objc_library( + name = "MPPImageSegmenterOptions", + srcs = ["sources/MPPImageSegmenterOptions.m"], + hdrs = ["sources/MPPImageSegmenterOptions.h"], + deps = [ + ":MPPImageSegmenterResult", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/vision/core:MPPRunningMode", + ], +) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h new file mode 100644 index 000000000..65a822c1e --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h @@ -0,0 +1,99 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h" + +NS_ASSUME_NONNULL_BEGIN + +@class MPPImageSegmenter; + +/** + * This protocol defines an interface for the delegates of `MPPImageSegmenter` object to receive + * results of performing asynchronous segmentation on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). + * + * The delegate of `MPPImageSegmenter` must adopt `MPPImageSegmenterLiveStreamDelegate` protocol. + * The methods in this protocol are optional. + */ +NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) +@protocol MPPImageSegmenterLiveStreamDelegate + +@optional + +/** + * This method notifies a delegate that the results of asynchronous segmentation of + * an image submitted to the `MPPImageSegmenter` is available. + * + * This method is called on a private serial dispatch queue created by the `MPPImageSegmenter` + * for performing the asynchronous delegates calls. + * + * @param imageSegmenter The image segmenter which performed the segmentation. + * This is useful to test equality when there are multiple instances of `MPPImageSegmenter`. + * @param result The `MPPImageSegmenterResult` object that contains a list of category or confidence + * masks and optional quality scores. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image was sent to the image segmenter. + * @param error An optional error parameter populated when there is an error in performing + * segmentation on the input live stream image data. + */ +- (void)imageSegmenter:(MPPImageSegmenter *)imageSegmenter + didFinishSegmentationWithResult:(nullable MPPImageSegmenterResult *)result + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(nullable NSError *)error + NS_SWIFT_NAME(imageSegmenter(_:didFinishSegmentation:timestampInMilliseconds:error:)); +@end + +/** Options for setting up a `MPPImageSegmenter`. */ +NS_SWIFT_NAME(ObjectDetectorOptions) +@interface MPPImageSegmenterOptions : MPPTaskOptions + +/** + * Running mode of the image segmenter task. Defaults to `MPPRunningModeImage`. + * `MPPImageSegmenter` can be created with one of the following running modes: + * 1. `MPPRunningModeImage`: The mode for performing segmentation on single image inputs. + * 2. `MPPRunningModeVideo`: The mode for performing segmentation on the decoded frames of a + * video. + * 3. `MPPRunningModeLiveStream`: The mode for performing segmentation on a live stream of + * input data, such as from the camera. + */ +@property(nonatomic) MPPRunningMode runningMode; + +/** + * An object that confirms to `MPPImageSegmenterLiveStreamDelegate` protocol. This object must + * implement `imageSegmenter:didFinishSegmentationWithResult:timestampInMilliseconds:error:` to + * receive the results of performing asynchronous segmentation on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). + */ +@property(nonatomic, weak, nullable) id + imageSegmenterLiveStreamDelegate; + +/** + * The locale to use for display names specified through the TFLite Model Metadata, if any. Defaults + * to English. + */ +@property(nonatomic, copy) NSString *displayNamesLocale; + +/** Represents whether to output confidence masks. */ +@property(nonatomic) BOOL shouldOutputConfidenceMasks; + +/** Represents whether to output category mask. */ +@property(nonatomic) BOOL shouldOutputCategoryMasks; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m new file mode 100644 index 000000000..282a729bb --- /dev/null +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m @@ -0,0 +1,40 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h" + +@implementation MPPImageSegmenterOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _displayNamesLocale = @"en"; + _shouldOutputConfidenceMasks = YES; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPImageSegmenterOptions *imageSegmenterOptions = [super copyWithZone:zone]; + + imageSegmenterOptions.runningMode = self.runningMode; + imageSegmenterOptions.shouldOutputConfidenceMasks = self.shouldOutputConfidenceMasks; + imageSegmenterOptions.shouldOutputCategoryMasks = self.shouldOutputConfidenceMasks; + imageSegmenterOptions.displayNamesLocale = self.displayNamesLocale; + imageSegmenterOptions.imageSegmenterLiveStreamDelegate = self.imageSegmenterLiveStreamDelegate; + + return imageSegmenterOptions; +} + +@end From 5dce8f283dbd6be4cff8521866b707b2bd25113f Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Fri, 23 Jun 2023 20:10:42 +0530 Subject: [PATCH 3/3] Updated image segmenter delegate method to be required --- .../vision/image_segmenter/sources/MPPImageSegmenterOptions.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h index 65a822c1e..31ae45b6c 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h @@ -33,7 +33,7 @@ NS_ASSUME_NONNULL_BEGIN NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) @protocol MPPImageSegmenterLiveStreamDelegate -@optional +@required /** * This method notifies a delegate that the results of asynchronous segmentation of