Merge pull request #4566 from priankakariatyml:ios-image-segmenter-containers

PiperOrigin-RevId: 542931120

mediapipe/tasks/ios/vision/image_segmenter/BUILD

@@ -0,0 +1,37 @@
+# Copyright 2023 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+objc_library(
+    name = "MPPImageSegmenterResult",
+    srcs = ["sources/MPPImageSegmenterResult.m"],
+    hdrs = ["sources/MPPImageSegmenterResult.h"],
+    deps = [
+        "//mediapipe/tasks/ios/core:MPPTaskResult",
+        "//mediapipe/tasks/ios/vision/core:MPPMask",
+    ],
+)
+
+objc_library(
+    name = "MPPImageSegmenterOptions",
+    srcs = ["sources/MPPImageSegmenterOptions.m"],
+    hdrs = ["sources/MPPImageSegmenterOptions.h"],
+    deps = [
+        ":MPPImageSegmenterResult",
+        "//mediapipe/tasks/ios/core:MPPTaskOptions",
+        "//mediapipe/tasks/ios/vision/core:MPPRunningMode",
+    ],
+)

mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h

@@ -0,0 +1,99 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+
+#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h"
+#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h"
+#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@class MPPImageSegmenter;
+
+/**
+ * This protocol defines an interface for the delegates of `MPPImageSegmenter` object to receive
+ * results of performing asynchronous segmentation on images (i.e, when `runningMode` =
+ * `MPPRunningModeLiveStream`).
+ *
+ * The delegate of `MPPImageSegmenter` must adopt `MPPImageSegmenterLiveStreamDelegate` protocol.
+ * The methods in this protocol are optional.
+ */
+NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate)
+@protocol MPPImageSegmenterLiveStreamDelegate <NSObject>
+
+@required
+
+/**
+ * This method notifies a delegate that the results of asynchronous segmentation of
+ * an image submitted to the `MPPImageSegmenter` is available.
+ *
+ * This method is called on a private serial dispatch queue created by the `MPPImageSegmenter`
+ * for performing the asynchronous delegates calls.
+ *
+ * @param imageSegmenter The image segmenter which performed the segmentation. This is useful to
+ * test equality when there are multiple instances of `MPPImageSegmenter`.
+ * @param result The `MPPImageSegmenterResult` object that contains a list of category or confidence
+ * masks and optional quality scores.
+ * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input
+ * image was sent to the image segmenter.
+ * @param error An optional error parameter populated when there is an error in performing
+ * segmentation on the input live stream image data.
+ */
+- (void)imageSegmenter:(MPPImageSegmenter *)imageSegmenter
+    didFinishSegmentationWithResult:(nullable MPPImageSegmenterResult *)result
+            timestampInMilliseconds:(NSInteger)timestampInMilliseconds
+                              error:(nullable NSError *)error
+    NS_SWIFT_NAME(imageSegmenter(_:didFinishSegmentation:timestampInMilliseconds:error:));
+@end
+
+/** Options for setting up a `MPPImageSegmenter`. */
+NS_SWIFT_NAME(ImageSegmenterOptions)
+@interface MPPImageSegmenterOptions : MPPTaskOptions <NSCopying>
+
+/**
+ * Running mode of the image segmenter task. Defaults to `MPPRunningModeImage`.
+ * `MPPImageSegmenter` can be created with one of the following running modes:
+ *  1. `MPPRunningModeImage`: The mode for performing segmentation on single image inputs.
+ *  2. `MPPRunningModeVideo`: The mode for performing segmentation on the decoded frames of a
+ *      video.
+ *  3. `MPPRunningModeLiveStream`: The mode for performing segmentation on a live stream of
+ *      input data, such as from the camera.
+ */
+@property(nonatomic) MPPRunningMode runningMode;
+
+/**
+ * An object that confirms to `MPPImageSegmenterLiveStreamDelegate` protocol. This object must
+ * implement `imageSegmenter:didFinishSegmentationWithResult:timestampInMilliseconds:error:` to
+ * receive the results of performing asynchronous segmentation on images (i.e, when `runningMode` =
+ * `MPPRunningModeLiveStream`).
+ */
+@property(nonatomic, weak, nullable) id<MPPImageSegmenterLiveStreamDelegate>
+    imageSegmenterLiveStreamDelegate;
+
+/**
+ * The locale to use for display names specified through the TFLite Model Metadata, if any. Defaults
+ * to English.
+ */
+@property(nonatomic, copy) NSString *displayNamesLocale;
+
+/** Represents whether to output confidence masks. */
+@property(nonatomic) BOOL shouldOutputConfidenceMasks;
+
+/** Represents whether to output category mask. */
+@property(nonatomic) BOOL shouldOutputCategoryMasks;
+
+@end
+
+NS_ASSUME_NONNULL_END

mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.m

@@ -0,0 +1,40 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h"
+
+@implementation MPPImageSegmenterOptions
+
+- (instancetype)init {
+  self = [super init];
+  if (self) {
+    _displayNamesLocale = @"en";
+    _shouldOutputConfidenceMasks = YES;
+  }
+  return self;
+}
+
+- (id)copyWithZone:(NSZone *)zone {
+  MPPImageSegmenterOptions *imageSegmenterOptions = [super copyWithZone:zone];
+
+  imageSegmenterOptions.runningMode = self.runningMode;
+  imageSegmenterOptions.shouldOutputConfidenceMasks = self.shouldOutputConfidenceMasks;
+  imageSegmenterOptions.shouldOutputCategoryMasks = self.shouldOutputConfidenceMasks;
+  imageSegmenterOptions.displayNamesLocale = self.displayNamesLocale;
+  imageSegmenterOptions.imageSegmenterLiveStreamDelegate = self.imageSegmenterLiveStreamDelegate;
+
+  return imageSegmenterOptions;
+}
+
+@end

mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h

@@ -0,0 +1,68 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h"
+#import "mediapipe/tasks/ios/vision/core/sources/MPPMask.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+/** Represents the segmentation results generated by `MPPImageSegmenter`. */
+NS_SWIFT_NAME(ImageSegmenterResult)
+@interface MPPImageSegmenterResult : MPPTaskResult
+
+/**
+ * An optional array of `MPPMask` objects. Each `MPPMask` in the array holds a 32 bit float array of
+ * size `image width` * `image height` which represents the confidence mask for each category. Each
+ * element of the float array represents the confidence with which the model predicted that the
+ * corresponding pixel belongs to the category that the mask represents, usually in the range [0,1].
+ */
+@property(nonatomic, readonly, nullable) NSArray<MPPMask *> *confidenceMasks;
+
+/**
+ * An optional `MPPMask` that holds a`UInt8` array of size `image width` * `image height`. Each
+ * element of this array represents the class to which the pixel in the original image was predicted
+ * to belong to.
+ */
+@property(nonatomic, readonly, nullable) MPPMask *categoryMask;
+
+/**
+ * The quality scores of the result masks, in the range of [0, 1]. Defaults to `1` if the model
+ * doesn't output quality scores. Each element corresponds to the score of the category in the model
+ * outputs.
+ */
+@property(nonatomic, readonly, nullable) NSArray<NSNumber *> *qualityScores;
+
+/**
+ * Initializes a new `MPPImageSegmenterResult` with the given array of confidence masks, category
+ * mask, quality scores and timestamp (in milliseconds).
+ *
+ * @param confidenceMasks An optional array of `MPPMask` objects. Each `MPPMask` in the array must
+ * be of type `MPPMaskDataTypeFloat32`.
+ * @param categoryMask An optional `MPMask` object of type `MPPMaskDataTypeUInt8`.
+ * @param qualityScores The quality scores of the result masks of type NSArray<NSNumber *> *. Each
+ * `NSNumber` in the array holds a `float`.
+ * @param timestampInMilliseconds The timestamp (in milliseconds) for this result.
+ *
+ * @return An instance of `MPPImageSegmenterResult` initialized with the given array of confidence
+ * masks, category mask, quality scores and timestamp (in milliseconds).
+ */
+- (instancetype)initWithConfidenceMasks:(nullable NSArray<MPPMask *> *)confidenceMasks
+                           categoryMask:(nullable MPPMask *)categoryMask
+                          qualityScores:(nullable NSArray<NSNumber *> *)qualityScores
+                timestampInMilliseconds:(NSInteger)timestampInMilliseconds;
+
+@end
+
+NS_ASSUME_NONNULL_END

mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.m

@@ -0,0 +1,32 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h"
+
+@implementation MPPImageSegmenterResult
+
+- (instancetype)initWithConfidenceMasks:(NSArray<MPPMask *> *)confidenceMasks
+                           categoryMask:(MPPMask *)categoryMask
+                          qualityScores:(NSArray<NSNumber *> *)qualityScores
+                timestampInMilliseconds:(NSInteger)timestampInMilliseconds {
+  self = [super initWithTimestampInMilliseconds:timestampInMilliseconds];
+  if (self) {
+    _confidenceMasks = confidenceMasks;
+    _categoryMask = categoryMask;
+    _qualityScores = qualityScores;
+  }
+  return self;
+}
+
+@end