From 3532503354e4ea7c09e9456c200754a10cd4f388 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Nov 2023 02:05:23 +0530 Subject: [PATCH 1/5] Added iOS interactive segmenter options --- .../ios/vision/interactive_segmenter/BUILD | 26 ++++++++++++ .../sources/MPPInteractiveSegmenterOptions.h | 41 +++++++++++++++++++ .../sources/MPPInteractiveSegmenterOptions.m | 38 +++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/interactive_segmenter/BUILD create mode 100644 mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h create mode 100644 mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.m diff --git a/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD b/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD new file mode 100644 index 000000000..f3cd98833 --- /dev/null +++ b/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD @@ -0,0 +1,26 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPInteractiveSegmenterOptions", + srcs = ["sources/MPPInteractiveSegmenterOptions.m"], + hdrs = ["sources/MPPInteractiveSegmenterOptions.h"], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskOptions", + ], +) diff --git a/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h new file mode 100644 index 000000000..9ae45b13f --- /dev/null +++ b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h @@ -0,0 +1,41 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@class MPPInteractiveSegmenter; + +/** Options for setting up a `InteractiveSegmenter`. */ +NS_SWIFT_NAME(InteractiveSegmenterOptions) +@interface MPPInteractiveSegmenterOptions : MPPTaskOptions + +/** + * The locale to use for display names specified through the TFLite Model Metadata, if any. Defaults + * to English. + */ +@property(nonatomic, copy) NSString *displayNamesLocale; + +/** Represents whether to output confidence masks. */ +@property(nonatomic) BOOL shouldOutputConfidenceMasks; + +/** Represents whether to output category mask. */ +@property(nonatomic) BOOL shouldOutputCategoryMask; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.m b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.m new file mode 100644 index 000000000..798ac11ce --- /dev/null +++ b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.m @@ -0,0 +1,38 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h" + +@implementation MPPInteractiveSegmenterOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _displayNamesLocale = @"en"; + _shouldOutputConfidenceMasks = YES; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPInteractiveSegmenterOptions *interactiveSegmenterOptions = [super copyWithZone:zone]; + + interactiveSegmenterOptions.shouldOutputConfidenceMasks = self.shouldOutputConfidenceMasks; + interactiveSegmenterOptions.shouldOutputCategoryMask = self.shouldOutputCategoryMask; + interactiveSegmenterOptions.displayNamesLocale = self.displayNamesLocale; + + return interactiveSegmenterOptions; +} + +@end From 4137dbcbf5e69e7b3254010b54996e445d39fb09 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Nov 2023 02:30:42 +0530 Subject: [PATCH 2/5] Added iOS region of interest --- .../tasks/ios/components/containers/BUILD | 7 ++ .../containers/sources/MPPRegionOfInterest.h | 67 +++++++++++++++++++ .../containers/sources/MPPRegionOfInterest.m | 35 ++++++++++ 3 files changed, 109 insertions(+) create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.m diff --git a/mediapipe/tasks/ios/components/containers/BUILD b/mediapipe/tasks/ios/components/containers/BUILD index 0477d288a..4effb74b2 100644 --- a/mediapipe/tasks/ios/components/containers/BUILD +++ b/mediapipe/tasks/ios/components/containers/BUILD @@ -66,3 +66,10 @@ objc_library( srcs = ["sources/MPPConnection.m"], hdrs = ["sources/MPPConnection.h"], ) + +objc_library( + name = "MPPRegionOfInterest", + srcs = ["sources/MPPRegionOfInterest.m"], + hdrs = ["sources/MPPRegionOfInterest.h"], + deps = [":MPPDetection"], +) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h b/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h new file mode 100644 index 000000000..a6b30269d --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h @@ -0,0 +1,67 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPDetection.h" + +NS_ASSUME_NONNULL_BEGIN + +/** The Region-Of-Interest (ROI) to interact with in an interactive segmentation inference. */ +/** An instance can contain erither a single normalized point pointing to the object that the user + * wants to segment or array of normalized key points that make up scribbles over the object that + * the user wants to segment.*/ +NS_SWIFT_NAME(RegionOfInterest) +@interface MPPRegionOfInterest : NSObject + +/** The normalized point pointing to the object that the user wants to segment. `nil if `scribbles` + * is not `nil` */ +@property(nonatomic, readonly, nullable) MPPNormalizedKeypoint *keypoint; + +/** The array of normalized key points that make up scribbles over the object that the user wants to + * segment. `nil if `keypoint` is not `nil` */ +@property(nonatomic, readonly, nullable) NSArray *scribbles; + +/** + * Initializes a new `RegionOfInterest` that represents a single normalized point pointing to the + * object that the user wants to segment. + * + * @param normalizedKeypoint The normalized key point pointing to the object that the user wants to + * segment. + * + * @return An instance of `RegionOfInterest` initialized with the given normalized key point + * pointing to the object that the user wants to segment. + */ +- (instancetype)initWithNormalizedKeyPoint:(MPPNormalizedKeypoint *)normalizedKeypoint + NS_DESIGNATED_INITIALIZER; + +/** + * Initializes a new `RegionOfInterest` that represents scribbles over the object that the user + * wants to segment. + * + * @param scribbles The array of normalized key points that make up scribbles over the object that + * the user wants to segment. + * + * @return An instance of `RegionOfInterest` initialized with the given normalized key points that + * make up scribbles over the object that the user wants to segment. + */ +- (instancetype)initWitScribbles:(NSArray *)scribbles + NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.m b/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.m new file mode 100644 index 000000000..0dfa0e5b4 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.m @@ -0,0 +1,35 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h" + +@implementation MPPRegionOfInterest + +- (instancetype)initWithNormalizedKeyPoint:(MPPNormalizedKeypoint *)normalizedKeypoint { + self = [super init]; + if (self) { + _keypoint = normalizedKeypoint; + } + return self; +} + +- (instancetype)initWitScribbles:(NSArray *)scribbles { + self = [super init]; + if (self) { + _scribbles = scribbles; + } + return self; +} + +@end From 4c02980b3f62d10ae409b629468012b7de8988c0 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Nov 2023 02:31:11 +0530 Subject: [PATCH 3/5] Added iOS region of interest helpers --- .../ios/components/containers/utils/BUILD | 13 +++++ .../sources/MPPRegionOfInterest+Helpers.h | 35 ++++++++++++ .../sources/MPPRegionOfInterest+Helpers.mm | 56 +++++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h create mode 100644 mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.mm diff --git a/mediapipe/tasks/ios/components/containers/utils/BUILD b/mediapipe/tasks/ios/components/containers/utils/BUILD index 5f0311f51..6da949858 100644 --- a/mediapipe/tasks/ios/components/containers/utils/BUILD +++ b/mediapipe/tasks/ios/components/containers/utils/BUILD @@ -16,6 +16,7 @@ package(default_visibility = ["//mediapipe/tasks:internal"]) licenses(["notice"]) + objc_library( name = "MPPCategoryHelpers", srcs = ["sources/MPPCategory+Helpers.mm"], @@ -84,3 +85,15 @@ objc_library( "//mediapipe/tasks/ios/components/containers:MPPLandmark", ], ) + +objc_library( + name = "MPPRegionOfInterestHelpers", + srcs = ["sources/MPPRegionOfInterest+Helpers.mm"], + hdrs = ["sources/MPPRegionOfInterest+Helpers.h"], + deps = [ + "//mediapipe/util:render_data_cc_proto", + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/components/containers:MPPRegionOfInterest", + ], +) diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h b/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h new file mode 100644 index 000000000..026d4f2e9 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h @@ -0,0 +1,35 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/util/render_data.pb.h" + +#import "mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPRegionOfInterest (Helpers) + +/** + * Creates a `RenderData` from the region of interest. + * + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return A `RenderData1 proto created from the region of interest. + */ +- (std::optional)getRenderDataWithError:(NSError **)error; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.mm new file mode 100644 index 000000000..c7f0bc9a0 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.mm @@ -0,0 +1,56 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h" + +namespace { +using RenderData = ::mediapipe::RenderData; +using RenderAnnotation = ::mediapipe::RenderAnnotation; + +} // namespace + +@implementation MPPRegionOfInterest (Helpers) + +- (std::optional)getRenderDataWithError:(NSError**)error { + RenderData result; + if (self.keypoint) { + auto* annotation = result.add_render_annotations(); + annotation->mutable_color()->set_r(255); + auto* point = annotation->mutable_point(); + point->set_normalized(true); + point->set_x(self.keypoint.location.x); + point->set_y(self.keypoint.location.y); + return result; + } else if (self.scribbles) { + auto* annotation = result.add_render_annotations(); + annotation->mutable_color()->set_r(255); + for (MPPNormalizedKeypoint* keypoint in self.scribbles) { + auto* point = annotation->mutable_scribble()->add_point(); + point->set_normalized(true); + point->set_x(self.keypoint.location.x); + point->set_y(self.keypoint.location.y); + } + return result; + } + + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"RegionOfInterest does not include a valid user interaction."]; + + return std::nullopt; +} + +@end From f5ac0637a2112350f4a698635bcef113585ca3e9 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Nov 2023 02:31:48 +0530 Subject: [PATCH 4/5] Updated iOS vision/core to add methods for processing region of interest --- mediapipe/tasks/ios/vision/core/BUILD | 2 ++ .../core/sources/MPPVisionPacketCreator.h | 15 +++++++++++++ .../core/sources/MPPVisionPacketCreator.mm | 14 ++++++++++++ .../vision/core/sources/MPPVisionTaskRunner.h | 22 +++++++++++++++++++ 4 files changed, 53 insertions(+) diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index 711b4ff95..edb202316 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -55,6 +55,8 @@ objc_library( "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/tasks/ios/vision/core/utils:MPPImageUtils", + "//mediapipe/tasks/ios/components/containers:MPPRegionOfInterest", + "//mediapipe/tasks/ios/components/containers/utils:MPPRegionOfInterestHelpers", ], ) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h index ed07c6d90..9d6ed34c3 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h @@ -14,6 +14,7 @@ #import +#import "mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" #include "mediapipe/framework/formats/rect.pb.h" @@ -73,4 +74,18 @@ + (mediapipe::Packet)createPacketWithNormalizedRect:(mediapipe::NormalizedRect &)normalizedRect timestampInMilliseconds:(NSInteger)timestampInMilliseconds; +/** + * Creates a MediapPipe Packet wrapping a `RenderData` constructed from an `MPPRegionOfInterest`. + * + * @param regionOfInterest The `MPPRegionOfInterest` to send to the MediaPipe graph. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return The MediaPipe packet containing the `RenderData` constructed from the given + * `MPPRegionOfInterest`. + */ ++ (std::optional)createRenderDataPacketWithRegionOfInterest: + (MPPRegionOfInterest *)regionOfInterest + error:(NSError **)error; + @end diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm index 92c33e09b..ae6efaa35 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h" #import "mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h" #include "mediapipe/framework/formats/image.h" @@ -26,6 +27,7 @@ using ::mediapipe::ImageFrame; using ::mediapipe::MakePacket; using ::mediapipe::NormalizedRect; using ::mediapipe::Packet; +using ::mediapipe::RenderData; using ::mediapipe::Timestamp; } // namespace @@ -64,4 +66,16 @@ using ::mediapipe::Timestamp; .At(Timestamp(int64(timestampInMilliseconds * kMicrosecondsPerMillisecond))); } ++ (std::optional)createRenderDataPacketWithRegionOfInterest: + (MPPRegionOfInterest *)regionOfInterest + error:(NSError **)error { + std::optional renderData = [regionOfInterest getRenderDataWithError:error]; + + if (!renderData.has_value()) { + return std::nullopt; + } + + return MakePacket(std::move(renderData.value())); +} + @end diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index aa0307d71..8c18260c1 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -20,6 +20,8 @@ #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#include "mediapipe/framework/packet.h" + NS_ASSUME_NONNULL_BEGIN /** @@ -190,6 +192,26 @@ NS_ASSUME_NONNULL_BEGIN timestampInMilliseconds:(NSInteger)timeStampInMilliseconds error:(NSError **)error; +/** + * This method creates an input packet map to the C++ task runner with the image and normalized rect + * calculated from the region of interest specified within the bounds of an image. Tasks which need + * to add more entries to the input packet map and build their own custom logic for processing + * images can use this method. + * + * @param image An `MPPImage` input to the task. + * @param regionOfInterest A `CGRect` specifying the region of interest within the given image data + * of type `MPPImage`, on which inference should be performed. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return A `BOOL` indicating if the creation of the input packet map with the image and the + * normalized rect calculated from the region of interest was successful. + */ +- (std::optional>) + inputPacketMapWithMPPImage:(MPPImage *)image + regionOfInterest:(CGRect)roi + error:(NSError **)error; + /** * This method returns a unique dispatch queue name by adding the given suffix and a `UUID` to the * pre-defined queue name prefix for vision tasks. The vision tasks can use this method to get From 90622475a29457acfc4b13097e4f211acd463b95 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Nov 2023 02:32:08 +0530 Subject: [PATCH 5/5] Added iOS interactive segmenter header --- .../ios/vision/interactive_segmenter/BUILD | 12 ++ .../sources/MPPInteractiveSegmenter.h | 138 ++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenter.h diff --git a/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD b/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD index f3cd98833..9e435485d 100644 --- a/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD +++ b/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD @@ -24,3 +24,15 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskOptions", ], ) + +objc_library( + name = "MPPInteractiveSegmenter", + hdrs = ["sources/MPPInteractiveSegmenter.h"], + module_name = "MPPInteractiveSegmenter", + deps = [ + ":MPPInteractiveSegmenterOptions", + "//mediapipe/tasks/ios/vision/components/containers:MPPRegionOfInterest", + "//mediapipe/tasks/ios/vision/core:MPPImage", + "//mediapipe/tasks/ios/vision/image_segmenter:MPPImageSegmenterResult", + ], +) diff --git a/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenter.h b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenter.h new file mode 100644 index 000000000..214deaec0 --- /dev/null +++ b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenter.h @@ -0,0 +1,138 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h" +#import "mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Class that performs interactive segmentation on images. + * + * Users can represent user interaction through `RegionOfInterest`, which gives a hint to + * `InteractiveSegmenter` to perform segmentation focusing on the given region of interest. + * + * The API expects a TFLite model with mandatory TFLite Model Metadata. + * + * Input tensor: + * (kTfLiteUInt8/kTfLiteFloat32) + * - image input of size `[batch x height x width x channels]`. + * - batch inference is not supported (`batch` is required to be 1). + * - RGB and greyscale inputs are supported (`channels` is required to be + * 1 or 3). + * - if type is kTfLiteFloat32, NormalizationOptions are required to be attached to the metadata + * for input normalization. Output tensors: (kTfLiteUInt8/kTfLiteFloat32) + * - list of segmented masks. + * - if `output_type` is CATEGORY_MASK, uint8 Image, Image vector of size 1. + * - if `output_type` is CONFIDENCE_MASK, float32 Image list of size `channels`. + * - batch is always 1. + * + * An example of such model can be found at: + * https://tfhub.dev/tensorflow/lite-model/deeplabv3/1/metadata/2 + */ +NS_SWIFT_NAME(InteractiveSegmenter) +@interface MPPInteractiveSegmenter : NSObject + +/** + * Get the category label list of the `InteractiveSegmenter` can recognize. For CATEGORY_MASK type, + * the index in the category mask corresponds to the category in the label list. For CONFIDENCE_MASK + * type, the output mask list at index corresponds to the category in the label list. If there is no + * labelmap provided in the model file, empty array is returned. + */ +@property(nonatomic, readonly) NSArray *labels; + +/** + * Creates a new instance of `InteractiveSegmenter` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `InteractiveSegmenterOptions`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * + * @return A new instance of `InteractiveSegmenter` with the given model path. `nil` if there is an + * error in initializing the interactive segmenter. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `InteractiveSegmenter` from the given `InteractiveSegmenterOptions`. + * + * @param options The options of type `InteractiveSegmenterOptions` to use for configuring the + * `InteractiveSegmenter`. + * + * @return A new instance of `InteractiveSegmenter` with the given options. `nil` if there is an + * error in initializing the interactive segmenter. + */ +- (nullable instancetype)initWithOptions:(MPPInteractiveSegmenterOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs segmentation on the provided MPPImage using the specified user's region of interest. + * Rotation will be applied according to the `orientation` property of the provided `MPImage`. Only + * use this method when the `InteractiveSegmenter` is created with running mode, `image`. + * + * This method supports RGBA images. If your `MPImage` has a source type of `pixelBuffer` or + * `sampleBuffer`, the underlying pixel buffer must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. + * + * @param image The `MPImage` on which segmentation is to be performed. + * + * @return An `ImageSegmenterResult` that contains the segmented masks. + */ +- (nullable MPPImageSegmenterResult *)segmentImage:(MPPImage *)image + regionOfInterest:(MPPRegionOfInterest *)regionOfInterest + error:(NSError **)error + NS_SWIFT_NAME(segment(image:regionOfInterest:)); + +/** + * Performs segmentation on the provided MPPImage using the specified user's region of interest and + * invokes the given completion handler block with the response. The method returns synchronously + * once the completion handler returns. + * + * Rotation will be applied according to the `orientation` property of the provided `MPImage`. Only + * use this method when the `InteractiveSegmenter` is created with running mode, `image`. + * + * This method supports RGBA images. If your `MPImage` has a source type of `pixelBuffer` or + * `sampleBuffer`, the underlying pixel buffer must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPImage` has a source type of `image` ensure that the color space is RGB with an Alpha + * channel. + * + * @param image The `MPImage` on which segmentation is to be performed. + * @param completionHandler A block to be invoked with the results of performing segmentation on the + * image. The block takes two arguments, the optional `ImageSegmenterResult` that contains the + * segmented masks if the segmentation was successful and an optional error populated upon failure. + * The lifetime of the returned masks is only guaranteed for the duration of the block. + */ +- (void)segmentImage:(MPPImage *)image + regionOfInterest:(MPPRegionOfInterest *)regionOfInterest + withCompletionHandler:(void (^)(MPPImageSegmenterResult *_Nullable result, + NSError *_Nullable error))completionHandler + NS_SWIFT_NAME(segment(image:regionOfInterest:completion:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END