From f5ac0637a2112350f4a698635bcef113585ca3e9 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Nov 2023 02:31:48 +0530 Subject: [PATCH] Updated iOS vision/core to add methods for processing region of interest --- mediapipe/tasks/ios/vision/core/BUILD | 2 ++ .../core/sources/MPPVisionPacketCreator.h | 15 +++++++++++++ .../core/sources/MPPVisionPacketCreator.mm | 14 ++++++++++++ .../vision/core/sources/MPPVisionTaskRunner.h | 22 +++++++++++++++++++ 4 files changed, 53 insertions(+) diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index 711b4ff95..edb202316 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -55,6 +55,8 @@ objc_library( "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/tasks/ios/vision/core/utils:MPPImageUtils", + "//mediapipe/tasks/ios/components/containers:MPPRegionOfInterest", + "//mediapipe/tasks/ios/components/containers/utils:MPPRegionOfInterestHelpers", ], ) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h index ed07c6d90..9d6ed34c3 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h @@ -14,6 +14,7 @@ #import +#import "mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" #include "mediapipe/framework/formats/rect.pb.h" @@ -73,4 +74,18 @@ + (mediapipe::Packet)createPacketWithNormalizedRect:(mediapipe::NormalizedRect &)normalizedRect timestampInMilliseconds:(NSInteger)timestampInMilliseconds; +/** + * Creates a MediapPipe Packet wrapping a `RenderData` constructed from an `MPPRegionOfInterest`. + * + * @param regionOfInterest The `MPPRegionOfInterest` to send to the MediaPipe graph. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return The MediaPipe packet containing the `RenderData` constructed from the given + * `MPPRegionOfInterest`. + */ ++ (std::optional)createRenderDataPacketWithRegionOfInterest: + (MPPRegionOfInterest *)regionOfInterest + error:(NSError **)error; + @end diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm index 92c33e09b..ae6efaa35 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h" #import "mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h" #include "mediapipe/framework/formats/image.h" @@ -26,6 +27,7 @@ using ::mediapipe::ImageFrame; using ::mediapipe::MakePacket; using ::mediapipe::NormalizedRect; using ::mediapipe::Packet; +using ::mediapipe::RenderData; using ::mediapipe::Timestamp; } // namespace @@ -64,4 +66,16 @@ using ::mediapipe::Timestamp; .At(Timestamp(int64(timestampInMilliseconds * kMicrosecondsPerMillisecond))); } ++ (std::optional)createRenderDataPacketWithRegionOfInterest: + (MPPRegionOfInterest *)regionOfInterest + error:(NSError **)error { + std::optional renderData = [regionOfInterest getRenderDataWithError:error]; + + if (!renderData.has_value()) { + return std::nullopt; + } + + return MakePacket(std::move(renderData.value())); +} + @end diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index aa0307d71..8c18260c1 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -20,6 +20,8 @@ #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#include "mediapipe/framework/packet.h" + NS_ASSUME_NONNULL_BEGIN /** @@ -190,6 +192,26 @@ NS_ASSUME_NONNULL_BEGIN timestampInMilliseconds:(NSInteger)timeStampInMilliseconds error:(NSError **)error; +/** + * This method creates an input packet map to the C++ task runner with the image and normalized rect + * calculated from the region of interest specified within the bounds of an image. Tasks which need + * to add more entries to the input packet map and build their own custom logic for processing + * images can use this method. + * + * @param image An `MPPImage` input to the task. + * @param regionOfInterest A `CGRect` specifying the region of interest within the given image data + * of type `MPPImage`, on which inference should be performed. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return A `BOOL` indicating if the creation of the input packet map with the image and the + * normalized rect calculated from the region of interest was successful. + */ +- (std::optional>) + inputPacketMapWithMPPImage:(MPPImage *)image + regionOfInterest:(CGRect)roi + error:(NSError **)error; + /** * This method returns a unique dispatch queue name by adding the given suffix and a `UUID` to the * pre-defined queue name prefix for vision tasks. The vision tasks can use this method to get