diff --git a/mediapipe/tasks/ios/BUILD b/mediapipe/tasks/ios/BUILD index d9be847f0..fae60e243 100644 --- a/mediapipe/tasks/ios/BUILD +++ b/mediapipe/tasks/ios/BUILD @@ -49,11 +49,12 @@ OBJC_TASK_COMMON_DEPS = [ ] CALCULATORS_AND_GRAPHS = [ - "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", - "//mediapipe/tasks/cc/vision/object_detector:object_detector_graph", + "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/tasks/cc/text/text_classifier:text_classifier_graph", "//mediapipe/tasks/cc/text/text_embedder:text_embedder_graph", - "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph", + "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", + "//mediapipe/tasks/cc/vision/object_detector:object_detector_graph", ] strip_api_include_path_prefix( @@ -76,6 +77,9 @@ strip_api_include_path_prefix( "//mediapipe/tasks/ios/text/text_embedder:sources/MPPTextEmbedderResult.h", "//mediapipe/tasks/ios/vision/core:sources/MPPRunningMode.h", "//mediapipe/tasks/ios/vision/core:sources/MPPImage.h", + "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetector.h", + "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetectorOptions.h", + "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetectorResult.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifier.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierOptions.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierResult.h", @@ -157,6 +161,9 @@ apple_static_xcframework( ":MPPTaskResult.h", ":MPPImage.h", ":MPPRunningMode.h", + ":MPPFaceDetector.h", + ":MPPFaceDetectorOptions.h", + ":MPPFaceDetectorResult.h", ":MPPImageClassifier.h", ":MPPImageClassifierOptions.h", ":MPPImageClassifierResult.h", @@ -165,6 +172,7 @@ apple_static_xcframework( ":MPPObjectDetectorResult.h", ], deps = [ + "//mediapipe/tasks/ios/vision/face_detector:MPPFaceDetector", "//mediapipe/tasks/ios/vision/image_classifier:MPPImageClassifier", "//mediapipe/tasks/ios/vision/object_detector:MPPObjectDetector", ], diff --git a/mediapipe/tasks/ios/vision/face_detector/BUILD b/mediapipe/tasks/ios/vision/face_detector/BUILD new file mode 100644 index 000000000..e4fc15616 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/BUILD @@ -0,0 +1,62 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPFaceDetectorResult", + srcs = ["sources/MPPFaceDetectorResult.m"], + hdrs = ["sources/MPPFaceDetectorResult.h"], + deps = [ + "//mediapipe/tasks/ios/components/containers:MPPDetection", + "//mediapipe/tasks/ios/core:MPPTaskResult", + ], +) + +objc_library( + name = "MPPFaceDetectorOptions", + srcs = ["sources/MPPFaceDetectorOptions.m"], + hdrs = ["sources/MPPFaceDetectorOptions.h"], + deps = [ + ":MPPFaceDetectorResult", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/vision/core:MPPRunningMode", + ], +) + +objc_library( + name = "MPPFaceDetector", + srcs = ["sources/MPPFaceDetector.mm"], + hdrs = ["sources/MPPFaceDetector.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + ":MPPFaceDetectorOptions", + ":MPPFaceDetectorResult", + "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/vision/core:MPPImage", + "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/face_detector/utils:MPPFaceDetectorOptionsHelpers", + "//mediapipe/tasks/ios/vision/face_detector/utils:MPPFaceDetectorResultHelpers", + ], +) diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h new file mode 100644 index 000000000..78f2fafbf --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h @@ -0,0 +1,190 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Class that performs face detection on images. + * + * The API expects a TFLite model with mandatory TFLite Model Metadata. + * + * The API supports models with one image input tensor and one or more output tensors. To be more + * specific, here are the requirements: + * + * Input tensor + * (kTfLiteUInt8/kTfLiteFloat32) + * - image input of size `[batch x height x width x channels]`. + * - batch inference is not supported (`batch` is required to be 1). + * - only RGB inputs are supported (`channels` is required to be 3). + * - if type is kTfLiteFloat32, NormalizationOptions are required to be attached to the metadata + * for input normalization. + * + * Output tensors must be the 4 outputs of a `DetectionPostProcess` op, i.e:(kTfLiteFloat32) + * (kTfLiteUInt8/kTfLiteFloat32) + * - locations tensor of size `[num_results x 4]`, the inner array representing bounding boxes + * in the form [top, left, right, bottom]. + * - BoundingBoxProperties are required to be attached to the metadata and must specify + * type=BOUNDARIES and coordinate_type=RATIO. + * (kTfLiteFloat32) + * - classes tensor of size `[num_results]`, each value representing the integer index of a + * class. + * - scores tensor of size `[num_results]`, each value representing the score of the detected + * face. + * - optional score calibration can be attached using ScoreCalibrationOptions and an + * AssociatedFile with type TENSOR_AXIS_SCORE_CALIBRATION. See metadata_schema.fbs [1] for more + * details. + * (kTfLiteFloat32) + * - integer num_results as a tensor of size `[1]` + */ +NS_SWIFT_NAME(FaceDetector) +@interface MPPFaceDetector : NSObject + +/** + * Creates a new instance of `MPPFaceDetector` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `MPPFaceDetector`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * @param error An optional error parameter populated when there is an error in initializing the + * face detector. + * + * @return A new instance of `MPPFaceDetector` with the given model path. `nil` if there is an + * error in initializing the face detector. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `MPPFaceDetector` from the given `MPPFaceDetectorOptions`. + * + * @param options The options of type `MPPFaceDetectorOptions` to use for configuring the + * `MPPFaceDetector`. + * @param error An optional error parameter populated when there is an error in initializing the + * face detector. + * + * @return A new instance of `MPPFaceDetector` with the given options. `nil` if there is an error + * in initializing the face detector. + */ +- (nullable instancetype)initWithOptions:(MPPFaceDetectorOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs face detection on the provided MPPImage using the whole image as region of + * interest. Rotation will be applied according to the `orientation` property of the provided + * `MPPImage`. Only use this method when the `MPPFaceDetector` is created with + * `MPPRunningModeImage`. + * + * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which face detection is to be performed. + * @param error An optional error parameter populated when there is an error in performing face + * detection on the input image. + * + * @return An `MPPFaceDetectorResult` face that contains a list of detections, each detection + * has a bounding box that is expressed in the unrotated input frame of reference coordinates + * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying + * image data. + */ +- (nullable MPPFaceDetectorResult *)detectInImage:(MPPImage *)image + error:(NSError **)error NS_SWIFT_NAME(detect(image:)); + +/** + * Performs face detection on the provided video frame of type `MPPImage` using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPFaceDetector` is created with + * `MPPRunningModeVideo`. + * + * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which face detection is to be performed. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing face + * detection on the input image. + * + * @return An `MPPFaceDetectorResult` face that contains a list of detections, each detection + * has a bounding box that is expressed in the unrotated input frame of reference coordinates + * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying + * image data. + */ +- (nullable MPPFaceDetectorResult *)detectInVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); + +/** + * Sends live stream image data of type `MPPImage` to perform face detection using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPFaceDetector` is created with + * `MPPRunningModeLiveStream`. + * + * The object which needs to be continuously notified of the available results of face + * detection must confirm to `MPPFaceDetectorLiveStreamDelegate` protocol and implement the + * `faceDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` delegate method. + * + * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent + * to the face detector. The input timestamps must be monotonically increasing. + * + * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color + * space is RGB with an Alpha channel. + * + * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you + * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its + * `videoSettings` property. + * + * @param image A live stream image data of type `MPPImage` on which face detection is to be + * performed. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image is sent to the face detector. The input timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing face + * detection on the input live stream image data. + * + * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + */ +- (BOOL)detectAsyncInImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(detectAsync(image:timestampInMilliseconds:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm new file mode 100644 index 000000000..ceb5c957d --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm @@ -0,0 +1,259 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h" + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h" +#import "mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h" + +using ::mediapipe::NormalizedRect; +using ::mediapipe::Packet; +using ::mediapipe::Timestamp; +using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; + +static constexpr int kMicrosecondsPerMillisecond = 1000; + +// Constants for the underlying MP Tasks Graph. See +// https://github.com/google/mediapipe/tree/master/mediapipe/tasks/cc/vision/face_detector/face_detector_graph.cc +static NSString *const kDetectionsStreamName = @"detections_out"; +static NSString *const kDetectionsTag = @"DETECTIONS"; +static NSString *const kImageInStreamName = @"image_in"; +static NSString *const kImageOutStreamName = @"image_out"; +static NSString *const kImageTag = @"IMAGE"; +static NSString *const kNormRectStreamName = @"norm_rect_in"; +static NSString *const kNormRectTag = @"NORM_RECT"; +static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.face_detector.FaceDetectorGraph"; +static NSString *const kTaskName = @"faceDetector"; + +#define InputPacketMap(imagePacket, normalizedRectPacket) \ + { \ + {kImageInStreamName.cppString, imagePacket}, { \ + kNormRectStreamName.cppString, normalizedRectPacket \ + } \ + } + +@interface MPPFaceDetector () { + /** iOS Vision Task Runner */ + MPPVisionTaskRunner *_visionTaskRunner; + dispatch_queue_t _callbackQueue; +} +@property(nonatomic, weak) id faceDetectorLiveStreamDelegate; + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult; +@end + +@implementation MPPFaceDetector + +- (instancetype)initWithOptions:(MPPFaceDetectorOptions *)options error:(NSError **)error { + self = [super init]; + if (self) { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] + initWithTaskGraphName:kTaskGraphName + inputStreams:@[ + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageInStreamName], + [NSString stringWithFormat:@"%@:%@", kNormRectTag, kNormRectStreamName] + ] + outputStreams:@[ + [NSString stringWithFormat:@"%@:%@", kDetectionsTag, kDetectionsStreamName], + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageOutStreamName] + ] + taskOptions:options + enableFlowLimiting:options.runningMode == MPPRunningModeLiveStream + error:error]; + + if (!taskInfo) { + return nil; + } + + PacketsCallback packetsCallback = nullptr; + + if (options.faceDetectorLiveStreamDelegate) { + _faceDetectorLiveStreamDelegate = options.faceDetectorLiveStreamDelegate; + + // Create a private serial dispatch queue in which the delegate method will be called + // asynchronously. This is to ensure that if the client performs a long running operation in + // the delegate method, the queue on which the C++ callbacks is invoked is not blocked and is + // freed up to continue with its operations. + _callbackQueue = dispatch_queue_create( + [MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName], NULL); + + // Capturing `self` as weak in order to avoid `self` being kept in memory + // and cause a retain cycle, after self is set to `nil`. + MPPFaceDetector *__weak weakSelf = self; + packetsCallback = [=](absl::StatusOr liveStreamResult) { + [weakSelf processLiveStreamResult:liveStreamResult]; + }; + } + + _visionTaskRunner = + [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] + runningMode:options.runningMode + packetsCallback:std::move(packetsCallback) + error:error]; + + if (!_visionTaskRunner) { + return nil; + } + } + + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPFaceDetectorOptions *options = [[MPPFaceDetectorOptions alloc] init]; + + options.baseOptions.modelAssetPath = modelPath; + + return [self initWithOptions:options error:error]; +} + +- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectFromRegionOfInterest:CGRectZero + imageSize:CGSizeMake(image.width, image.height) + imageOrientation:image.orientation + ROIAllowed:NO + error:error]; + if (!rect.has_value()) { + return std::nullopt; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (imagePacket.IsEmpty()) { + return std::nullopt; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() + timestampInMilliseconds:timestampInMilliseconds]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + return inputPacketMap; +} + +- (nullable MPPFaceDetectorResult *)detectInImage:(MPPImage *)image error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectFromRegionOfInterest:CGRectZero + imageSize:CGSizeMake(image.width, image.height) + imageOrientation:image.orientation + ROIAllowed:NO + error:error]; + if (!rect.has_value()) { + return nil; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; + if (imagePacket.IsEmpty()) { + return nil; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + + std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap + error:error]; + if (!outputPacketMap.has_value()) { + return nil; + } + + return [MPPFaceDetectorResult + faceDetectorResultWithDetectionsPacket:outputPacketMap + .value()[kDetectionsStreamName.cppString]]; +} + +- (nullable MPPFaceDetectorResult *)detectInVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (!inputPacketMap.has_value()) { + return nil; + } + + std::optional outputPacketMap = + [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + + if (!outputPacketMap.has_value()) { + return nil; + } + + return [MPPFaceDetectorResult + faceDetectorResultWithDetectionsPacket:outputPacketMap + .value()[kDetectionsStreamName.cppString]]; +} + +- (BOOL)detectAsyncInImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (!inputPacketMap.has_value()) { + return NO; + } + + return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; +} + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.faceDetectorLiveStreamDelegate + respondsToSelector:@selector(faceDetector: + didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { + return; + } + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.faceDetectorLiveStreamDelegate faceDetector:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPFaceDetectorResult *result = [MPPFaceDetectorResult + faceDetectorResultWithDetectionsPacket:liveStreamResult + .value()[kDetectionsStreamName.cppString]]; + + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicrosecondsPerMillisecond; + dispatch_async(_callbackQueue, ^{ + [self.faceDetectorLiveStreamDelegate faceDetector:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h new file mode 100644 index 000000000..b5d652683 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h @@ -0,0 +1,101 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h" + +NS_ASSUME_NONNULL_BEGIN + +@class MPPFaceDetector; + +/** + * This protocol defines an interface for the delegates of `MPPFaceDetector` face to receive + * results of performing asynchronous face detection on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). + * + * The delegate of `MPPFaceDetector` must adopt `MPPFaceDetectorLiveStreamDelegate` protocol. + * The methods in this protocol are optional. + */ +NS_SWIFT_NAME(FaceDetectorLiveStreamDelegate) +@protocol MPPFaceDetectorLiveStreamDelegate + +@optional + +/** + * This method notifies a delegate that the results of asynchronous face detection of + * an image submitted to the `MPPFaceDetector` is available. + * + * This method is called on a private serial dispatch queue created by the `MPPFaceDetector` + * for performing the asynchronous delegates calls. + * + * @param faceDetector The face detector which performed the face detection. + * This is useful to test equality when there are multiple instances of `MPPFaceDetector`. + * @param result The `MPPFaceDetectorResult` object that contains a list of detections, each + * detection has a bounding box that is expressed in the unrotated input frame of reference + * coordinates system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the + * underlying image data. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image was sent to the face detector. + * @param error An optional error parameter populated when there is an error in performing face + * detection on the input live stream image data. + */ +- (void)faceDetector:(MPPFaceDetector *)faceDetector + didFinishDetectionWithResult:(nullable MPPFaceDetectorResult *)result + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(nullable NSError *)error + NS_SWIFT_NAME(faceDetector(_:didFinishDetection:timestampInMilliseconds:error:)); +@end + +/** Options for setting up a `MPPFaceDetector`. */ +NS_SWIFT_NAME(FaceDetectorOptions) +@interface MPPFaceDetectorOptions : MPPTaskOptions + +/** + * Running mode of the face detector task. Defaults to `MPPRunningModeImage`. + * `MPPFaceDetector` can be created with one of the following running modes: + * 1. `MPPRunningModeImage`: The mode for performing face detection on single image inputs. + * 2. `MPPRunningModeVideo`: The mode for performing face detection on the decoded frames of a + * video. + * 3. `MPPRunningModeLiveStream`: The mode for performing face detection on a live stream of + * input data, such as from the camera. + */ +@property(nonatomic) MPPRunningMode runningMode; + +/** + * An object that confirms to `MPPFaceDetectorLiveStreamDelegate` protocol. This object must + * implement `faceDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` to receive + * the results of performing asynchronous face detection on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). + */ +@property(nonatomic, weak, nullable) id + faceDetectorLiveStreamDelegate; + +/** + * The minimum confidence score for the face detection to be considered successful. Defaults to + * 0.5. + */ +@property(nonatomic) float minDetectionConfidence; + +/** + * The minimum non-maximum-suppression threshold for face detection to be considered overlapped. + * Defaults to 0.3. + */ +@property(nonatomic) float minSuppressionThreshold; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.m b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.m new file mode 100644 index 000000000..7d990aa69 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.m @@ -0,0 +1,38 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h" + +@implementation MPPFaceDetectorOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _minDetectionConfidence = 0.5; + _minSuppressionThreshold = 0.3; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPFaceDetectorOptions *faceDetectorOptions = [super copyWithZone:zone]; + + faceDetectorOptions.minDetectionConfidence = self.minDetectionConfidence; + faceDetectorOptions.minSuppressionThreshold = self.minSuppressionThreshold; + faceDetectorOptions.faceDetectorLiveStreamDelegate = self.faceDetectorLiveStreamDelegate; + + return faceDetectorOptions; +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h new file mode 100644 index 000000000..67a9082af --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h @@ -0,0 +1,49 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPDetection.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Represents the detection results generated by `MPPFaceDetector`. */ +NS_SWIFT_NAME(FaceDetectorResult) +@interface MPPFaceDetectorResult : MPPTaskResult + +/** + * The array of `MPPDetection` objects each of which has a bounding box that is expressed in the + * unrotated input frame of reference coordinates system, i.e. in `[0,image_width) x + * [0,image_height)`, which are the dimensions of the underlying image data. + */ +@property(nonatomic, readonly) NSArray *detections; + +/** + * Initializes a new `MPPFaceDetectorResult` with the given array of detections and timestamp (in + * milliseconds). + * + * @param detections An array of `MPPDetection` objects each of which has a bounding box that is + * expressed in the unrotated input frame of reference coordinates system, i.e. in `[0,image_width) + * x [0,image_height)`, which are the dimensions of the underlying image data. + * @param timestampInMilliseconds The timestamp (in milliseconds) for this result. + * + * @return An instance of `MPPFaceDetectorResult` initialized with the given array of detections + * and timestamp (in milliseconds). + */ +- (instancetype)initWithDetections:(NSArray *)detections + timestampInMilliseconds:(NSInteger)timestampInMilliseconds; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.m b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.m new file mode 100644 index 000000000..e6ec7fabe --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.m @@ -0,0 +1,28 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h" + +@implementation MPPFaceDetectorResult + +- (instancetype)initWithDetections:(NSArray *)detections + timestampInMilliseconds:(NSInteger)timestampInMilliseconds { + self = [super initWithTimestampInMilliseconds:timestampInMilliseconds]; + if (self) { + _detections = [detections copy]; + } + return self; +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_detector/utils/BUILD b/mediapipe/tasks/ios/vision/face_detector/utils/BUILD new file mode 100644 index 000000000..2659fcca6 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/utils/BUILD @@ -0,0 +1,42 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPFaceDetectorOptionsHelpers", + srcs = ["sources/MPPFaceDetectorOptions+Helpers.mm"], + hdrs = ["sources/MPPFaceDetectorOptions+Helpers.h"], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol", + "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", + "//mediapipe/tasks/ios/vision/face_detector:MPPFaceDetectorOptions", + ], +) + +objc_library( + name = "MPPFaceDetectorResultHelpers", + srcs = ["sources/MPPFaceDetectorResult+Helpers.mm"], + hdrs = ["sources/MPPFaceDetectorResult+Helpers.h"], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/tasks/ios/components/containers/utils:MPPDetectionHelpers", + "//mediapipe/tasks/ios/vision/face_detector:MPPFaceDetectorResult", + ], +) diff --git a/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h new file mode 100644 index 000000000..16be4a3f1 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h @@ -0,0 +1,36 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __cplusplus +#error "This file requires Objective-C++." +#endif // __cplusplus + +#include "mediapipe/framework/calculator_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPFaceDetectorOptions (Helpers) + +/** + * Populates the provided `CalculatorOptions` proto container with the current settings. + * + * @param optionsProto The `CalculatorOptions` proto object to copy the settings to. + */ +- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.mm b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.mm new file mode 100644 index 000000000..50335c8e5 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.mm @@ -0,0 +1,39 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h" + +using CalculatorOptionsProto = ::mediapipe::CalculatorOptions; +using FaceDetectorGraphOptionsProto = + ::mediapipe::tasks::vision::face_detector::proto::FaceDetectorGraphOptions; + +@implementation MPPFaceDetectorOptions (Helpers) + +- (void)copyToProto:(CalculatorOptionsProto *)optionsProto { + FaceDetectorGraphOptionsProto *graphOptions = + optionsProto->MutableExtension(FaceDetectorGraphOptionsProto::ext); + + graphOptions->Clear(); + + [self.baseOptions copyToProto:graphOptions->mutable_base_options()]; + graphOptions->set_min_detection_confidence(self.minDetectionConfidence); + graphOptions->set_min_suppression_threshold(self.minSuppressionThreshold); +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h new file mode 100644 index 000000000..8e077d393 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h @@ -0,0 +1,39 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __cplusplus +#error "This file requires Objective-C++." +#endif // __cplusplus + +#include "mediapipe/framework/packet.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPFaceDetectorResult (Helpers) + +/** + * Creates an `MPPFaceDetectorResult` from a MediaPipe packet containing a + * `std::vector`. + * + * @param packet a MediaPipe packet wrapping a `std::vector`. + * + * @return An `MPPFaceDetectorResult` object that contains a list of detections. + */ ++ (nullable MPPFaceDetectorResult *)faceDetectorResultWithDetectionsPacket: + (const ::mediapipe::Packet &)packet; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.mm b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.mm new file mode 100644 index 000000000..67fb20937 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.mm @@ -0,0 +1,45 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h" + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.h" + +using DetectionProto = ::mediapipe::Detection; +using ::mediapipe::Packet; + +static constexpr int kMicrosecondsPerMillisecond = 1000; + +@implementation MPPFaceDetectorResult (Helpers) + ++ (nullable MPPFaceDetectorResult *)faceDetectorResultWithDetectionsPacket:(const Packet &)packet { + NSMutableArray *detections; + + if (packet.ValidateAsType>().ok()) { + const std::vector &detectionProtos = packet.Get>(); + detections = [NSMutableArray arrayWithCapacity:(NSUInteger)detectionProtos.size()]; + for (const auto &detectionProto : detectionProtos) { + [detections addObject:[MPPDetection detectionWithProto:detectionProto]]; + } + } else { + detections = [NSMutableArray arrayWithCapacity:0]; + } + + return + [[MPPFaceDetectorResult alloc] initWithDetections:detections + timestampInMilliseconds:(NSInteger)(packet.Timestamp().Value() / + kMicrosecondsPerMillisecond)]; +} + +@end