From 45addac249b065c5dd0bdf4b348990f9a981b101 Mon Sep 17 00:00:00 2001 From: kinaryml Date: Wed, 17 May 2023 10:55:40 +0530 Subject: [PATCH 01/22] Testing MediaPipe Python with GPU support --- mediapipe/tasks/python/core/BUILD | 2 + mediapipe/tasks/python/core/base_options.py | 27 ++++-- setup.py | 102 +++++++++++++++----- 3 files changed, 96 insertions(+), 35 deletions(-) diff --git a/mediapipe/tasks/python/core/BUILD b/mediapipe/tasks/python/core/BUILD index f90826354..c0abefdde 100644 --- a/mediapipe/tasks/python/core/BUILD +++ b/mediapipe/tasks/python/core/BUILD @@ -34,6 +34,8 @@ py_library( ], deps = [ ":optional_dependencies", + "//mediapipe/calculators/tensor/inference_calculator_py_pb2", + "//mediapipe/tasks/cc/core/proto:acceleration_py_pb2", "//mediapipe/tasks/cc/core/proto:base_options_py_pb2", "//mediapipe/tasks/cc/core/proto:external_file_py_pb2", ], diff --git a/mediapipe/tasks/python/core/base_options.py b/mediapipe/tasks/python/core/base_options.py index 90ef045b8..8247f15bd 100644 --- a/mediapipe/tasks/python/core/base_options.py +++ b/mediapipe/tasks/python/core/base_options.py @@ -14,13 +14,18 @@ """Base options for MediaPipe Task APIs.""" import dataclasses +import enum import os from typing import Any, Optional +from mediapipe.calculators.tensor import inference_calculator_pb2 +from mediapipe.tasks.cc.core.proto import acceleration_pb2 from mediapipe.tasks.cc.core.proto import base_options_pb2 from mediapipe.tasks.cc.core.proto import external_file_pb2 from mediapipe.tasks.python.core.optional_dependencies import doc_controls +_DelegateProto = inference_calculator_pb2.InferenceCalculatorOptions.Delegate +_AccelerationProto = acceleration_pb2.Acceleration _BaseOptionsProto = base_options_pb2.BaseOptions _ExternalFileProto = external_file_pb2.ExternalFile @@ -42,10 +47,13 @@ class BaseOptions: model_asset_path: Path to the model asset file. model_asset_buffer: The model asset file contents as bytes. """ + class Delegate(enum.Enum): + CPU = 0 + GPU = 1 model_asset_path: Optional[str] = None model_asset_buffer: Optional[bytes] = None - # TODO: Allow Python API to specify acceleration settings. + delegate: Optional[Delegate] = None @doc_controls.do_not_generate_docs def to_pb2(self) -> _BaseOptionsProto: @@ -55,17 +63,16 @@ class BaseOptions: else: full_path = None + if self.delegate == BaseOptions.Delegate.GPU: + acceleration_proto = _AccelerationProto(gpu=_DelegateProto.Gpu()) + else: + acceleration_proto = _AccelerationProto(tflite=_DelegateProto.TfLite()) + return _BaseOptionsProto( model_asset=_ExternalFileProto( - file_name=full_path, file_content=self.model_asset_buffer)) - - @classmethod - @doc_controls.do_not_generate_docs - def create_from_pb2(cls, pb2_obj: _BaseOptionsProto) -> 'BaseOptions': - """Creates a `BaseOptions` object from the given protobuf object.""" - return BaseOptions( - model_asset_path=pb2_obj.model_asset.file_name, - model_asset_buffer=pb2_obj.model_asset.file_content) + file_name=full_path, file_content=self.model_asset_buffer), + acceleration=acceleration_proto + ) def __eq__(self, other: Any) -> bool: """Checks if this object is equal to the given object. diff --git a/setup.py b/setup.py index 892c6dca7..0712a95b0 100644 --- a/setup.py +++ b/setup.py @@ -245,15 +245,28 @@ class BuildModules(build_ext.build_ext): sys.stderr.write('downloading file: %s\n' % external_file) self._download_external_file(external_file) + # CPU binary graphs + # binary_graphs = [ + # 'face_detection/face_detection_short_range_cpu', + # 'face_detection/face_detection_full_range_cpu', + # 'face_landmark/face_landmark_front_cpu', + # 'hand_landmark/hand_landmark_tracking_cpu', + # 'holistic_landmark/holistic_landmark_cpu', 'objectron/objectron_cpu', + # 'pose_landmark/pose_landmark_cpu', + # 'selfie_segmentation/selfie_segmentation_cpu' + # ] + + # GPU binary graphs binary_graphs = [ - 'face_detection/face_detection_short_range_cpu', - 'face_detection/face_detection_full_range_cpu', - 'face_landmark/face_landmark_front_cpu', - 'hand_landmark/hand_landmark_tracking_cpu', - 'holistic_landmark/holistic_landmark_cpu', 'objectron/objectron_cpu', - 'pose_landmark/pose_landmark_cpu', - 'selfie_segmentation/selfie_segmentation_cpu' + 'face_detection/face_detection_short_range_gpu', + 'face_detection/face_detection_full_range_gpu', + 'face_landmark/face_landmark_front_gpu', + 'hand_landmark/hand_landmark_tracking_gpu', + 'holistic_landmark/holistic_landmark_gpu', 'objectron/objectron_gpu', + 'pose_landmark/pose_landmark_gpu', + 'selfie_segmentation/selfie_segmentation_gpu' ] + for elem in binary_graphs: binary_graph = os.path.join('mediapipe/modules/', elem) sys.stderr.write('generating binarypb: %s\n' % binary_graph) @@ -271,23 +284,42 @@ class BuildModules(build_ext.build_ext): sys.exit(-1) _copy_to_build_lib_dir(self.build_lib, external_file) - def _generate_binary_graph(self, binary_graph_target): - """Generate binary graph for a particular MediaPipe binary graph target.""" + # def _generate_binary_graph(self, binary_graph_target): + # """Generate binary graph for a particular MediaPipe binary graph target.""" + # + # bazel_command = [ + # 'bazel', + # 'build', + # '--compilation_mode=opt', + # '--copt=-DNDEBUG', + # '--define=MEDIAPIPE_DISABLE_GPU=1', + # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), + # binary_graph_target, + # ] + # if not self.link_opencv and not IS_WINDOWS: + # bazel_command.append('--define=OPENCV=source') + # if subprocess.call(bazel_command) != 0: + # sys.exit(-1) + # _copy_to_build_lib_dir(self.build_lib, binary_graph_target + '.binarypb') - bazel_command = [ - 'bazel', - 'build', - '--compilation_mode=opt', - '--copt=-DNDEBUG', - '--define=MEDIAPIPE_DISABLE_GPU=1', - '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), - binary_graph_target, - ] - if not self.link_opencv and not IS_WINDOWS: - bazel_command.append('--define=OPENCV=source') - if subprocess.call(bazel_command) != 0: - sys.exit(-1) - _copy_to_build_lib_dir(self.build_lib, binary_graph_target + '.binarypb') + def _generate_binary_graph(self, binary_graph_target): + """Generate binary graph for a particular MediaPipe binary graph target.""" + + bazel_command = [ + 'bazel', + 'build', + '--compilation_mode=opt', + '--copt=-DNDEBUG', + '--copt=-DMESA_EGL_NO_X11_HEADERS', + '--copt=-DEGL_NO_X11', + '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), + binary_graph_target, + ] + if not self.link_opencv and not IS_WINDOWS: + bazel_command.append('--define=OPENCV=source') + if subprocess.call(bazel_command) != 0: + sys.exit(-1) + _copy_to_build_lib_dir(self.build_lib, binary_graph_target + '.binarypb') class GenerateMetadataSchema(build_ext.build_ext): @@ -300,11 +332,21 @@ class GenerateMetadataSchema(build_ext.build_ext): 'object_detector_metadata_schema_py', 'schema_py', ]: + # bazel_command = [ + # 'bazel', + # 'build', + # '--compilation_mode=opt', + # '--define=MEDIAPIPE_DISABLE_GPU=1', + # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), + # '//mediapipe/tasks/metadata:' + target, + # ] + bazel_command = [ 'bazel', 'build', '--compilation_mode=opt', - '--define=MEDIAPIPE_DISABLE_GPU=1', + '--copt=-DMESA_EGL_NO_X11_HEADERS', + '--copt=-DEGL_NO_X11', '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), '//mediapipe/tasks/metadata:' + target, ] @@ -385,12 +427,22 @@ class BuildExtension(build_ext.build_ext): def _build_binary(self, ext, extra_args=None): if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) + # bazel_command = [ + # 'bazel', + # 'build', + # '--compilation_mode=opt', + # '--copt=-DNDEBUG', + # '--define=MEDIAPIPE_DISABLE_GPU=1', + # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), + # str(ext.bazel_target + '.so'), + # ] bazel_command = [ 'bazel', 'build', '--compilation_mode=opt', '--copt=-DNDEBUG', - '--define=MEDIAPIPE_DISABLE_GPU=1', + '--copt=-DMESA_EGL_NO_X11_HEADERS', + '--copt=-DEGL_NO_X11', '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), str(ext.bazel_target + '.so'), ] From 126df20658c11e6ef97f7c64bd781b9b3d768e9b Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 11:00:12 +0530 Subject: [PATCH 02/22] Included CPU binary graphs in setup.py --- setup.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 0712a95b0..ce162b594 100644 --- a/setup.py +++ b/setup.py @@ -246,18 +246,18 @@ class BuildModules(build_ext.build_ext): self._download_external_file(external_file) # CPU binary graphs - # binary_graphs = [ - # 'face_detection/face_detection_short_range_cpu', - # 'face_detection/face_detection_full_range_cpu', - # 'face_landmark/face_landmark_front_cpu', - # 'hand_landmark/hand_landmark_tracking_cpu', - # 'holistic_landmark/holistic_landmark_cpu', 'objectron/objectron_cpu', - # 'pose_landmark/pose_landmark_cpu', - # 'selfie_segmentation/selfie_segmentation_cpu' - # ] + binary_graphs = [ + 'face_detection/face_detection_short_range_cpu', + 'face_detection/face_detection_full_range_cpu', + 'face_landmark/face_landmark_front_cpu', + 'hand_landmark/hand_landmark_tracking_cpu', + 'holistic_landmark/holistic_landmark_cpu', 'objectron/objectron_cpu', + 'pose_landmark/pose_landmark_cpu', + 'selfie_segmentation/selfie_segmentation_cpu' + ] # GPU binary graphs - binary_graphs = [ + binary_graphs += [ 'face_detection/face_detection_short_range_gpu', 'face_detection/face_detection_full_range_gpu', 'face_landmark/face_landmark_front_gpu', From 620ff3508a1331c192a3103622da55b71fb37d6d Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 12:51:08 +0530 Subject: [PATCH 03/22] Update BUILD --- third_party/BUILD | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/third_party/BUILD b/third_party/BUILD index f6107106d..041812b74 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -167,6 +167,8 @@ cmake_external( "BUILD_PERF_TESTS": "OFF", "BUILD_EXAMPLES": "OFF", "BUILD_SHARED_LIBS": "ON" if OPENCV_SHARED_LIBS else "OFF", + # Disable IPP + "WITH_IPP": "OFF", "WITH_ITT": "OFF", "WITH_JASPER": "OFF", "WITH_JPEG": "ON", @@ -174,6 +176,9 @@ cmake_external( "WITH_TIFF": "ON", "WITH_OPENCL": "OFF", "WITH_WEBP": "OFF", + # Disable carotene_o4t + "ENABLE_NEON": "OFF", + "WITH_TENGINE": "OFF", # Optimization flags "CV_ENABLE_INTRINSICS": "ON", "WITH_EIGEN": "ON", From 14dba421c4406115f7dc059ca71c93a40cf91bf4 Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 13:23:44 +0530 Subject: [PATCH 04/22] Update BUILD --- third_party/BUILD | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/third_party/BUILD b/third_party/BUILD index 041812b74..68f203ccf 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -150,7 +150,7 @@ OPENCV_MODULES = [ # still only builds the shared libraries, so we have to choose one or the # other. We build shared libraries by default, but this variable can be used # to switch to static libraries. -OPENCV_SHARED_LIBS = True +OPENCV_SHARED_LIBS = False OPENCV_SO_VERSION = "3.4" @@ -168,7 +168,7 @@ cmake_external( "BUILD_EXAMPLES": "OFF", "BUILD_SHARED_LIBS": "ON" if OPENCV_SHARED_LIBS else "OFF", # Disable IPP - "WITH_IPP": "OFF", + # "WITH_IPP": "OFF", "WITH_ITT": "OFF", "WITH_JASPER": "OFF", "WITH_JPEG": "ON", @@ -177,8 +177,8 @@ cmake_external( "WITH_OPENCL": "OFF", "WITH_WEBP": "OFF", # Disable carotene_o4t - "ENABLE_NEON": "OFF", - "WITH_TENGINE": "OFF", + # "ENABLE_NEON": "OFF", + # "WITH_TENGINE": "OFF", # Optimization flags "CV_ENABLE_INTRINSICS": "ON", "WITH_EIGEN": "ON", From b5072c59e79fa2d8c9b24dd875ef928f1884144c Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 15:41:08 +0530 Subject: [PATCH 05/22] Update BUILD --- third_party/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/BUILD b/third_party/BUILD index 68f203ccf..6250088a8 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -168,7 +168,7 @@ cmake_external( "BUILD_EXAMPLES": "OFF", "BUILD_SHARED_LIBS": "ON" if OPENCV_SHARED_LIBS else "OFF", # Disable IPP - # "WITH_IPP": "OFF", + "WITH_IPP": "OFF", "WITH_ITT": "OFF", "WITH_JASPER": "OFF", "WITH_JPEG": "ON", From f63baaf8d24bb0b48715ff57944bb5a057a8cfb4 Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 18:08:04 +0530 Subject: [PATCH 06/22] Update BUILD --- third_party/BUILD | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/third_party/BUILD b/third_party/BUILD index 6250088a8..fbec0b351 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -150,7 +150,7 @@ OPENCV_MODULES = [ # still only builds the shared libraries, so we have to choose one or the # other. We build shared libraries by default, but this variable can be used # to switch to static libraries. -OPENCV_SHARED_LIBS = False +OPENCV_SHARED_LIBS = True OPENCV_SO_VERSION = "3.4" @@ -167,8 +167,6 @@ cmake_external( "BUILD_PERF_TESTS": "OFF", "BUILD_EXAMPLES": "OFF", "BUILD_SHARED_LIBS": "ON" if OPENCV_SHARED_LIBS else "OFF", - # Disable IPP - "WITH_IPP": "OFF", "WITH_ITT": "OFF", "WITH_JASPER": "OFF", "WITH_JPEG": "ON", @@ -176,6 +174,9 @@ cmake_external( "WITH_TIFF": "ON", "WITH_OPENCL": "OFF", "WITH_WEBP": "OFF", + # Disable IPP + "WITH_IPP": "OFF", + "WITH_OPENEXR": "OFF", # Disable carotene_o4t # "ENABLE_NEON": "OFF", # "WITH_TENGINE": "OFF", From 6100f0e76eb6667521ebf423ae7750e1cbe6dbf5 Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 18:09:39 +0530 Subject: [PATCH 07/22] Update base_options.py --- mediapipe/tasks/python/core/base_options.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/python/core/base_options.py b/mediapipe/tasks/python/core/base_options.py index 8247f15bd..2d74b16eb 100644 --- a/mediapipe/tasks/python/core/base_options.py +++ b/mediapipe/tasks/python/core/base_options.py @@ -16,6 +16,7 @@ import dataclasses import enum import os +import platform from typing import Any, Optional from mediapipe.calculators.tensor import inference_calculator_pb2 @@ -63,10 +64,22 @@ class BaseOptions: else: full_path = None - if self.delegate == BaseOptions.Delegate.GPU: - acceleration_proto = _AccelerationProto(gpu=_DelegateProto.Gpu()) + platform = platform.system() + + if self.delegate is not None: + if platform == "Linux": + if self.delegate == BaseOptions.Delegate.GPU: + acceleration_proto = _AccelerationProto(gpu=_DelegateProto.Gpu()) + else: + acceleration_proto = _AccelerationProto(tflite=_DelegateProto.TfLite()) + elif platform == "Windows": + raise Exception("Delegate is unsupported for Windows.") + elif platform == "Darwin": + raise Exception("Delegate is unsupported for MacOS.") + else: + raise Exception("Unidentified system") else: - acceleration_proto = _AccelerationProto(tflite=_DelegateProto.TfLite()) + acceleration_proto = None return _BaseOptionsProto( model_asset=_ExternalFileProto( From 01fdeaf1e18e74c352d18ed65e5213eec4845f29 Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 18:29:22 +0530 Subject: [PATCH 08/22] Update Dockerfile --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 03b335823..c7c459190 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:20.04 - +FROM nvidia/cudagl:11.3.0-devel-ubuntu20.04 MAINTAINER WORKDIR /io From b5148c8ce36701a6ecd1df65b5823cde21fa6399 Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 18:32:45 +0530 Subject: [PATCH 09/22] Update setup.py --- setup.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/setup.py b/setup.py index ce162b594..c49169191 100644 --- a/setup.py +++ b/setup.py @@ -284,24 +284,6 @@ class BuildModules(build_ext.build_ext): sys.exit(-1) _copy_to_build_lib_dir(self.build_lib, external_file) - # def _generate_binary_graph(self, binary_graph_target): - # """Generate binary graph for a particular MediaPipe binary graph target.""" - # - # bazel_command = [ - # 'bazel', - # 'build', - # '--compilation_mode=opt', - # '--copt=-DNDEBUG', - # '--define=MEDIAPIPE_DISABLE_GPU=1', - # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), - # binary_graph_target, - # ] - # if not self.link_opencv and not IS_WINDOWS: - # bazel_command.append('--define=OPENCV=source') - # if subprocess.call(bazel_command) != 0: - # sys.exit(-1) - # _copy_to_build_lib_dir(self.build_lib, binary_graph_target + '.binarypb') - def _generate_binary_graph(self, binary_graph_target): """Generate binary graph for a particular MediaPipe binary graph target.""" @@ -332,14 +314,6 @@ class GenerateMetadataSchema(build_ext.build_ext): 'object_detector_metadata_schema_py', 'schema_py', ]: - # bazel_command = [ - # 'bazel', - # 'build', - # '--compilation_mode=opt', - # '--define=MEDIAPIPE_DISABLE_GPU=1', - # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), - # '//mediapipe/tasks/metadata:' + target, - # ] bazel_command = [ 'bazel', @@ -427,15 +401,6 @@ class BuildExtension(build_ext.build_ext): def _build_binary(self, ext, extra_args=None): if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - # bazel_command = [ - # 'bazel', - # 'build', - # '--compilation_mode=opt', - # '--copt=-DNDEBUG', - # '--define=MEDIAPIPE_DISABLE_GPU=1', - # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), - # str(ext.bazel_target + '.so'), - # ] bazel_command = [ 'bazel', 'build', From 69017381af8d647b14242320e3a2c208c2b42d8f Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 24 May 2023 19:57:38 +0530 Subject: [PATCH 10/22] Updated MPPObjectDetectorResult Helpers to return empty result instead of nil --- .../utils/sources/MPPObjectDetectorResult+Helpers.mm | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/ios/vision/object_detector/utils/sources/MPPObjectDetectorResult+Helpers.mm b/mediapipe/tasks/ios/vision/object_detector/utils/sources/MPPObjectDetectorResult+Helpers.mm index b2f9cfc08..3a8a72f71 100644 --- a/mediapipe/tasks/ios/vision/object_detector/utils/sources/MPPObjectDetectorResult+Helpers.mm +++ b/mediapipe/tasks/ios/vision/object_detector/utils/sources/MPPObjectDetectorResult+Helpers.mm @@ -25,8 +25,12 @@ using ::mediapipe::Packet; + (nullable MPPObjectDetectorResult *)objectDetectorResultWithDetectionsPacket: (const Packet &)packet { + + NSInteger timestampInMilliseconds = (NSInteger)(packet.Timestamp().Value() / + kMicroSecondsPerMilliSecond); if (!packet.ValidateAsType>().ok()) { - return nil; + return [[MPPObjectDetectorResult alloc] initWithDetections:@[] + timestampInMilliseconds:timestampInMilliseconds]; } const std::vector &detectionProtos = packet.Get>(); @@ -39,8 +43,7 @@ using ::mediapipe::Packet; return [[MPPObjectDetectorResult alloc] initWithDetections:detections - timestampInMilliseconds:(NSInteger)(packet.Timestamp().Value() / - kMicroSecondsPerMilliSecond)]; + timestampInMilliseconds:timestampInMilliseconds]; } @end From 1e1693d9aaf348fdf1d2fa9d2836bd76056b1c54 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 24 May 2023 20:24:34 +0530 Subject: [PATCH 11/22] Added support to set delegates in MPPBaseOptions --- .../tasks/ios/core/sources/MPPBaseOptions.h | 21 ++++++++++++++++++- .../tasks/ios/core/sources/MPPBaseOptions.m | 1 + .../utils/sources/MPPBaseOptions+Helpers.mm | 13 ++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h index bef6bb9ee..9f61d872a 100644 --- a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h +++ b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h @@ -16,6 +16,17 @@ NS_ASSUME_NONNULL_BEGIN +/** + * MediaPipe Tasks delegate. + */ + typedef NS_ENUM(NSUInteger, MPPDelegate) { + /** CPU. */ + MPPDelegateCPU, + + /** GPU. */ + MPPDelegateGPU + } NS_SWIFT_NAME(Delegate); + /** * Holds the base options that is used for creation of any type of task. It has fields with * important information acceleration configuration, TFLite model source etc. @@ -23,9 +34,17 @@ NS_ASSUME_NONNULL_BEGIN NS_SWIFT_NAME(BaseOptions) @interface MPPBaseOptions : NSObject -/** The path to the model asset to open and mmap in memory. */ +/** + * The absolute path to a model asset file (a tflite model or a model asset bundle file) stored in the app bundle. + */ @property(nonatomic, copy) NSString *modelAssetPath; +/** + * Device delegate to run the MediaPipe pipeline. If the delegate is not set, the default + * delegate CPU is used. + */ +@property(nonatomic) MPPDelegate delegate; + @end NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m index a43119ad8..c3571c4b4 100644 --- a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m +++ b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m @@ -28,6 +28,7 @@ MPPBaseOptions *baseOptions = [[MPPBaseOptions alloc] init]; baseOptions.modelAssetPath = self.modelAssetPath; + baseOptions.delegate = self.delegate; return baseOptions; } diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm index 73bcac49d..a97487cd9 100644 --- a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm @@ -33,6 +33,19 @@ using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions; if (self.modelAssetPath) { baseOptionsProto->mutable_model_asset()->set_file_name(self.modelAssetPath.UTF8String); } + + switch (self.delegate) { + case MPPDelegateCPU: { + baseOptionsProto->mutable_acceleration()->mutable_tflite(); + break; + } + case MPPDelegateGPU: { + baseOptionsProto->mutable_acceleration()->mutable_gpu(); + break; + } + default: + break; + } } @end From 8f1a56f3c2ee36737a6511e7fc951eb65fd1d243 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 24 May 2023 20:24:41 +0530 Subject: [PATCH 12/22] Fixed typos --- .../object_detector/sources/MPPObjectDetector.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h index d3f946bbe..f8cfcc916 100644 --- a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetector.h @@ -80,7 +80,7 @@ NS_SWIFT_NAME(ObjectDetector) * Creates a new instance of `MPPObjectDetector` from the given `MPPObjectDetectorOptions`. * * @param options The options of type `MPPObjectDetectorOptions` to use for configuring the - * `MPPImageClassifMPPObjectDetectorier`. + * `MPPObjectDetector`. * @param error An optional error parameter populated when there is an error in initializing the * object detector. * @@ -96,7 +96,7 @@ NS_SWIFT_NAME(ObjectDetector) * `MPPImage`. Only use this method when the `MPPObjectDetector` is created with * `MPPRunningModeImage`. * - * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * This method supports detecting objects in RGBA images. If your `MPPImage` has a source type of * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer * must have one of the following pixel format types: * 1. kCVPixelFormatType_32BGRA @@ -123,7 +123,7 @@ NS_SWIFT_NAME(ObjectDetector) * the provided `MPPImage`. Only use this method when the `MPPObjectDetector` is created with * `MPPRunningModeVideo`. * - * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * This method supports detecting objects in of RGBA images. If your `MPPImage` has a source type of * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer * must have one of the following pixel format types: * 1. kCVPixelFormatType_32BGRA @@ -161,7 +161,7 @@ NS_SWIFT_NAME(ObjectDetector) * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent * to the object detector. The input timestamps must be monotonically increasing. * - * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * This method supports detecting objects in RGBA images. If your `MPPImage` has a source type of * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer * must have one of the following pixel format types: * 1. kCVPixelFormatType_32BGRA @@ -170,8 +170,8 @@ NS_SWIFT_NAME(ObjectDetector) * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color * space is RGB with an Alpha channel. * - * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you - * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its + * If this method is used for detecting objects in live camera frames using `AVFoundation`, ensure + * that you request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its * `videoSettings` property. * * @param image A live stream image data of type `MPPImage` on which object detection is to be From 1aa44abcaba07ab2e5ba768d6f3e9882a3e694f9 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Wed, 24 May 2023 20:25:45 +0530 Subject: [PATCH 13/22] Revert "Added support to set delegates in MPPBaseOptions" This reverts commit 1e1693d9aaf348fdf1d2fa9d2836bd76056b1c54. --- .../tasks/ios/core/sources/MPPBaseOptions.h | 21 +------------------ .../tasks/ios/core/sources/MPPBaseOptions.m | 1 - .../utils/sources/MPPBaseOptions+Helpers.mm | 13 ------------ 3 files changed, 1 insertion(+), 34 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h index 9f61d872a..bef6bb9ee 100644 --- a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h +++ b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h @@ -16,17 +16,6 @@ NS_ASSUME_NONNULL_BEGIN -/** - * MediaPipe Tasks delegate. - */ - typedef NS_ENUM(NSUInteger, MPPDelegate) { - /** CPU. */ - MPPDelegateCPU, - - /** GPU. */ - MPPDelegateGPU - } NS_SWIFT_NAME(Delegate); - /** * Holds the base options that is used for creation of any type of task. It has fields with * important information acceleration configuration, TFLite model source etc. @@ -34,17 +23,9 @@ NS_ASSUME_NONNULL_BEGIN NS_SWIFT_NAME(BaseOptions) @interface MPPBaseOptions : NSObject -/** - * The absolute path to a model asset file (a tflite model or a model asset bundle file) stored in the app bundle. - */ +/** The path to the model asset to open and mmap in memory. */ @property(nonatomic, copy) NSString *modelAssetPath; -/** - * Device delegate to run the MediaPipe pipeline. If the delegate is not set, the default - * delegate CPU is used. - */ -@property(nonatomic) MPPDelegate delegate; - @end NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m index c3571c4b4..a43119ad8 100644 --- a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m +++ b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m @@ -28,7 +28,6 @@ MPPBaseOptions *baseOptions = [[MPPBaseOptions alloc] init]; baseOptions.modelAssetPath = self.modelAssetPath; - baseOptions.delegate = self.delegate; return baseOptions; } diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm index a97487cd9..73bcac49d 100644 --- a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm @@ -33,19 +33,6 @@ using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions; if (self.modelAssetPath) { baseOptionsProto->mutable_model_asset()->set_file_name(self.modelAssetPath.UTF8String); } - - switch (self.delegate) { - case MPPDelegateCPU: { - baseOptionsProto->mutable_acceleration()->mutable_tflite(); - break; - } - case MPPDelegateGPU: { - baseOptionsProto->mutable_acceleration()->mutable_gpu(); - break; - } - default: - break; - } } @end From 2017fcc9ab593da1717a79353ea706a365f907ea Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 24 May 2023 08:52:13 -0700 Subject: [PATCH 14/22] Add FaceDetector iOS API PiperOrigin-RevId: 534858193 --- mediapipe/tasks/ios/BUILD | 14 +- .../tasks/ios/vision/face_detector/BUILD | 62 +++++ .../face_detector/sources/MPPFaceDetector.h | 190 +++++++++++++ .../face_detector/sources/MPPFaceDetector.mm | 259 ++++++++++++++++++ .../sources/MPPFaceDetectorOptions.h | 101 +++++++ .../sources/MPPFaceDetectorOptions.m | 38 +++ .../sources/MPPFaceDetectorResult.h | 49 ++++ .../sources/MPPFaceDetectorResult.m | 28 ++ .../ios/vision/face_detector/utils/BUILD | 42 +++ .../sources/MPPFaceDetectorOptions+Helpers.h | 36 +++ .../sources/MPPFaceDetectorOptions+Helpers.mm | 39 +++ .../sources/MPPFaceDetectorResult+Helpers.h | 39 +++ .../sources/MPPFaceDetectorResult+Helpers.mm | 45 +++ 13 files changed, 939 insertions(+), 3 deletions(-) create mode 100644 mediapipe/tasks/ios/vision/face_detector/BUILD create mode 100644 mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h create mode 100644 mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm create mode 100644 mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h create mode 100644 mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.m create mode 100644 mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h create mode 100644 mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.m create mode 100644 mediapipe/tasks/ios/vision/face_detector/utils/BUILD create mode 100644 mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.mm create mode 100644 mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h create mode 100644 mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.mm diff --git a/mediapipe/tasks/ios/BUILD b/mediapipe/tasks/ios/BUILD index d9be847f0..fae60e243 100644 --- a/mediapipe/tasks/ios/BUILD +++ b/mediapipe/tasks/ios/BUILD @@ -49,11 +49,12 @@ OBJC_TASK_COMMON_DEPS = [ ] CALCULATORS_AND_GRAPHS = [ - "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", - "//mediapipe/tasks/cc/vision/object_detector:object_detector_graph", + "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/tasks/cc/text/text_classifier:text_classifier_graph", "//mediapipe/tasks/cc/text/text_embedder:text_embedder_graph", - "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph", + "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", + "//mediapipe/tasks/cc/vision/object_detector:object_detector_graph", ] strip_api_include_path_prefix( @@ -76,6 +77,9 @@ strip_api_include_path_prefix( "//mediapipe/tasks/ios/text/text_embedder:sources/MPPTextEmbedderResult.h", "//mediapipe/tasks/ios/vision/core:sources/MPPRunningMode.h", "//mediapipe/tasks/ios/vision/core:sources/MPPImage.h", + "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetector.h", + "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetectorOptions.h", + "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetectorResult.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifier.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierOptions.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierResult.h", @@ -157,6 +161,9 @@ apple_static_xcframework( ":MPPTaskResult.h", ":MPPImage.h", ":MPPRunningMode.h", + ":MPPFaceDetector.h", + ":MPPFaceDetectorOptions.h", + ":MPPFaceDetectorResult.h", ":MPPImageClassifier.h", ":MPPImageClassifierOptions.h", ":MPPImageClassifierResult.h", @@ -165,6 +172,7 @@ apple_static_xcframework( ":MPPObjectDetectorResult.h", ], deps = [ + "//mediapipe/tasks/ios/vision/face_detector:MPPFaceDetector", "//mediapipe/tasks/ios/vision/image_classifier:MPPImageClassifier", "//mediapipe/tasks/ios/vision/object_detector:MPPObjectDetector", ], diff --git a/mediapipe/tasks/ios/vision/face_detector/BUILD b/mediapipe/tasks/ios/vision/face_detector/BUILD new file mode 100644 index 000000000..e4fc15616 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/BUILD @@ -0,0 +1,62 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPFaceDetectorResult", + srcs = ["sources/MPPFaceDetectorResult.m"], + hdrs = ["sources/MPPFaceDetectorResult.h"], + deps = [ + "//mediapipe/tasks/ios/components/containers:MPPDetection", + "//mediapipe/tasks/ios/core:MPPTaskResult", + ], +) + +objc_library( + name = "MPPFaceDetectorOptions", + srcs = ["sources/MPPFaceDetectorOptions.m"], + hdrs = ["sources/MPPFaceDetectorOptions.h"], + deps = [ + ":MPPFaceDetectorResult", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/vision/core:MPPRunningMode", + ], +) + +objc_library( + name = "MPPFaceDetector", + srcs = ["sources/MPPFaceDetector.mm"], + hdrs = ["sources/MPPFaceDetector.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + ":MPPFaceDetectorOptions", + ":MPPFaceDetectorResult", + "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/vision/core:MPPImage", + "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/face_detector/utils:MPPFaceDetectorOptionsHelpers", + "//mediapipe/tasks/ios/vision/face_detector/utils:MPPFaceDetectorResultHelpers", + ], +) diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h new file mode 100644 index 000000000..78f2fafbf --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h @@ -0,0 +1,190 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Class that performs face detection on images. + * + * The API expects a TFLite model with mandatory TFLite Model Metadata. + * + * The API supports models with one image input tensor and one or more output tensors. To be more + * specific, here are the requirements: + * + * Input tensor + * (kTfLiteUInt8/kTfLiteFloat32) + * - image input of size `[batch x height x width x channels]`. + * - batch inference is not supported (`batch` is required to be 1). + * - only RGB inputs are supported (`channels` is required to be 3). + * - if type is kTfLiteFloat32, NormalizationOptions are required to be attached to the metadata + * for input normalization. + * + * Output tensors must be the 4 outputs of a `DetectionPostProcess` op, i.e:(kTfLiteFloat32) + * (kTfLiteUInt8/kTfLiteFloat32) + * - locations tensor of size `[num_results x 4]`, the inner array representing bounding boxes + * in the form [top, left, right, bottom]. + * - BoundingBoxProperties are required to be attached to the metadata and must specify + * type=BOUNDARIES and coordinate_type=RATIO. + * (kTfLiteFloat32) + * - classes tensor of size `[num_results]`, each value representing the integer index of a + * class. + * - scores tensor of size `[num_results]`, each value representing the score of the detected + * face. + * - optional score calibration can be attached using ScoreCalibrationOptions and an + * AssociatedFile with type TENSOR_AXIS_SCORE_CALIBRATION. See metadata_schema.fbs [1] for more + * details. + * (kTfLiteFloat32) + * - integer num_results as a tensor of size `[1]` + */ +NS_SWIFT_NAME(FaceDetector) +@interface MPPFaceDetector : NSObject + +/** + * Creates a new instance of `MPPFaceDetector` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `MPPFaceDetector`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * @param error An optional error parameter populated when there is an error in initializing the + * face detector. + * + * @return A new instance of `MPPFaceDetector` with the given model path. `nil` if there is an + * error in initializing the face detector. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `MPPFaceDetector` from the given `MPPFaceDetectorOptions`. + * + * @param options The options of type `MPPFaceDetectorOptions` to use for configuring the + * `MPPFaceDetector`. + * @param error An optional error parameter populated when there is an error in initializing the + * face detector. + * + * @return A new instance of `MPPFaceDetector` with the given options. `nil` if there is an error + * in initializing the face detector. + */ +- (nullable instancetype)initWithOptions:(MPPFaceDetectorOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs face detection on the provided MPPImage using the whole image as region of + * interest. Rotation will be applied according to the `orientation` property of the provided + * `MPPImage`. Only use this method when the `MPPFaceDetector` is created with + * `MPPRunningModeImage`. + * + * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which face detection is to be performed. + * @param error An optional error parameter populated when there is an error in performing face + * detection on the input image. + * + * @return An `MPPFaceDetectorResult` face that contains a list of detections, each detection + * has a bounding box that is expressed in the unrotated input frame of reference coordinates + * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying + * image data. + */ +- (nullable MPPFaceDetectorResult *)detectInImage:(MPPImage *)image + error:(NSError **)error NS_SWIFT_NAME(detect(image:)); + +/** + * Performs face detection on the provided video frame of type `MPPImage` using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPFaceDetector` is created with + * `MPPRunningModeVideo`. + * + * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is + * RGB with an Alpha channel. + * + * @param image The `MPPImage` on which face detection is to be performed. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing face + * detection on the input image. + * + * @return An `MPPFaceDetectorResult` face that contains a list of detections, each detection + * has a bounding box that is expressed in the unrotated input frame of reference coordinates + * system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the underlying + * image data. + */ +- (nullable MPPFaceDetectorResult *)detectInVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); + +/** + * Sends live stream image data of type `MPPImage` to perform face detection using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPPImage`. Only use this method when the `MPPFaceDetector` is created with + * `MPPRunningModeLiveStream`. + * + * The object which needs to be continuously notified of the available results of face + * detection must confirm to `MPPFaceDetectorLiveStreamDelegate` protocol and implement the + * `faceDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` delegate method. + * + * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent + * to the face detector. The input timestamps must be monotonically increasing. + * + * This method supports classification of RGBA images. If your `MPPImage` has a source type of + * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer + * must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color + * space is RGB with an Alpha channel. + * + * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you + * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its + * `videoSettings` property. + * + * @param image A live stream image data of type `MPPImage` on which face detection is to be + * performed. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image is sent to the face detector. The input timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing face + * detection on the input live stream image data. + * + * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + */ +- (BOOL)detectAsyncInImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(detectAsync(image:timestampInMilliseconds:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm new file mode 100644 index 000000000..ceb5c957d --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.mm @@ -0,0 +1,259 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h" + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h" +#import "mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h" + +using ::mediapipe::NormalizedRect; +using ::mediapipe::Packet; +using ::mediapipe::Timestamp; +using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; + +static constexpr int kMicrosecondsPerMillisecond = 1000; + +// Constants for the underlying MP Tasks Graph. See +// https://github.com/google/mediapipe/tree/master/mediapipe/tasks/cc/vision/face_detector/face_detector_graph.cc +static NSString *const kDetectionsStreamName = @"detections_out"; +static NSString *const kDetectionsTag = @"DETECTIONS"; +static NSString *const kImageInStreamName = @"image_in"; +static NSString *const kImageOutStreamName = @"image_out"; +static NSString *const kImageTag = @"IMAGE"; +static NSString *const kNormRectStreamName = @"norm_rect_in"; +static NSString *const kNormRectTag = @"NORM_RECT"; +static NSString *const kTaskGraphName = @"mediapipe.tasks.vision.face_detector.FaceDetectorGraph"; +static NSString *const kTaskName = @"faceDetector"; + +#define InputPacketMap(imagePacket, normalizedRectPacket) \ + { \ + {kImageInStreamName.cppString, imagePacket}, { \ + kNormRectStreamName.cppString, normalizedRectPacket \ + } \ + } + +@interface MPPFaceDetector () { + /** iOS Vision Task Runner */ + MPPVisionTaskRunner *_visionTaskRunner; + dispatch_queue_t _callbackQueue; +} +@property(nonatomic, weak) id faceDetectorLiveStreamDelegate; + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult; +@end + +@implementation MPPFaceDetector + +- (instancetype)initWithOptions:(MPPFaceDetectorOptions *)options error:(NSError **)error { + self = [super init]; + if (self) { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] + initWithTaskGraphName:kTaskGraphName + inputStreams:@[ + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageInStreamName], + [NSString stringWithFormat:@"%@:%@", kNormRectTag, kNormRectStreamName] + ] + outputStreams:@[ + [NSString stringWithFormat:@"%@:%@", kDetectionsTag, kDetectionsStreamName], + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageOutStreamName] + ] + taskOptions:options + enableFlowLimiting:options.runningMode == MPPRunningModeLiveStream + error:error]; + + if (!taskInfo) { + return nil; + } + + PacketsCallback packetsCallback = nullptr; + + if (options.faceDetectorLiveStreamDelegate) { + _faceDetectorLiveStreamDelegate = options.faceDetectorLiveStreamDelegate; + + // Create a private serial dispatch queue in which the delegate method will be called + // asynchronously. This is to ensure that if the client performs a long running operation in + // the delegate method, the queue on which the C++ callbacks is invoked is not blocked and is + // freed up to continue with its operations. + _callbackQueue = dispatch_queue_create( + [MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName], NULL); + + // Capturing `self` as weak in order to avoid `self` being kept in memory + // and cause a retain cycle, after self is set to `nil`. + MPPFaceDetector *__weak weakSelf = self; + packetsCallback = [=](absl::StatusOr liveStreamResult) { + [weakSelf processLiveStreamResult:liveStreamResult]; + }; + } + + _visionTaskRunner = + [[MPPVisionTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] + runningMode:options.runningMode + packetsCallback:std::move(packetsCallback) + error:error]; + + if (!_visionTaskRunner) { + return nil; + } + } + + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPFaceDetectorOptions *options = [[MPPFaceDetectorOptions alloc] init]; + + options.baseOptions.modelAssetPath = modelPath; + + return [self initWithOptions:options error:error]; +} + +- (std::optional)inputPacketMapWithMPPImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectFromRegionOfInterest:CGRectZero + imageSize:CGSizeMake(image.width, image.height) + imageOrientation:image.orientation + ROIAllowed:NO + error:error]; + if (!rect.has_value()) { + return std::nullopt; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (imagePacket.IsEmpty()) { + return std::nullopt; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value() + timestampInMilliseconds:timestampInMilliseconds]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + return inputPacketMap; +} + +- (nullable MPPFaceDetectorResult *)detectInImage:(MPPImage *)image error:(NSError **)error { + std::optional rect = + [_visionTaskRunner normalizedRectFromRegionOfInterest:CGRectZero + imageSize:CGSizeMake(image.width, image.height) + imageOrientation:image.orientation + ROIAllowed:NO + error:error]; + if (!rect.has_value()) { + return nil; + } + + Packet imagePacket = [MPPVisionPacketCreator createPacketWithMPPImage:image error:error]; + if (imagePacket.IsEmpty()) { + return nil; + } + + Packet normalizedRectPacket = + [MPPVisionPacketCreator createPacketWithNormalizedRect:rect.value()]; + + PacketMap inputPacketMap = InputPacketMap(imagePacket, normalizedRectPacket); + + std::optional outputPacketMap = [_visionTaskRunner processImagePacketMap:inputPacketMap + error:error]; + if (!outputPacketMap.has_value()) { + return nil; + } + + return [MPPFaceDetectorResult + faceDetectorResultWithDetectionsPacket:outputPacketMap + .value()[kDetectionsStreamName.cppString]]; +} + +- (nullable MPPFaceDetectorResult *)detectInVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (!inputPacketMap.has_value()) { + return nil; + } + + std::optional outputPacketMap = + [_visionTaskRunner processVideoFramePacketMap:inputPacketMap.value() error:error]; + + if (!outputPacketMap.has_value()) { + return nil; + } + + return [MPPFaceDetectorResult + faceDetectorResultWithDetectionsPacket:outputPacketMap + .value()[kDetectionsStreamName.cppString]]; +} + +- (BOOL)detectAsyncInImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional inputPacketMap = [self inputPacketMapWithMPPImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + if (!inputPacketMap.has_value()) { + return NO; + } + + return [_visionTaskRunner processLiveStreamPacketMap:inputPacketMap.value() error:error]; +} + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.faceDetectorLiveStreamDelegate + respondsToSelector:@selector(faceDetector: + didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { + return; + } + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.faceDetectorLiveStreamDelegate faceDetector:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPFaceDetectorResult *result = [MPPFaceDetectorResult + faceDetectorResultWithDetectionsPacket:liveStreamResult + .value()[kDetectionsStreamName.cppString]]; + + NSInteger timeStampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicrosecondsPerMillisecond; + dispatch_async(_callbackQueue, ^{ + [self.faceDetectorLiveStreamDelegate faceDetector:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timeStampInMilliseconds + error:callbackError]; + }); +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h new file mode 100644 index 000000000..b5d652683 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h @@ -0,0 +1,101 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h" + +NS_ASSUME_NONNULL_BEGIN + +@class MPPFaceDetector; + +/** + * This protocol defines an interface for the delegates of `MPPFaceDetector` face to receive + * results of performing asynchronous face detection on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). + * + * The delegate of `MPPFaceDetector` must adopt `MPPFaceDetectorLiveStreamDelegate` protocol. + * The methods in this protocol are optional. + */ +NS_SWIFT_NAME(FaceDetectorLiveStreamDelegate) +@protocol MPPFaceDetectorLiveStreamDelegate + +@optional + +/** + * This method notifies a delegate that the results of asynchronous face detection of + * an image submitted to the `MPPFaceDetector` is available. + * + * This method is called on a private serial dispatch queue created by the `MPPFaceDetector` + * for performing the asynchronous delegates calls. + * + * @param faceDetector The face detector which performed the face detection. + * This is useful to test equality when there are multiple instances of `MPPFaceDetector`. + * @param result The `MPPFaceDetectorResult` object that contains a list of detections, each + * detection has a bounding box that is expressed in the unrotated input frame of reference + * coordinates system, i.e. in `[0,image_width) x [0,image_height)`, which are the dimensions of the + * underlying image data. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image was sent to the face detector. + * @param error An optional error parameter populated when there is an error in performing face + * detection on the input live stream image data. + */ +- (void)faceDetector:(MPPFaceDetector *)faceDetector + didFinishDetectionWithResult:(nullable MPPFaceDetectorResult *)result + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(nullable NSError *)error + NS_SWIFT_NAME(faceDetector(_:didFinishDetection:timestampInMilliseconds:error:)); +@end + +/** Options for setting up a `MPPFaceDetector`. */ +NS_SWIFT_NAME(FaceDetectorOptions) +@interface MPPFaceDetectorOptions : MPPTaskOptions + +/** + * Running mode of the face detector task. Defaults to `MPPRunningModeImage`. + * `MPPFaceDetector` can be created with one of the following running modes: + * 1. `MPPRunningModeImage`: The mode for performing face detection on single image inputs. + * 2. `MPPRunningModeVideo`: The mode for performing face detection on the decoded frames of a + * video. + * 3. `MPPRunningModeLiveStream`: The mode for performing face detection on a live stream of + * input data, such as from the camera. + */ +@property(nonatomic) MPPRunningMode runningMode; + +/** + * An object that confirms to `MPPFaceDetectorLiveStreamDelegate` protocol. This object must + * implement `faceDetector:didFinishDetectionWithResult:timestampInMilliseconds:error:` to receive + * the results of performing asynchronous face detection on images (i.e, when `runningMode` = + * `MPPRunningModeLiveStream`). + */ +@property(nonatomic, weak, nullable) id + faceDetectorLiveStreamDelegate; + +/** + * The minimum confidence score for the face detection to be considered successful. Defaults to + * 0.5. + */ +@property(nonatomic) float minDetectionConfidence; + +/** + * The minimum non-maximum-suppression threshold for face detection to be considered overlapped. + * Defaults to 0.3. + */ +@property(nonatomic) float minSuppressionThreshold; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.m b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.m new file mode 100644 index 000000000..7d990aa69 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.m @@ -0,0 +1,38 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h" + +@implementation MPPFaceDetectorOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _minDetectionConfidence = 0.5; + _minSuppressionThreshold = 0.3; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPFaceDetectorOptions *faceDetectorOptions = [super copyWithZone:zone]; + + faceDetectorOptions.minDetectionConfidence = self.minDetectionConfidence; + faceDetectorOptions.minSuppressionThreshold = self.minSuppressionThreshold; + faceDetectorOptions.faceDetectorLiveStreamDelegate = self.faceDetectorLiveStreamDelegate; + + return faceDetectorOptions; +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h new file mode 100644 index 000000000..67a9082af --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h @@ -0,0 +1,49 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPDetection.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Represents the detection results generated by `MPPFaceDetector`. */ +NS_SWIFT_NAME(FaceDetectorResult) +@interface MPPFaceDetectorResult : MPPTaskResult + +/** + * The array of `MPPDetection` objects each of which has a bounding box that is expressed in the + * unrotated input frame of reference coordinates system, i.e. in `[0,image_width) x + * [0,image_height)`, which are the dimensions of the underlying image data. + */ +@property(nonatomic, readonly) NSArray *detections; + +/** + * Initializes a new `MPPFaceDetectorResult` with the given array of detections and timestamp (in + * milliseconds). + * + * @param detections An array of `MPPDetection` objects each of which has a bounding box that is + * expressed in the unrotated input frame of reference coordinates system, i.e. in `[0,image_width) + * x [0,image_height)`, which are the dimensions of the underlying image data. + * @param timestampInMilliseconds The timestamp (in milliseconds) for this result. + * + * @return An instance of `MPPFaceDetectorResult` initialized with the given array of detections + * and timestamp (in milliseconds). + */ +- (instancetype)initWithDetections:(NSArray *)detections + timestampInMilliseconds:(NSInteger)timestampInMilliseconds; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.m b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.m new file mode 100644 index 000000000..e6ec7fabe --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.m @@ -0,0 +1,28 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h" + +@implementation MPPFaceDetectorResult + +- (instancetype)initWithDetections:(NSArray *)detections + timestampInMilliseconds:(NSInteger)timestampInMilliseconds { + self = [super initWithTimestampInMilliseconds:timestampInMilliseconds]; + if (self) { + _detections = [detections copy]; + } + return self; +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_detector/utils/BUILD b/mediapipe/tasks/ios/vision/face_detector/utils/BUILD new file mode 100644 index 000000000..2659fcca6 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/utils/BUILD @@ -0,0 +1,42 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPFaceDetectorOptionsHelpers", + srcs = ["sources/MPPFaceDetectorOptions+Helpers.mm"], + hdrs = ["sources/MPPFaceDetectorOptions+Helpers.h"], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol", + "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", + "//mediapipe/tasks/ios/vision/face_detector:MPPFaceDetectorOptions", + ], +) + +objc_library( + name = "MPPFaceDetectorResultHelpers", + srcs = ["sources/MPPFaceDetectorResult+Helpers.mm"], + hdrs = ["sources/MPPFaceDetectorResult+Helpers.h"], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/tasks/ios/components/containers/utils:MPPDetectionHelpers", + "//mediapipe/tasks/ios/vision/face_detector:MPPFaceDetectorResult", + ], +) diff --git a/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h new file mode 100644 index 000000000..16be4a3f1 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h @@ -0,0 +1,36 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __cplusplus +#error "This file requires Objective-C++." +#endif // __cplusplus + +#include "mediapipe/framework/calculator_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPFaceDetectorOptions (Helpers) + +/** + * Populates the provided `CalculatorOptions` proto container with the current settings. + * + * @param optionsProto The `CalculatorOptions` proto object to copy the settings to. + */ +- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.mm b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.mm new file mode 100644 index 000000000..50335c8e5 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.mm @@ -0,0 +1,39 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorOptions+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h" + +using CalculatorOptionsProto = ::mediapipe::CalculatorOptions; +using FaceDetectorGraphOptionsProto = + ::mediapipe::tasks::vision::face_detector::proto::FaceDetectorGraphOptions; + +@implementation MPPFaceDetectorOptions (Helpers) + +- (void)copyToProto:(CalculatorOptionsProto *)optionsProto { + FaceDetectorGraphOptionsProto *graphOptions = + optionsProto->MutableExtension(FaceDetectorGraphOptionsProto::ext); + + graphOptions->Clear(); + + [self.baseOptions copyToProto:graphOptions->mutable_base_options()]; + graphOptions->set_min_detection_confidence(self.minDetectionConfidence); + graphOptions->set_min_suppression_threshold(self.minSuppressionThreshold); +} + +@end diff --git a/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h new file mode 100644 index 000000000..8e077d393 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h @@ -0,0 +1,39 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __cplusplus +#error "This file requires Objective-C++." +#endif // __cplusplus + +#include "mediapipe/framework/packet.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPFaceDetectorResult (Helpers) + +/** + * Creates an `MPPFaceDetectorResult` from a MediaPipe packet containing a + * `std::vector`. + * + * @param packet a MediaPipe packet wrapping a `std::vector`. + * + * @return An `MPPFaceDetectorResult` object that contains a list of detections. + */ ++ (nullable MPPFaceDetectorResult *)faceDetectorResultWithDetectionsPacket: + (const ::mediapipe::Packet &)packet; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.mm b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.mm new file mode 100644 index 000000000..67fb20937 --- /dev/null +++ b/mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.mm @@ -0,0 +1,45 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/face_detector/utils/sources/MPPFaceDetectorResult+Helpers.h" + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.h" + +using DetectionProto = ::mediapipe::Detection; +using ::mediapipe::Packet; + +static constexpr int kMicrosecondsPerMillisecond = 1000; + +@implementation MPPFaceDetectorResult (Helpers) + ++ (nullable MPPFaceDetectorResult *)faceDetectorResultWithDetectionsPacket:(const Packet &)packet { + NSMutableArray *detections; + + if (packet.ValidateAsType>().ok()) { + const std::vector &detectionProtos = packet.Get>(); + detections = [NSMutableArray arrayWithCapacity:(NSUInteger)detectionProtos.size()]; + for (const auto &detectionProto : detectionProtos) { + [detections addObject:[MPPDetection detectionWithProto:detectionProto]]; + } + } else { + detections = [NSMutableArray arrayWithCapacity:0]; + } + + return + [[MPPFaceDetectorResult alloc] initWithDetections:detections + timestampInMilliseconds:(NSInteger)(packet.Timestamp().Value() / + kMicrosecondsPerMillisecond)]; +} + +@end From d9f316e12a1e59b6d9daa45c64431d29d8700310 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 24 May 2023 08:53:19 -0700 Subject: [PATCH 15/22] Rename ObjectDetctionResult to ObjectDetectorResult PiperOrigin-RevId: 534858600 --- .../com/google/mediapipe/tasks/vision/BUILD | 1 + .../objectdetector/ObjectDetectionResult.java | 22 ++++---- .../vision/objectdetector/ObjectDetector.java | 34 +++++++------ .../objectdetector/ObjectDetectorResult.java | 44 ++++++++++++++++ .../objectdetector/ObjectDetectorTest.java | 50 +++++++++++-------- 5 files changed, 104 insertions(+), 47 deletions(-) create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectorResult.java diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD index 399156da3..cbb1797e2 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD @@ -71,6 +71,7 @@ android_library( srcs = [ "objectdetector/ObjectDetectionResult.java", "objectdetector/ObjectDetector.java", + "objectdetector/ObjectDetectorResult.java", ], javacopts = [ "-Xep:AndroidJdkLibsChecker:OFF", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectionResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectionResult.java index 120cddd46..49ab0ae2b 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectionResult.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectionResult.java @@ -14,15 +14,16 @@ package com.google.mediapipe.tasks.vision.objectdetector; -import com.google.auto.value.AutoValue; import com.google.mediapipe.tasks.core.TaskResult; import com.google.mediapipe.formats.proto.DetectionProto.Detection; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -/** Represents the detection results generated by {@link ObjectDetector}. */ -@AutoValue +/** + * Represents the detection results generated by {@link ObjectDetector}. + * + * @deprecated Use {@link ObjectDetectorResult} instead. + */ +@Deprecated public abstract class ObjectDetectionResult implements TaskResult { @Override @@ -36,15 +37,10 @@ public abstract class ObjectDetectionResult implements TaskResult { * * @param detectionList a list of {@link DetectionOuterClass.Detection} protobuf messages. * @param timestampMs a timestamp for this result. + * @deprecated Use {@link ObjectDetectorResult#create} instead. */ + @Deprecated public static ObjectDetectionResult create(List detectionList, long timestampMs) { - List detections = new ArrayList<>(); - for (Detection detectionProto : detectionList) { - detections.add( - com.google.mediapipe.tasks.components.containers.Detection.createFromProto( - detectionProto)); - } - return new AutoValue_ObjectDetectionResult( - timestampMs, Collections.unmodifiableList(detections)); + return ObjectDetectorResult.create(detectionList, timestampMs); } } diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetector.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetector.java index d9a36cce7..0c70a119d 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetector.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetector.java @@ -99,11 +99,16 @@ public final class ObjectDetector extends BaseVisionTaskApi { private static final String TAG = ObjectDetector.class.getSimpleName(); private static final String IMAGE_IN_STREAM_NAME = "image_in"; private static final String NORM_RECT_IN_STREAM_NAME = "norm_rect_in"; + + @SuppressWarnings("ConstantCaseForConstants") private static final List INPUT_STREAMS = Collections.unmodifiableList( Arrays.asList("IMAGE:" + IMAGE_IN_STREAM_NAME, "NORM_RECT:" + NORM_RECT_IN_STREAM_NAME)); + + @SuppressWarnings("ConstantCaseForConstants") private static final List OUTPUT_STREAMS = Collections.unmodifiableList(Arrays.asList("DETECTIONS:detections_out", "IMAGE:image_out")); + private static final int DETECTIONS_OUT_STREAM_INDEX = 0; private static final int IMAGE_OUT_STREAM_INDEX = 1; private static final String TASK_GRAPH_NAME = "mediapipe.tasks.vision.ObjectDetectorGraph"; @@ -166,19 +171,19 @@ public final class ObjectDetector extends BaseVisionTaskApi { public static ObjectDetector createFromOptions( Context context, ObjectDetectorOptions detectorOptions) { // TODO: Consolidate OutputHandler and TaskRunner. - OutputHandler handler = new OutputHandler<>(); + OutputHandler handler = new OutputHandler<>(); handler.setOutputPacketConverter( - new OutputHandler.OutputPacketConverter() { + new OutputHandler.OutputPacketConverter() { @Override - public ObjectDetectionResult convertToTaskResult(List packets) { + public ObjectDetectorResult convertToTaskResult(List packets) { // If there is no object detected in the image, just returns empty lists. if (packets.get(DETECTIONS_OUT_STREAM_INDEX).isEmpty()) { - return ObjectDetectionResult.create( + return ObjectDetectorResult.create( new ArrayList<>(), BaseVisionTaskApi.generateResultTimestampMs( detectorOptions.runningMode(), packets.get(DETECTIONS_OUT_STREAM_INDEX))); } - return ObjectDetectionResult.create( + return ObjectDetectorResult.create( PacketGetter.getProtoVector( packets.get(DETECTIONS_OUT_STREAM_INDEX), Detection.parser()), BaseVisionTaskApi.generateResultTimestampMs( @@ -235,7 +240,7 @@ public final class ObjectDetector extends BaseVisionTaskApi { * @param image a MediaPipe {@link MPImage} object for processing. * @throws MediaPipeException if there is an internal error. */ - public ObjectDetectionResult detect(MPImage image) { + public ObjectDetectorResult detect(MPImage image) { return detect(image, ImageProcessingOptions.builder().build()); } @@ -258,10 +263,9 @@ public final class ObjectDetector extends BaseVisionTaskApi { * region-of-interest. * @throws MediaPipeException if there is an internal error. */ - public ObjectDetectionResult detect( - MPImage image, ImageProcessingOptions imageProcessingOptions) { + public ObjectDetectorResult detect(MPImage image, ImageProcessingOptions imageProcessingOptions) { validateImageProcessingOptions(imageProcessingOptions); - return (ObjectDetectionResult) processImageData(image, imageProcessingOptions); + return (ObjectDetectorResult) processImageData(image, imageProcessingOptions); } /** @@ -282,7 +286,7 @@ public final class ObjectDetector extends BaseVisionTaskApi { * @param timestampMs the input timestamp (in milliseconds). * @throws MediaPipeException if there is an internal error. */ - public ObjectDetectionResult detectForVideo(MPImage image, long timestampMs) { + public ObjectDetectorResult detectForVideo(MPImage image, long timestampMs) { return detectForVideo(image, ImageProcessingOptions.builder().build(), timestampMs); } @@ -309,10 +313,10 @@ public final class ObjectDetector extends BaseVisionTaskApi { * region-of-interest. * @throws MediaPipeException if there is an internal error. */ - public ObjectDetectionResult detectForVideo( + public ObjectDetectorResult detectForVideo( MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { validateImageProcessingOptions(imageProcessingOptions); - return (ObjectDetectionResult) processVideoData(image, imageProcessingOptions, timestampMs); + return (ObjectDetectorResult) processVideoData(image, imageProcessingOptions, timestampMs); } /** @@ -435,7 +439,7 @@ public final class ObjectDetector extends BaseVisionTaskApi { * object detector is in the live stream mode. */ public abstract Builder setResultListener( - ResultListener value); + ResultListener value); /** Sets an optional {@link ErrorListener}}. */ public abstract Builder setErrorListener(ErrorListener value); @@ -476,11 +480,13 @@ public final class ObjectDetector extends BaseVisionTaskApi { abstract Optional scoreThreshold(); + @SuppressWarnings("AutoValueImmutableFields") abstract List categoryAllowlist(); + @SuppressWarnings("AutoValueImmutableFields") abstract List categoryDenylist(); - abstract Optional> resultListener(); + abstract Optional> resultListener(); abstract Optional errorListener(); diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectorResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectorResult.java new file mode 100644 index 000000000..a17dc90c0 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectorResult.java @@ -0,0 +1,44 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.vision.objectdetector; + +import com.google.auto.value.AutoValue; +import com.google.mediapipe.formats.proto.DetectionProto.Detection; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Represents the detection results generated by {@link ObjectDetector}. */ +@AutoValue +@SuppressWarnings("deprecation") +public abstract class ObjectDetectorResult extends ObjectDetectionResult { + /** + * Creates an {@link ObjectDetectorResult} instance from a list of {@link Detection} protobuf + * messages. + * + * @param detectionList a list of {@link DetectionOuterClass.Detection} protobuf messages. + * @param timestampMs a timestamp for this result. + */ + public static ObjectDetectorResult create(List detectionList, long timestampMs) { + List detections = new ArrayList<>(); + for (Detection detectionProto : detectionList) { + detections.add( + com.google.mediapipe.tasks.components.containers.Detection.createFromProto( + detectionProto)); + } + return new AutoValue_ObjectDetectorResult( + timestampMs, Collections.unmodifiableList(detections)); + } +} diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectorTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectorTest.java index 20ddfcef6..fb83723c5 100644 --- a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectorTest.java +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/objectdetector/ObjectDetectorTest.java @@ -69,7 +69,7 @@ public class ObjectDetectorTest { .build(); ObjectDetector objectDetector = ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options); - ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + ObjectDetectorResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE); } @@ -77,7 +77,7 @@ public class ObjectDetectorTest { public void detect_successWithNoOptions() throws Exception { ObjectDetector objectDetector = ObjectDetector.createFromFile(ApplicationProvider.getApplicationContext(), MODEL_FILE); - ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + ObjectDetectorResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); // Check if the object with the highest score is cat. assertIsCat(results.detections().get(0).categories().get(0), CAT_SCORE); } @@ -91,7 +91,7 @@ public class ObjectDetectorTest { .build(); ObjectDetector objectDetector = ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options); - ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + ObjectDetectorResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); // results should have 8 detected objects because maxResults was set to 8. assertThat(results.detections()).hasSize(8); } @@ -105,7 +105,7 @@ public class ObjectDetectorTest { .build(); ObjectDetector objectDetector = ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options); - ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + ObjectDetectorResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); // The score threshold should block all other other objects, except cat. assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE); } @@ -119,7 +119,7 @@ public class ObjectDetectorTest { .build(); ObjectDetector objectDetector = ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options); - ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + ObjectDetectorResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); // The score threshold should block objects. assertThat(results.detections()).isEmpty(); } @@ -133,7 +133,7 @@ public class ObjectDetectorTest { .build(); ObjectDetector objectDetector = ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options); - ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + ObjectDetectorResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); // Because of the allowlist, results should only contain cat, and there are 6 detected // bounding boxes of cats in CAT_AND_DOG_IMAGE. assertThat(results.detections()).hasSize(5); @@ -148,7 +148,7 @@ public class ObjectDetectorTest { .build(); ObjectDetector objectDetector = ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options); - ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + ObjectDetectorResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); // Because of the denylist, the highest result is not cat anymore. assertThat(results.detections().get(0).categories().get(0).categoryName()) .isNotEqualTo("cat"); @@ -160,7 +160,7 @@ public class ObjectDetectorTest { ObjectDetector.createFromFile( ApplicationProvider.getApplicationContext(), TestUtils.loadFile(ApplicationProvider.getApplicationContext(), MODEL_FILE)); - ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + ObjectDetectorResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); // Check if the object with the highest score is cat. assertIsCat(results.detections().get(0).categories().get(0), CAT_SCORE); } @@ -172,7 +172,7 @@ public class ObjectDetectorTest { ApplicationProvider.getApplicationContext(), TestUtils.loadToDirectByteBuffer( ApplicationProvider.getApplicationContext(), MODEL_FILE)); - ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + ObjectDetectorResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); // Check if the object with the highest score is cat. assertIsCat(results.detections().get(0).categories().get(0), CAT_SCORE); } @@ -191,7 +191,7 @@ public class ObjectDetectorTest { .build(); ObjectDetector objectDetector = ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options); - ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + ObjectDetectorResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE); } @@ -256,7 +256,7 @@ public class ObjectDetectorTest { ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options); ImageProcessingOptions imageProcessingOptions = ImageProcessingOptions.builder().setRotationDegrees(-90).build(); - ObjectDetectionResult results = + ObjectDetectorResult results = objectDetector.detect( getImageFromAsset(CAT_AND_DOG_ROTATED_IMAGE), imageProcessingOptions); @@ -302,7 +302,7 @@ public class ObjectDetectorTest { ObjectDetectorOptions.builder() .setBaseOptions(BaseOptions.builder().setModelAssetPath(MODEL_FILE).build()) .setRunningMode(mode) - .setResultListener((objectDetectionResult, inputImage) -> {}) + .setResultListener((ObjectDetectorResult, inputImage) -> {}) .build()); assertThat(exception) .hasMessageThat() @@ -381,7 +381,7 @@ public class ObjectDetectorTest { ObjectDetectorOptions.builder() .setBaseOptions(BaseOptions.builder().setModelAssetPath(MODEL_FILE).build()) .setRunningMode(RunningMode.LIVE_STREAM) - .setResultListener((objectDetectionResult, inputImage) -> {}) + .setResultListener((ObjectDetectorResult, inputImage) -> {}) .build(); ObjectDetector objectDetector = @@ -411,7 +411,7 @@ public class ObjectDetectorTest { .build(); ObjectDetector objectDetector = ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options); - ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + ObjectDetectorResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE); } @@ -426,7 +426,7 @@ public class ObjectDetectorTest { ObjectDetector objectDetector = ObjectDetector.createFromOptions(ApplicationProvider.getApplicationContext(), options); for (int i = 0; i < 3; i++) { - ObjectDetectionResult results = + ObjectDetectorResult results = objectDetector.detectForVideo( getImageFromAsset(CAT_AND_DOG_IMAGE), /* timestampsMs= */ i); assertContainsOnlyCat(results, CAT_BOUNDING_BOX, CAT_SCORE); @@ -441,8 +441,8 @@ public class ObjectDetectorTest { .setBaseOptions(BaseOptions.builder().setModelAssetPath(MODEL_FILE).build()) .setRunningMode(RunningMode.LIVE_STREAM) .setResultListener( - (objectDetectionResult, inputImage) -> { - assertContainsOnlyCat(objectDetectionResult, CAT_BOUNDING_BOX, CAT_SCORE); + (ObjectDetectorResult, inputImage) -> { + assertContainsOnlyCat(ObjectDetectorResult, CAT_BOUNDING_BOX, CAT_SCORE); assertImageSizeIsExpected(inputImage); }) .setMaxResults(1) @@ -468,8 +468,8 @@ public class ObjectDetectorTest { .setBaseOptions(BaseOptions.builder().setModelAssetPath(MODEL_FILE).build()) .setRunningMode(RunningMode.LIVE_STREAM) .setResultListener( - (objectDetectionResult, inputImage) -> { - assertContainsOnlyCat(objectDetectionResult, CAT_BOUNDING_BOX, CAT_SCORE); + (ObjectDetectorResult, inputImage) -> { + assertContainsOnlyCat(ObjectDetectorResult, CAT_BOUNDING_BOX, CAT_SCORE); assertImageSizeIsExpected(inputImage); }) .setMaxResults(1) @@ -483,6 +483,16 @@ public class ObjectDetectorTest { } } + @Test + @SuppressWarnings("deprecation") + public void detect_canUseDeprecatedApi() throws Exception { + ObjectDetector objectDetector = + ObjectDetector.createFromFile(ApplicationProvider.getApplicationContext(), MODEL_FILE); + ObjectDetectionResult results = objectDetector.detect(getImageFromAsset(CAT_AND_DOG_IMAGE)); + // Check if the object with the highest score is cat. + assertIsCat(results.detections().get(0).categories().get(0), CAT_SCORE); + } + private static MPImage getImageFromAsset(String filePath) throws Exception { AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets(); InputStream istr = assetManager.open(filePath); @@ -491,7 +501,7 @@ public class ObjectDetectorTest { // Checks if results has one and only detection result, which is a cat. private static void assertContainsOnlyCat( - ObjectDetectionResult result, RectF expectedBoundingBox, float expectedScore) { + ObjectDetectorResult result, RectF expectedBoundingBox, float expectedScore) { assertThat(result.detections()).hasSize(1); Detection catResult = result.detections().get(0); assertApproximatelyEqualBoundingBoxes(catResult.boundingBox(), expectedBoundingBox); From acfaf3f1b6e1fcbd73ef6b0b4568e2c7b4dca15a Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 24 May 2023 09:37:45 -0700 Subject: [PATCH 16/22] Add unit test for FaceDetector iOS PiperOrigin-RevId: 534874603 --- .../tasks/ios/test/vision/face_detector/BUILD | 64 +++ .../face_detector/MPPFaceDetectorTests.mm | 522 ++++++++++++++++++ 2 files changed, 586 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/face_detector/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm diff --git a/mediapipe/tasks/ios/test/vision/face_detector/BUILD b/mediapipe/tasks/ios/test/vision/face_detector/BUILD new file mode 100644 index 000000000..49e2fe1bf --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/face_detector/BUILD @@ -0,0 +1,64 @@ +load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "//mediapipe/framework/tool:ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPFaceDetectorObjcTestLibrary", + testonly = 1, + srcs = ["MPPFaceDetectorTests.mm"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + "//mediapipe/tasks/testdata/vision:test_protos", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/components/containers/utils:MPPDetectionHelpers", + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + "//mediapipe/tasks/ios/vision/face_detector:MPPFaceDetector", + "//mediapipe/tasks/ios/vision/face_detector:MPPFaceDetectorResult", + "//third_party/apple_frameworks:UIKit", + ] + select({ + "//third_party:opencv_ios_sim_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_x86_64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//conditions:default": ["@ios_opencv//:OpencvFramework"], + }), +) + +ios_unit_test( + name = "MPPFaceDetectorObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPFaceDetectorObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm b/mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm new file mode 100644 index 000000000..ea0664409 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/face_detector/MPPFaceDetectorTests.mm @@ -0,0 +1,522 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import +#import + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetector.h" +#import "mediapipe/tasks/ios/vision/face_detector/sources/MPPFaceDetectorResult.h" + +static NSDictionary *const kPortraitImage = + @{@"name" : @"portrait", @"type" : @"jpg", @"orientation" : @(UIImageOrientationUp)}; +static NSDictionary *const kPortraitRotatedImage = + @{@"name" : @"portrait_rotated", @"type" : @"jpg", @"orientation" : @(UIImageOrientationRight)}; +static NSDictionary *const kCatImage = @{@"name" : @"cat", @"type" : @"jpg"}; +static NSString *const kShortRangeBlazeFaceModel = @"face_detection_short_range"; +static NSArray *const kPortraitExpectedKeypoints = @[ + @[ @0.44416f, @0.17643f ], @[ @0.55514f, @0.17731f ], @[ @0.50467f, @0.22657f ], + @[ @0.50227f, @0.27199f ], @[ @0.36063f, @0.20143f ], @[ @0.60841f, @0.20409f ] +]; +static NSArray *const kPortraitRotatedExpectedKeypoints = @[ + @[ @0.82075f, @0.44679f ], @[ @0.81965f, @0.56261f ], @[ @0.76194f, @0.51719f ], + @[ @0.71993f, @0.51719f ], @[ @0.80700f, @0.36298f ], @[ @0.80882f, @0.61204f ] +]; +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; +static NSString *const kLiveStreamTestsDictFaceDetectorKey = @"face_detector"; +static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; + +static const float kKeypointErrorThreshold = 1e-2; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertEqualObjects(error.localizedDescription, expectedError.localizedDescription) + +@interface MPPFaceDetectorTests : XCTestCase { + NSDictionary *liveStreamSucceedsTestDict; + NSDictionary *outOfOrderTimestampTestDict; +} +@end + +@implementation MPPFaceDetectorTests + +#pragma mark General Tests + +- (void)testCreateFaceDetectorWithMissingModelPathFails { + NSString *modelPath = [MPPFaceDetectorTests filePathWithName:@"" extension:@""]; + + NSError *error = nil; + MPPFaceDetector *faceDetector = [[MPPFaceDetector alloc] initWithModelPath:modelPath + error:&error]; + XCTAssertNil(faceDetector); + + NSError *expectedError = [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: ExternalFile must specify at least one of 'file_content', " + @"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'." + }]; + AssertEqualErrors(error, expectedError); +} + +#pragma mark Image Mode Tests + +- (void)testDetectWithImageModeAndPotraitSucceeds { + NSString *modelPath = [MPPFaceDetectorTests filePathWithName:kShortRangeBlazeFaceModel + extension:@"tflite"]; + MPPFaceDetector *faceDetector = [[MPPFaceDetector alloc] initWithModelPath:modelPath error:nil]; + + [self assertResultsOfDetectInImageWithFileInfo:kPortraitImage + usingFaceDetector:faceDetector + containsExpectedKeypoints:kPortraitExpectedKeypoints]; +} + +- (void)testDetectWithImageModeAndRotatedPotraitSucceeds { + NSString *modelPath = [MPPFaceDetectorTests filePathWithName:kShortRangeBlazeFaceModel + extension:@"tflite"]; + MPPFaceDetector *faceDetector = [[MPPFaceDetector alloc] initWithModelPath:modelPath error:nil]; + XCTAssertNotNil(faceDetector); + + MPPImage *image = [self imageWithFileInfo:kPortraitRotatedImage]; + [self assertResultsOfDetectInImage:image + usingFaceDetector:faceDetector + containsExpectedKeypoints:kPortraitRotatedExpectedKeypoints]; +} + +- (void)testDetectWithImageModeAndNoFaceSucceeds { + NSString *modelPath = [MPPFaceDetectorTests filePathWithName:kShortRangeBlazeFaceModel + extension:@"tflite"]; + MPPFaceDetector *faceDetector = [[MPPFaceDetector alloc] initWithModelPath:modelPath error:nil]; + XCTAssertNotNil(faceDetector); + + NSError *error; + MPPImage *mppImage = [self imageWithFileInfo:kCatImage]; + MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectInImage:mppImage error:&error]; + XCTAssertNil(error); + XCTAssertNotNil(faceDetectorResult); + XCTAssertEqual(faceDetectorResult.detections.count, 0); +} + +#pragma mark Video Mode Tests + +- (void)testDetectWithVideoModeAndPotraitSucceeds { + MPPFaceDetectorOptions *options = + [self faceDetectorOptionsWithModelName:kShortRangeBlazeFaceModel]; + options.runningMode = MPPRunningModeVideo; + MPPFaceDetector *faceDetector = [self faceDetectorWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + for (int i = 0; i < 3; i++) { + MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectInVideoFrame:image + timestampInMilliseconds:i + error:nil]; + [self assertFaceDetectorResult:faceDetectorResult + containsExpectedKeypoints:kPortraitExpectedKeypoints]; + } +} + +- (void)testDetectWithVideoModeAndRotatedPotraitSucceeds { + MPPFaceDetectorOptions *options = + [self faceDetectorOptionsWithModelName:kShortRangeBlazeFaceModel]; + options.runningMode = MPPRunningModeVideo; + MPPFaceDetector *faceDetector = [self faceDetectorWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kPortraitRotatedImage]; + for (int i = 0; i < 3; i++) { + MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectInVideoFrame:image + timestampInMilliseconds:i + error:nil]; + [self assertFaceDetectorResult:faceDetectorResult + containsExpectedKeypoints:kPortraitRotatedExpectedKeypoints]; + } +} + +#pragma mark Live Stream Mode Tests + +- (void)testDetectWithLiveStreamModeAndPotraitSucceeds { + NSInteger iterationCount = 100; + + // Because of flow limiting, the callback might be invoked fewer than `iterationCount` times. An + // normal expectation will fail if expectation.fullfill() is not called + // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will + // only succeed if expectation is not fullfilled for the specified `expectedFulfillmentCount`. + // Since it is not possible to predict how many times the expectation is supposed to be + // fullfilled, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and + // `expectation.isInverted = true` ensures that test succeeds if expectation is fullfilled <= + // `iterationCount` times. + XCTestExpectation *expectation = [[XCTestExpectation alloc] + initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; + expectation.expectedFulfillmentCount = iterationCount + 1; + expectation.inverted = YES; + + MPPFaceDetectorOptions *options = + [self faceDetectorOptionsWithModelName:kShortRangeBlazeFaceModel]; + options.runningMode = MPPRunningModeLiveStream; + options.faceDetectorLiveStreamDelegate = self; + + MPPFaceDetector *faceDetector = [self faceDetectorWithOptionsSucceeds:options]; + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + + liveStreamSucceedsTestDict = @{ + kLiveStreamTestsDictFaceDetectorKey : faceDetector, + kLiveStreamTestsDictExpectationKey : expectation + }; + + for (int i = 0; i < iterationCount; i++) { + XCTAssertTrue([faceDetector detectAsyncInImage:image timestampInMilliseconds:i error:nil]); + } + + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; +} + +- (void)testDetectWithOutOfOrderTimestampsAndLiveStreamModeFails { + MPPFaceDetectorOptions *options = + [self faceDetectorOptionsWithModelName:kShortRangeBlazeFaceModel]; + options.runningMode = MPPRunningModeLiveStream; + options.faceDetectorLiveStreamDelegate = self; + + XCTestExpectation *expectation = [[XCTestExpectation alloc] + initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; + expectation.expectedFulfillmentCount = 1; + + MPPFaceDetector *faceDetector = [self faceDetectorWithOptionsSucceeds:options]; + liveStreamSucceedsTestDict = @{ + kLiveStreamTestsDictFaceDetectorKey : faceDetector, + kLiveStreamTestsDictExpectationKey : expectation + }; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + XCTAssertTrue([faceDetector detectAsyncInImage:image timestampInMilliseconds:1 error:nil]); + + NSError *error; + XCTAssertFalse([faceDetector detectAsyncInImage:image timestampInMilliseconds:0 error:&error]); + + NSError *expectedError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: Input timestamp must be monotonically increasing." + }]; + AssertEqualErrors(error, expectedError); + + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; +} + +#pragma mark Running Mode Tests + +- (void)testCreateFaceDetectorFailsWithDelegateInNonLiveStreamMode { + MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo}; + for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) { + MPPFaceDetectorOptions *options = + [self faceDetectorOptionsWithModelName:kShortRangeBlazeFaceModel]; + + options.runningMode = runningModesToTest[i]; + options.faceDetectorLiveStreamDelegate = self; + + [self assertCreateFaceDetectorWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in image or video mode. The " + @"delegate must not be set in the task's options." + }]]; + } +} + +- (void)testCreateFaceDetectorFailsWithMissingDelegateInLiveStreamMode { + MPPFaceDetectorOptions *options = + [self faceDetectorOptionsWithModelName:kShortRangeBlazeFaceModel]; + + options.runningMode = MPPRunningModeLiveStream; + + [self assertCreateFaceDetectorWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in live stream mode. An " + @"object must be set as the delegate of the task " + @"in its options to ensure asynchronous delivery " + @"of results." + }]]; +} + +- (void)testDetectFailsWithCallingWrongApiInImageMode { + MPPFaceDetectorOptions *options = + [self faceDetectorOptionsWithModelName:kShortRangeBlazeFaceModel]; + + MPPFaceDetector *faceDetector = [self faceDetectorWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + + NSError *liveStreamApiCallError; + XCTAssertFalse([faceDetector detectAsyncInImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Image" + }]; + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([faceDetector detectInVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Image" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testDetectFailsWithCallingWrongApiInVideoMode { + MPPFaceDetectorOptions *options = + [self faceDetectorOptionsWithModelName:kShortRangeBlazeFaceModel]; + options.runningMode = MPPRunningModeVideo; + + MPPFaceDetector *faceDetector = [self faceDetectorWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + + NSError *liveStreamApiCallError; + XCTAssertFalse([faceDetector detectAsyncInImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Video" + }]; + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *imageApiCallError; + XCTAssertFalse([faceDetector detectInImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Video" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); +} + +- (void)testDetectFailsWithCallingWrongApiInLiveStreamMode { + MPPFaceDetectorOptions *options = + [self faceDetectorOptionsWithModelName:kShortRangeBlazeFaceModel]; + + options.runningMode = MPPRunningModeLiveStream; + options.faceDetectorLiveStreamDelegate = self; + + MPPFaceDetector *faceDetector = [self faceDetectorWithOptionsSucceeds:options]; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + + NSError *imageApiCallError; + XCTAssertFalse([faceDetector detectInImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([faceDetector detectInVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testDetectWithLiveStreamModeSucceeds { + MPPFaceDetectorOptions *options = + [self faceDetectorOptionsWithModelName:kShortRangeBlazeFaceModel]; + options.runningMode = MPPRunningModeLiveStream; + options.faceDetectorLiveStreamDelegate = self; + + NSInteger iterationCount = 100; + + // Because of flow limiting, the callback might be invoked fewer than `iterationCount` times. An + // normal expectation will fail if expectation.fullfill() is not called times. An normal + // expectation will fail if expectation.fullfill() is not called + // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will + // only succeed if expectation is not fullfilled for the specified `expectedFulfillmentCount`. + // Since it it not possible to determine how many times the expectation is supposed to be + // fullfilled, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and + // `expectation.isInverted = true` ensures that test succeeds if expectation is fullfilled <= + // `iterationCount` times. + XCTestExpectation *expectation = [[XCTestExpectation alloc] + initWithDescription:@"detectWithOutOfOrderTimestampsAndLiveStream"]; + expectation.expectedFulfillmentCount = iterationCount + 1; + expectation.inverted = YES; + + MPPFaceDetector *faceDetector = [self faceDetectorWithOptionsSucceeds:options]; + + liveStreamSucceedsTestDict = @{ + kLiveStreamTestsDictFaceDetectorKey : faceDetector, + kLiveStreamTestsDictExpectationKey : expectation + }; + + MPPImage *image = [self imageWithFileInfo:kPortraitImage]; + for (int i = 0; i < iterationCount; i++) { + XCTAssertTrue([faceDetector detectAsyncInImage:image timestampInMilliseconds:i error:nil]); + } + + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; +} + +#pragma mark MPPFaceDetectorLiveStreamDelegate Methods +- (void)faceDetector:(MPPFaceDetector *)faceDetector + didFinishDetectionWithResult:(MPPFaceDetectorResult *)faceDetectorResult + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError *)error { + [self assertFaceDetectorResult:faceDetectorResult + containsExpectedKeypoints:kPortraitExpectedKeypoints]; + + if (faceDetector == outOfOrderTimestampTestDict[kLiveStreamTestsDictFaceDetectorKey]) { + [outOfOrderTimestampTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } else if (faceDetector == liveStreamSucceedsTestDict[kLiveStreamTestsDictFaceDetectorKey]) { + [liveStreamSucceedsTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } +} + ++ (NSString *)filePathWithName:(NSString *)fileName extension:(NSString *)extension { + NSString *filePath = + [[NSBundle bundleForClass:[MPPFaceDetectorTests class]] pathForResource:fileName + ofType:extension]; + return filePath; +} + +- (void)assertKeypoints:(NSArray *)keypoints + areEqualToExpectedKeypoints:(NSArray *)expectedKeypoint { + XCTAssertEqual(keypoints.count, expectedKeypoint.count); + for (int i = 0; i < keypoints.count; ++i) { + XCTAssertEqualWithAccuracy(keypoints[i].location.x, [expectedKeypoint[i][0] floatValue], + kKeypointErrorThreshold, @"index i = %d", i); + XCTAssertEqualWithAccuracy(keypoints[i].location.y, [expectedKeypoint[i][1] floatValue], + kKeypointErrorThreshold, @"index i = %d", i); + } +} + +- (void)assertDetections:(NSArray *)detections + containExpectedKeypoints:(NSArray *)expectedKeypoints { + XCTAssertEqual(detections.count, 1); + MPPDetection *detection = detections[0]; + XCTAssertNotNil(detection); + [self assertKeypoints:detections[0].keypoints areEqualToExpectedKeypoints:expectedKeypoints]; +} + +- (void)assertFaceDetectorResult:(MPPFaceDetectorResult *)faceDetectorResult + containsExpectedKeypoints:(NSArray *)expectedKeypoints { + [self assertDetections:faceDetectorResult.detections containExpectedKeypoints:expectedKeypoints]; +} + +#pragma mark Face Detector Initializers + +- (MPPFaceDetectorOptions *)faceDetectorOptionsWithModelName:(NSString *)modelName { + NSString *modelPath = [MPPFaceDetectorTests filePathWithName:modelName extension:@"tflite"]; + MPPFaceDetectorOptions *faceDetectorOptions = [[MPPFaceDetectorOptions alloc] init]; + faceDetectorOptions.baseOptions.modelAssetPath = modelPath; + + return faceDetectorOptions; +} + +- (void)assertCreateFaceDetectorWithOptions:(MPPFaceDetectorOptions *)faceDetectorOptions + failsWithExpectedError:(NSError *)expectedError { + NSError *error = nil; + MPPFaceDetector *faceDetector = [[MPPFaceDetector alloc] initWithOptions:faceDetectorOptions + error:&error]; + XCTAssertNil(faceDetector); + AssertEqualErrors(error, expectedError); +} + +- (MPPFaceDetector *)faceDetectorWithOptionsSucceeds:(MPPFaceDetectorOptions *)faceDetectorOptions { + MPPFaceDetector *faceDetector = [[MPPFaceDetector alloc] initWithOptions:faceDetectorOptions + error:nil]; + XCTAssertNotNil(faceDetector); + + return faceDetector; +} + +#pragma mark Assert Detection Results + +- (MPPImage *)imageWithFileInfo:(NSDictionary *)fileInfo { + UIImageOrientation orientation = (UIImageOrientation)[fileInfo[@"orientation"] intValue]; + MPPImage *image = [MPPImage imageFromBundleWithClass:[MPPFaceDetectorTests class] + fileName:fileInfo[@"name"] + ofType:fileInfo[@"type"] + orientation:orientation]; + XCTAssertNotNil(image); + return image; +} + +- (void)assertResultsOfDetectInImage:(MPPImage *)mppImage + usingFaceDetector:(MPPFaceDetector *)faceDetector + containsExpectedKeypoints:(NSArray *)expectedKeypoints { + NSError *error; + MPPFaceDetectorResult *faceDetectorResult = [faceDetector detectInImage:mppImage error:&error]; + XCTAssertNil(error); + XCTAssertNotNil(faceDetectorResult); + [self assertFaceDetectorResult:faceDetectorResult containsExpectedKeypoints:expectedKeypoints]; +} + +- (void)assertResultsOfDetectInImageWithFileInfo:(NSDictionary *)fileInfo + usingFaceDetector:(MPPFaceDetector *)faceDetector + containsExpectedKeypoints:(NSArray *)expectedKeypoints { + MPPImage *mppImage = [self imageWithFileInfo:fileInfo]; + + [self assertResultsOfDetectInImage:mppImage + usingFaceDetector:faceDetector + containsExpectedKeypoints:expectedKeypoints]; +} + +@end From 7facc925ba57ead64b2ad9ceee920868d3bea97f Mon Sep 17 00:00:00 2001 From: Jiuqiang Tang Date: Wed, 24 May 2023 11:12:20 -0700 Subject: [PATCH 17/22] Allow FaceStylizerGraph to miss base options. Fix the color issue when the graph is running on gpu and "face alignment only" mode. PiperOrigin-RevId: 534912498 --- .../face_stylizer/face_stylizer_graph.cc | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/mediapipe/tasks/cc/vision/face_stylizer/face_stylizer_graph.cc b/mediapipe/tasks/cc/vision/face_stylizer/face_stylizer_graph.cc index d7265a146..d4057467b 100644 --- a/mediapipe/tasks/cc/vision/face_stylizer/face_stylizer_graph.cc +++ b/mediapipe/tasks/cc/vision/face_stylizer/face_stylizer_graph.cc @@ -231,18 +231,26 @@ class FaceStylizerGraph : public core::ModelTaskGraph { SubgraphContext* sc) override { bool output_stylized = HasOutput(sc->OriginalNode(), kStylizedImageTag); bool output_alignment = HasOutput(sc->OriginalNode(), kFaceAlignmentTag); - ASSIGN_OR_RETURN( - const auto* model_asset_bundle_resources, - CreateModelAssetBundleResources(sc)); - // Copies the file content instead of passing the pointer of file in - // memory if the subgraph model resource service is not available. auto face_stylizer_external_file = absl::make_unique(); - MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( - *model_asset_bundle_resources, - sc->MutableOptions(), - output_stylized ? face_stylizer_external_file.get() : nullptr, - !sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService) - .IsAvailable())); + if (sc->Options().has_base_options()) { + ASSIGN_OR_RETURN( + const auto* model_asset_bundle_resources, + CreateModelAssetBundleResources(sc)); + // Copies the file content instead of passing the pointer of file in + // memory if the subgraph model resource service is not available. + MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( + *model_asset_bundle_resources, + sc->MutableOptions(), + output_stylized ? face_stylizer_external_file.get() : nullptr, + !sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService) + .IsAvailable())); + } else if (output_stylized) { + return CreateStatusWithPayload( + absl::StatusCode::kInvalidArgument, + "Face stylizer must specify its base options when the " + "\"STYLIZED_IMAGE\" output stream is connected.", + MediaPipeTasksStatus::kInvalidArgumentError); + } Graph graph; ASSIGN_OR_RETURN( auto face_landmark_lists, @@ -347,7 +355,7 @@ class FaceStylizerGraph : public core::ModelTaskGraph { auto& image_to_tensor = graph.AddNode("ImageToTensorCalculator"); auto& image_to_tensor_options = image_to_tensor.GetOptions(); - image_to_tensor_options.mutable_output_tensor_float_range()->set_min(-1); + image_to_tensor_options.mutable_output_tensor_float_range()->set_min(0); image_to_tensor_options.mutable_output_tensor_float_range()->set_max(1); image_to_tensor_options.set_output_tensor_width(kFaceAlignmentOutputSize); image_to_tensor_options.set_output_tensor_height( @@ -363,7 +371,7 @@ class FaceStylizerGraph : public core::ModelTaskGraph { graph.AddNode("mediapipe.tasks.TensorsToImageCalculator"); auto& tensors_to_image_options = tensors_to_image.GetOptions(); - tensors_to_image_options.mutable_input_tensor_float_range()->set_min(-1); + tensors_to_image_options.mutable_input_tensor_float_range()->set_min(0); tensors_to_image_options.mutable_input_tensor_float_range()->set_max(1); face_alignment_image >> tensors_to_image.In(kTensorsTag); face_alignment = tensors_to_image.Out(kImageTag).Cast(); From a7b81c7d108efaa595a3f9dc11c3c121a6e45fd0 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 24 May 2023 11:52:27 -0700 Subject: [PATCH 18/22] Change "numberOfHands" property name to "numHands". PiperOrigin-RevId: 534927982 --- .../gesture_recognizer/sources/MPPGestureRecognizerOptions.h | 2 +- .../gesture_recognizer/sources/MPPGestureRecognizerOptions.m | 4 ++-- .../utils/sources/MPPGestureRecognizerOptions+Helpers.mm | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h index b896e3756..323763cb5 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.h @@ -87,7 +87,7 @@ NS_SWIFT_NAME(GestureRecognizerOptions) gestureRecognizerLiveStreamDelegate; /** Sets the maximum number of hands can be detected by the GestureRecognizer. */ -@property(nonatomic) NSInteger numberOfHands NS_SWIFT_NAME(numHands); +@property(nonatomic) NSInteger numHands; /** Sets minimum confidence score for the hand detection to be considered successful */ @property(nonatomic) float minHandDetectionConfidence; diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m index 099ab33e1..71b85cc83 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/sources/MPPGestureRecognizerOptions.m @@ -19,7 +19,7 @@ - (instancetype)init { self = [super init]; if (self) { - _numberOfHands = 1; + _numHands = 1; _minHandDetectionConfidence = 0.5f; _minHandPresenceConfidence = 0.5f; _minTrackingConfidence = 0.5f; @@ -33,7 +33,7 @@ gestureRecognizerOptions.runningMode = self.runningMode; gestureRecognizerOptions.gestureRecognizerLiveStreamDelegate = self.gestureRecognizerLiveStreamDelegate; - gestureRecognizerOptions.numberOfHands = self.numberOfHands; + gestureRecognizerOptions.numHands = self.numHands; gestureRecognizerOptions.minHandDetectionConfidence = self.minHandDetectionConfidence; gestureRecognizerOptions.minHandPresenceConfidence = self.minHandPresenceConfidence; gestureRecognizerOptions.minTrackingConfidence = self.minTrackingConfidence; diff --git a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm index d234d86a0..5f8d25d7c 100644 --- a/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm +++ b/mediapipe/tasks/ios/vision/gesture_recognizer/utils/sources/MPPGestureRecognizerOptions+Helpers.mm @@ -60,7 +60,7 @@ using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto HandDetectorGraphOptionsProto *handDetectorGraphOptionsProto = handLandmarkerGraphOptionsProto->mutable_hand_detector_graph_options(); handDetectorGraphOptionsProto->Clear(); - handDetectorGraphOptionsProto->set_num_hands(self.numberOfHands); + handDetectorGraphOptionsProto->set_num_hands(self.numHands); handDetectorGraphOptionsProto->set_min_detection_confidence(self.minHandDetectionConfidence); HandLandmarksDetectorGraphOptionsProto *handLandmarksDetectorGraphOptionsProto = From c8ee09796bd36766550d2d6df69de8369644ecfb Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 24 May 2023 13:24:32 -0700 Subject: [PATCH 19/22] Internal change PiperOrigin-RevId: 534959867 --- mediapipe/calculators/tensor/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index 4893b3b79..59102585c 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -394,7 +394,7 @@ mediapipe_proto_library( # If you want to have precise control of which implementations to include (e.g. for strict binary # size concerns), depend on those implementations directly, and do not depend on # :inference_calculator. -# In all cases, use "InferenceCalulator" in your graphs. +# In all cases, use "InferenceCalculator" in your graphs. cc_library_with_tflite( name = "inference_calculator_interface", srcs = ["inference_calculator.cc"], From 7800d238e916d7b74ba019c254c33999cae834e1 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 24 May 2023 20:15:30 -0700 Subject: [PATCH 20/22] add needed enum type for choose fuse pipeline. PiperOrigin-RevId: 535076733 --- mediapipe/calculators/core/BUILD | 4 ++-- mediapipe/calculators/core/begin_loop_calculator.cc | 4 ++++ mediapipe/calculators/core/end_loop_calculator.cc | 4 ++++ mediapipe/calculators/core/split_vector_calculator.cc | 9 +++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index 3294c0383..4e32ed59f 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -194,6 +194,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework:packet", "//mediapipe/framework/formats:detection_cc_proto", + "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:matrix", @@ -225,10 +226,8 @@ cc_library( "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", "//mediapipe/gpu:gpu_buffer", "//mediapipe/util:render_data_cc_proto", - "@com_google_absl//absl/memory", "@com_google_absl//absl/status", "@org_tensorflow//tensorflow/lite:framework", ], @@ -907,6 +906,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/formats:classification_cc_proto", "//mediapipe/framework/formats:detection_cc_proto", + "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:landmark_cc_proto", "//mediapipe/framework/formats:matrix", "//mediapipe/framework/formats:rect_cc_proto", diff --git a/mediapipe/calculators/core/begin_loop_calculator.cc b/mediapipe/calculators/core/begin_loop_calculator.cc index ac74bb382..7da90989b 100644 --- a/mediapipe/calculators/core/begin_loop_calculator.cc +++ b/mediapipe/calculators/core/begin_loop_calculator.cc @@ -17,6 +17,7 @@ #include #include "mediapipe/framework/formats/detection.pb.h" +#include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/matrix.h" @@ -72,4 +73,7 @@ typedef BeginLoopCalculator> BeginLoopGpuBufferCalculator; REGISTER_CALCULATOR(BeginLoopGpuBufferCalculator); +// A calculator to process std::vector. +typedef BeginLoopCalculator> BeginLoopImageCalculator; +REGISTER_CALCULATOR(BeginLoopImageCalculator); } // namespace mediapipe diff --git a/mediapipe/calculators/core/end_loop_calculator.cc b/mediapipe/calculators/core/end_loop_calculator.cc index dea2b6cad..752580cfd 100644 --- a/mediapipe/calculators/core/end_loop_calculator.cc +++ b/mediapipe/calculators/core/end_loop_calculator.cc @@ -80,4 +80,8 @@ typedef EndLoopCalculator> EndLoopImageCalculator; REGISTER_CALCULATOR(EndLoopImageCalculator); +typedef EndLoopCalculator>> + EndLoopAffineMatrixCalculator; +REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator); + } // namespace mediapipe diff --git a/mediapipe/calculators/core/split_vector_calculator.cc b/mediapipe/calculators/core/split_vector_calculator.cc index b76722de9..67fc38ce9 100644 --- a/mediapipe/calculators/core/split_vector_calculator.cc +++ b/mediapipe/calculators/core/split_vector_calculator.cc @@ -18,6 +18,7 @@ #include "mediapipe/framework/formats/classification.pb.h" #include "mediapipe/framework/formats/detection.pb.h" +#include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/landmark.pb.h" #include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/rect.pb.h" @@ -86,4 +87,12 @@ REGISTER_CALCULATOR(SplitUint64tVectorCalculator); typedef SplitVectorCalculator SplitFloatVectorCalculator; REGISTER_CALCULATOR(SplitFloatVectorCalculator); +typedef SplitVectorCalculator + SplitImageVectorCalculator; +REGISTER_CALCULATOR(SplitImageVectorCalculator); + +typedef SplitVectorCalculator, false> + SplitAffineMatrixVectorCalculator; +REGISTER_CALCULATOR(SplitAffineMatrixVectorCalculator); + } // namespace mediapipe From 1baf72e46b3c38cb3b39093ccf13aa577a5a5f15 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 24 May 2023 23:42:59 -0700 Subject: [PATCH 21/22] Internal change PiperOrigin-RevId: 535131938 --- .../vision/face_landmarker/face_landmarker.ts | 22 ++++++++++--------- .../face_landmarker_result.d.ts | 4 ++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/mediapipe/tasks/web/vision/face_landmarker/face_landmarker.ts b/mediapipe/tasks/web/vision/face_landmarker/face_landmarker.ts index 9dc8626c9..e2e25c71b 100644 --- a/mediapipe/tasks/web/vision/face_landmarker/face_landmarker.ts +++ b/mediapipe/tasks/web/vision/face_landmarker/face_landmarker.ts @@ -59,7 +59,11 @@ const DEFAULT_SCORE_THRESHOLD = 0.5; * This API expects a pre-trained face landmarker model asset bundle. */ export class FaceLandmarker extends VisionTaskRunner { - private result: FaceLandmarkerResult = {faceLandmarks: []}; + private result: FaceLandmarkerResult = { + faceLandmarks: [], + faceBlendshapes: [], + facialTransformationMatrixes: [] + }; private outputFaceBlendshapes = false; private outputFacialTransformationMatrixes = false; @@ -256,13 +260,11 @@ export class FaceLandmarker extends VisionTaskRunner { } private resetResults(): void { - this.result = {faceLandmarks: []}; - if (this.outputFaceBlendshapes) { - this.result.faceBlendshapes = []; - } - if (this.outputFacialTransformationMatrixes) { - this.result.facialTransformationMatrixes = []; - } + this.result = { + faceLandmarks: [], + faceBlendshapes: [], + facialTransformationMatrixes: [] + }; } /** Sets the default values for the graph. */ @@ -286,7 +288,7 @@ export class FaceLandmarker extends VisionTaskRunner { /** Adds new blendshapes from the given proto. */ private addBlenshape(data: Uint8Array[]): void { - if (!this.result.faceBlendshapes) { + if (!this.outputFaceBlendshapes) { return; } @@ -300,7 +302,7 @@ export class FaceLandmarker extends VisionTaskRunner { /** Adds new transformation matrixes from the given proto. */ private addFacialTransformationMatrixes(data: Uint8Array[]): void { - if (!this.result.facialTransformationMatrixes) { + if (!this.outputFacialTransformationMatrixes) { return; } diff --git a/mediapipe/tasks/web/vision/face_landmarker/face_landmarker_result.d.ts b/mediapipe/tasks/web/vision/face_landmarker/face_landmarker_result.d.ts index 4bbe3b0c9..4af483ab3 100644 --- a/mediapipe/tasks/web/vision/face_landmarker/face_landmarker_result.d.ts +++ b/mediapipe/tasks/web/vision/face_landmarker/face_landmarker_result.d.ts @@ -29,8 +29,8 @@ export declare interface FaceLandmarkerResult { faceLandmarks: NormalizedLandmark[][]; /** Optional face blendshapes results. */ - faceBlendshapes?: Classifications[]; + faceBlendshapes: Classifications[]; /** Optional facial transformation matrix. */ - facialTransformationMatrixes?: Matrix[]; + facialTransformationMatrixes: Matrix[]; } From 3ac903787a16422c054a1f9c83214f0ef096df40 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 25 May 2023 01:47:16 -0700 Subject: [PATCH 22/22] Internal change PiperOrigin-RevId: 535162357 --- mediapipe/calculators/tensor/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index 59102585c..4893b3b79 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -394,7 +394,7 @@ mediapipe_proto_library( # If you want to have precise control of which implementations to include (e.g. for strict binary # size concerns), depend on those implementations directly, and do not depend on # :inference_calculator. -# In all cases, use "InferenceCalculator" in your graphs. +# In all cases, use "InferenceCalulator" in your graphs. cc_library_with_tflite( name = "inference_calculator_interface", srcs = ["inference_calculator.cc"],