From 45addac249b065c5dd0bdf4b348990f9a981b101 Mon Sep 17 00:00:00 2001 From: kinaryml Date: Wed, 17 May 2023 10:55:40 +0530 Subject: [PATCH 1/9] Testing MediaPipe Python with GPU support --- mediapipe/tasks/python/core/BUILD | 2 + mediapipe/tasks/python/core/base_options.py | 27 ++++-- setup.py | 102 +++++++++++++++----- 3 files changed, 96 insertions(+), 35 deletions(-) diff --git a/mediapipe/tasks/python/core/BUILD b/mediapipe/tasks/python/core/BUILD index f90826354..c0abefdde 100644 --- a/mediapipe/tasks/python/core/BUILD +++ b/mediapipe/tasks/python/core/BUILD @@ -34,6 +34,8 @@ py_library( ], deps = [ ":optional_dependencies", + "//mediapipe/calculators/tensor/inference_calculator_py_pb2", + "//mediapipe/tasks/cc/core/proto:acceleration_py_pb2", "//mediapipe/tasks/cc/core/proto:base_options_py_pb2", "//mediapipe/tasks/cc/core/proto:external_file_py_pb2", ], diff --git a/mediapipe/tasks/python/core/base_options.py b/mediapipe/tasks/python/core/base_options.py index 90ef045b8..8247f15bd 100644 --- a/mediapipe/tasks/python/core/base_options.py +++ b/mediapipe/tasks/python/core/base_options.py @@ -14,13 +14,18 @@ """Base options for MediaPipe Task APIs.""" import dataclasses +import enum import os from typing import Any, Optional +from mediapipe.calculators.tensor import inference_calculator_pb2 +from mediapipe.tasks.cc.core.proto import acceleration_pb2 from mediapipe.tasks.cc.core.proto import base_options_pb2 from mediapipe.tasks.cc.core.proto import external_file_pb2 from mediapipe.tasks.python.core.optional_dependencies import doc_controls +_DelegateProto = inference_calculator_pb2.InferenceCalculatorOptions.Delegate +_AccelerationProto = acceleration_pb2.Acceleration _BaseOptionsProto = base_options_pb2.BaseOptions _ExternalFileProto = external_file_pb2.ExternalFile @@ -42,10 +47,13 @@ class BaseOptions: model_asset_path: Path to the model asset file. model_asset_buffer: The model asset file contents as bytes. """ + class Delegate(enum.Enum): + CPU = 0 + GPU = 1 model_asset_path: Optional[str] = None model_asset_buffer: Optional[bytes] = None - # TODO: Allow Python API to specify acceleration settings. + delegate: Optional[Delegate] = None @doc_controls.do_not_generate_docs def to_pb2(self) -> _BaseOptionsProto: @@ -55,17 +63,16 @@ class BaseOptions: else: full_path = None + if self.delegate == BaseOptions.Delegate.GPU: + acceleration_proto = _AccelerationProto(gpu=_DelegateProto.Gpu()) + else: + acceleration_proto = _AccelerationProto(tflite=_DelegateProto.TfLite()) + return _BaseOptionsProto( model_asset=_ExternalFileProto( - file_name=full_path, file_content=self.model_asset_buffer)) - - @classmethod - @doc_controls.do_not_generate_docs - def create_from_pb2(cls, pb2_obj: _BaseOptionsProto) -> 'BaseOptions': - """Creates a `BaseOptions` object from the given protobuf object.""" - return BaseOptions( - model_asset_path=pb2_obj.model_asset.file_name, - model_asset_buffer=pb2_obj.model_asset.file_content) + file_name=full_path, file_content=self.model_asset_buffer), + acceleration=acceleration_proto + ) def __eq__(self, other: Any) -> bool: """Checks if this object is equal to the given object. diff --git a/setup.py b/setup.py index 892c6dca7..0712a95b0 100644 --- a/setup.py +++ b/setup.py @@ -245,15 +245,28 @@ class BuildModules(build_ext.build_ext): sys.stderr.write('downloading file: %s\n' % external_file) self._download_external_file(external_file) + # CPU binary graphs + # binary_graphs = [ + # 'face_detection/face_detection_short_range_cpu', + # 'face_detection/face_detection_full_range_cpu', + # 'face_landmark/face_landmark_front_cpu', + # 'hand_landmark/hand_landmark_tracking_cpu', + # 'holistic_landmark/holistic_landmark_cpu', 'objectron/objectron_cpu', + # 'pose_landmark/pose_landmark_cpu', + # 'selfie_segmentation/selfie_segmentation_cpu' + # ] + + # GPU binary graphs binary_graphs = [ - 'face_detection/face_detection_short_range_cpu', - 'face_detection/face_detection_full_range_cpu', - 'face_landmark/face_landmark_front_cpu', - 'hand_landmark/hand_landmark_tracking_cpu', - 'holistic_landmark/holistic_landmark_cpu', 'objectron/objectron_cpu', - 'pose_landmark/pose_landmark_cpu', - 'selfie_segmentation/selfie_segmentation_cpu' + 'face_detection/face_detection_short_range_gpu', + 'face_detection/face_detection_full_range_gpu', + 'face_landmark/face_landmark_front_gpu', + 'hand_landmark/hand_landmark_tracking_gpu', + 'holistic_landmark/holistic_landmark_gpu', 'objectron/objectron_gpu', + 'pose_landmark/pose_landmark_gpu', + 'selfie_segmentation/selfie_segmentation_gpu' ] + for elem in binary_graphs: binary_graph = os.path.join('mediapipe/modules/', elem) sys.stderr.write('generating binarypb: %s\n' % binary_graph) @@ -271,23 +284,42 @@ class BuildModules(build_ext.build_ext): sys.exit(-1) _copy_to_build_lib_dir(self.build_lib, external_file) - def _generate_binary_graph(self, binary_graph_target): - """Generate binary graph for a particular MediaPipe binary graph target.""" + # def _generate_binary_graph(self, binary_graph_target): + # """Generate binary graph for a particular MediaPipe binary graph target.""" + # + # bazel_command = [ + # 'bazel', + # 'build', + # '--compilation_mode=opt', + # '--copt=-DNDEBUG', + # '--define=MEDIAPIPE_DISABLE_GPU=1', + # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), + # binary_graph_target, + # ] + # if not self.link_opencv and not IS_WINDOWS: + # bazel_command.append('--define=OPENCV=source') + # if subprocess.call(bazel_command) != 0: + # sys.exit(-1) + # _copy_to_build_lib_dir(self.build_lib, binary_graph_target + '.binarypb') - bazel_command = [ - 'bazel', - 'build', - '--compilation_mode=opt', - '--copt=-DNDEBUG', - '--define=MEDIAPIPE_DISABLE_GPU=1', - '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), - binary_graph_target, - ] - if not self.link_opencv and not IS_WINDOWS: - bazel_command.append('--define=OPENCV=source') - if subprocess.call(bazel_command) != 0: - sys.exit(-1) - _copy_to_build_lib_dir(self.build_lib, binary_graph_target + '.binarypb') + def _generate_binary_graph(self, binary_graph_target): + """Generate binary graph for a particular MediaPipe binary graph target.""" + + bazel_command = [ + 'bazel', + 'build', + '--compilation_mode=opt', + '--copt=-DNDEBUG', + '--copt=-DMESA_EGL_NO_X11_HEADERS', + '--copt=-DEGL_NO_X11', + '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), + binary_graph_target, + ] + if not self.link_opencv and not IS_WINDOWS: + bazel_command.append('--define=OPENCV=source') + if subprocess.call(bazel_command) != 0: + sys.exit(-1) + _copy_to_build_lib_dir(self.build_lib, binary_graph_target + '.binarypb') class GenerateMetadataSchema(build_ext.build_ext): @@ -300,11 +332,21 @@ class GenerateMetadataSchema(build_ext.build_ext): 'object_detector_metadata_schema_py', 'schema_py', ]: + # bazel_command = [ + # 'bazel', + # 'build', + # '--compilation_mode=opt', + # '--define=MEDIAPIPE_DISABLE_GPU=1', + # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), + # '//mediapipe/tasks/metadata:' + target, + # ] + bazel_command = [ 'bazel', 'build', '--compilation_mode=opt', - '--define=MEDIAPIPE_DISABLE_GPU=1', + '--copt=-DMESA_EGL_NO_X11_HEADERS', + '--copt=-DEGL_NO_X11', '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), '//mediapipe/tasks/metadata:' + target, ] @@ -385,12 +427,22 @@ class BuildExtension(build_ext.build_ext): def _build_binary(self, ext, extra_args=None): if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) + # bazel_command = [ + # 'bazel', + # 'build', + # '--compilation_mode=opt', + # '--copt=-DNDEBUG', + # '--define=MEDIAPIPE_DISABLE_GPU=1', + # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), + # str(ext.bazel_target + '.so'), + # ] bazel_command = [ 'bazel', 'build', '--compilation_mode=opt', '--copt=-DNDEBUG', - '--define=MEDIAPIPE_DISABLE_GPU=1', + '--copt=-DMESA_EGL_NO_X11_HEADERS', + '--copt=-DEGL_NO_X11', '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), str(ext.bazel_target + '.so'), ] From 126df20658c11e6ef97f7c64bd781b9b3d768e9b Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 11:00:12 +0530 Subject: [PATCH 2/9] Included CPU binary graphs in setup.py --- setup.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 0712a95b0..ce162b594 100644 --- a/setup.py +++ b/setup.py @@ -246,18 +246,18 @@ class BuildModules(build_ext.build_ext): self._download_external_file(external_file) # CPU binary graphs - # binary_graphs = [ - # 'face_detection/face_detection_short_range_cpu', - # 'face_detection/face_detection_full_range_cpu', - # 'face_landmark/face_landmark_front_cpu', - # 'hand_landmark/hand_landmark_tracking_cpu', - # 'holistic_landmark/holistic_landmark_cpu', 'objectron/objectron_cpu', - # 'pose_landmark/pose_landmark_cpu', - # 'selfie_segmentation/selfie_segmentation_cpu' - # ] + binary_graphs = [ + 'face_detection/face_detection_short_range_cpu', + 'face_detection/face_detection_full_range_cpu', + 'face_landmark/face_landmark_front_cpu', + 'hand_landmark/hand_landmark_tracking_cpu', + 'holistic_landmark/holistic_landmark_cpu', 'objectron/objectron_cpu', + 'pose_landmark/pose_landmark_cpu', + 'selfie_segmentation/selfie_segmentation_cpu' + ] # GPU binary graphs - binary_graphs = [ + binary_graphs += [ 'face_detection/face_detection_short_range_gpu', 'face_detection/face_detection_full_range_gpu', 'face_landmark/face_landmark_front_gpu', From 620ff3508a1331c192a3103622da55b71fb37d6d Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 12:51:08 +0530 Subject: [PATCH 3/9] Update BUILD --- third_party/BUILD | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/third_party/BUILD b/third_party/BUILD index f6107106d..041812b74 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -167,6 +167,8 @@ cmake_external( "BUILD_PERF_TESTS": "OFF", "BUILD_EXAMPLES": "OFF", "BUILD_SHARED_LIBS": "ON" if OPENCV_SHARED_LIBS else "OFF", + # Disable IPP + "WITH_IPP": "OFF", "WITH_ITT": "OFF", "WITH_JASPER": "OFF", "WITH_JPEG": "ON", @@ -174,6 +176,9 @@ cmake_external( "WITH_TIFF": "ON", "WITH_OPENCL": "OFF", "WITH_WEBP": "OFF", + # Disable carotene_o4t + "ENABLE_NEON": "OFF", + "WITH_TENGINE": "OFF", # Optimization flags "CV_ENABLE_INTRINSICS": "ON", "WITH_EIGEN": "ON", From 14dba421c4406115f7dc059ca71c93a40cf91bf4 Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 13:23:44 +0530 Subject: [PATCH 4/9] Update BUILD --- third_party/BUILD | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/third_party/BUILD b/third_party/BUILD index 041812b74..68f203ccf 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -150,7 +150,7 @@ OPENCV_MODULES = [ # still only builds the shared libraries, so we have to choose one or the # other. We build shared libraries by default, but this variable can be used # to switch to static libraries. -OPENCV_SHARED_LIBS = True +OPENCV_SHARED_LIBS = False OPENCV_SO_VERSION = "3.4" @@ -168,7 +168,7 @@ cmake_external( "BUILD_EXAMPLES": "OFF", "BUILD_SHARED_LIBS": "ON" if OPENCV_SHARED_LIBS else "OFF", # Disable IPP - "WITH_IPP": "OFF", + # "WITH_IPP": "OFF", "WITH_ITT": "OFF", "WITH_JASPER": "OFF", "WITH_JPEG": "ON", @@ -177,8 +177,8 @@ cmake_external( "WITH_OPENCL": "OFF", "WITH_WEBP": "OFF", # Disable carotene_o4t - "ENABLE_NEON": "OFF", - "WITH_TENGINE": "OFF", + # "ENABLE_NEON": "OFF", + # "WITH_TENGINE": "OFF", # Optimization flags "CV_ENABLE_INTRINSICS": "ON", "WITH_EIGEN": "ON", From b5072c59e79fa2d8c9b24dd875ef928f1884144c Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 15:41:08 +0530 Subject: [PATCH 5/9] Update BUILD --- third_party/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/BUILD b/third_party/BUILD index 68f203ccf..6250088a8 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -168,7 +168,7 @@ cmake_external( "BUILD_EXAMPLES": "OFF", "BUILD_SHARED_LIBS": "ON" if OPENCV_SHARED_LIBS else "OFF", # Disable IPP - # "WITH_IPP": "OFF", + "WITH_IPP": "OFF", "WITH_ITT": "OFF", "WITH_JASPER": "OFF", "WITH_JPEG": "ON", From f63baaf8d24bb0b48715ff57944bb5a057a8cfb4 Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 18:08:04 +0530 Subject: [PATCH 6/9] Update BUILD --- third_party/BUILD | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/third_party/BUILD b/third_party/BUILD index 6250088a8..fbec0b351 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -150,7 +150,7 @@ OPENCV_MODULES = [ # still only builds the shared libraries, so we have to choose one or the # other. We build shared libraries by default, but this variable can be used # to switch to static libraries. -OPENCV_SHARED_LIBS = False +OPENCV_SHARED_LIBS = True OPENCV_SO_VERSION = "3.4" @@ -167,8 +167,6 @@ cmake_external( "BUILD_PERF_TESTS": "OFF", "BUILD_EXAMPLES": "OFF", "BUILD_SHARED_LIBS": "ON" if OPENCV_SHARED_LIBS else "OFF", - # Disable IPP - "WITH_IPP": "OFF", "WITH_ITT": "OFF", "WITH_JASPER": "OFF", "WITH_JPEG": "ON", @@ -176,6 +174,9 @@ cmake_external( "WITH_TIFF": "ON", "WITH_OPENCL": "OFF", "WITH_WEBP": "OFF", + # Disable IPP + "WITH_IPP": "OFF", + "WITH_OPENEXR": "OFF", # Disable carotene_o4t # "ENABLE_NEON": "OFF", # "WITH_TENGINE": "OFF", From 6100f0e76eb6667521ebf423ae7750e1cbe6dbf5 Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 18:09:39 +0530 Subject: [PATCH 7/9] Update base_options.py --- mediapipe/tasks/python/core/base_options.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/python/core/base_options.py b/mediapipe/tasks/python/core/base_options.py index 8247f15bd..2d74b16eb 100644 --- a/mediapipe/tasks/python/core/base_options.py +++ b/mediapipe/tasks/python/core/base_options.py @@ -16,6 +16,7 @@ import dataclasses import enum import os +import platform from typing import Any, Optional from mediapipe.calculators.tensor import inference_calculator_pb2 @@ -63,10 +64,22 @@ class BaseOptions: else: full_path = None - if self.delegate == BaseOptions.Delegate.GPU: - acceleration_proto = _AccelerationProto(gpu=_DelegateProto.Gpu()) + platform = platform.system() + + if self.delegate is not None: + if platform == "Linux": + if self.delegate == BaseOptions.Delegate.GPU: + acceleration_proto = _AccelerationProto(gpu=_DelegateProto.Gpu()) + else: + acceleration_proto = _AccelerationProto(tflite=_DelegateProto.TfLite()) + elif platform == "Windows": + raise Exception("Delegate is unsupported for Windows.") + elif platform == "Darwin": + raise Exception("Delegate is unsupported for MacOS.") + else: + raise Exception("Unidentified system") else: - acceleration_proto = _AccelerationProto(tflite=_DelegateProto.TfLite()) + acceleration_proto = None return _BaseOptionsProto( model_asset=_ExternalFileProto( From 01fdeaf1e18e74c352d18ed65e5213eec4845f29 Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 18:29:22 +0530 Subject: [PATCH 8/9] Update Dockerfile --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 03b335823..c7c459190 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM ubuntu:20.04 - +FROM nvidia/cudagl:11.3.0-devel-ubuntu20.04 MAINTAINER WORKDIR /io From b5148c8ce36701a6ecd1df65b5823cde21fa6399 Mon Sep 17 00:00:00 2001 From: Kinar R <42828719+kinaryml@users.noreply.github.com> Date: Thu, 18 May 2023 18:32:45 +0530 Subject: [PATCH 9/9] Update setup.py --- setup.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/setup.py b/setup.py index ce162b594..c49169191 100644 --- a/setup.py +++ b/setup.py @@ -284,24 +284,6 @@ class BuildModules(build_ext.build_ext): sys.exit(-1) _copy_to_build_lib_dir(self.build_lib, external_file) - # def _generate_binary_graph(self, binary_graph_target): - # """Generate binary graph for a particular MediaPipe binary graph target.""" - # - # bazel_command = [ - # 'bazel', - # 'build', - # '--compilation_mode=opt', - # '--copt=-DNDEBUG', - # '--define=MEDIAPIPE_DISABLE_GPU=1', - # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), - # binary_graph_target, - # ] - # if not self.link_opencv and not IS_WINDOWS: - # bazel_command.append('--define=OPENCV=source') - # if subprocess.call(bazel_command) != 0: - # sys.exit(-1) - # _copy_to_build_lib_dir(self.build_lib, binary_graph_target + '.binarypb') - def _generate_binary_graph(self, binary_graph_target): """Generate binary graph for a particular MediaPipe binary graph target.""" @@ -332,14 +314,6 @@ class GenerateMetadataSchema(build_ext.build_ext): 'object_detector_metadata_schema_py', 'schema_py', ]: - # bazel_command = [ - # 'bazel', - # 'build', - # '--compilation_mode=opt', - # '--define=MEDIAPIPE_DISABLE_GPU=1', - # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), - # '//mediapipe/tasks/metadata:' + target, - # ] bazel_command = [ 'bazel', @@ -427,15 +401,6 @@ class BuildExtension(build_ext.build_ext): def _build_binary(self, ext, extra_args=None): if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) - # bazel_command = [ - # 'bazel', - # 'build', - # '--compilation_mode=opt', - # '--copt=-DNDEBUG', - # '--define=MEDIAPIPE_DISABLE_GPU=1', - # '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), - # str(ext.bazel_target + '.so'), - # ] bazel_command = [ 'bazel', 'build',