From 1c40ecf8a54a8915e65592eb50fb377f6ba41b5f Mon Sep 17 00:00:00 2001 From: Kinar Date: Tue, 19 Sep 2023 03:08:01 -0700 Subject: [PATCH 001/157] Added files for Face Stylizer Unit Tests --- mediapipe/tasks/python/test/vision/BUILD | 17 ++ .../python/test/vision/face_stylizer_test.py | 182 ++++++++++++++++++ mediapipe/tasks/testdata/vision/BUILD | 1 + 3 files changed, 200 insertions(+) create mode 100644 mediapipe/tasks/python/test/vision/face_stylizer_test.py diff --git a/mediapipe/tasks/python/test/vision/BUILD b/mediapipe/tasks/python/test/vision/BUILD index ae3d53d61..c6fae0e6c 100644 --- a/mediapipe/tasks/python/test/vision/BUILD +++ b/mediapipe/tasks/python/test/vision/BUILD @@ -211,3 +211,20 @@ py_test( "//mediapipe/tasks/python/vision/core:image_processing_options", ], ) + +py_test( + name = "face_stylizer_test", + srcs = ["face_stylizer_test.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/components/containers:rect", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/test:test_utils", + "//mediapipe/tasks/python/vision:face_stylizer", + "//mediapipe/tasks/python/vision/core:image_processing_options", + ], +) diff --git a/mediapipe/tasks/python/test/vision/face_stylizer_test.py b/mediapipe/tasks/python/test/vision/face_stylizer_test.py new file mode 100644 index 000000000..a94f92db7 --- /dev/null +++ b/mediapipe/tasks/python/test/vision/face_stylizer_test.py @@ -0,0 +1,182 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for face stylizer.""" + +import enum +import os + +from absl.testing import absltest +from absl.testing import parameterized + +from mediapipe.python._framework_bindings import image as image_module +from mediapipe.tasks.python.core import base_options as base_options_module +from mediapipe.tasks.python.components.containers import rect +from mediapipe.tasks.python.test import test_utils +from mediapipe.tasks.python.vision import face_stylizer +from mediapipe.tasks.python.vision.core import image_processing_options as image_processing_options_module + + +_BaseOptions = base_options_module.BaseOptions +_Rect = rect.Rect +_Image = image_module.Image +_FaceStylizer = face_stylizer.FaceStylizer +_FaceStylizerOptions = face_stylizer.FaceStylizerOptions +_ImageProcessingOptions = image_processing_options_module.ImageProcessingOptions + +_MODEL = 'face_stylizer_color_ink.task' +_LARGE_FACE_IMAGE = "portrait.jpg" +_MODEL_IMAGE_SIZE = 256 +_TEST_DATA_DIR = 'mediapipe/tasks/testdata/vision' + + +class ModelFileType(enum.Enum): + FILE_CONTENT = 1 + FILE_NAME = 2 + + +class FaceStylizerTest(parameterized.TestCase): + + def setUp(self): + super().setUp() + self.test_image = _Image.create_from_file( + test_utils.get_test_data_path( + os.path.join(_TEST_DATA_DIR, _LARGE_FACE_IMAGE))) + self.model_path = test_utils.get_test_data_path( + os.path.join(_TEST_DATA_DIR, _MODEL)) + + def test_create_from_file_succeeds_with_valid_model_path(self): + # Creates with default option and valid model file successfully. + with _FaceStylizer.create_from_model_path(self.model_path) as stylizer: + self.assertIsInstance(stylizer, _FaceStylizer) + + def test_create_from_options_succeeds_with_valid_model_path(self): + # Creates with options containing model file successfully. + base_options = _BaseOptions(model_asset_path=self.model_path) + options = _FaceStylizerOptions(base_options=base_options) + with _FaceStylizer.create_from_options(options) as stylizer: + self.assertIsInstance(stylizer, _FaceStylizer) + + def test_create_from_options_fails_with_invalid_model_path(self): + with self.assertRaisesRegex( + RuntimeError, 'Unable to open file at /path/to/invalid/model.tflite'): + base_options = _BaseOptions( + model_asset_path='/path/to/invalid/model.tflite') + options = _FaceStylizerOptions(base_options=base_options) + _FaceStylizer.create_from_options(options) + + def test_create_from_options_succeeds_with_valid_model_content(self): + # Creates with options containing model content successfully. + with open(self.model_path, 'rb') as f: + base_options = _BaseOptions(model_asset_buffer=f.read()) + options = _FaceStylizerOptions(base_options=base_options) + stylizer = _FaceStylizer.create_from_options(options) + self.assertIsInstance(stylizer, _FaceStylizer) + + @parameterized.parameters( + (ModelFileType.FILE_NAME, _LARGE_FACE_IMAGE), + (ModelFileType.FILE_CONTENT, _LARGE_FACE_IMAGE) + ) + def test_stylize(self, model_file_type, image_file_name): + # Load the test image. + self.test_image = _Image.create_from_file( + test_utils.get_test_data_path( + os.path.join(_TEST_DATA_DIR, image_file_name))) + # Creates stylizer. + if model_file_type is ModelFileType.FILE_NAME: + base_options = _BaseOptions(model_asset_path=self.model_path) + elif model_file_type is ModelFileType.FILE_CONTENT: + with open(self.model_path, 'rb') as f: + model_content = f.read() + base_options = _BaseOptions(model_asset_buffer=model_content) + else: + # Should never happen + raise ValueError('model_file_type is invalid.') + + options = _FaceStylizerOptions(base_options=base_options) + stylizer = _FaceStylizer.create_from_options(options) + + # Performs face stylization on the input. + stylized_image = stylizer.stylize(self.test_image) + self.assertIsInstance(stylized_image, _Image) + # Closes the stylizer explicitly when the stylizer is not used in + # a context. + stylizer.close() + + @parameterized.parameters( + (ModelFileType.FILE_NAME, _LARGE_FACE_IMAGE), + (ModelFileType.FILE_CONTENT, _LARGE_FACE_IMAGE) + ) + def test_stylize_in_context(self, model_file_type, image_file_name): + # Load the test image. + self.test_image = _Image.create_from_file( + test_utils.get_test_data_path( + os.path.join(_TEST_DATA_DIR, image_file_name))) + # Creates stylizer. + if model_file_type is ModelFileType.FILE_NAME: + base_options = _BaseOptions(model_asset_path=self.model_path) + elif model_file_type is ModelFileType.FILE_CONTENT: + with open(self.model_path, 'rb') as f: + model_content = f.read() + base_options = _BaseOptions(model_asset_buffer=model_content) + else: + # Should never happen + raise ValueError('model_file_type is invalid.') + + options = _FaceStylizerOptions(base_options=base_options) + with _FaceStylizer.create_from_options(options) as stylizer: + # Performs face stylization on the input. + stylized_image = stylizer.stylize(self.test_image) + self.assertIsInstance(stylized_image, _Image) + self.assertEqual(stylized_image.width, _MODEL_IMAGE_SIZE) + self.assertEqual(stylized_image.height, _MODEL_IMAGE_SIZE) + + def test_stylize_succeeds_with_region_of_interest(self): + base_options = _BaseOptions(model_asset_path=self.model_path) + options = _FaceStylizerOptions(base_options=base_options) + with _FaceStylizer.create_from_options(options) as stylizer: + # Load the test image. + test_image = _Image.create_from_file( + test_utils.get_test_data_path( + os.path.join(_TEST_DATA_DIR, _LARGE_FACE_IMAGE) + ) + ) + # Region-of-interest around the face. + roi = _Rect(left=0.32, top=0.02, right=0.67, bottom=0.32) + image_processing_options = _ImageProcessingOptions(roi) + # Performs face stylization on the input. + stylized_image = stylizer.stylize(test_image, image_processing_options) + self.assertIsInstance(stylized_image, _Image) + self.assertEqual(stylized_image.width, _MODEL_IMAGE_SIZE) + self.assertEqual(stylized_image.height, _MODEL_IMAGE_SIZE) + + def test_stylize_succeeds_with_no_face_detected(self): + base_options = _BaseOptions(model_asset_path=self.model_path) + options = _FaceStylizerOptions(base_options=base_options) + with _FaceStylizer.create_from_options(options) as stylizer: + # Load the test image. + test_image = _Image.create_from_file( + test_utils.get_test_data_path( + os.path.join(_TEST_DATA_DIR, _LARGE_FACE_IMAGE) + ) + ) + # Region-of-interest that doesn't contain a human face. + roi = _Rect(left=0.1, top=0.1, right=0.2, bottom=0.2) + image_processing_options = _ImageProcessingOptions(roi) + # Performs face stylization on the input. + stylized_image = stylizer.stylize(test_image, image_processing_options) + self.assertIsNone(stylized_image) + + +if __name__ == '__main__': + absltest.main() diff --git a/mediapipe/tasks/testdata/vision/BUILD b/mediapipe/tasks/testdata/vision/BUILD index 19c9e6132..4fec8df9f 100644 --- a/mediapipe/tasks/testdata/vision/BUILD +++ b/mediapipe/tasks/testdata/vision/BUILD @@ -186,6 +186,7 @@ filegroup( "face_detection_short_range.tflite", "face_landmarker.task", "face_landmarker_v2.task", + "face_stylizer_color_ink.task", "hair_segmentation.tflite", "hand_landmark_full.tflite", "hand_landmark_lite.tflite", From 308f4f0e7364eccbfb42dcfb063b447daa24e972 Mon Sep 17 00:00:00 2001 From: Mark McDonald Date: Wed, 20 Sep 2023 16:45:22 +0800 Subject: [PATCH 002/157] Adds an empty skeleton project for iOS docgen. Dependencies tracked in `Podfile` are used to generate reference docs. --- .../project.pbxproj | 342 ++++++++++++++++++ .../contents.xcworkspacedata | 7 + .../xcshareddata/IDEWorkspaceChecks.plist | 8 + .../UserInterfaceState.xcuserstate | Bin 0 -> 12311 bytes .../xcschemes/xcschememanagement.plist | 14 + .../MediaPipeTasksDocGen.h | 18 + docs/MediaPipeTasksDocGen/Podfile | 11 + docs/MediaPipeTasksDocGen/README.md | 7 + 8 files changed, 407 insertions(+) create mode 100644 docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.pbxproj create mode 100644 docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/contents.xcworkspacedata create mode 100644 docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist create mode 100644 docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcuserdata/macd.xcuserdatad/UserInterfaceState.xcuserstate create mode 100644 docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/xcuserdata/macd.xcuserdatad/xcschemes/xcschememanagement.plist create mode 100644 docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen/MediaPipeTasksDocGen.h create mode 100644 docs/MediaPipeTasksDocGen/Podfile create mode 100644 docs/MediaPipeTasksDocGen/README.md diff --git a/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.pbxproj b/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.pbxproj new file mode 100644 index 000000000..ea38695b4 --- /dev/null +++ b/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.pbxproj @@ -0,0 +1,342 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 56; + objects = { + +/* Begin PBXBuildFile section */ + 8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */ = {isa = PBXBuildFile; fileRef = 8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */; settings = {ATTRIBUTES = (Public, ); }; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + 8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = MediaPipeTasksDocGen.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + 8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MediaPipeTasksDocGen.h; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 8566B5562ABABF9A00AAB22A /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 8566B54F2ABABF9A00AAB22A = { + isa = PBXGroup; + children = ( + 8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */, + 8566B55A2ABABF9A00AAB22A /* Products */, + ); + sourceTree = ""; + }; + 8566B55A2ABABF9A00AAB22A /* Products */ = { + isa = PBXGroup; + children = ( + 8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */, + ); + name = Products; + sourceTree = ""; + }; + 8566B55B2ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = { + isa = PBXGroup; + children = ( + 8566B55C2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h */, + ); + path = MediaPipeTasksDocGen; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXHeadersBuildPhase section */ + 8566B5542ABABF9A00AAB22A /* Headers */ = { + isa = PBXHeadersBuildPhase; + buildActionMask = 2147483647; + files = ( + 8566B55D2ABABF9A00AAB22A /* MediaPipeTasksDocGen.h in Headers */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXHeadersBuildPhase section */ + +/* Begin PBXNativeTarget section */ + 8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */ = { + isa = PBXNativeTarget; + buildConfigurationList = 8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */; + buildPhases = ( + 8566B5542ABABF9A00AAB22A /* Headers */, + 8566B5552ABABF9A00AAB22A /* Sources */, + 8566B5562ABABF9A00AAB22A /* Frameworks */, + 8566B5572ABABF9A00AAB22A /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = MediaPipeTasksDocGen; + productName = MediaPipeTasksDocGen; + productReference = 8566B5592ABABF9A00AAB22A /* MediaPipeTasksDocGen.framework */; + productType = "com.apple.product-type.framework"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 8566B5502ABABF9A00AAB22A /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastUpgradeCheck = 1430; + TargetAttributes = { + 8566B5582ABABF9A00AAB22A = { + CreatedOnToolsVersion = 14.3.1; + }; + }; + }; + buildConfigurationList = 8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */; + compatibilityVersion = "Xcode 14.0"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 8566B54F2ABABF9A00AAB22A; + productRefGroup = 8566B55A2ABABF9A00AAB22A /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 8566B5582ABABF9A00AAB22A /* MediaPipeTasksDocGen */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 8566B5572ABABF9A00AAB22A /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 8566B5552ABABF9A00AAB22A /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin XCBuildConfiguration section */ + 8566B55E2ABABF9A00AAB22A /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + CURRENT_PROJECT_VERSION = 1; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 16.4; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + VERSIONING_SYSTEM = "apple-generic"; + VERSION_INFO_PREFIX = ""; + }; + name = Debug; + }; + 8566B55F2ABABF9A00AAB22A /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_QUOTED_INCLUDE_IN_FRAMEWORK_HEADER = YES; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + COPY_PHASE_STRIP = NO; + CURRENT_PROJECT_VERSION = 1; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 16.4; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + VALIDATE_PRODUCT = YES; + VERSIONING_SYSTEM = "apple-generic"; + VERSION_INFO_PREFIX = ""; + }; + name = Release; + }; + 8566B5612ABABF9A00AAB22A /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEFINES_MODULE = YES; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + ENABLE_MODULE_VERIFIER = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++"; + MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20"; + PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen; + PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)"; + SKIP_INSTALL = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 8566B5622ABABF9A00AAB22A /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + DEFINES_MODULE = YES; + DYLIB_COMPATIBILITY_VERSION = 1; + DYLIB_CURRENT_VERSION = 1; + DYLIB_INSTALL_NAME_BASE = "@rpath"; + ENABLE_MODULE_VERIFIER = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + MARKETING_VERSION = 1.0; + MODULE_VERIFIER_SUPPORTED_LANGUAGES = "objective-c objective-c++"; + MODULE_VERIFIER_SUPPORTED_LANGUAGE_STANDARDS = "gnu11 gnu++20"; + PRODUCT_BUNDLE_IDENTIFIER = com.google.mediapipe.MediaPipeTasksDocGen; + PRODUCT_NAME = "$(TARGET_NAME:c99extidentifier)"; + SKIP_INSTALL = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 8566B5532ABABF9A00AAB22A /* Build configuration list for PBXProject "MediaPipeTasksDocGen" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 8566B55E2ABABF9A00AAB22A /* Debug */, + 8566B55F2ABABF9A00AAB22A /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 8566B5602ABABF9A00AAB22A /* Build configuration list for PBXNativeTarget "MediaPipeTasksDocGen" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 8566B5612ABABF9A00AAB22A /* Debug */, + 8566B5622ABABF9A00AAB22A /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 8566B5502ABABF9A00AAB22A /* Project object */; +} diff --git a/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 000000000..919434a62 --- /dev/null +++ b/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 000000000..18d981003 --- /dev/null +++ b/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcuserdata/macd.xcuserdatad/UserInterfaceState.xcuserstate b/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen.xcodeproj/project.xcworkspace/xcuserdata/macd.xcuserdatad/UserInterfaceState.xcuserstate new file mode 100644 index 0000000000000000000000000000000000000000..d667b462e8e7b9ec7cbb7ee36ffabf187121d251 GIT binary patch literal 12311 zcmd6N2UwF=`}a8~gDfQ|i4wv}LWU9u5CW*R0trf0M3xLO4del$A&E(F)Yp32T05+5 zwY#=&1*~ngyLL}IZEJ_K!?t$XW%su0e?QMd0x0dj*Y|$!cYS!7JURQ^=e~dU@4nB| z9`gGlQJd`m!iXRN5+V^2qXZ;tGAv`l5nnLSZ199Tx_$7g-VhCjnhe4A#Y{&uG7jN& zTinXT>blv?O2%6@*k~`3AZbHg)DvYWzC-ydk|PD0fF`08l#dEfA+n<)qT1bvD=L!YB#=nM2E`U-uG zzCquj@6h+?2lNa26AQ2yCtw*Ki!*T+)?qz1;B0KfCTzwz*oMn-1$JT=uEaC(EL?@_ za6N9ojkq0mU@vBHCti$~;H5Z>m*M63U-)c%4ql5d#Ov|J_!4{z-i~+RTk&o9c6>Tv=I+kOqP;fvXY!Z zR*^O20&*c)Pc9;tk}JrSOD9;p{p*X1r@Rz_Ojdr*ZM4IO?@B+`$P&^St{pdTe86&2EY>ToNQ ztDLR{!SIqu$kV|%gB`s+Od#r38j4Cv$_i~IMb?V^@(QcH%u!@5t0;C@U4;dOc6(u= zqs-=rO{_uL$hIA&A}vZo=_mtbqAa9CdSsv?Dy9ikLZwtjr zFW`xS0fLRiw!gp^@CKK|NE-gBEFAU(x}a+y>SWq`yQ%`6LANraHW*=|QP0R>8hsI8 zyU*{7u7m;W7(duBGO4Lo=zG>h`14KwV~|IcP4bW)-I06Jc86 zJIwe&7RwllsEQ`Ll?yw4er8HuUS2~4P%5v-)8Wly*OJ%6czvFbFT_~A!HzB_kT;ht zsDXldPh?31b~f`b3UgPm1B74~`_Oz;i|SB)3$LF6PY=_?KQfVq>N*fhKU6m~RoBhq zurRit1*jF-22c}fMlEy#ok&v#&_dLP7Euk&rbbp|IU|I{3n_LOF|p4Hc5nv|=R8{T znlQ7>7wnDHd%|7JprTcdkF!cI>g+=dO{Inds2lmvVi5fjZ3PgGK+SuLo4o(-Ue1fpRto+ox$%OOvTyU-nGf>9qwBV@v`|Q54zwQ5Z#NI?d=u zy=WQDq*+wg$ZxfgUxr&b?&MX0`j2*4{AsL4XS$W+tLx_WMm_C*#_0o-3kE#ll>=xs zT7w!ER@V)c@m_AFX4@ z8f9xS@mz@3BioiOtfKSCcrn_9)C1@ebSc_M&D26C4WP@=<>(4JndZ?#76;P-2T#Hc z9xx){dELsQ(HhM0M516D-jNtkzLM|A^)rsvZ)AIRg4j|wqw85?y^iMep?^~=hk>J? zS3fAy0NTphFRMMQlDJ>m=iNwid68{HH#Nd>0bur{ZKxqO!7XS9JHd7eAGgx{znLHr z-HGl(cQ?ZXk!aW#=%NK~<bRi?&`xw8XY2g$*lHg@53#F#klOpu z!?cK9?SHlcR%(x%8Zp}XylhzvaWIwCnv5B8X&mwgndWM$tq338Rb+o}bRs%%w z>-!5UiuK-uUPJ2b=q2V?EhqHOcjVd|eKo9_w(wv1ML7jLWdF!Q_Bbk_hS+%?N58RZ2xWcfcRG*ixP};E z!Z)npR2vJiXi*$IIj|bQLKd!+pa4M8umqDW6dJS7a&{JQ%JErnBA$S3H(@1C!sGCG zoQzdiO&e$?Gi{-*Hz6ZVL5FcFRHdQApdSlCL)xIq4c}lKDWIi1+o%r){q3G` zRfJR*hFR0xbev<^@NEjnpACCdPZiJ zPH)IIn#`6-lXE6d&a+Ln0evn4f1|0c<3(8s489HwlnK;$!oV=AoNncW>bjxg!Dj5C zwZ%KJzkspTMOXTnxj?zVhM|uy90AH32?YaQR@TN?^SPdYhv%;~Y_u7Pvh#T1!`JC! z8;-J=dSHG)Rsji=aM=7B=6F^Hd!rFhasGF8Z%;ehJt^LTR|>Waw$Kph;pQErznt0? zKbsmPYw#4q~suq6$Z;&FQihl(Dg@uMY864peaPZ-E7~B&S#z^N7QtfwD&YV?s+FbfBdY^ZVt*kH(8%XBhe`W;OFy!k`XPF=mL zVL>yCSSQ5ueT`Jv*93vV#Qm)cU2TipfP4T29ER{F+R#wtjPaClBdhAbDrgb}mgCtV z-{mUEw>t~0pb4ecN>{1NT2We1W-TfxFE4NumsVC3ITx;Cb!zZDVQU>;FyY~o^>@ra z(Ankc_SqH#3}wXZh+oO@hJ$_wWH1~~05V1xD)_LG4fw`fX9!*=P}Sc6{a>mZEIb&3 zg%6*(KR6$X!;uj|O@Da)BHIdw4gQLX3mI>z4Ki&>NP`TJuqc3pS{0 zVkxAsN+Dsj5YkfT;tlu;NIUJrU*ew#A_|g23dwX*1!<$XkTR-;lu;vTA}x?Kazomv zop=dDI!QMqkd}}h5`;8Tgha_kNE$r`$)G<3*@8I&MsTKJt>7D>SeP!%7Ul?xgmZ+| z!Wv<%uvxfB=o5y8mkF;F-XPp2yj}R9@L}Pj!pDWXgii@y5Pl>)F8odShX{#GBD<(m zG)pvFR4-a4x=M79=w8wNq6b9>L|=%$6=ShbEETK8DTwSq5Oc(Jq$BJ{o z6UAagaHj*fvIjk8wLA*Ol^Z6i}?G)R}X}B6v4tO@6gXhw2>Z6MX@I3fA zpDux)%h{!7av^Xobf4{u=EmpAje8MTXDkH~uLZ_CGaT#<@f8qiM`ANBzzdOWD{jKg zxCOUTKkcCb8r+K8@FMKSZ8Sue(lC6&%*o&(qYR$~D__YHY%cIjVuRE8Ol&b&M?(No zI-@k6$dAEz8>%=u7au3Ko-W*t)DTkxbzvFUh;i$JAbLm=#M=6C0AeTHLnD1SNTYw- zXCDx=2#(@jU?Bj8ET61lbLF*=BVr;zQ8@2Sm(kvNNqF~2&NeY6quCc_dbkzCkwbPS zTGr74Ar0eY1ZK0ka(bY^3f(@R)jnOHjU3MrH~87kN^5444)+L9|ur7>VSVyyblQEKc3}R zt`(Gf9YpzIG=m!%Y;Q`$@^Hn8Eums+NDBD_7q4E z)}RK|0xqyjwz#H)OtR9MqNgDttA* z249Oe(>0XRv*_9M9J-dCOV`o!==t=59r$`=#W&zBxDWT^0lXF8h_~UJ=!KA$zld(2 z7t>4Ve)deQBBy_1Ng&J4q$XAtb;o|NOR~ze7jgR zci#r=YdF$CrU2k=@OpvE_VdmobFgTLzK!e`Sv1$P!q)>-#ybc{NnGzP0A_ZHtWxjC z_u~5i@9u}Yum`{$Ku8jk&>SYv#ks9vWsM$xFQdeCJ#Q@!;fEn)iTCKokD!L(1ugSJ zOrR&@(kXCY}-INp2NHG^Y{h) zBB;bhx`|#!FQ->*!!O~N@hkAT7cvf4(yQph@VSrAHze`0gh?TTFu3vYeAz5OaZpq! z>{t?MuFE2WQ*!tYFdb7%sN}@P@BbsnyWpdAGc~-9KgFNn z&+#!hlP`d@eTBcq-{5cYcldk!HFBZpz(r%}cwQv|kJ!QakTDBfDZ}PHSmYQ?k&aXf zaPJZJigqYx?pCG-I4(0d(uwVpM%1$N48IV7-m&!z*?&Barw`Iw=yrM?eF&b$)1*WA z7knK53Y+^4|BnB_e*!1Pgb)FUCKm9-GJA`LP>nZORh+lf27^&x2Mk{hJ$oRE9%|0! zoLDWM*BKjgsE!WMf72W3&GdTO?^fo;A#sE-`2mM`QXCw)<04`>E|NfRfHWDA5jjdE z3S=c?TZU$7;O3`WKo$D_!mDu>M3Trjh>F3yhrA0PC2<4Pczof0GM@D@f7@pKnm_r! z{oGxS4^@*1Xbyy9TPbAu;7*HZNb2Z!8T~}d4nO)eMlXWh4Bu=uLE+TiNj6 zyL}^iVrT{T7(3p6k`0(LtbpH-iJ18;;D~b@65>cD?k5&@lWf-TAGL>EQAe;R*AoIX z=Whpd+3P_d?t_L8VU!KnH?fjjl1F#YTj^~BB%c(JLV7#BpFY3>z-TvkG4qe z6O7jU|I&pYDPby-Y+SCh>n z6#qJMJ$;G3%u$Au)%P#7^>0;ufNbSeorlVs$jy94afD8D9H^h%!YVbJZaldXm_m#b zx%t`=-kaP?ZW~m;yXfomz@YlQ#;e~wtl)RjgG1_fKc{|&{zLWKMV^Hm1bLD?MV==sWa7dX#=lbxkbEWEpNV$1~Zd7M_0cgr41VoCU{8b?lJ# zl0)n?`^bKBfE=Vp=)3ei`u%UQ-4sr4-9 zh%%tMW@PQ)%Qtl$!4RZAGDlQ$-Qfiqcmb#*>S&Ie ziGGIf^bvFC2y7?y%}0j?h4dHDmE&x>DG_?d!Y?*{)6ZD=1UCkxH&Wo$v>bR#atgd@ zH6PxET!I-BSmYN4MAwVtW8*- zup!}+gpCPTCtRCwUBbT;wj}II_(h_V9?2t;XC=F(Vril@OKO(pOQ%cAq!rRi=`87K((|PorPoTYlin!Z zF1=s+kn|Dhf zPL-9(Dr7F%Oj(tzUbaNGR(8GYPT6y^w`50UzskqUv*kv4iF~D;%CD4PFTYuSmwc!E zKKTRkXXLx(FUVh#zarl&e^dUp{D}NL`DgN96i7i7a)m-MRxv@LQD_zEiph#8ifM}J ziW!P>g;P=x=!7oUZ8GP zx2nVHb?O_`52>G1?^Zvreo=i?{hj(p_0Q^G)xWF%oS>PYpD<%W`2=T*G9@LYJf$in zobp=AhbbSYe427B<;#?>H4`;inp{o3rcmS1lxU`C7HPUP-I~Rkh=yve(p;n2thruu zgQib2pt(_Vljau94$WsXQ(LEP(t5Q)?NaSJ?Iqex+RL?9Y6rA;YVX$W z)ZV9kK>LvP5$*HZ7qu^GU(xQ>?$;jFzNY;=4W}ihrKU|yJ0!(+{Wrl>STluj#+1|Cxa^1Q|sc(=*C4Dl%LdwHa+09T`kUSB5WxW?YbQbH;5M zdom7Yyr1!5#?g#VGQP=7$W&zNGACy`GD|Y2WNymboOx5`t(gyG?#eut`AwEK%aK)) zbz0V(tm>?qtOZ%kS*=-ZS?;WhvmVVln)OxIw^`q3{isXONp*7F7+s<+pj)n6tGhtA zUbjKFNq4#KO5N4E8+6-r59{{nKGuD&`$6}k?q_|Xexg24Z`aS(*XtMPoAs@FuYQgG zEd4q9t@>^HoAulEx9V@#->JV_zf=E`{uTXR{eJyH{cHL+^zZ9G(toV~RR5#?XZ>;g zZ-(&(t-)fLY_J;g4ATr12A5%`p~_HWs5R6Z8VyZ`^9=VHju<{Md~W!{@KrX-CfUMl zakeD8D?60EI{WPGwb|>k*Jp3Yz9f5N_O|Rtv-f2m$$mHcz3dONj~RtVsWHn~Xq;oL zHr5zxjrGPxW0TQi^cp*j-Ns(ya^p(lD&r>OX5-Dq?Z#V;w;LZeK5cx~xZC)GagXt! z@ipTc#>2+9jYo{17{4|CZc>?arpYF&DbJK|Dl|FC9X7pVde`*6=|{7`EH+Eba`PB-qFHawG3T2L%|+&7v(r4=Jl8zWJm1`8 zZZR)3FEXEQ4wzS&FE(Fg-fG@qzRi4x`7ZMV=7-FWm>)AgVSe6x$o!i5b@Q9%x6GfK zkD0$Te`Eg6{Db)qi@+kXBv_;twI$7xVac-SEt4!cmRyU?Qec^8aan3D&6ZY6o5gME bv~*h*Tl|)Qh4L + + + + SchemeUserState + + MediaPipeTasksDocGen.xcscheme_^#shared#^_ + + orderHint + 0 + + + + diff --git a/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen/MediaPipeTasksDocGen.h b/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen/MediaPipeTasksDocGen.h new file mode 100644 index 000000000..ab21ced9e --- /dev/null +++ b/docs/MediaPipeTasksDocGen/MediaPipeTasksDocGen/MediaPipeTasksDocGen.h @@ -0,0 +1,18 @@ +// +// MediaPipeTasksDocGen.h +// MediaPipeTasksDocGen +// +// Created by Mark McDonald on 20/9/2023. +// + +#import + +//! Project version number for MediaPipeTasksDocGen. +FOUNDATION_EXPORT double MediaPipeTasksDocGenVersionNumber; + +//! Project version string for MediaPipeTasksDocGen. +FOUNDATION_EXPORT const unsigned char MediaPipeTasksDocGenVersionString[]; + +// In this header, you should import all the public headers of your framework using statements like #import + + diff --git a/docs/MediaPipeTasksDocGen/Podfile b/docs/MediaPipeTasksDocGen/Podfile new file mode 100644 index 000000000..3d8f4f16a --- /dev/null +++ b/docs/MediaPipeTasksDocGen/Podfile @@ -0,0 +1,11 @@ +# Uncomment the next line to define a global platform for your project +platform :ios, '15.0' + +target 'MediaPipeTasksDocGen' do + # Comment the next line if you don't want to use dynamic frameworks + use_frameworks! + + # Pods for MediaPipeTasksDocGen + pod 'MediaPipeTasksText', '0.10.5' + pod 'MediaPipeTasksVision', '0.10.5' +end diff --git a/docs/MediaPipeTasksDocGen/README.md b/docs/MediaPipeTasksDocGen/README.md new file mode 100644 index 000000000..77fc6abad --- /dev/null +++ b/docs/MediaPipeTasksDocGen/README.md @@ -0,0 +1,7 @@ +# MediaPipeTasksDocGen + +This empty project is used to generate reference documentation for the ObjectiveC and Swift libraries. + +Docs are generated using [Jazzy](https://github.com/realm/jazzy) and published to [the developer site](https://developers.devsite.corp.google.com/mediapipe/api/solutions/). + +To bump the API version used, edit [`Podfile`](./Podfile). From acca31503ab1a4ad26559b254db2b4b46cbee67a Mon Sep 17 00:00:00 2001 From: Mark McDonald Date: Thu, 21 Sep 2023 13:21:45 +0800 Subject: [PATCH 003/157] Remove pinned versions from deps --- docs/MediaPipeTasksDocGen/Podfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/MediaPipeTasksDocGen/Podfile b/docs/MediaPipeTasksDocGen/Podfile index 3d8f4f16a..3c8d8f09d 100644 --- a/docs/MediaPipeTasksDocGen/Podfile +++ b/docs/MediaPipeTasksDocGen/Podfile @@ -6,6 +6,6 @@ target 'MediaPipeTasksDocGen' do use_frameworks! # Pods for MediaPipeTasksDocGen - pod 'MediaPipeTasksText', '0.10.5' - pod 'MediaPipeTasksVision', '0.10.5' + pod 'MediaPipeTasksText' + pod 'MediaPipeTasksVision' end From 8d5cf33ca47be346874cda14f2e21b6046c0ef0c Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 3 Oct 2023 19:17:49 +0530 Subject: [PATCH 004/157] Added iOS Pose Landmarker Result --- .../tasks/ios/vision/pose_landmarker/BUILD | 29 +++++++++ .../sources/MPPPoseLandmarkerResult.h | 60 +++++++++++++++++++ .../sources/MPPPoseLandmarkerResult.m | 34 +++++++++++ 3 files changed, 123 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/BUILD create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.m diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/BUILD b/mediapipe/tasks/ios/vision/pose_landmarker/BUILD new file mode 100644 index 000000000..8aa0068cc --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/BUILD @@ -0,0 +1,29 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPPoseLandmarkerResult", + srcs = ["sources/MPPPoseLandmarkerResult.m"], + hdrs = ["sources/MPPPoseLandmarkerResult.h"], + deps = [ + "//mediapipe/tasks/ios/vision/core:MPPMask", + "//mediapipe/tasks/ios/components/containers:MPPLandmark", + "//mediapipe/tasks/ios/core:MPPTaskResult", + ], +) + diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h new file mode 100644 index 000000000..293459a6b --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h @@ -0,0 +1,60 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPLandmark.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPMask.h" + +NS_ASSUME_NONNULL_BEGIN + +/** /** Represents the pose landmarks deection results generated by `PoseLandmarker`. */ +NS_SWIFT_NAME(PoseLandmarkerResult) +@interface MPPPoseLandmarkerResult : MPPTaskResult + +/** Pose landmarks of detected poses. */ +@property(nonatomic, readonly) NSArray *> *landmarks; + +/** Pose landmarks in world coordniates of detected poses. */ +@property(nonatomic, readonly) NSArray *> *worldLandmarks; + +/** Pose segmentation masks. */ +@property(nonatomic, readonly) NSArray *segmentationMasks; + +/** + * Initializes a new `PoseLandmarkerResult` with the given array of landmarks, world landmarks, + * segmentation masks of the detected poses and timestamp (in milliseconds). + * + * @param landmarks An array of `NormalizedLandmark` objects. + * @param worldLandmarks An array of `Landmark` objects. + * @param segmentationMasks An array of `Mask` objects. + * @param timestampInMilliseconds The timestamp (in milliseconds) for this result. + * + * @return An instance of `PoseLandmarkerResult` initialized with the given array of landmarks, + * world landmarks, segmentation masks of the detected poses and timestamp (in milliseconds). + */ +- (instancetype)initWithLandmarks:(NSArray *> *)landmarks + worldLandmarks:(NSArray *> *)worldLandmarks + segmentationMasks:(NSArray *)segmentationMasks + timestampInMilliseconds:(NSInteger)timestampInMilliseconds NS_DESIGNATED_INITIALIZER; + +- (instancetype)initWithTimestampInMilliseconds:(NSInteger)timestampInMilliseconds NS_UNAVAILABLE; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.m b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.m new file mode 100644 index 000000000..4a0324508 --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.m @@ -0,0 +1,34 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h" + +@implementation MPPPoseLandmarkerResult + +- (instancetype)initWithLandmarks:(NSArray *> *)landmarks + worldLandmarks:(NSArray *> *)worldLandmarks + segmentationMasks:(NSArray *)segmentationMasks + timestampInMilliseconds:(NSInteger)timestampInMilliseconds { + self = [super initWithTimestampInMilliseconds:timestampInMilliseconds]; + if (self) { + _landmarks = [landmarks copy]; + _worldLandmarks = [worldLandmarks copy]; + _segmentationMasks = [segmentationMasks copy]; + } + return self; +} + +@end From c560032a9198e90f9045d2296942bbf16dcada21 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 3 Oct 2023 19:18:06 +0530 Subject: [PATCH 005/157] Added iOS pose landmarker options --- .../tasks/ios/vision/pose_landmarker/BUILD | 10 ++ .../sources/MPPPoseLandmarkerOptions.h | 107 ++++++++++++++++++ .../sources/MPPPoseLandmarkerOptions.m | 45 ++++++++ 3 files changed, 162 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.m diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/BUILD b/mediapipe/tasks/ios/vision/pose_landmarker/BUILD index 8aa0068cc..e45113758 100644 --- a/mediapipe/tasks/ios/vision/pose_landmarker/BUILD +++ b/mediapipe/tasks/ios/vision/pose_landmarker/BUILD @@ -27,3 +27,13 @@ objc_library( ], ) +objc_library( + name = "MPPPoseLandmarkerOptions", + srcs = ["sources/MPPPoseLandmarkerOptions.m"], + hdrs = ["sources/MPPPoseLandmarkerOptions.h"], + deps = [ + ":MPPPoseLandmarkerResult", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/vision/core:MPPRunningMode", + ], +) diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h new file mode 100644 index 000000000..e0cb4d7bf --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h @@ -0,0 +1,107 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h" + +NS_ASSUME_NONNULL_BEGIN + +@class MPPPoseLandmarker; + +/** + * This protocol defines an interface for the delegates of `PoseLandmarker` to receive + * results of performing asynchronous pose landmark detection on images (i.e, when `runningMode` = + * `.liveStream`). + * + * The delegate of `PoseLandmarker` must adopt `PoseLandmarkerLiveStreamDelegate` protocol. + * The methods in this protocol are optional. + */ +NS_SWIFT_NAME(PoseLandmarkerLiveStreamDelegate) +@protocol MPPPoseLandmarkerLiveStreamDelegate + +/** + * This method notifies a delegate that the results of asynchronous pose landmark detection of an + * image submitted to the `PoseLandmarker` is available. + * + * This method is called on a private serial dispatch queue created by the `PoseLandmarker` + * for performing the asynchronous delegates calls. + * + * @param poseLandmarker The pose landmarker which performed the pose landmark detection. + * This is useful to test equality when there are multiple instances of `PoseLandmarker`. + * @param result The `PoseLandmarkerResult` object that contains a list of landmark. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image was sent to the pose landmarker. + * @param error An optional error parameter populated when there is an error in performing pose + * landmark detection on the input live stream image data. + */ +- (void)poseLandmarker:(MPPPoseLandmarker *)poseLandmarker + didFinishDetectionWithResult:(nullable MPPPoseLandmarkerResult *)result + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(nullable NSError *)error + NS_SWIFT_NAME(poseLandmarker(_:didFinishDetection:timestampInMilliseconds:error:)); +@end + +/** Options for setting up a `PoseLandmarker`. */ +NS_SWIFT_NAME(PoseLandmarkerOptions) +@interface MPPPoseLandmarkerOptions : MPPTaskOptions + +/** + * Running mode of the pose landmark dection task. Defaults to `.image`. `PoseLandmarker` can be + * created with one of the following running modes: + * 1. `.image`: The mode for performing pose landmark detection on single image inputs. + * 2. `.video`: The mode for performing pose landmark detection on the decoded frames of a video. + * 3. `.liveStream`: The mode for performing pose landmark detection on a live stream of input + * data, such as from the camera. + */ +@property(nonatomic) MPPRunningMode runningMode; + +/** + * An object that confirms to `PoseLandmarkerLiveStreamDelegate` protocol. This object must + * implement `poseLandmarker(_:didFinishDetectionWithResult:timestampInMilliseconds:error:)` to + * receive the results of performing asynchronous pose landmark detection on images (i.e, when + * `runningMode` = `.liveStream`). + */ +@property(nonatomic, weak, nullable) id + poseLandmarkerLiveStreamDelegate; + +/** The maximum number of poses that can be detected by the `PoseLandmarker`. Defaults to 1. */ +@property(nonatomic) NSInteger numPoses; + +/** + * The minimum confidence score for pose detection to be considered successful. Defaults to 0.5. + */ +@property(nonatomic) float minPoseDetectionConfidence; + +/** + * The minimum confidence score of pose presence score in the pose landmark detection. Defaults to + * 0.5. + */ +@property(nonatomic) float minPosePresenceConfidence; + +/** + * The minimum confidence score for pose tracking to be considered successful. Defaults to 0.5. + */ +@property(nonatomic) float minTrackingConfidence; + +/** + * Whether to output segmentation masks. Defaults to `false`. + */ +@property(nonatomic) BOOL shouldOutputSegmentationMasks; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.m b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.m new file mode 100644 index 000000000..0a838693c --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.m @@ -0,0 +1,45 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h" + +@implementation MPPPoseLandmarkerOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _numPoses = 1; + _minPoseDetectionConfidence = 0.5f; + _minPosePresenceConfidence = 0.5f; + _minTrackingConfidence = 0.5f; + _shouldOutputSegmentationMasks = NO; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPPoseLandmarkerOptions *poseLandmarkerOptions = [super copyWithZone:zone]; + + poseLandmarkerOptions.runningMode = self.runningMode; + poseLandmarkerOptions.numPoses = self.numPoses; + poseLandmarkerOptions.minPoseDetectionConfidence = self.minPoseDetectionConfidence; + poseLandmarkerOptions.minPosePresenceConfidence = self.minPosePresenceConfidence; + poseLandmarkerOptions.minTrackingConfidence = self.minTrackingConfidence; + poseLandmarkerOptions.shouldOutputSegmentationMasks = self.shouldOutputSegmentationMasks; + poseLandmarkerOptions.poseLandmarkerLiveStreamDelegate = self.poseLandmarkerLiveStreamDelegate; + + return poseLandmarkerOptions; +} + +@end From 3067c20955395d90ea9abb8d5609df66d7a9fbc5 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 3 Oct 2023 19:18:29 +0530 Subject: [PATCH 006/157] Added iOS pose landmarker result helpers --- .../ios/vision/pose_landmarker/utils/BUILD | 37 ++++++++++++++ .../MPPPoseLandmarkerOptions+Helpers.h | 36 +++++++++++++ .../MPPPoseLandmarkerOptions+Helpers.mm | 51 +++++++++++++++++++ 3 files changed, 124 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/utils/BUILD create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.mm diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/utils/BUILD b/mediapipe/tasks/ios/vision/pose_landmarker/utils/BUILD new file mode 100644 index 000000000..bcee773e8 --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/utils/BUILD @@ -0,0 +1,37 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPPoseLandmarkerOptionsHelpers", + srcs = ["sources/MPPPoseLandmarkerOptions+Helpers.mm"], + hdrs = ["sources/MPPPoseLandmarkerOptions+Helpers.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarker_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarks_detector_graph_options_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol", + "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", + "//mediapipe/tasks/ios/vision/pose_landmarker:MPPPoseLandmarkerOptions", + ], +) diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.h b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.h new file mode 100644 index 000000000..9a60c195b --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.h @@ -0,0 +1,36 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __cplusplus +#error "This file requires Objective-C++." +#endif // __cplusplus + +#include "mediapipe/framework/calculator_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" +#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPPoseLandmarkerOptions (Helpers) + +/** + * Populates the provided `CalculatorOptions` proto container with the current settings. + * + * @param optionsProto The `CalculatorOptions` proto object to copy the settings to. + */ +- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.mm b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.mm new file mode 100644 index 000000000..3f5baaa39 --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.mm @@ -0,0 +1,51 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarker_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.pb.h" + +using CalculatorOptionsProto = ::mediapipe::CalculatorOptions; +using PoseDetectorGraphOptionsProto = + ::mediapipe::tasks::vision::pose_detector::proto::PoseDetectorGraphOptions; +using PoseLandmarksDetectorGraphOptionsProto = + ::mediapipe::tasks::vision::pose_landmarker::proto::PoseLandmarksDetectorGraphOptions; +using PoseLandmarkerGraphOptionsProto = + ::mediapipe::tasks::vision::pose_landmarker::proto::PoseLandmarkerGraphOptions; + +@implementation MPPPoseLandmarkerOptions (Helpers) + +- (void)copyToProto:(CalculatorOptionsProto *)optionsProto { + PoseLandmarkerGraphOptionsProto *poseLandmarkerGraphOptions = + optionsProto->MutableExtension(PoseLandmarkerGraphOptionsProto::ext); + poseLandmarkerGraphOptions->Clear(); + + [self.baseOptions copyToProto:poseLandmarkerGraphOptions->mutable_base_options()]; + poseLandmarkerGraphOptions->set_min_tracking_confidence(self.minTrackingConfidence); + + PoseLandmarksDetectorGraphOptionsProto *poseLandmarksDetectorGraphOptions = + poseLandmarkerGraphOptions->mutable_pose_landmarks_detector_graph_options(); + poseLandmarksDetectorGraphOptions->set_min_detection_confidence(self.minPosePresenceConfidence); + + PoseDetectorGraphOptionsProto *poseDetectorGraphOptions = + poseLandmarkerGraphOptions->mutable_pose_detector_graph_options(); + poseDetectorGraphOptions->set_num_poses(self.numPoses); + poseDetectorGraphOptions->set_min_detection_confidence(self.minPoseDetectionConfidence); +} + +@end From 3c13e4b6d6220091659d52b88507ec3b1b67b17d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 3 Oct 2023 19:20:33 +0530 Subject: [PATCH 007/157] Added iOS language detector options --- .../tasks/ios/text/language_detector/BUILD | 27 ++++++++ .../sources/MPPLanguageDetectorOptions.h | 61 +++++++++++++++++++ .../sources/MPPLanguageDetectorOptions.m | 40 ++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 mediapipe/tasks/ios/text/language_detector/BUILD create mode 100644 mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorOptions.h create mode 100644 mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorOptions.m diff --git a/mediapipe/tasks/ios/text/language_detector/BUILD b/mediapipe/tasks/ios/text/language_detector/BUILD new file mode 100644 index 000000000..7e9d6e68b --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/BUILD @@ -0,0 +1,27 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = [ + "//mediapipe/tasks:internal", +]) + +licenses(["notice"]) + +objc_library( + name = "MPPLanguageDetectorOptions", + srcs = ["sources/MPPLanguageDetectorOptions.m"], + hdrs = ["sources/MPPLanguageDetectorOptions.h"], + deps = ["//mediapipe/tasks/ios/core:MPPTaskOptions"], +) + diff --git a/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorOptions.h b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorOptions.h new file mode 100644 index 000000000..9674c482b --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorOptions.h @@ -0,0 +1,61 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Options for setting up a `LanguageDetector`. + */ +NS_SWIFT_NAME(LanguageDetectorOptions) +@interface MPPLanguageDetectorOptions : MPPTaskOptions + +/** + * The locale to use for display names specified through the TFLite Model Metadata, if any. Defaults + * to English. + */ +@property(nonatomic, copy) NSString *displayNamesLocale; + +/** + * The maximum number of top-scored classification results to return. If < 0, all available results + * will be returned. If 0, an invalid argument error is returned. + */ +@property(nonatomic) NSInteger maxResults; + +/** + * Score threshold to override the one provided in the model metadata (if any). Results below this + * value are rejected. + */ +@property(nonatomic) float scoreThreshold; + +/** + * The allowlist of category names. If non-empty, detection results whose category name is not in + * this set will be filtered out. Duplicate or unknown category names are ignored. Mutually + * exclusive with categoryDenylist. + */ +@property(nonatomic, copy) NSArray *categoryAllowlist; + +/** + * The denylist of category names. If non-empty, detection results whose category name is in this + * set will be filtered out. Duplicate or unknown category names are ignored. Mutually exclusive + * with categoryAllowlist. + */ +@property(nonatomic, copy) NSArray *categoryDenylist; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorOptions.m b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorOptions.m new file mode 100644 index 000000000..df36493ef --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorOptions.m @@ -0,0 +1,40 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorOptions.h" + +@implementation MPPLanguageDetectorOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _maxResults = -1; + _scoreThreshold = 0; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPLanguageDetectorOptions *languageDetectorOptions = [super copyWithZone:zone]; + + languageDetectorOptions.scoreThreshold = self.scoreThreshold; + languageDetectorOptions.maxResults = self.maxResults; + languageDetectorOptions.categoryDenylist = self.categoryDenylist; + languageDetectorOptions.categoryAllowlist = self.categoryAllowlist; + languageDetectorOptions.displayNamesLocale = self.displayNamesLocale; + + return languageDetectorOptions; +} + +@end From 38de7493dfcd6ccfd726d5dc423471f7d211f6bd Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 3 Oct 2023 19:20:45 +0530 Subject: [PATCH 008/157] Added iOS language detector results --- .../tasks/ios/text/language_detector/BUILD | 8 +++ .../sources/MPPLanguageDetectorResult.h | 70 +++++++++++++++++++ .../sources/MPPLanguageDetectorResult.m | 41 +++++++++++ 3 files changed, 119 insertions(+) create mode 100644 mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorResult.h create mode 100644 mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorResult.m diff --git a/mediapipe/tasks/ios/text/language_detector/BUILD b/mediapipe/tasks/ios/text/language_detector/BUILD index 7e9d6e68b..cf2fbacb6 100644 --- a/mediapipe/tasks/ios/text/language_detector/BUILD +++ b/mediapipe/tasks/ios/text/language_detector/BUILD @@ -25,3 +25,11 @@ objc_library( deps = ["//mediapipe/tasks/ios/core:MPPTaskOptions"], ) +objc_library( + name = "MPPLanguageDetectorResult", + srcs = ["sources/MPPLanguageDetectorResult.m"], + hdrs = ["sources/MPPLanguageDetectorResult.h"], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskResult", + ], +) diff --git a/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorResult.h b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorResult.h new file mode 100644 index 000000000..a8b9fe735 --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorResult.h @@ -0,0 +1,70 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h" + +NS_ASSUME_NONNULL_BEGIN + +NS_SWIFT_NAME(LanguagePrediction) +@interface MPPLanguagePrediction : NSObject + +/** The i18n language / locale code for the prediction. */ +@property(nonatomic, readonly) NSString *languageCode; + +/** The probability for the prediction. */ +@property(nonatomic, readonly) float probability; + +/** + * Initializes a new `LanguagePrediction` with the given language code and probability. + * + * @param languageCode The i18n language / locale code for the prediction. + * @param probability The probability for the prediction. + * + * @return An instance of `LanguagePrediction` initialized with the given language code and + * probability. + */ +- (instancetype)initWithLanguageCode:(NSString *)languageCode probability:(float)probability; + +@end + +/** Represents the results generated by `LanguageDetector`. **/ +NS_SWIFT_NAME(LanguageDetectorResult) +@interface MPPLanguageDetectorResult : MPPTaskResult + +/** A list of language predictions. */ +@property(nonatomic, readonly) NSArray *languagePredictions; + +/** + * Initializes a new `LanguageDetectorResult` with the given array of language predictions and + * timestamp (in milliseconds). + * + * @param languagePrediction The array of language predictions in this result. + * @param timestampInMilliseconds The timestamp (in milliseconds) for this result. + * + * @return An instance of `LanguageDetectorResult` initialized with the given array of language + * predictions and timestamp (in milliseconds). + */ +- (instancetype)initWithLanguagePredictions:(NSArray *)languagePredictions + timestampInMilliseconds:(NSInteger)timestampInMilliseconds; + +- (instancetype)initWithTimestampInMilliseconds:(NSInteger)timestampInMilliseconds NS_UNAVAILABLE; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorResult.m b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorResult.m new file mode 100644 index 000000000..126cf6c67 --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorResult.m @@ -0,0 +1,41 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorResult.h" + +@implementation MPPLanguagePrediction + +- (instancetype)initWithLanguageCode:(NSString *)languageCode probability:(float)probability { + self = [super init]; + if (self) { + _languageCode = languageCode; + _probability = probability; + } + return self; +} + +@end + +@implementation MPPLanguageDetectorResult + +- (instancetype)initWithLanguagePredictions:(NSArray *)languagePredictions + timestampInMilliseconds:(NSInteger)timestampInMilliseconds { + self = [super initWithTimestampInMilliseconds:timestampInMilliseconds]; + if (self) { + _languagePredictions = languagePredictions; + } + return self; +} + +@end From 0ee9b7f86ef69553e2629a3e7928401404e27f75 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 3 Oct 2023 19:21:05 +0530 Subject: [PATCH 009/157] Added iOS language detector options helpers --- .../ios/text/language_detector/utils/BUILD | 32 +++++++++++ .../MPPLanguageDetectorOptions+Helpers.h | 27 +++++++++ .../MPPLanguageDetectorOptions+Helpers.mm | 56 +++++++++++++++++++ 3 files changed, 115 insertions(+) create mode 100644 mediapipe/tasks/ios/text/language_detector/utils/BUILD create mode 100644 mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorOptions+Helpers.h create mode 100644 mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorOptions+Helpers.mm diff --git a/mediapipe/tasks/ios/text/language_detector/utils/BUILD b/mediapipe/tasks/ios/text/language_detector/utils/BUILD new file mode 100644 index 000000000..00a37e940 --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/utils/BUILD @@ -0,0 +1,32 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPLanguageDetectorOptionsHelpers", + srcs = ["sources/MPPLanguageDetectorOptions+Helpers.mm"], + hdrs = ["sources/MPPLanguageDetectorOptions+Helpers.h"], + deps = [ + "//mediapipe/framework:calculator_options_cc_proto", + "//mediapipe/tasks/cc/components/processors/proto:classifier_options_cc_proto", + "//mediapipe/tasks/cc/text/text_classifier/proto:text_classifier_graph_options_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskOptionsProtocol", + "//mediapipe/tasks/ios/core/utils:MPPBaseOptionsHelpers", + "//mediapipe/tasks/ios/text/language_detector:MPPLanguageDetectorOptions", + ], +) diff --git a/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorOptions+Helpers.h b/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorOptions+Helpers.h new file mode 100644 index 000000000..5406e901d --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorOptions+Helpers.h @@ -0,0 +1,27 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/calculator_options.pb.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptionsProtocol.h" +#import "mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPLanguageDetectorOptions (Helpers) + +- (void)copyToProto:(::mediapipe::CalculatorOptions *)optionsProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorOptions+Helpers.mm b/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorOptions+Helpers.mm new file mode 100644 index 000000000..9d75105b4 --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorOptions+Helpers.mm @@ -0,0 +1,56 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorOptions+Helpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" + +#include "mediapipe/tasks/cc/components/processors/proto/classifier_options.pb.h" +#include "mediapipe/tasks/cc/text/text_classifier/proto/text_classifier_graph_options.pb.h" + +namespace { +using CalculatorOptionsProto = ::mediapipe::CalculatorOptions; +using TextClassifierGraphOptionsProto = + ::mediapipe::tasks::text::text_classifier::proto::TextClassifierGraphOptions; +using ClassifierOptionsProto = ::mediapipe::tasks::components::processors::proto::ClassifierOptions; +} // namespace + +@implementation MPPLanguageDetectorOptions (Helpers) + +- (void)copyToProto:(CalculatorOptionsProto *)optionsProto { + TextClassifierGraphOptionsProto *graphOptions = + optionsProto->MutableExtension(TextClassifierGraphOptionsProto::ext); + [self.baseOptions copyToProto:graphOptions->mutable_base_options()]; + + ClassifierOptionsProto *classifierOptionsProto = graphOptions->mutable_classifier_options(); + classifierOptionsProto->Clear(); + + if (self.displayNamesLocale) { + classifierOptionsProto->set_display_names_locale(self.displayNamesLocale.cppString); + } + + classifierOptionsProto->set_max_results((int)self.maxResults); + classifierOptionsProto->set_score_threshold(self.scoreThreshold); + + for (NSString *category in self.categoryAllowlist) { + classifierOptionsProto->add_category_allowlist(category.cppString); + } + + for (NSString *category in self.categoryDenylist) { + classifierOptionsProto->add_category_denylist(category.cppString); + } +} + +@end From dd823d16f821838fd7011c991889d390fb6094e1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 9 Oct 2023 17:46:34 +0530 Subject: [PATCH 010/157] Added property to get labels from iOS Image Segmenter --- .../tasks/ios/core/sources/MPPTaskRunner.h | 6 + .../tasks/ios/core/sources/MPPTaskRunner.mm | 4 + .../tasks/ios/vision/image_segmenter/BUILD | 2 + .../sources/MPPImageSegmenter.h | 129 +++++++++--------- .../sources/MPPImageSegmenter.mm | 49 ++++++- 5 files changed, 121 insertions(+), 69 deletions(-) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 41515571a..c4da9789e 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -36,6 +36,12 @@ NS_ASSUME_NONNULL_BEGIN */ @interface MPPTaskRunner : NSObject +/** + * The canonicalized `CalculatorGraphConfig` of the underlying graph managed by the C++ task + * runner. + */ +@property(nonatomic, readonly) const mediapipe::CalculatorGraphConfig &graphConfig; + /** * Initializes a new `MPPTaskRunner` with the MediaPipe calculator configuration proto and an * optional C++ packets callback. diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm index 0813760c2..3e9fb61ea 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm @@ -33,6 +33,10 @@ using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; @implementation MPPTaskRunner +- (const CalculatorGraphConfig &)graphConfig { + return _cppTaskRunner->GetGraphConfig(); +} + - (instancetype)initWithCalculatorGraphConfig:(CalculatorGraphConfig)graphConfig packetsCallback:(PacketsCallback)packetsCallback error:(NSError **)error { diff --git a/mediapipe/tasks/ios/vision/image_segmenter/BUILD b/mediapipe/tasks/ios/vision/image_segmenter/BUILD index c3bb897ea..7dd75eaf1 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/BUILD +++ b/mediapipe/tasks/ios/vision/image_segmenter/BUILD @@ -51,6 +51,8 @@ objc_library( ":MPPImageSegmenterOptions", ":MPPImageSegmenterResult", "//mediapipe/tasks/cc/vision/image_segmenter:image_segmenter_graph", + "//mediapipe/tasks/cc/vision/image_segmenter/calculators:tensors_to_segmentation_calculator", + "//mediapipe/tasks/ios/common:MPPCommon", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", "//mediapipe/tasks/ios/common/utils:NSStringHelpers", "//mediapipe/tasks/ios/core:MPPTaskInfo", diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h index f75e7575e..9af9586fb 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h @@ -29,27 +29,31 @@ NS_SWIFT_NAME(ImageSegmenter) @interface MPPImageSegmenter : NSObject /** - * Creates a new instance of `MPPImageSegmenter` from an absolute path to a TensorFlow Lite model - * file stored locally on the device and the default `MPPImageSegmenterOptions`. + * Get the category label list of the `ImageSegmenter` can recognize. For CATEGORY_MASK type, the + * index in the category mask corresponds to the category in the label list. For CONFIDENCE_MASK + * type, the output mask list at index corresponds to the category in the label list. If there is no + * labelmap provided in the model file, empty array is returned. + */ +@property(nonatomic, readonly) NSArray *labels; + +/** + * Creates a new instance of `ImageSegmenter` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `ImageSegmenterOptions`. * * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. - * @param error An optional error parameter populated when there is an error in initializing the - * image segmenter. * - * @return A new instance of `MPPImageSegmenter` with the given model path. `nil` if there is an + * @return A new instance of `ImageSegmenter` with the given model path. `nil` if there is an * error in initializing the image segmenter. */ - (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; /** - * Creates a new instance of `MPPImageSegmenter` from the given `MPPImageSegmenterOptions`. + * Creates a new instance of `ImageSegmenter` from the given `ImageSegmenterOptions`. * - * @param options The options of type `MPPImageSegmenterOptions` to use for configuring the - * `MPPImageSegmenter`. - * @param error An optional error parameter populated when there is an error in initializing the - * image segmenter. + * @param options The options of type `ImageSegmenterOptions` to use for configuring the + * `ImageSegmenter`. * - * @return A new instance of `MPPImageSegmenter` with the given options. `nil` if there is an error + * @return A new instance of `ImageSegmenter` with the given options. `nil` if there is an error * in initializing the image segmenter. */ - (nullable instancetype)initWithOptions:(MPPImageSegmenterOptions *)options @@ -57,23 +61,20 @@ NS_SWIFT_NAME(ImageSegmenter) /** * Performs segmentation on the provided MPPImage using the whole image as region of interest. - * Rotation will be applied according to the `orientation` property of the provided `MPPImage`. Only - * use this method when the `MPPImageSegmenter` is created with `MPPRunningModeImage`. + * Rotation will be applied according to the `orientation` property of the provided `MPImage`. Only + * use this method when the `ImageSegmenter` is created with running mode, `image`. * - * This method supports RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports RGBA images. If your `MPImage` has a source type of `pixelBuffer` or + * `sampleBuffer`, the underlying pixel buffer must have one of the following pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. * - * @param image The `MPPImage` on which segmentation is to be performed. - * @param error An optional error parameter populated when there is an error in performing - * segmentation on the input image. + * @param image The `MPImage` on which segmentation is to be performed. * - * @return An `MPPImageSegmenterResult` that contains the segmented masks. + * @return An `ImageSegmenterResult` that contains the segmented masks. */ - (nullable MPPImageSegmenterResult *)segmentImage:(MPPImage *)image error:(NSError **)error NS_SWIFT_NAME(segment(image:)); @@ -83,22 +84,20 @@ NS_SWIFT_NAME(ImageSegmenter) * invokes the given completion handler block with the response. The method returns synchronously * once the completion handler returns. * - * Rotation will be applied according to the `orientation` property of the provided - * `MPPImage`. Only use this method when the `MPPImageSegmenter` is created with - * `MPPRunningModeImage`. + * Rotation will be applied according to the `orientation` property of the provided `MPImage`. Only + * use this method when the `ImageSegmenter` is created with running mode, `image`. * - * This method supports RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports RGBA images. If your `MPImage` has a source type of `pixelBuffer` or + * `sampleBuffer`, the underlying pixel buffer must have one of the following pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `image` ensure that the color space is RGB with an Alpha + * channel. * - * @param image The `MPPImage` on which segmentation is to be performed. + * @param image The `MPImage` on which segmentation is to be performed. * @param completionHandler A block to be invoked with the results of performing segmentation on the - * image. The block takes two arguments, the optional `MPPImageSegmenterResult` that contains the + * image. The block takes two arguments, the optional `ImageSegmenterResult` that contains the * segmented masks if the segmentation was successful and an optional error populated upon failure. * The lifetime of the returned masks is only guaranteed for the duration of the block. */ @@ -108,28 +107,25 @@ NS_SWIFT_NAME(ImageSegmenter) NS_SWIFT_NAME(segment(image:completion:)); /** - * Performs segmentation on the provided video frame of type `MPPImage` using the whole image as + * Performs segmentation on the provided video frame of type `MPImage` using the whole image as * region of interest. * - * Rotation will be applied according to the `orientation` property of the provided `MPPImage`. Only - * use this method when the `MPPImageSegmenter` is created with `MPPRunningModeVideo`. + * Rotation will be applied according to the `orientation` property of the provided `MPImage`. Only + * use this method when the `ImageSegmenter` is created with `video`. * - * This method supports RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports RGBA images. If your `MPImage` has a source type of `pixelBuffer` or + * `sampleBuffer`, the underlying pixel buffer must have one of the following pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `image` ensure that the color space is RGB with an Alpha + * channel. * - * @param image The `MPPImage` on which segmentation is to be performed. + * @param image The `MPImage` on which segmentation is to be performed. * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input * timestamps must be monotonically increasing. - * @param error An optional error parameter populated when there is an error in performing - * segmentation on the input image. * - * @return An `MPPImageSegmenterResult` that contains a the segmented masks. + * @return An `ImageSegmenterResult` that contains a the segmented masks. */ - (nullable MPPImageSegmenterResult *)segmentVideoFrame:(MPPImage *)image timestampInMilliseconds:(NSInteger)timestampInMilliseconds @@ -137,27 +133,26 @@ NS_SWIFT_NAME(ImageSegmenter) NS_SWIFT_NAME(segment(videoFrame:timestampInMilliseconds:)); /** - * Performs segmentation on the provided video frame of type `MPPImage` using the whole image as + * Performs segmentation on the provided video frame of type `MPImage` using the whole image as * region of interest invokes the given completion handler block with the response. The method * returns synchronously once the completion handler returns. * - * Rotation will be applied according to the `orientation` property of the provided `MPPImage`. Only - * use this method when the `MPPImageSegmenter` is created with `MPPRunningModeVideo`. + * Rotation will be applied according to the `orientation` property of the provided `MPImage`. Only + * use this method when the `ImageSegmenter` is created with running mode, `video`. * - * This method supports RGBA images. If your `MPPImage` has a source type of - * `MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports RGBA images. If your `MPImage` has a source type of `pixelBuffer` or + * `sampleBuffer`, the underlying pixel buffer must have one of the following pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If your `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color space is - * RGB with an Alpha channel. + * If your `MPImage` has a source type of `image` ensure that the color space is RGB with an Alpha + * channel. * - * @param image The `MPPImage` on which segmentation is to be performed. + * @param image The `MPImage` on which segmentation is to be performed. * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input * timestamps must be monotonically increasing. * @param completionHandler A block to be invoked with the results of performing segmentation on the - * image. The block takes two arguments, the optional `MPPImageSegmenterResult` that contains the + * image. The block takes two arguments, the optional `ImageSegmenterResult` that contains the * segmented masks if the segmentation was successful and an optional error only populated upon * failure. The lifetime of the returned masks is only guaranteed for the duration of the block. */ @@ -168,38 +163,36 @@ NS_SWIFT_NAME(ImageSegmenter) NS_SWIFT_NAME(segment(videoFrame:timestampInMilliseconds:completion:)); /** - * Sends live stream image data of type `MPPImage` to perform segmentation using the whole image as - * region of interest. + * Sends live stream image data of type `MPImage` to perform segmentation using the whole image as + *region of interest. * - * Rotation will be applied according to the `orientation` property of the provided `MPPImage`. Only - * use this method when the `MPPImageSegmenter` is created with`MPPRunningModeLiveStream`. + * Rotation will be applied according to the `orientation` property of the provided `MPImage`. Only + *use this method when the `ImageSegmenter` is created with running mode, `liveStream`. * * The object which needs to be continuously notified of the available results of image segmentation - * must confirm to `MPPImageSegmenterLiveStreamDelegate` protocol and implement the - *`imageSegmenter:didFinishSegmentationWithResult:timestampInMilliseconds:error:` delegate method. + * must confirm to `ImageSegmenterLiveStreamDelegate` protocol and implement the + * `imageSegmenter(_:didFinishSegmentationWithResult:timestampInMilliseconds:error:)` delegate + * method. * * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent * to the segmenter. The input timestamps must be monotonically increasing. * - * This method supports RGBA images. If your `MPPImage` has a source type of - *`MPPImageSourceTypePixelBuffer` or `MPPImageSourceTypeSampleBuffer`, the underlying pixel buffer - * must have one of the following pixel format types: + * This method supports RGBA images. If your `MPImage` has a source type of `pixelBuffer` or + *`sampleBuffer`, the underlying pixel buffer must have one of the following pixel format types: * 1. kCVPixelFormatType_32BGRA * 2. kCVPixelFormatType_32RGBA * - * If the input `MPPImage` has a source type of `MPPImageSourceTypeImage` ensure that the color - * space is RGB with an Alpha channel. + * If the input `MPImage` has a source type of `image` ensure that the color space is RGB with an + * Alpha channel. * * If this method is used for classifying live camera frames using `AVFoundation`, ensure that you * request `AVCaptureVideoDataOutput` to output frames in `kCMPixelFormat_32RGBA` using its * `videoSettings` property. * - * @param image A live stream image data of type `MPPImage` on which segmentation is to be + * @param image A live stream image data of type `MPImage` on which segmentation is to be * performed. * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input * image is sent to the segmenter. The input timestamps must be monotonically increasing. - * @param error An optional error parameter populated when there is an error when sending the input - * image to the graph. * * @return `YES` if the image was sent to the task successfully, otherwise `NO`. */ diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm index 8fad36671..9752880f3 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.mm @@ -14,6 +14,7 @@ #import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenter.h" +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" #import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" #import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" @@ -21,6 +22,8 @@ #import "mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterOptions+Helpers.h" #import "mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.h" +#include "mediapipe/tasks/cc/vision/image_segmenter/calculators/tensors_to_segmentation_calculator.pb.h" + static constexpr int kMicrosecondsPerMillisecond = 1000; // Constants for the underlying MP Tasks Graph. See @@ -48,7 +51,9 @@ static NSString *const kTaskName = @"imageSegmenter"; } namespace { +using ::mediapipe::CalculatorGraphConfig; using ::mediapipe::Timestamp; +using ::mediapipe::tasks::TensorsToSegmentationCalculatorOptions; using ::mediapipe::tasks::core::PacketMap; using ::mediapipe::tasks::core::PacketsCallback; } // anonymous namespace @@ -125,10 +130,15 @@ using ::mediapipe::tasks::core::PacketsCallback; imageInputStreamName:kImageInStreamName normRectInputStreamName:kNormRectStreamName error:error]; - if (!_visionTaskRunner) { return nil; } + + _labels = [MPPImageSegmenter populateLabelsWithGraphConfig:_visionTaskRunner.graphConfig + error:error]; + if (!_labels) { + return nil; + } } return self; @@ -197,6 +207,43 @@ using ::mediapipe::tasks::core::PacketsCallback; #pragma mark - Private ++ (NSArray *)populateLabelsWithGraphConfig:(const CalculatorGraphConfig &)graphConfig + error:(NSError **)error { + bool found_tensor_to_segmentation_calculator = false; + + NSMutableArray *labels = [NSMutableArray arrayWithCapacity:(NSUInteger)graphConfig.node_size()]; + for (const auto &node : graphConfig.node()) { + if (node.calculator() == "mediapipe.tasks.TensorsToSegmentationCalculator") { + if (!found_tensor_to_segmentation_calculator) { + found_tensor_to_segmentation_calculator = true; + } else { + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeFailedPreconditionError + description:@"The graph has more than one " + @"`mediapipe.tasks.TensorsToSegmentationCalculator`."]; + return nil; + } + TensorsToSegmentationCalculatorOptions options = + node.options().GetExtension(TensorsToSegmentationCalculatorOptions::ext); + if (!options.label_items().empty()) { + for (int i = 0; i < options.label_items_size(); ++i) { + if (!options.label_items().contains(i)) { + [MPPCommonUtils + createCustomError:error + withCode:MPPTasksErrorCodeFailedPreconditionError + description:[NSString + stringWithFormat:@"The lablemap has no expected key %d.", i]]; + + return nil; + } + [labels addObject:[NSString stringWithCppString:options.label_items().at(i).name()]]; + } + } + } + } + return labels; +} + + (nullable MPPImageSegmenterResult *) imageSegmenterResultWithOptionalOutputPacketMap:(std::optional &)outputPacketMap shouldCopyMaskPacketData:(BOOL)shouldCopyMaskPacketData { From fce7b19ad70f513fdbff8f276d3c8d1d51d5eeec Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 9 Oct 2023 17:47:03 +0530 Subject: [PATCH 011/157] Added a test for getting labels from iOS image segmenter --- .../image_segmenter/MPPImageSegmenterTests.mm | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm index 4954555e5..ba40db422 100644 --- a/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm +++ b/mediapipe/tasks/ios/test/vision/image_segmenter/MPPImageSegmenterTests.mm @@ -507,6 +507,22 @@ double softIOU(const float *mask1, const float *mask2, size_t size) { }]; } +#pragma mark GetLabelsTest + +- (void)testGetLabelsSucceeds { + MPPImageSegmenterOptions *options = + [self imageSegmenterOptionsWithModelFileInfo:kImageSegmenterModelFileInfo]; + + MPPImageSegmenter *imageSegmenter = [self createImageSegmenterWithOptionsSucceeds:options]; + + NSArray *expectedLabels = @[@"background", @"aeroplane", @"bicycle", + @"bird", @"boat", @"bottle", @"bus", @"car",@"cat", + @"chair", @"cow", @"dining table", @"dog", @"horse", @"motorbike", + @"person", @"potted plant", @"sheep", @"sofa", @"train", @"tv"]; + + XCTAssertEqualObjects(imageSegmenter.labels, expectedLabels); +} + #pragma mark - Image Segmenter Initializers - (MPPImageSegmenterOptions *)imageSegmenterOptionsWithModelFileInfo:(MPPFileInfo *)fileInfo { From 6c4b4469ae49a306168c11fd7556d32c18c4a151 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 9 Oct 2023 17:48:39 +0530 Subject: [PATCH 012/157] Updated iOS Image Segmenter documentation to use Swift names --- .../tasks/ios/vision/core/sources/MPPMask.h | 32 +++++++++---------- .../tasks/ios/vision/core/sources/MPPMask.mm | 1 - .../sources/MPPImageSegmenterOptions.h | 32 +++++++++---------- .../sources/MPPImageSegmenterResult.h | 24 +++++++------- 4 files changed, 43 insertions(+), 46 deletions(-) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPMask.h b/mediapipe/tasks/ios/vision/core/sources/MPPMask.h index 5d3a6910d..6cb08fdb1 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPMask.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPMask.h @@ -33,15 +33,15 @@ typedef NS_ENUM(NSUInteger, MPPMaskDataType) { * Masks are stored as `UInt8 *` or `float *` objects. * Every mask has an underlying type which can be accessed using `dataType`. You can access the * mask as any other type using the appropriate properties. For example, if the underlying type is - * `MPPMaskDataTypeUInt8`, in addition to accessing the mask using `uint8Data`, you can access - * `float32Data` to get the 32 bit float data (with values ranging from 0.0 to 1.0). The first - * time you access the data as a type different from the underlying type, an expensive type - * conversion is performed. Subsequent accesses return a pointer to the memory location for the same - * type converted array. As type conversions can be expensive, it is recommended to limit the - * accesses to data of types different from the underlying type. + * `uInt8`, in addition to accessing the mask using `uint8Data`, you can access `float32Data` to get + * the 32 bit float data (with values ranging from 0.0 to 1.0). The first time you access the data + * as a type different from the underlying type, an expensive type conversion is performed. + * Subsequent accesses return a pointer to the memory location for the same type converted array. As + * type conversions can be expensive, it is recommended to limit the accesses to data of types + * different from the underlying type. * * Masks that are returned from a MediaPipe Tasks are owned by by the underlying C++ Task. If you - * need to extend the lifetime of these objects, you can invoke the `[MPPMask copy:]` method. + * need to extend the lifetime of these objects, you can invoke the `copy()` method. */ NS_SWIFT_NAME(Mask) @interface MPPMask : NSObject @@ -68,19 +68,18 @@ NS_SWIFT_NAME(Mask) @property(nonatomic, readonly, assign) const float *float32Data; /** - * Initializes an `MPPMask` object of type `MPPMaskDataTypeUInt8` with the given `UInt8*` data, - * width and height. + * Initializes an `Mask` object of type `uInt8` with the given `UInt8*` data, width and height. * - * If `shouldCopy` is set to `YES`, the newly created `MPPMask` stores a reference to a deep copied + * If `shouldCopy` is set to `true`, the newly created `Mask` stores a reference to a deep copied * `uint8Data`. Since deep copies are expensive, it is recommended to not set `shouldCopy` unless - * the `MPPMask` must outlive the passed in `uint8Data`. + * the `Mask` must outlive the passed in `uint8Data`. * * @param uint8Data A pointer to the memory location of the `UInt8` data array. * @param width The width of the mask. * @param height The height of the mask. * @param shouldCopy The height of the mask. * - * @return A new `MPPMask` instance with the given `UInt8*` data, width and height. + * @return A new `Mask` instance with the given `UInt8*` data, width and height. */ - (nullable instancetype)initWithUInt8Data:(const UInt8 *)uint8Data width:(NSInteger)width @@ -88,18 +87,17 @@ NS_SWIFT_NAME(Mask) shouldCopy:(BOOL)shouldCopy NS_DESIGNATED_INITIALIZER; /** - * Initializes an `MPPMask` object of type `MPPMaskDataTypeFloat32` with the given `float*` data, - * width and height. + * Initializes an `Mask` object of type `float32` with the given `float*` data, width and height. * - * If `shouldCopy` is set to `YES`, the newly created `MPPMask` stores a reference to a deep copied + * If `shouldCopy` is set to `true`, the newly created `Mask` stores a reference to a deep copied * `float32Data`. Since deep copies are expensive, it is recommended to not set `shouldCopy` unless - * the `MPPMask` must outlive the passed in `float32Data`. + * the `Mask` must outlive the passed in `float32Data`. * * @param float32Data A pointer to the memory location of the `float` data array. * @param width The width of the mask. * @param height The height of the mask. * - * @return A new `MPPMask` instance with the given `float*` data, width and height. + * @return A new `Mask` instance with the given `float*` data, width and height. */ - (nullable instancetype)initWithFloat32Data:(const float *)float32Data width:(NSInteger)width diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPMask.mm b/mediapipe/tasks/ios/vision/core/sources/MPPMask.mm index 0d78e11d3..262f7fc97 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPMask.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPMask.mm @@ -30,7 +30,6 @@ width:(NSInteger)width height:(NSInteger)height shouldCopy:(BOOL)shouldCopy { - self = [super init]; if (self) { _width = width; diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h index b089ac7d3..9699b6175 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h @@ -23,11 +23,11 @@ NS_ASSUME_NONNULL_BEGIN @class MPPImageSegmenter; /** - * This protocol defines an interface for the delegates of `MPPImageSegmenter` object to receive + * This protocol defines an interface for the delegates of `ImageSegmenter` object to receive * results of performing asynchronous segmentation on images (i.e, when `runningMode` = - * `MPPRunningModeLiveStream`). + * `liveStream`). * - * The delegate of `MPPImageSegmenter` must adopt `MPPImageSegmenterLiveStreamDelegate` protocol. + * The delegate of `ImageSegmenter` must adopt `ImageSegmenterLiveStreamDelegate` protocol. * The methods in this protocol are optional. */ NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) @@ -37,14 +37,14 @@ NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) /** * This method notifies a delegate that the results of asynchronous segmentation of - * an image submitted to the `MPPImageSegmenter` is available. + * an image submitted to the `ImageSegmenter` is available. * - * This method is called on a private serial dispatch queue created by the `MPPImageSegmenter` + * This method is called on a private serial dispatch queue created by the `ImageSegmenter` * for performing the asynchronous delegates calls. * * @param imageSegmenter The image segmenter which performed the segmentation. This is useful to - * test equality when there are multiple instances of `MPPImageSegmenter`. - * @param result The `MPPImageSegmenterResult` object that contains a list of category or confidence + * test equality when there are multiple instances of `ImageSegmenter`. + * @param result The `ImageSegmenterResult` object that contains a list of category or confidence * masks and optional quality scores. * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input * image was sent to the image segmenter. @@ -58,26 +58,26 @@ NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) NS_SWIFT_NAME(imageSegmenter(_:didFinishSegmentation:timestampInMilliseconds:error:)); @end -/** Options for setting up a `MPPImageSegmenter`. */ +/** Options for setting up a `ImageSegmenter`. */ NS_SWIFT_NAME(ImageSegmenterOptions) @interface MPPImageSegmenterOptions : MPPTaskOptions /** - * Running mode of the image segmenter task. Defaults to `MPPRunningModeImage`. - * `MPPImageSegmenter` can be created with one of the following running modes: - * 1. `MPPRunningModeImage`: The mode for performing segmentation on single image inputs. - * 2. `MPPRunningModeVideo`: The mode for performing segmentation on the decoded frames of a + * Running mode of the image segmenter task. Defaults to `image`. + * `ImageSegmenter` can be created with one of the following running modes: + * 1. `image`: The mode for performing segmentation on single image inputs. + * 2. `video`: The mode for performing segmentation on the decoded frames of a * video. - * 3. `MPPRunningModeLiveStream`: The mode for performing segmentation on a live stream of + * 3. `liveStream`: The mode for performing segmentation on a live stream of * input data, such as from the camera. */ @property(nonatomic) MPPRunningMode runningMode; /** - * An object that confirms to `MPPImageSegmenterLiveStreamDelegate` protocol. This object must - * implement `imageSegmenter:didFinishSegmentationWithResult:timestampInMilliseconds:error:` to + * An object that confirms to `ImageSegmenterLiveStreamDelegate` protocol. This object must + * implement `imageSegmenter(_:didFinishSegmentationWithResult:timestampInMilliseconds:error:)` to * receive the results of performing asynchronous segmentation on images (i.e, when `runningMode` = - * `MPPRunningModeLiveStream`). + * `liveStream`). */ @property(nonatomic, weak, nullable) id imageSegmenterLiveStreamDelegate; diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h index 20bf3fefe..7b8277d2a 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h @@ -18,22 +18,22 @@ NS_ASSUME_NONNULL_BEGIN -/** Represents the segmentation results generated by `MPPImageSegmenter`. */ +/** Represents the segmentation results generated by `ImageSegmenter`. */ NS_SWIFT_NAME(ImageSegmenterResult) @interface MPPImageSegmenterResult : MPPTaskResult /** - * An optional array of `MPPMask` objects. Each `MPPMask` in the array holds a 32 bit float array of - * size `image width` * `image height` which represents the confidence mask for each category. Each + * An optional array of `Mask` objects. Each `Mask` in the array holds a 32 bit float array of size + * `image width` * `image height` which represents the confidence mask for each category. Each * element of the float array represents the confidence with which the model predicted that the * corresponding pixel belongs to the category that the mask represents, usually in the range [0,1]. */ @property(nonatomic, readonly, nullable) NSArray *confidenceMasks; /** - * An optional `MPPMask` that holds a`UInt8` array of size `image width` * `image height`. Each - * element of this array represents the class to which the pixel in the original image was predicted - * to belong to. + * An optional `Mask` that holds a`UInt8` array of size `image width` * `image height`. Each element + * of this array represents the class to which the pixel in the original image was predicted to + * belong to. */ @property(nonatomic, readonly, nullable) MPPMask *categoryMask; @@ -45,17 +45,17 @@ NS_SWIFT_NAME(ImageSegmenterResult) @property(nonatomic, readonly, nullable) NSArray *qualityScores; /** - * Initializes a new `MPPImageSegmenterResult` with the given array of confidence masks, category - * mask, quality scores and timestamp (in milliseconds). + * Initializes a new `ImageSegmenterResult` with the given array of confidence masks, category mask, + * quality scores and timestamp (in milliseconds). * - * @param confidenceMasks An optional array of `MPPMask` objects. Each `MPPMask` in the array must - * be of type `MPPMaskDataTypeFloat32`. - * @param categoryMask An optional `MPMask` object of type `MPPMaskDataTypeUInt8`. + * @param confidenceMasks An optional array of `Mask` objects. Each `Mask` in the array must + * be of type `float32`. + * @param categoryMask An optional `Mask` object of type `uInt8`. * @param qualityScores The quality scores of the result masks of type NSArray *. Each * `NSNumber` in the array holds a `float`. * @param timestampInMilliseconds The timestamp (in milliseconds) for this result. * - * @return An instance of `MPPImageSegmenterResult` initialized with the given array of confidence + * @return An instance of `ImageSegmenterResult` initialized with the given array of confidence * masks, category mask, quality scores and timestamp (in milliseconds). */ - (instancetype)initWithConfidenceMasks:(nullable NSArray *)confidenceMasks From 3a9776256986b1e19c34cd7ef44335a9590ddc3d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 9 Oct 2023 17:48:57 +0530 Subject: [PATCH 013/157] Fixed typo in iOS image segmenter Swift delegate name --- .../vision/image_segmenter/sources/MPPImageSegmenterOptions.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h index 9699b6175..484f3324b 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h +++ b/mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterOptions.h @@ -30,7 +30,7 @@ NS_ASSUME_NONNULL_BEGIN * The delegate of `ImageSegmenter` must adopt `ImageSegmenterLiveStreamDelegate` protocol. * The methods in this protocol are optional. */ -NS_SWIFT_NAME(ObjectDetectorLiveStreamDelegate) +NS_SWIFT_NAME(ImageSegmenterLiveStreamDelegate) @protocol MPPImageSegmenterLiveStreamDelegate @optional From 69b7a2136809a36b2a2db2d10d7a08f7e0165052 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 19 Oct 2023 20:42:06 +0530 Subject: [PATCH 014/157] Updated deletion in FreeDataProviderReleaseCallback --- mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm index 440b321b9..0569ad783 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm @@ -37,7 +37,7 @@ vImage_Buffer allocatedVImageBuffer(vImagePixelCount width, vImagePixelCount hei } static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size_t size) { - delete (vImage_Buffer *)buffer; + delete[] buffer; } } // namespace From 06d893a9f9c77e955fa7ad92f609921b19f88a52 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 19 Oct 2023 20:45:32 +0530 Subject: [PATCH 015/157] Revert "Updated deletion in FreeDataProviderReleaseCallback" This reverts commit 69b7a2136809a36b2a2db2d10d7a08f7e0165052. --- mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm index 0569ad783..440b321b9 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm @@ -37,7 +37,7 @@ vImage_Buffer allocatedVImageBuffer(vImagePixelCount width, vImagePixelCount hei } static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size_t size) { - delete[] buffer; + delete (vImage_Buffer *)buffer; } } // namespace From f9fa7cfbeb99b93a1625fb08bb1d5096d4704616 Mon Sep 17 00:00:00 2001 From: Fergus Henderson Date: Thu, 19 Oct 2023 10:08:09 -0700 Subject: [PATCH 016/157] No public description PiperOrigin-RevId: 574913082 --- mediapipe/util/tflite/BUILD | 4 +++- mediapipe/util/tflite/tflite_model_loader.h | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mediapipe/util/tflite/BUILD b/mediapipe/util/tflite/BUILD index 350ac230c..b34d0e080 100644 --- a/mediapipe/util/tflite/BUILD +++ b/mediapipe/util/tflite/BUILD @@ -129,7 +129,9 @@ cc_library_with_tflite( name = "tflite_model_loader", srcs = ["tflite_model_loader.cc"], hdrs = ["tflite_model_loader.h"], - tflite_deps = ["@org_tensorflow//tensorflow/lite:framework_stable"], + tflite_deps = [ + "@org_tensorflow//tensorflow/lite:framework_stable", + ], visibility = ["//visibility:public"], deps = [ "//mediapipe/framework/api2:packet", diff --git a/mediapipe/util/tflite/tflite_model_loader.h b/mediapipe/util/tflite/tflite_model_loader.h index 65bd9ba72..f1a021f4d 100644 --- a/mediapipe/util/tflite/tflite_model_loader.h +++ b/mediapipe/util/tflite/tflite_model_loader.h @@ -25,6 +25,7 @@ #include "tensorflow/lite/model.h" namespace mediapipe { + // Represents a TfLite model as a FlatBuffer. using TfLiteModelPtr = std::unique_ptr Date: Thu, 19 Oct 2023 11:09:35 -0700 Subject: [PATCH 017/157] No public description PiperOrigin-RevId: 574938418 --- .bazelrc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.bazelrc b/.bazelrc index 44bc3d0a1..7774473c8 100644 --- a/.bazelrc +++ b/.bazelrc @@ -98,6 +98,9 @@ build:darwin_arm64 --apple_platform_type=macos build:darwin_arm64 --macos_minimum_os=10.16 build:darwin_arm64 --cpu=darwin_arm64 +# Turn off maximum stdout size +build --experimental_ui_max_stdouterr_bytes=-1 + # This bazelrc file is meant to be written by a setup script. try-import %workspace%/.configure.bazelrc From 2d0d258403faa417aa1c6d445e88db86e2c7ffb5 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 19 Oct 2023 11:10:53 -0700 Subject: [PATCH 018/157] Delete arm64 only file in Mac wheel Fixes https://github.com/google/mediapipe/issues/4888#issuecomment-1768861583 PiperOrigin-RevId: 574938905 --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index f080503fe..b5b75b73c 100644 --- a/setup.py +++ b/setup.py @@ -404,6 +404,8 @@ class BuildExtension(build_ext.build_ext): arm64_name, ] _invoke_shell_command(lipo_command) + # Delete the arm64 file (the x86 file was overwritten by lipo) + _invoke_shell_command(['rm', arm64_name]) else: for ext in self.extensions: self._build_binary(ext) From 5779f5e9daf2de586550ae8f4d2a977943538793 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 19 Oct 2023 12:27:47 -0700 Subject: [PATCH 019/157] Allow GPU Origin Proto to be build by Maven PiperOrigin-RevId: 574966597 --- mediapipe/gpu/gpu_origin.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mediapipe/gpu/gpu_origin.proto b/mediapipe/gpu/gpu_origin.proto index 9d4ae2aa1..2c116cbde 100644 --- a/mediapipe/gpu/gpu_origin.proto +++ b/mediapipe/gpu/gpu_origin.proto @@ -16,6 +16,9 @@ syntax = "proto3"; package mediapipe; +option java_package = "com.google.mediapipe.gpu"; +option java_outer_classname = "GpuOriginProto"; + message GpuOrigin { enum Mode { DEFAULT = 0; From 305d7abec40aa70907270f4195df7f44d6924e53 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 19 Oct 2023 15:27:38 -0700 Subject: [PATCH 020/157] Add a field to GPUBuffer C struct so FFIGen can handle it PiperOrigin-RevId: 575020084 --- .../tasks/c/vision/image_classifier/image_classifier.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/c/vision/image_classifier/image_classifier.h b/mediapipe/tasks/c/vision/image_classifier/image_classifier.h index 9b59e4127..60dc4a2c4 100644 --- a/mediapipe/tasks/c/vision/image_classifier/image_classifier.h +++ b/mediapipe/tasks/c/vision/image_classifier/image_classifier.h @@ -57,14 +57,17 @@ struct ImageFrame { }; // TODO: Add GPU buffer declaration and proccessing logic for it. -struct GpuBuffer {}; +struct GpuBuffer { + int width; + int height; +}; // The object to contain an image, realizes `OneOf` concept. struct MpImage { enum { IMAGE_FRAME, GPU_BUFFER } type; union { - ImageFrame image_frame; - GpuBuffer gpu_buffer; + struct ImageFrame image_frame; + struct GpuBuffer gpu_buffer; }; }; From 8fc3a0473f11f9bda7fdb656671719a2a67ba801 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 20 Oct 2023 01:06:10 -0700 Subject: [PATCH 021/157] Add scaling support to surface view renderer. PiperOrigin-RevId: 575134648 --- .../components/GlSurfaceViewRenderer.java | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java index 4376aeb58..2ecb9d1e6 100644 --- a/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java +++ b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java @@ -25,6 +25,7 @@ import android.opengl.GLES31; import android.opengl.GLSurfaceView; import android.opengl.Matrix; import android.util.Log; +import android.util.Pair; import com.google.mediapipe.framework.TextureFrame; import com.google.mediapipe.glutil.CommonShaders; import com.google.mediapipe.glutil.ShaderUtil; @@ -92,6 +93,9 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { private boolean shouldClampToBorder = false; private Scale scale = Scale.FILL; + private float zoomFactor = 1.0f; + private Pair zoomLocation = new Pair<>(0.5f, 0.5f); + /** * Sets the {@link BitmapCaptureListener}. */ @@ -294,6 +298,15 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { float textureBottom = (1.0f - scaleHeight) * alignmentVertical; float textureTop = textureBottom + scaleHeight; + Pair currentZoomLocation = this.zoomLocation; + float zoomLocationX = currentZoomLocation.first; + float zoomLocationY = currentZoomLocation.second; + + textureLeft = (textureLeft - 0.5f) / zoomFactor + zoomLocationX; + textureRight = (textureRight - 0.5f) / zoomFactor + zoomLocationX; + textureBottom = (textureBottom - 0.5f) / zoomFactor + zoomLocationY; + textureTop = (textureTop - 0.5f) / zoomFactor + zoomLocationY; + return new float[] {textureLeft, textureRight, textureBottom, textureTop}; } @@ -380,6 +393,22 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { this.scale = scale; } + /** Zoom factor applied to the frame, must not be 0. */ + public void setZoomFactor(float zoomFactor) { + if (zoomFactor == 0.f) { + return; + } + this.zoomFactor = zoomFactor; + } + + /** + * Location where to apply the zooming of the frame to. Default is 0.5, 0.5 (scaling is applied to + * the center). + */ + public void setZoomLocation(float zoomLocationX, float zoomLocationY) { + this.zoomLocation = new Pair<>(zoomLocationX, zoomLocationY); + } + private boolean isExternalTexture() { return textureTarget == GLES11Ext.GL_TEXTURE_EXTERNAL_OES; } From c4315c500d8d11a301ce91784ac1acbdf26fd8f9 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Sat, 21 Oct 2023 03:52:46 +0530 Subject: [PATCH 022/157] Added pose landmarker result helpers --- .../sources/MPPPoseLandmarkerResult.h | 2 +- .../ios/vision/pose_landmarker/utils/BUILD | 17 +++ .../sources/MPPPoseLandmarkerResult+Helpers.h | 65 ++++++++++ .../MPPPoseLandmarkerResult+Helpers.mm | 122 ++++++++++++++++++ 4 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.mm diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h index ff9d001b2..b3dc72e8f 100644 --- a/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h +++ b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h @@ -46,7 +46,7 @@ NS_SWIFT_NAME(PoseLandmarkerResult) */ - (instancetype)initWithLandmarks:(NSArray *> *)landmarks worldLandmarks:(NSArray *> *)worldLandmarks - segmentationMasks:(NSArray *)segmentationMasks + segmentationMasks:(nullable NSArray *)segmentationMasks timestampInMilliseconds:(NSInteger)timestampInMilliseconds NS_DESIGNATED_INITIALIZER; - (instancetype)initWithTimestampInMilliseconds:(NSInteger)timestampInMilliseconds NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/utils/BUILD b/mediapipe/tasks/ios/vision/pose_landmarker/utils/BUILD index 94c945bc1..346f7e67d 100644 --- a/mediapipe/tasks/ios/vision/pose_landmarker/utils/BUILD +++ b/mediapipe/tasks/ios/vision/pose_landmarker/utils/BUILD @@ -36,3 +36,20 @@ objc_library( "//mediapipe/tasks/ios/vision/pose_landmarker:MPPPoseLandmarkerOptions", ], ) +objc_library( + name = "MPPPoseLandmarkerResultHelpers", + srcs = ["sources/MPPPoseLandmarkerResult+Helpers.mm"], + hdrs = ["sources/MPPPoseLandmarkerResult+Helpers.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/tasks/ios/components/containers/utils:MPPLandmarkHelpers", + "//mediapipe/tasks/ios/vision/pose_landmarker:MPPPoseLandmarkerResult", + ], +) diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h new file mode 100644 index 000000000..bf8e5495d --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h @@ -0,0 +1,65 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h" + +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/packet.h" + +NS_ASSUME_NONNULL_BEGIN + +static const int kMicroSecondsPerMilliSecond = 1000; + +@interface MPPPoseLandmarkerResult (Helpers) + +/** + * Creates an `MPPPoseLandmarkerResult` from landmarks, world landmarks and segmentation mask + * packets. + * + * @param landmarksPacket A MediaPipe packet wrapping a `std::vector`. + * @param worldLandmarksPacket A MediaPipe packet wrapping a `std::vector`. + * @param segmentationMasksPacket a MediaPipe packet wrapping a `std::vector`. + * + * @return An `MPPPoseLandmarkerResult` object that contains the hand landmark detection + * results. + */ ++ (MPPPoseLandmarkerResult *) + poseLandmarkerResultWithLandmarksPacket:(const mediapipe::Packet &)landmarksPacket + worldLandmarksPacket:(const mediapipe::Packet &)worldLandmarksPacket + segmentationMasksPacket:(const mediapipe::Packet *)segmentationMasksPacket; + +/** + * Creates an `MPPPoseLandmarkerResult` from landmarks, world landmarks and segmentation mask + * images. + * + * @param landmarksProto A vector of protos of type `std::vector`. + * @param worldLandmarksPacket A vector of protos of type `std::vector`. + * @param segmentationMasks A vector of type `std::vector`. + * @param timestampInMilliSeconds The timestamp of the Packet that contained the result. + * + * @return An `MPPPoseLandmarkerResult` object that contains the pose landmark detection + * results. + */ ++ (MPPPoseLandmarkerResult *) + poseLandmarkerResultWithLandmarksProto: + (const std::vector &)landmarksProto + worldLandmarksProto: + (const std::vector &)worldLandmarksProto + segmentationMasks:(const std::vector *)segmentationMasks + timestampInMilliSeconds:(NSInteger)timestampInMilliseconds; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.mm b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.mm new file mode 100644 index 000000000..8995504e1 --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.mm @@ -0,0 +1,122 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h" + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPLandmark+Helpers.h" + +namespace { +using LandmarkListProto = ::mediapipe::LandmarkList; +using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList; +using ::mediapipe::Image; +using ::mediapipe::Packet; +} // namespace + +@implementation MPPPoseLandmarkerResult (Helpers) + ++ (MPPPoseLandmarkerResult *)emptyPoseLandmarkerResultWithTimestampInMilliseconds: + (NSInteger)timestampInMilliseconds { + return [[MPPPoseLandmarkerResult alloc] initWithLandmarks:@[] + worldLandmarks:@[] + segmentationMasks:@[] + timestampInMilliseconds:timestampInMilliseconds]; +} + ++ (MPPPoseLandmarkerResult *) + poseLandmarkerResultWithLandmarksProto: + (const std::vector &)landmarksProto + worldLandmarksProto: + (const std::vector &)worldLandmarksProto + segmentationMasks:(const std::vector *)segmentationMasks + timestampInMilliSeconds:(NSInteger)timestampInMilliseconds { + NSMutableArray *> *multiplePoseLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)landmarksProto.size()]; + + for (const auto &landmarkListProto : landmarksProto) { + NSMutableArray *landmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)landmarkListProto.landmark().size()]; + for (const auto &normalizedLandmarkProto : landmarkListProto.landmark()) { + MPPNormalizedLandmark *normalizedLandmark = + [MPPNormalizedLandmark normalizedLandmarkWithProto:normalizedLandmarkProto]; + [landmarks addObject:normalizedLandmark]; + } + [multiplePoseLandmarks addObject:landmarks]; + } + + NSMutableArray *> *multiplePoseWorldLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarksProto.size()]; + + for (const auto &worldLandmarkListProto : worldLandmarksProto) { + NSMutableArray *worldLandmarks = + [NSMutableArray arrayWithCapacity:(NSUInteger)worldLandmarkListProto.landmark().size()]; + for (const auto &landmarkProto : worldLandmarkListProto.landmark()) { + MPPLandmark *landmark = [MPPLandmark landmarkWithProto:landmarkProto]; + [worldLandmarks addObject:landmark]; + } + [multiplePoseWorldLandmarks addObject:worldLandmarks]; + } + + NSMutableArray *confidenceMasks = + [NSMutableArray arrayWithCapacity:(NSUInteger)segmentationMasks->size()]; + + for (const auto &segmentationMask : *segmentationMasks) { + [confidenceMasks addObject:[[MPPMask alloc] initWithFloat32Data:(float *)segmentationMask + .GetImageFrameSharedPtr() + .get() + ->PixelData() + width:segmentationMask.width() + height:segmentationMask.height() + /** Always deep copy */ + shouldCopy:YES]]; + } + + MPPPoseLandmarkerResult *poseLandmarkerResult = + [[MPPPoseLandmarkerResult alloc] initWithLandmarks:multiplePoseLandmarks + worldLandmarks:multiplePoseWorldLandmarks + segmentationMasks:confidenceMasks + timestampInMilliseconds:timestampInMilliseconds]; + return poseLandmarkerResult; +} + ++ (MPPPoseLandmarkerResult *) + poseLandmarkerResultWithLandmarksPacket:(const Packet &)landmarksPacket + worldLandmarksPacket:(const Packet &)worldLandmarksPacket + segmentationMasksPacket:(const Packet *)segmentationMasksPacket { + NSInteger timestampInMilliseconds = + (NSInteger)(landmarksPacket.Timestamp().Value() / kMicroSecondsPerMilliSecond); + + if (landmarksPacket.IsEmpty()) { + return [MPPPoseLandmarkerResult + emptyPoseLandmarkerResultWithTimestampInMilliseconds:timestampInMilliseconds]; + } + + if (!landmarksPacket.ValidateAsType>().ok() || + !worldLandmarksPacket.ValidateAsType>().ok()) { + return [MPPPoseLandmarkerResult + emptyPoseLandmarkerResultWithTimestampInMilliseconds:timestampInMilliseconds]; + } + + const std::vector *segmentationMasks = + segmentationMasksPacket ? &(segmentationMasksPacket->Get>()) : nullptr; + + return [MPPPoseLandmarkerResult + poseLandmarkerResultWithLandmarksProto:landmarksPacket + .Get>() + worldLandmarksProto:worldLandmarksPacket + .Get>() + segmentationMasks:segmentationMasks + timestampInMilliSeconds:timestampInMilliseconds]; +} + +@end From 3622ff9bff9c2cfca61a17f1fb281e7c907c2567 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Sat, 21 Oct 2023 03:53:54 +0530 Subject: [PATCH 023/157] Added iOS pose landmarks connections --- .../tasks/ios/vision/pose_landmarker/BUILD | 8 ++++ .../sources/MPPPoseLandmarksConnections.h | 40 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarksConnections.h diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/BUILD b/mediapipe/tasks/ios/vision/pose_landmarker/BUILD index 16e791a97..951e0522f 100644 --- a/mediapipe/tasks/ios/vision/pose_landmarker/BUILD +++ b/mediapipe/tasks/ios/vision/pose_landmarker/BUILD @@ -37,3 +37,11 @@ objc_library( "//mediapipe/tasks/ios/vision/core:MPPRunningMode", ], ) + +objc_library( + name = "MPPPoseLandmarksConnections", + hdrs = ["sources/MPPPoseLandmarksConnections.h"], + module_name = "MPPPoseLandmarksConnections", + deps = ["//mediapipe/tasks/ios/components/containers:MPPConnection"], +) + diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarksConnections.h b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarksConnections.h new file mode 100644 index 000000000..71dcad6b8 --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarksConnections.h @@ -0,0 +1,40 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPConnection.h" + +NS_ASSUME_NONNULL_BEGIN + +NSArray *const MPPPoseLandmarksConnections = @[ + [[MPPConnection alloc] initWithStart:0 end:1], [[MPPConnection alloc] initWithStart:1 end:2], + [[MPPConnection alloc] initWithStart:2 end:3], [[MPPConnection alloc] initWithStart:3 end:7], + [[MPPConnection alloc] initWithStart:0 end:4], [[MPPConnection alloc] initWithStart:4 end:5], + [[MPPConnection alloc] initWithStart:5 end:6], [[MPPConnection alloc] initWithStart:6 end:8], + [[MPPConnection alloc] initWithStart:9 end:10], [[MPPConnection alloc] initWithStart:11 end:12], + [[MPPConnection alloc] initWithStart:11 end:13], [[MPPConnection alloc] initWithStart:13 end:15], + [[MPPConnection alloc] initWithStart:15 end:17], [[MPPConnection alloc] initWithStart:15 end:19], + [[MPPConnection alloc] initWithStart:15 end:21], [[MPPConnection alloc] initWithStart:17 end:19], + [[MPPConnection alloc] initWithStart:12 end:14], [[MPPConnection alloc] initWithStart:14 end:16], + [[MPPConnection alloc] initWithStart:16 end:18], [[MPPConnection alloc] initWithStart:16 end:20], + [[MPPConnection alloc] initWithStart:16 end:22], [[MPPConnection alloc] initWithStart:18 end:20], + [[MPPConnection alloc] initWithStart:11 end:23], [[MPPConnection alloc] initWithStart:12 end:24], + [[MPPConnection alloc] initWithStart:23 end:24], [[MPPConnection alloc] initWithStart:23 end:25], + [[MPPConnection alloc] initWithStart:26 end:28], [[MPPConnection alloc] initWithStart:27 end:29], + [[MPPConnection alloc] initWithStart:28 end:30], [[MPPConnection alloc] initWithStart:29 end:31], + [[MPPConnection alloc] initWithStart:30 end:32], [[MPPConnection alloc] initWithStart:27 end:31], + [[MPPConnection alloc] initWithStart:28 end:32] +]; + +NS_ASSUME_NONNULL_END From 96ed3a7422647eacb7794fec8f919b971112b32d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Sat, 21 Oct 2023 03:55:00 +0530 Subject: [PATCH 024/157] Added iOS pose landmarker header --- .../tasks/ios/vision/pose_landmarker/BUILD | 12 ++ .../sources/MPPPoseLandmarker.h | 161 ++++++++++++++++++ 2 files changed, 173 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarker.h diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/BUILD b/mediapipe/tasks/ios/vision/pose_landmarker/BUILD index 951e0522f..a7b612bce 100644 --- a/mediapipe/tasks/ios/vision/pose_landmarker/BUILD +++ b/mediapipe/tasks/ios/vision/pose_landmarker/BUILD @@ -45,3 +45,15 @@ objc_library( deps = ["//mediapipe/tasks/ios/components/containers:MPPConnection"], ) +objc_library( + name = "MPPPoseLandmarker", + hdrs = ["sources/MPPPoseLandmarker.h"], + module_name = "MPPPoseLandmarker", + deps = [ + ":MPPPoseLandmarkerOptions", + ":MPPPoseLandmarkerResult", + ":MPPPoseLandmarksConnections", + "//mediapipe/tasks/ios/components/containers:MPPConnection", + "//mediapipe/tasks/ios/vision/core:MPPImage", + ], +) diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarker.h b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarker.h new file mode 100644 index 000000000..4c3cb4062 --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarker.h @@ -0,0 +1,161 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/components/containers/sources/MPPConnection.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerOptions.h" +#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Performs pose landmarks detection on images. + * + * This API expects a pre-trained pose landmarks model asset bundle. + */ +NS_SWIFT_NAME(PoseLandmarker) +@interface MPPPoseLandmarker : NSObject + +/** The array of connections between all the landmarks in the detected pose. */ +@property(class, nonatomic, readonly) NSArray *poseLandmarks; + +/** + * Creates a new instance of `PoseLandmarker` from an absolute path to a model asset bundle stored + * locally on the device and the default `PoseLandmarkerOptions`. + * + * @param modelPath An absolute path to a model asset bundle stored locally on the device. + * @param error An optional error parameter populated when there is an error in initializing the + * pose landmarker. + * + * @return A new instance of `PoseLandmarker` with the given model path. `nil` if there is an error + * in initializing the pose landmarker. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `PoseLandmarker` from the given `PoseLandmarkerOptions`. + * + * @param options The options of type `PoseLandmarkerOptions` to use for configuring the + * `PoseLandmarker`. + * @param error An optional error parameter populated when there is an error in initializing the + * pose landmarker. + * + * @return A new instance of `PoseLandmarker` with the given options. `nil` if there is an error in + * initializing the pose landmarker. + */ +- (nullable instancetype)initWithOptions:(MPPPoseLandmarkerOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs pose landmarks detection on the provided `MPImage` using the whole image as region of + * interest. Rotation will be applied according to the `orientation` property of the provided + * `MPImage`. Only use this method when the `PoseLandmarker` is created with running mode, `.image`. + * + * This method supports performing pose landmarks detection on RGBA images. If your `MPImage` has a + * source type of `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of + * the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. + * + * @param image The `MPImage` on which pose landmarks detection is to be performed. + * @param error An optional error parameter populated when there is an error in performing pose + * landmark detection on the input image. + * + * @return An `PoseLandmarkerResult` object that contains the pose landmarks detection + * results. + */ +- (nullable MPPPoseLandmarkerResult *)detectImage:(MPPImage *)image + error:(NSError **)error NS_SWIFT_NAME(detect(image:)); + +/** + * Performs pose landmarks detection on the provided video frame of type `MPImage` using the whole + * image as region of interest. Rotation will be applied according to the `orientation` property of + * the provided `MPImage`. Only use this method when the `PoseLandmarker` is created with running + * mode, `.video`. + * + * It's required to provide the video frame's timestamp (in milliseconds). The input timestamps must + * be monotonically increasing. + * + * This method supports performing pose landmarks detection on RGBA images. If your `MPImage` has a + * source type of `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of + * the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. + * + * @param image The `MPImage` on which pose landmarks detection is to be performed. + * @param timestampInMilliseconds The video frame's timestamp (in milliseconds). The input + * timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing pose + * landmark detection on the input video frame. + * + * @return An `PoseLandmarkerResult` object that contains the pose landmarks detection + * results. + */ +- (nullable MPPPoseLandmarkerResult *)detectVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(detect(videoFrame:timestampInMilliseconds:)); + +/** + * Sends live stream image data of type `MPImage` to perform pose landmarks detection using the + * whole image as region of interest. Rotation will be applied according to the `orientation` + * property of the provided `MPImage`. Only use this method when the `PoseLandmarker` is created + * with running mode, `.liveStream`. + * + * The object which needs to be continuously notified of the available results of pose landmark + * detection must confirm to `PoseLandmarkerLiveStreamDelegate` protocol and implement the + * `poseLandmarker(_:didFinishDetectionWithResult:timestampInMilliseconds:error:)` delegate method. + * + * It's required to provide a timestamp (in milliseconds) to indicate when the input image is sent + * to the pose landmarker. The input timestamps must be monotonically increasing. + * + * This method supports performing pose landmarks detection on RGBA images. If your `MPImage` has a + * source type of `.pixelBuffer` or `.sampleBuffer`, the underlying pixel buffer must have one of + * the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * + * If the input `MPImage` has a source type of `.image` ensure that the color space is RGB with an + * Alpha channel. + * + * If this method is used for performing pose landmarks detection on live camera frames using + * `AVFoundation`, ensure that you request `AVCaptureVideoDataOutput` to output frames in + * `kCMPixelFormat_32BGRA` using its `videoSettings` property. + * + * @param image A live stream image data of type `MPImage` on which pose landmarks detection is to + * be performed. + * @param timestampInMilliseconds The timestamp (in milliseconds) which indicates when the input + * image is sent to the pose landmarker. The input timestamps must be monotonically increasing. + * @param error An optional error parameter populated when there is an error in performing pose + * landmark detection on the input live stream image data. + * + * @return `YES` if the image was sent to the task successfully, otherwise `NO`. + */ +- (BOOL)detectAsyncImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error + NS_SWIFT_NAME(detectAsync(image:timestampInMilliseconds:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END From c48a5668b867b786713c1a5f0770f203f4c93d6d Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Sat, 21 Oct 2023 03:57:55 +0530 Subject: [PATCH 025/157] Updated documentation --- .../utils/sources/MPPPoseLandmarkerResult+Helpers.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h index bf8e5495d..351de2f35 100644 --- a/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h +++ b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h @@ -45,7 +45,7 @@ static const int kMicroSecondsPerMilliSecond = 1000; * images. * * @param landmarksProto A vector of protos of type `std::vector`. - * @param worldLandmarksPacket A vector of protos of type `std::vector`. + * @param worldLandmarksProto A vector of protos of type `std::vector`. * @param segmentationMasks A vector of type `std::vector`. * @param timestampInMilliSeconds The timestamp of the Packet that contained the result. * From f185bc6635f253b82f7da5b331c2d71f8dda6b39 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Sat, 21 Oct 2023 10:42:32 +0530 Subject: [PATCH 026/157] Added language detector result helpers --- .../ios/text/language_detector/utils/BUILD | 12 ++++ .../MPPLanguageDetectorResult+Helpers.h | 28 +++++++++ .../MPPLanguageDetectorResult+Helpers.mm | 61 +++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorResult+Helpers.h create mode 100644 mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorResult+Helpers.mm diff --git a/mediapipe/tasks/ios/text/language_detector/utils/BUILD b/mediapipe/tasks/ios/text/language_detector/utils/BUILD index 00a37e940..74de385c0 100644 --- a/mediapipe/tasks/ios/text/language_detector/utils/BUILD +++ b/mediapipe/tasks/ios/text/language_detector/utils/BUILD @@ -30,3 +30,15 @@ objc_library( "//mediapipe/tasks/ios/text/language_detector:MPPLanguageDetectorOptions", ], ) + +objc_library( + name = "MPPLanguageDetectorResultHelpers", + srcs = ["sources/MPPLanguageDetectorResult+Helpers.mm"], + hdrs = ["sources/MPPLanguageDetectorResult+Helpers.h"], + deps = [ + "//mediapipe/framework:packet", + "//mediapipe/tasks/cc/components/containers/proto:classifications_cc_proto", + "//mediapipe/tasks/ios/components/containers/utils:MPPClassificationResultHelpers", + "//mediapipe/tasks/ios/text/language_detector:MPPLanguageDetectorResult", + ], +) diff --git a/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorResult+Helpers.h b/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorResult+Helpers.h new file mode 100644 index 000000000..93fbfed48 --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorResult+Helpers.h @@ -0,0 +1,28 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/language_Detector/sources/MPPLanguageDetectorResult.h" + +#include "mediapipe/framework/packet.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPLanguageDetectorResult (Helpers) + ++ (MPPLanguageDetectorResult *)languageDetectorResultWithClassificationsPacket: + (const mediapipe::Packet &)packet; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorResult+Helpers.mm b/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorResult+Helpers.mm new file mode 100644 index 000000000..a55fdb89e --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorResult+Helpers.mm @@ -0,0 +1,61 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPClassificationResult+Helpers.h" +#import "mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorResult+Helpers.h" + +#include "mediapipe/tasks/cc/components/containers/proto/classifications.pb.h" + +static const int kMicroSecondsPerMilliSecond = 1000; + +namespace { +using ClassificationResultProto = + ::mediapipe::tasks::components::containers::proto::ClassificationResult; +using ::mediapipe::Packet; +} // namespace + +#define int kMicroSecondsPerMilliSecond = 1000; + +@implementation MPPLanguageDetectorResult (Helpers) + ++ (MPPLanguageDetectorResult *)languageDetectorResultWithClassificationsPacket: + (const mediapipe::Packet &)packet { + MPPClassificationResult *classificationResult = [MPPClassificationResult + classificationResultWithProto:packet.Get()]; + + return [MPPLanguageDetectorResult + languageDetectorResultWithClassificationResult:classificationResult + timestampInMilliseconds:(NSInteger)(packet.Timestamp().Value() / + kMicroSecondsPerMilliSecond)]; +} + ++ (MPPLanguageDetectorResult *) + languageDetectorResultWithClassificationResult:(MPPClassificationResult *)classificationResult + timestampInMilliseconds:(NSInteger)timestampInMilliseconds { + NSArray *languagePredictions = + [NSMutableArray arrayWithCapacity:classificationResult.classifications.count]; + + if (classificationResult.classifications.count > 0) { + for (MPPCategory *category in classificationResult.classifications[0].categories) { + MPPLanguagePrediction *languagePrediction = + [[MPPLanguagePrediction alloc] initWithLanguageCode:category.categoryName + probability:category.score]; + } + } + + return [[MPPLanguageDetectorResult alloc] initWithLanguagePredictions:languagePredictions + timestampInMilliseconds:timestampInMilliseconds]; +} + +@end From 3a43aff13cb9695888ff24b9d23a89c90e312877 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Sat, 21 Oct 2023 10:42:48 +0530 Subject: [PATCH 027/157] Added iOS language detector implementation --- .../tasks/ios/text/language_detector/BUILD | 25 +++++ .../sources/MPPLanguageDetector.h | 88 +++++++++++++++++ .../sources/MPPLanguageDetector.mm | 96 +++++++++++++++++++ 3 files changed, 209 insertions(+) create mode 100644 mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.h create mode 100644 mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.mm diff --git a/mediapipe/tasks/ios/text/language_detector/BUILD b/mediapipe/tasks/ios/text/language_detector/BUILD index 3b59fbd59..4df278037 100644 --- a/mediapipe/tasks/ios/text/language_detector/BUILD +++ b/mediapipe/tasks/ios/text/language_detector/BUILD @@ -31,3 +31,28 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskResult", ], ) + +objc_library( + name = "MPPLanguageDetector", + srcs = ["sources/MPPLanguageDetector.mm"], + hdrs = ["sources/MPPLanguageDetector.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + module_name = "MPPLanguageDetector", + deps = [ + ":MPPLanguageDetectorOptions", + ":MPPLanguageDetectorResult", + "//mediapipe/tasks/cc/text/text_classifier:text_classifier_graph", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/core:MPPTaskInfo", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/core:MPPTextPacketCreator", + "//mediapipe/tasks/ios/text/core:MPPTextTaskRunner", + "//mediapipe/tasks/ios/text/language_detector/utils:MPPLanguageDetectorOptionsHelpers", + "//mediapipe/tasks/ios/text/language_detector/utils:MPPLanguageDetectorResultHelpers", + ], +) diff --git a/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.h b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.h new file mode 100644 index 000000000..7213a8e5f --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.h @@ -0,0 +1,88 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorOptions.h" +#import "mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetectorResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Predicts the language of an input text. + * + * This API expects a TFLite model with [TFLite Model + * Metadata](https://www.tensorflow.org/lite/convert/metadata")that contains the mandatory + * (described below) input tensor, output tensor, and the language codes in an AssociatedFile. + * + * Metadata is required for models with int32 input tensors because it contains the input + * process unit for the model's Tokenizer. No metadata is required for models with string + * input tensors. + * + * Input tensor + * - One input tensor (`kTfLiteString`) of shape `[1]` containing the input string. + * + * Output tensor + * - One output tensor (`kTfLiteFloat32`) of shape `[1 x N]` where `N` is the number of languages. + */ +NS_SWIFT_NAME(LanguageDetector) +@interface MPPLanguageDetector : NSObject + +/** + * Creates a new instance of `LanguageDetector` from an absolute path to a TensorFlow Lite + * model file stored locally on the device and the default `LanguageDetectorOptions`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * @param error An optional error parameter populated when there is an error in initializing the + * language detector. + * + * @return A new instance of `LanguageDetector` with the given model path. `nil` if there is an + * error in initializing the language detector. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `LanguageDetector` from the given `LanguageDetectorOptions`. + * + * @param options The options of type `LanguageDetectorOptions` to use for configuring the + * `LanguageDetector`. + * @param error An optional error parameter populated when there is an error in initializing the + * language detector. + * + * @return A new instance of `LanguageDetector` with the given options. `nil` if there is an + * error in initializing the language detector. + */ +- (nullable instancetype)initWithOptions:(MPPLanguageDetectorOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Predicts the language of the input text. + * + * @param text The `NSString` for which language is to be predicted. + * @param error An optional error parameter populated when there is an error in performing + * language prediction on the input text. + * + * @return A `LanguageDetectorResult` object that contains a list of language predictions. + */ +- (nullable MPPLanguageDetectorResult *)detectText:(NSString *)text + error:(NSError **)error NS_SWIFT_NAME(detect(text:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.mm b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.mm new file mode 100644 index 000000000..4c9628c82 --- /dev/null +++ b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.mm @@ -0,0 +1,96 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.h" + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/core/sources/MPPTextPacketCreator.h" +#import "mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h" +#import "mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorOptions+Helpers.h" +#import "mediapipe/tasks/ios/text/language_detector/utils/sources/MPPLanguageDetectorResult+Helpers.h" + +namespace { +using ::mediapipe::Packet; +using ::mediapipe::tasks::core::PacketMap; +} // namespace + +static NSString *const kClassificationsStreamName = @"classifications_out"; +static NSString *const kClassificationsTag = @"CLASSIFICATIONS"; +static NSString *const kTextInStreamName = @"text_in"; +static NSString *const kTextTag = @"TEXT"; +static NSString *const kTaskGraphName = + @"mediapipe.tasks.text.language_detector.LanguageDetectorGraph"; + +@interface MPPLanguageDetector () { + /** iOS Text Task Runner */ + MPPTextTaskRunner *_textTaskRunner; +} +@end + +@implementation MPPLanguageDetector + +- (instancetype)initWithOptions:(MPPLanguageDetectorOptions *)options error:(NSError **)error { + self = [super init]; + if (self) { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] + initWithTaskGraphName:kTaskGraphName + inputStreams:@[ [NSString stringWithFormat:@"%@:%@", kTextTag, kTextInStreamName] ] + outputStreams:@[ [NSString stringWithFormat:@"%@:%@", kClassificationsTag, + kClassificationsStreamName] ] + taskOptions:options + enableFlowLimiting:NO + error:error]; + + if (!taskInfo) { + return nil; + } + + _textTaskRunner = + [[MPPTextTaskRunner alloc] initWithCalculatorGraphConfig:[taskInfo generateGraphConfig] + error:error]; + + if (!_textTaskRunner) { + return nil; + } + } + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPLanguageDetectorOptions *options = [[MPPLanguageDetectorOptions alloc] init]; + + options.baseOptions.modelAssetPath = modelPath; + + return [self initWithOptions:options error:error]; +} + +- (nullable MPPLanguageDetectorResult *)detectText:(NSString *)text error:(NSError **)error { + Packet packet = [MPPTextPacketCreator createWithText:text]; + + std::map packetMap = {{kTextInStreamName.cppString, packet}}; + std::optional outputPacketMap = [_textTaskRunner processPacketMap:packetMap + error:error]; + + if (!outputPacketMap.has_value()) { + return nil; + } + + return + [MPPLanguageDetectorResult languageDetectorResultWithClassificationsPacket: + outputPacketMap.value()[kClassificationsStreamName.cppString]]; +} + +@end From 0dee33ccba37fcb9362a90b0042cd46730a7f9b5 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Sat, 21 Oct 2023 10:17:02 -0700 Subject: [PATCH 028/157] No public description PiperOrigin-RevId: 575477678 --- .../examples/desktop/media_sequence/run_graph_file_io_main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc b/mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc index a14c7734d..80e19a355 100644 --- a/mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc +++ b/mediapipe/examples/desktop/media_sequence/run_graph_file_io_main.cc @@ -55,7 +55,7 @@ absl::Status RunMPPGraph() { for (const std::string& kv_pair : kv_pairs) { std::vector name_and_value = absl::StrSplit(kv_pair, '='); RET_CHECK(name_and_value.size() == 2); - RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0])); + RET_CHECK(!input_side_packets.contains(name_and_value[0])); std::string input_side_packet_contents; MP_RETURN_IF_ERROR(mediapipe::file::GetContents( name_and_value[1], &input_side_packet_contents)); From 4b3cb5b7582392bbe8ad4a611d48386da35c7339 Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 23 Oct 2023 00:30:51 -0700 Subject: [PATCH 029/157] Added files for the Image Embedder C API and tests --- mediapipe/tasks/c/vision/image_embedder/BUILD | 70 +++++++++ .../c/vision/image_embedder/image_embedder.cc | 143 ++++++++++++++++++ .../c/vision/image_embedder/image_embedder.h | 133 ++++++++++++++++ .../image_embedder/image_embedder_test.cc | 119 +++++++++++++++ 4 files changed, 465 insertions(+) create mode 100644 mediapipe/tasks/c/vision/image_embedder/BUILD create mode 100644 mediapipe/tasks/c/vision/image_embedder/image_embedder.cc create mode 100644 mediapipe/tasks/c/vision/image_embedder/image_embedder.h create mode 100644 mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc diff --git a/mediapipe/tasks/c/vision/image_embedder/BUILD b/mediapipe/tasks/c/vision/image_embedder/BUILD new file mode 100644 index 000000000..3300b4a0a --- /dev/null +++ b/mediapipe/tasks/c/vision/image_embedder/BUILD @@ -0,0 +1,70 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +cc_library( + name = "image_embedder_lib", + srcs = ["image_embedder.cc"], + hdrs = ["image_embedder.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/tasks/c/components/containers:embedding_result", + "//mediapipe/tasks/c/components/containers:embedding_result_converter", + "//mediapipe/tasks/c/components/processors:embedder_options", + "//mediapipe/tasks/c/components/processors:embedder_options_converter", + "//mediapipe/tasks/c/core:base_options", + "//mediapipe/tasks/c/core:base_options_converter", + "//mediapipe/tasks/cc/vision/image_embedder", + "//mediapipe/tasks/cc/vision/utils:image_utils", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/time", + ], + alwayslink = 1, +) + +cc_test( + name = "image_embedder_test", + srcs = ["image_embedder_test.cc"], + data = [ + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + linkstatic = 1, + deps = [ + ":image_embedder_lib", + "//mediapipe/framework/deps:file_path", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:gtest", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/tasks/c/components/containers:category", + "//mediapipe/tasks/cc/vision/utils:image_utils", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc b/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc new file mode 100644 index 000000000..4b42034cb --- /dev/null +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc @@ -0,0 +1,143 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/image_embedder/image_embedder.h" + +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/tasks/c/components/containers/embedding_result_converter.h" +#include "mediapipe/tasks/c/components/processors/embedder_options_converter.h" +#include "mediapipe/tasks/c/core/base_options_converter.h" +#include "mediapipe/tasks/cc/vision/image_embedder/image_embedder.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace mediapipe::tasks::c::vision::image_embedder { + +namespace { + +using ::mediapipe::tasks::c::components::containers::CppCloseEmbeddingResult; +using ::mediapipe::tasks::c::components::containers:: + CppConvertToEmbeddingResult; +using ::mediapipe::tasks::c::components::processors:: + CppConvertToEmbedderOptions; +using ::mediapipe::tasks::c::core::CppConvertToBaseOptions; +using ::mediapipe::tasks::vision::CreateImageFromBuffer; +using ::mediapipe::tasks::vision::image_embedder::ImageEmbedder; + +int CppProcessError(absl::Status status, char** error_msg) { + if (error_msg) { + *error_msg = strdup(status.ToString().c_str()); + } + return status.raw_code(); +} + +} // namespace + +ImageEmbedder* CppImageEmbedderCreate(const ImageEmbedderOptions& options, + char** error_msg) { + auto cpp_options = std::make_unique< + ::mediapipe::tasks::vision::image_embedder::ImageEmbedderOptions>(); + + CppConvertToBaseOptions(options.base_options, &cpp_options->base_options); + CppConvertToEmbedderOptions(options.embedder_options, + &cpp_options->embedder_options); + + auto embedder = ImageEmbedder::Create(std::move(cpp_options)); + if (!embedder.ok()) { + ABSL_LOG(ERROR) << "Failed to create ImageEmbedder: " << embedder.status(); + CppProcessError(embedder.status(), error_msg); + return nullptr; + } + return embedder->release(); +} + +int CppImageEmbedderEmbed(void* embedder, const MpImage* image, + ImageEmbedderResult* result, char** error_msg) { + if (image->type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("gpu buffer not supported yet"); + + ABSL_LOG(ERROR) << "Classification failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image->image_frame.format), + image->image_frame.image_buffer, image->image_frame.width, + image->image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_embedder = static_cast(embedder); + auto cpp_result = cpp_embedder->Embed(*img); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Classification failed: " << cpp_result.status(); + return CppProcessError(cpp_result.status(), error_msg); + } + CppConvertToEmbeddingResult(*cpp_result, result); + return 0; +} + +void CppImageEmbedderCloseResult(ImageEmbedderResult* result) { + CppCloseEmbeddingResult(result); +} + +int CppImageEmbedderClose(void* embedder, char** error_msg) { + auto cpp_embedder = static_cast(embedder); + auto result = cpp_embedder->Close(); + if (!result.ok()) { + ABSL_LOG(ERROR) << "Failed to close ImageEmbedder: " << result; + return CppProcessError(result, error_msg); + } + delete cpp_embedder; + return 0; +} + +} // namespace mediapipe::tasks::c::vision::image_embedder + +extern "C" { + +void* image_embedder_create(struct ImageEmbedderOptions* options, + char** error_msg) { + return mediapipe::tasks::c::vision::image_embedder::CppImageEmbedderCreate( + *options, error_msg); +} + +int image_embedder_embed_image(void* embedder, const MpImage* image, + ImageEmbedderResult* result, char** error_msg) { + return mediapipe::tasks::c::vision::image_embedder::CppImageEmbedderEmbed( + embedder, image, result, error_msg); +} + +void image_embedder_close_result(ImageEmbedderResult* result) { + mediapipe::tasks::c::vision::image_embedder::CppImageEmbedderCloseResult( + result); +} + +int image_embedder_close(void* embedder, char** error_ms) { + return mediapipe::tasks::c::vision::image_embedder::CppImageEmbedderClose( + embedder, error_ms); +} + +} // extern "C" diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder.h b/mediapipe/tasks/c/vision/image_embedder/image_embedder.h new file mode 100644 index 000000000..cefe97b1b --- /dev/null +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder.h @@ -0,0 +1,133 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_VISION_IMAGE_EMBEDDER_IMAGE_EMBEDDER_H_ +#define MEDIAPIPE_TASKS_C_VISION_IMAGE_EMBEDDER_IMAGE_EMBEDDER_H_ + +#include + +#include "mediapipe/tasks/c/components/containers/embedding_result.h" +#include "mediapipe/tasks/c/components/processors/embedder_options.h" +#include "mediapipe/tasks/c/core/base_options.h" + +#ifndef MP_EXPORT +#define MP_EXPORT __attribute__((visibility("default"))) +#endif // MP_EXPORT + +#ifdef __cplusplus +extern "C" { +#endif + +typedef EmbeddingResult ImageEmbedderResult; + +// Supported image formats. +enum ImageFormat { + UNKNOWN = 0, + SRGB = 1, + SRGBA = 2, + GRAY8 = 3, + SBGRA = 11 // compatible with Flutter `bgra8888` format. +}; + +// Supported processing modes. +enum RunningMode { + IMAGE = 1, + VIDEO = 2, + LIVE_STREAM = 3, +}; + +// Structure to hold image frame. +struct ImageFrame { + enum ImageFormat format; + const uint8_t* image_buffer; + int width; + int height; +}; + +// TODO: Add GPU buffer declaration and proccessing logic for it. +struct GpuBuffer { + int width; + int height; +}; + +// The object to contain an image, realizes `OneOf` concept. +struct MpImage { + enum { IMAGE_FRAME, GPU_BUFFER } type; + union { + struct ImageFrame image_frame; + struct GpuBuffer gpu_buffer; + }; +}; + +// The options for configuring a MediaPipe image embedder task. +struct ImageEmbedderOptions { + // Base options for configuring MediaPipe Tasks, such as specifying the model + // file with metadata, accelerator options, op resolver, etc. + struct BaseOptions base_options; + + // The running mode of the task. Default to the image mode. + // Image embedder has three running modes: + // 1) The image mode for embedding image on single image inputs. + // 2) The video mode for embedding image on the decoded frames of a video. + // 3) The live stream mode for embedding image on the live stream of input + // data, such as from camera. In this mode, the "result_callback" below must + // be specified to receive the embedding results asynchronously. + RunningMode running_mode; + + // Options for configuring the embedder behavior, such as l2_normalize and + // quantize. + struct EmbedderOptions embedder_options; + + // The user-defined result callback for processing live stream data. + // The result callback should only be specified when the running mode is set + // to RunningMode::LIVE_STREAM. + typedef void (*result_callback_fn)(ImageEmbedderResult*, const MpImage*, + int64_t); + result_callback_fn result_callback; +}; + +// Creates an ImageEmbedder from provided `options`. +// Returns a pointer to the image embedder on success. +// If an error occurs, returns `nullptr` and sets the error parameter to an +// an error message (if `error_msg` is not nullptr). You must free the memory +// allocated for the error message. +MP_EXPORT void* image_embedder_create(struct ImageEmbedderOptions* options, + char** error_msg = nullptr); + +// Performs embedding extraction on the input `image`. Returns `0` on success. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not nullptr). You must free the memory +// allocated for the error message. +// +// TODO: Add API for video and live stream processing. +MP_EXPORT int image_embedder_embed_image(void* embedder, const MpImage* image, + ImageEmbedderResult* result, + char** error_msg = nullptr); + +// Frees the memory allocated inside a ImageEmbedderResult result. +// Does not free the result pointer itself. +MP_EXPORT void image_embedder_close_result(ImageEmbedderResult* result); + +// Frees image embedder. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not nullptr). You must free the memory +// allocated for the error message. +MP_EXPORT int image_embedder_close(void* embedder, char** error_msg = nullptr); + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_VISION_IMAGE_EMBEDDER_IMAGE_EMBEDDER_H_ diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc b/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc new file mode 100644 index 000000000..fb6b0b628 --- /dev/null +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc @@ -0,0 +1,119 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/image_embedder/image_embedder.h" + +#include +#include + +#include "absl/flags/flag.h" +#include "absl/strings/string_view.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/components/containers/category.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace { + +using ::mediapipe::file::JoinPath; +using ::mediapipe::tasks::vision::DecodeImageFromFile; +using testing::HasSubstr; + +constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; +constexpr char kModelName[] = "mobilenet_v3_small_100_224_embedder.tflite"; +constexpr char kImageFile[] = "burger.jpg"; +constexpr float kPrecision = 1e-6; + +std::string GetFullPath(absl::string_view file_name) { + return JoinPath("./", kTestDataDirectory, file_name); +} + +TEST(ImageEmbedderTest, SmokeTest) { + const auto image = DecodeImageFromFile(GetFullPath("burger.jpg")); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + ImageEmbedderOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* embedder_options= */ + {/* l2_normalize= */ true, + /* quantize= */ false}, + }; + + void* embedder = image_embedder_create(&options); + EXPECT_NE(embedder, nullptr); + + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = { + .format = static_cast( + image->GetImageFrameSharedPtr()->Format()), + .image_buffer = image->GetImageFrameSharedPtr()->PixelData(), + .width = image->GetImageFrameSharedPtr()->Width(), + .height = image->GetImageFrameSharedPtr()->Height()}}; + + ImageEmbedderResult result; + image_embedder_embed_image(embedder, &mp_image, &result); + EXPECT_EQ(result.embeddings_count, 1); + EXPECT_NEAR(result.embeddings[0].float_embedding[0], -0.0142344, kPrecision); + image_embedder_close_result(&result); + image_embedder_close(embedder); +} + +TEST(ImageEmbedderTest, InvalidArgumentHandling) { + // It is an error to set neither the asset buffer nor the path. + ImageEmbedderOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_path= */ nullptr}, + /* embedder_options= */ {}, + }; + + char* error_msg; + void* embedder = image_embedder_create(&options, &error_msg); + EXPECT_EQ(embedder, nullptr); + + EXPECT_THAT(error_msg, HasSubstr("ExternalFile must specify")); + + free(error_msg); +} + +TEST(ImageEmbedderTest, FailedEmbeddingHandling) { + const std::string model_path = GetFullPath(kModelName); + ImageEmbedderOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* embedder_options= */ + {/* l2_normalize= */ false, + /* quantize= */ false}, + }; + + void* embedder = image_embedder_create(&options); + EXPECT_NE(embedder, nullptr); + + const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}}; + ImageEmbedderResult result; + char* error_msg; + image_embedder_embed_image(embedder, &mp_image, &result, &error_msg); + EXPECT_THAT(error_msg, HasSubstr("gpu buffer not supported yet")); + free(error_msg); + image_embedder_close(embedder); +} + +} // namespace From 305f076c7f85b04bd7a059432bcd20e11a5feba5 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 23 Oct 2023 20:02:39 +0530 Subject: [PATCH 030/157] Fixed extra condition check in iOS Image Segmenter Result Helper --- .../utils/sources/MPPImageSegmenterResult+Helpers.mm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.mm b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.mm index 885df734d..c4c3d398c 100644 --- a/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.mm +++ b/mediapipe/tasks/ios/vision/image_segmenter/utils/sources/MPPImageSegmenterResult+Helpers.mm @@ -47,7 +47,7 @@ using ::mediapipe::Packet; ->PixelData() width:confidenceMask.width() height:confidenceMask.height() - shouldCopy:shouldCopyMaskPacketData ? YES : NO]]; + shouldCopy:shouldCopyMaskPacketData]]; } } @@ -57,7 +57,7 @@ using ::mediapipe::Packet; initWithUInt8Data:(UInt8 *)cppCategoryMask.GetImageFrameSharedPtr().get()->PixelData() width:cppCategoryMask.width() height:cppCategoryMask.height() - shouldCopy:shouldCopyMaskPacketData ? YES : NO]; + shouldCopy:shouldCopyMaskPacketData]; } if (qualityScoresPacket.ValidateAsType>().ok()) { From 7c45bc802fcc0a4b300d9b7d319867ad6ff89706 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 23 Oct 2023 20:02:57 +0530 Subject: [PATCH 031/157] Added iOS Image Segmenter to CocoaPods build --- mediapipe/tasks/ios/BUILD | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mediapipe/tasks/ios/BUILD b/mediapipe/tasks/ios/BUILD index 7f3db7f7a..88b99ffec 100644 --- a/mediapipe/tasks/ios/BUILD +++ b/mediapipe/tasks/ios/BUILD @@ -57,6 +57,7 @@ CALCULATORS_AND_GRAPHS = [ "//mediapipe/tasks/cc/vision/hand_landmarker:hand_landmarker_graph", "//mediapipe/tasks/cc/vision/gesture_recognizer:gesture_recognizer_graph", "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", + "//mediapipe/tasks/cc/vision/image_segmenter:image_segmenter_graph", "//mediapipe/tasks/cc/vision/object_detector:object_detector_graph", ] @@ -83,6 +84,7 @@ strip_api_include_path_prefix( "//mediapipe/tasks/ios/text/text_embedder:sources/MPPTextEmbedderResult.h", "//mediapipe/tasks/ios/vision/core:sources/MPPRunningMode.h", "//mediapipe/tasks/ios/vision/core:sources/MPPImage.h", + "//mediapipe/tasks/ios/vision/core:sources/MPPMask.h", "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetector.h", "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetectorOptions.h", "//mediapipe/tasks/ios/vision/face_detector:sources/MPPFaceDetectorResult.h", @@ -98,6 +100,9 @@ strip_api_include_path_prefix( "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifier.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierOptions.h", "//mediapipe/tasks/ios/vision/image_classifier:sources/MPPImageClassifierResult.h", + "//mediapipe/tasks/ios/vision/image_segmenter:sources/MPPImageSegmenter.h", + "//mediapipe/tasks/ios/vision/image_segmenter:sources/MPPImageSegmenterOptions.h", + "//mediapipe/tasks/ios/vision/image_segmenter:sources/MPPImageSegmenterResult.h", "//mediapipe/tasks/ios/vision/object_detector:sources/MPPObjectDetector.h", "//mediapipe/tasks/ios/vision/object_detector:sources/MPPObjectDetectorOptions.h", "//mediapipe/tasks/ios/vision/object_detector:sources/MPPObjectDetectorResult.h", @@ -178,6 +183,7 @@ apple_static_xcframework( ":MPPTaskOptions.h", ":MPPTaskResult.h", ":MPPImage.h", + ":MPPMask.h", ":MPPRunningMode.h", ":MPPFaceDetector.h", ":MPPFaceDetectorOptions.h", @@ -188,6 +194,9 @@ apple_static_xcframework( ":MPPImageClassifier.h", ":MPPImageClassifierOptions.h", ":MPPImageClassifierResult.h", + ":MPPImageSegmenter.h", + ":MPPImageSegmenterOptions.h", + ":MPPImageSegmenterResult.h", ":MPPHandLandmarker.h", ":MPPHandLandmarkerOptions.h", ":MPPHandLandmarkerResult.h", @@ -204,6 +213,7 @@ apple_static_xcframework( "//mediapipe/tasks/ios/vision/gesture_recognizer:MPPGestureRecognizer", "//mediapipe/tasks/ios/vision/hand_landmarker:MPPHandLandmarker", "//mediapipe/tasks/ios/vision/image_classifier:MPPImageClassifier", + "//mediapipe/tasks/ios/vision/image_segmenter:MPPImageSegmenter", "//mediapipe/tasks/ios/vision/object_detector:MPPObjectDetector", ], ) From d5a1bc03afa0ab1442552be244a6448120f4f330 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 23 Oct 2023 20:28:43 +0530 Subject: [PATCH 032/157] Fixed deletion of iOS output MPImage buffer in MPImage Utils --- mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm index 440b321b9..0569ad783 100644 --- a/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm +++ b/mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.mm @@ -37,7 +37,7 @@ vImage_Buffer allocatedVImageBuffer(vImagePixelCount width, vImagePixelCount hei } static void FreeDataProviderReleaseCallback(void *buffer, const void *data, size_t size) { - delete (vImage_Buffer *)buffer; + delete[] buffer; } } // namespace From 6aa27d9aebb1e2fa083da88f74364839ef6c2b74 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 23 Oct 2023 09:31:58 -0700 Subject: [PATCH 033/157] Initialize GPU support for Python Task API PiperOrigin-RevId: 575842513 --- mediapipe/tasks/cc/core/BUILD | 1 + mediapipe/tasks/cc/core/task_runner.cc | 22 +++++++++++++++++++ mediapipe/tasks/cc/core/task_runner.h | 15 +++++++++++++ mediapipe/tasks/python/core/pybind/BUILD | 2 ++ .../tasks/python/core/pybind/task_runner.cc | 21 ++++++++++++++++++ 5 files changed, 61 insertions(+) diff --git a/mediapipe/tasks/cc/core/BUILD b/mediapipe/tasks/cc/core/BUILD index fa61feb9d..bb0d4b001 100644 --- a/mediapipe/tasks/cc/core/BUILD +++ b/mediapipe/tasks/cc/core/BUILD @@ -264,6 +264,7 @@ cc_library_with_tflite( "//mediapipe/framework:executor", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:name_util", + "//mediapipe/gpu:gpu_shared_data_internal", "//mediapipe/tasks/cc:common", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/container:flat_hash_map", diff --git a/mediapipe/tasks/cc/core/task_runner.cc b/mediapipe/tasks/cc/core/task_runner.cc index 88c91bcdb..e3862ddd7 100644 --- a/mediapipe/tasks/cc/core/task_runner.cc +++ b/mediapipe/tasks/cc/core/task_runner.cc @@ -39,6 +39,10 @@ limitations under the License. #include "mediapipe/tasks/cc/common.h" #include "mediapipe/tasks/cc/core/model_resources_cache.h" +#if !MEDIAPIPE_DISABLE_GPU +#include "mediapipe/gpu/gpu_shared_data_internal.h" +#endif // !MEDIAPIPE_DISABLE_GPU + namespace mediapipe { namespace tasks { namespace core { @@ -88,16 +92,34 @@ absl::StatusOr GenerateOutputPacketMap( } // namespace /* static */ +#if !MEDIAPIPE_DISABLE_GPU +absl::StatusOr> TaskRunner::Create( + CalculatorGraphConfig config, + std::unique_ptr op_resolver, + PacketsCallback packets_callback, + std::shared_ptr default_executor, + std::optional input_side_packets, + std::shared_ptr<::mediapipe::GpuResources> resources) { +#else absl::StatusOr> TaskRunner::Create( CalculatorGraphConfig config, std::unique_ptr op_resolver, PacketsCallback packets_callback, std::shared_ptr default_executor, std::optional input_side_packets) { +#endif // !MEDIAPIPE_DISABLE_GPU auto task_runner = absl::WrapUnique(new TaskRunner(packets_callback)); MP_RETURN_IF_ERROR(task_runner->Initialize( std::move(config), std::move(op_resolver), std::move(default_executor), std::move(input_side_packets))); + +#if !MEDIAPIPE_DISABLE_GPU + if (resources) { + MP_RETURN_IF_ERROR( + task_runner->graph_.SetGpuResources(std::move(resources))); + } +#endif // !MEDIAPIPE_DISABLE_GPU + MP_RETURN_IF_ERROR(task_runner->Start()); return task_runner; } diff --git a/mediapipe/tasks/cc/core/task_runner.h b/mediapipe/tasks/cc/core/task_runner.h index 810063d4b..ef48bef55 100644 --- a/mediapipe/tasks/cc/core/task_runner.h +++ b/mediapipe/tasks/cc/core/task_runner.h @@ -42,6 +42,11 @@ limitations under the License. #include "tensorflow/lite/core/api/op_resolver.h" namespace mediapipe { + +#if !MEDIAPIPE_DISABLE_GPU +class GpuResources; +#endif // !MEDIAPIPE_DISABLE_GPU + namespace tasks { namespace core { @@ -72,12 +77,22 @@ class TaskRunner { // asynchronous method, Send(), to provide the input packets. If the packets // callback is absent, clients must use the synchronous method, Process(), to // provide the input packets and receive the output packets. +#if !MEDIAPIPE_DISABLE_GPU + static absl::StatusOr> Create( + CalculatorGraphConfig config, + std::unique_ptr op_resolver = nullptr, + PacketsCallback packets_callback = nullptr, + std::shared_ptr default_executor = nullptr, + std::optional input_side_packets = std::nullopt, + std::shared_ptr<::mediapipe::GpuResources> resources = nullptr); +#else static absl::StatusOr> Create( CalculatorGraphConfig config, std::unique_ptr op_resolver = nullptr, PacketsCallback packets_callback = nullptr, std::shared_ptr default_executor = nullptr, std::optional input_side_packets = std::nullopt); +#endif // !MEDIAPIPE_DISABLE_GPU // TaskRunner is neither copyable nor movable. TaskRunner(const TaskRunner&) = delete; diff --git a/mediapipe/tasks/python/core/pybind/BUILD b/mediapipe/tasks/python/core/pybind/BUILD index 88ea05f4f..391712f27 100644 --- a/mediapipe/tasks/python/core/pybind/BUILD +++ b/mediapipe/tasks/python/core/pybind/BUILD @@ -26,9 +26,11 @@ pybind_library( "//mediapipe/framework:calculator_cc_proto", "//mediapipe/framework/api2:builder", "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/gpu:gpu_shared_data_internal", "//mediapipe/python/pybind:util", "//mediapipe/tasks/cc/core:mediapipe_builtin_op_resolver", "//mediapipe/tasks/cc/core:task_runner", + "@com_google_absl//absl/log:absl_log", "@org_tensorflow//tensorflow/lite/core/api:op_resolver", "@pybind11_protobuf//pybind11_protobuf:native_proto_caster", ], diff --git a/mediapipe/tasks/python/core/pybind/task_runner.cc b/mediapipe/tasks/python/core/pybind/task_runner.cc index f95cddde8..0de7d24d8 100644 --- a/mediapipe/tasks/python/core/pybind/task_runner.cc +++ b/mediapipe/tasks/python/core/pybind/task_runner.cc @@ -14,6 +14,7 @@ #include "mediapipe/tasks/python/core/pybind/task_runner.h" +#include "absl/log/absl_log.h" #include "mediapipe/framework/calculator.pb.h" #include "mediapipe/python/pybind/util.h" #include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" @@ -21,6 +22,9 @@ #include "pybind11/stl.h" #include "pybind11_protobuf/native_proto_caster.h" #include "tensorflow/lite/core/api/op_resolver.h" +#if !MEDIAPIPE_DISABLE_GPU +#include "mediapipe/gpu/gpu_shared_data_internal.h" +#endif // MEDIAPIPE_DISABLE_GPU namespace mediapipe { namespace tasks { @@ -74,10 +78,27 @@ mode) or not (synchronous mode).)doc"); return absl::OkStatus(); }; } + +#if !MEDIAPIPE_DISABLE_GPU + auto gpu_resources_ = mediapipe::GpuResources::Create(); + if (!gpu_resources_.ok()) { + ABSL_LOG(INFO) << "GPU suport is not available: " + << gpu_resources_.status(); + gpu_resources_ = nullptr; + } + auto task_runner = TaskRunner::Create( + std::move(graph_config), + absl::make_unique(), + std::move(callback), + /* default_executor= */ nullptr, + /* input_side_packes= */ std::nullopt, std::move(*gpu_resources_)); +#else auto task_runner = TaskRunner::Create( std::move(graph_config), absl::make_unique(), std::move(callback)); +#endif // !MEDIAPIPE_DISABLE_GPU + RaisePyErrorIfNotOk(task_runner.status()); return std::move(*task_runner); }, From b904ade0cf906210c7c885a1866c40cfe9203ac0 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 23 Oct 2023 11:41:19 -0700 Subject: [PATCH 034/157] Allow Mac to use GPU Delegate PiperOrigin-RevId: 575882254 --- mediapipe/tasks/python/core/base_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/python/core/base_options.py b/mediapipe/tasks/python/core/base_options.py index 2d4258fed..da81bcd5d 100644 --- a/mediapipe/tasks/python/core/base_options.py +++ b/mediapipe/tasks/python/core/base_options.py @@ -70,7 +70,7 @@ class BaseOptions: platform_name = platform.system() if self.delegate == BaseOptions.Delegate.GPU: - if platform_name == 'Linux': + if platform_name in ['Linux', 'Darwin']: acceleration_proto = _AccelerationProto(gpu=_DelegateProto.Gpu()) else: raise NotImplementedError( From aedafd63f9e8b4f30614b83ec20f41fe2d3e05c7 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 23 Oct 2023 12:30:37 -0700 Subject: [PATCH 035/157] Remove objc_library from Python build path for Mac GPU build Addresses https://github.com/bazelbuild/bazel/issues/19912 PiperOrigin-RevId: 575896231 --- WORKSPACE | 3 + mediapipe/gpu/BUILD | 18 +- .../gpu/{MPPMetalUtil.mm => MPPMetalUtil.cc} | 8 +- ...resources.mm => metal_shared_resources.cc} | 7 +- ...pool_util.mm => pixel_buffer_pool_util.cc} | 66 ++++--- mediapipe/objc/BUILD | 11 +- .../objc/{MPPGraph.mm => DrishtiGraph.cc} | 174 +++++++++++------- ...verter.mm => DrishtiTimestampConverter.cc} | 12 +- mediapipe/objc/NSError+util_status.cc | 72 ++++++++ .../org_tensorflow_objc_build_fixes.diff | 86 +++++++++ 10 files changed, 338 insertions(+), 119 deletions(-) rename mediapipe/gpu/{MPPMetalUtil.mm => MPPMetalUtil.cc} (95%) rename mediapipe/gpu/{metal_shared_resources.mm => metal_shared_resources.cc} (85%) rename mediapipe/gpu/{pixel_buffer_pool_util.mm => pixel_buffer_pool_util.cc} (63%) rename mediapipe/objc/{MPPGraph.mm => DrishtiGraph.cc} (74%) rename mediapipe/objc/{MPPTimestampConverter.mm => DrishtiTimestampConverter.cc} (81%) create mode 100644 mediapipe/objc/NSError+util_status.cc create mode 100644 third_party/org_tensorflow_objc_build_fixes.diff diff --git a/WORKSPACE b/WORKSPACE index df2c4f93b..3a539569f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -513,6 +513,9 @@ http_archive( "@//third_party:org_tensorflow_system_python.diff", # Diff is generated with a script, don't update it manually. "@//third_party:org_tensorflow_custom_ops.diff", + # Works around Bazel issue with objc_library. + # See https://github.com/bazelbuild/bazel/issues/19912 + "@//third_party:org_tensorflow_objc_build_fixes.diff", ], patch_args = [ "-p1", diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 27770acaa..8a8a402f1 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -526,12 +526,14 @@ mediapipe_proto_library( visibility = ["//visibility:public"], ) -objc_library( +cc_library( name = "pixel_buffer_pool_util", - srcs = ["pixel_buffer_pool_util.mm"], + srcs = ["pixel_buffer_pool_util.cc"], hdrs = ["pixel_buffer_pool_util.h"], copts = [ + "-x objective-c++", "-Wno-shorten-64-to-32", + "-fobjc-arc", # enable reference-counting ], visibility = ["//visibility:public"], deps = [ @@ -542,13 +544,14 @@ objc_library( ], ) -objc_library( +cc_library( name = "metal_shared_resources", - srcs = ["metal_shared_resources.mm"], + srcs = ["metal_shared_resources.cc"], hdrs = ["metal_shared_resources.h"], copts = [ "-x objective-c++", "-Wno-shorten-64-to-32", + "-fobjc-arc", # enable reference-counting ], features = ["-layering_check"], visibility = ["//visibility:public"], @@ -557,15 +560,17 @@ objc_library( "@google_toolbox_for_mac//:GTM_Defines", ] + [ ], + alwayslink = 1, ) -objc_library( +cc_library( name = "MPPMetalUtil", - srcs = ["MPPMetalUtil.mm"], + srcs = ["MPPMetalUtil.cc"], hdrs = ["MPPMetalUtil.h"], copts = [ "-x objective-c++", "-Wno-shorten-64-to-32", + "-fobjc-arc", # enable reference-counting ], visibility = ["//visibility:public"], deps = [ @@ -575,6 +580,7 @@ objc_library( "@com_google_absl//absl/time", "@google_toolbox_for_mac//:GTM_Defines", ], + alwayslink = 1, ) mediapipe_proto_library( diff --git a/mediapipe/gpu/MPPMetalUtil.mm b/mediapipe/gpu/MPPMetalUtil.cc similarity index 95% rename from mediapipe/gpu/MPPMetalUtil.mm rename to mediapipe/gpu/MPPMetalUtil.cc index ba8be0dbd..c9bd6798d 100644 --- a/mediapipe/gpu/MPPMetalUtil.mm +++ b/mediapipe/gpu/MPPMetalUtil.cc @@ -69,10 +69,10 @@ while (!bufferCompleted) { auto duration = absl::Now() - start_time; // If the spin-lock takes more than 5 ms then go to blocking wait: - // - it frees the CPU core for another threads: increase the performance/decrease power - // consumption. - // - if a driver thread that notifies that the GPU buffer is completed has lower priority then - // the CPU core is allocated for the thread. + // - it frees the CPU core for another threads: increase the + // performance/decrease power consumption. + // - if a driver thread that notifies that the GPU buffer is completed has + // lower priority then the CPU core is allocated for the thread. if (duration >= absl::Milliseconds(5)) { [commandBuffer waitUntilCompleted]; break; diff --git a/mediapipe/gpu/metal_shared_resources.mm b/mediapipe/gpu/metal_shared_resources.cc similarity index 85% rename from mediapipe/gpu/metal_shared_resources.mm rename to mediapipe/gpu/metal_shared_resources.cc index 80d755a01..925c0f995 100644 --- a/mediapipe/gpu/metal_shared_resources.mm +++ b/mediapipe/gpu/metal_shared_resources.cc @@ -50,9 +50,10 @@ - (CVMetalTextureCacheRef)mtlTextureCache { @synchronized(self) { if (!_mtlTextureCache) { - CVReturn __unused err = - CVMetalTextureCacheCreate(NULL, NULL, self.mtlDevice, NULL, &_mtlTextureCache); - NSAssert(err == kCVReturnSuccess, @"Error at CVMetalTextureCacheCreate %d ; device %@", err, + CVReturn __unused err = CVMetalTextureCacheCreate( + NULL, NULL, self.mtlDevice, NULL, &_mtlTextureCache); + NSAssert(err == kCVReturnSuccess, + @"Error at CVMetalTextureCacheCreate %d ; device %@", err, self.mtlDevice); // TODO: register and flush metal caches too. } diff --git a/mediapipe/gpu/pixel_buffer_pool_util.mm b/mediapipe/gpu/pixel_buffer_pool_util.cc similarity index 63% rename from mediapipe/gpu/pixel_buffer_pool_util.mm rename to mediapipe/gpu/pixel_buffer_pool_util.cc index 0b13cb194..9980d0a5d 100644 --- a/mediapipe/gpu/pixel_buffer_pool_util.mm +++ b/mediapipe/gpu/pixel_buffer_pool_util.cc @@ -24,23 +24,27 @@ namespace mediapipe { -CVPixelBufferPoolRef CreateCVPixelBufferPool( - int width, int height, OSType pixelFormat, int keepCount, - CFTimeInterval maxAge) { +CVPixelBufferPoolRef CreateCVPixelBufferPool(int width, int height, + OSType pixelFormat, int keepCount, + CFTimeInterval maxAge) { CVPixelBufferPoolRef pool = NULL; NSMutableDictionary *sourcePixelBufferOptions = - [(__bridge NSDictionary*)GetCVPixelBufferAttributesForGlCompatibility() mutableCopy]; + [(__bridge NSDictionary *)GetCVPixelBufferAttributesForGlCompatibility() + mutableCopy]; [sourcePixelBufferOptions addEntriesFromDictionary:@{ (id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat), (id)kCVPixelBufferWidthKey : @(width), (id)kCVPixelBufferHeightKey : @(height), }]; - NSMutableDictionary *pixelBufferPoolOptions = [[NSMutableDictionary alloc] init]; - pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] = @(keepCount); + NSMutableDictionary *pixelBufferPoolOptions = + [[NSMutableDictionary alloc] init]; + pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] = + @(keepCount); if (maxAge > 0) { - pixelBufferPoolOptions[(id)kCVPixelBufferPoolMaximumBufferAgeKey] = @(maxAge); + pixelBufferPoolOptions[(id)kCVPixelBufferPoolMaximumBufferAgeKey] = + @(maxAge); } CVPixelBufferPoolCreate( @@ -50,8 +54,9 @@ CVPixelBufferPoolRef CreateCVPixelBufferPool( return pool; } -OSStatus PreallocateCVPixelBufferPoolBuffers( - CVPixelBufferPoolRef pool, int count, CFDictionaryRef auxAttributes) { +OSStatus PreallocateCVPixelBufferPoolBuffers(CVPixelBufferPoolRef pool, + int count, + CFDictionaryRef auxAttributes) { CVReturn err = kCVReturnSuccess; NSMutableArray *pixelBuffers = [[NSMutableArray alloc] init]; for (int i = 0; i < count && err == kCVReturnSuccess; i++) { @@ -68,30 +73,37 @@ OSStatus PreallocateCVPixelBufferPoolBuffers( return err; } -CFDictionaryRef CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold(int allocationThreshold) { +CFDictionaryRef CreateCVPixelBufferPoolAuxiliaryAttributesForThreshold( + int allocationThreshold) { if (allocationThreshold > 0) { - return (CFDictionaryRef)CFBridgingRetain( - @{(id)kCVPixelBufferPoolAllocationThresholdKey: @(allocationThreshold)}); + return (CFDictionaryRef)CFBridgingRetain(@{ + (id)kCVPixelBufferPoolAllocationThresholdKey : @(allocationThreshold) + }); } else { return nil; } } -CVReturn CreateCVPixelBufferWithPool( - CVPixelBufferPoolRef pool, CFDictionaryRef auxAttributes, - CVTextureCacheType textureCache, CVPixelBufferRef* outBuffer) { - return CreateCVPixelBufferWithPool(pool, auxAttributes, [textureCache](){ +CVReturn CreateCVPixelBufferWithPool(CVPixelBufferPoolRef pool, + CFDictionaryRef auxAttributes, + CVTextureCacheType textureCache, + CVPixelBufferRef *outBuffer) { + return CreateCVPixelBufferWithPool( + pool, auxAttributes, + [textureCache]() { #if TARGET_OS_OSX - CVOpenGLTextureCacheFlush(textureCache, 0); + CVOpenGLTextureCacheFlush(textureCache, 0); #else - CVOpenGLESTextureCacheFlush(textureCache, 0); + CVOpenGLESTextureCacheFlush(textureCache, 0); #endif // TARGET_OS_OSX - }, outBuffer); + }, + outBuffer); } -CVReturn CreateCVPixelBufferWithPool( - CVPixelBufferPoolRef pool, CFDictionaryRef auxAttributes, - std::function flush, CVPixelBufferRef* outBuffer) { +CVReturn CreateCVPixelBufferWithPool(CVPixelBufferPoolRef pool, + CFDictionaryRef auxAttributes, + std::function flush, + CVPixelBufferRef *outBuffer) { CVReturn err = CVPixelBufferPoolCreatePixelBufferWithAuxAttributes( kCFAllocatorDefault, pool, auxAttributes, outBuffer); if (err == kCVReturnWouldExceedAllocationThreshold) { @@ -103,11 +115,13 @@ CVReturn CreateCVPixelBufferWithPool( kCFAllocatorDefault, pool, auxAttributes, outBuffer); } if (err == kCVReturnWouldExceedAllocationThreshold) { - // TODO: allow the application to set the threshold. For now, disable it by - // default, since the threshold we are using is arbitrary and some graphs routinely cross it. + // TODO: allow the application to set the threshold. For now, disable it + // by default, since the threshold we are using is arbitrary and some + // graphs routinely cross it. #ifdef ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK - NSLog(@"Using more buffers than expected! This is a debug-only warning, " - "you can ignore it if your app works fine otherwise."); + NSLog( + @"Using more buffers than expected! This is a debug-only warning, " + "you can ignore it if your app works fine otherwise."); #ifdef DEBUG NSLog(@"Pool status: %@", ((__bridge NSObject *)pool).description); #endif // DEBUG diff --git a/mediapipe/objc/BUILD b/mediapipe/objc/BUILD index df6c8db08..481a60bb6 100644 --- a/mediapipe/objc/BUILD +++ b/mediapipe/objc/BUILD @@ -52,9 +52,9 @@ objc_library( ) MEDIAPIPE_IOS_SRCS = [ - "MPPGraph.mm", - "MPPTimestampConverter.mm", - "NSError+util_status.mm", + "MPPGraph.cc", + "MPPTimestampConverter.cc", + "NSError+util_status.cc", ] MEDIAPIPE_IOS_HDRS = [ @@ -63,11 +63,13 @@ MEDIAPIPE_IOS_HDRS = [ "NSError+util_status.h", ] -objc_library( +cc_library( name = "mediapipe_framework_ios", srcs = MEDIAPIPE_IOS_SRCS, hdrs = MEDIAPIPE_IOS_HDRS, copts = [ + "-x objective-c++", + "-fobjc-arc", # enable reference-counting "-Wno-shorten-64-to-32", ], # This build rule is public to allow external customers to build their own iOS apps. @@ -99,6 +101,7 @@ objc_library( "@com_google_absl//absl/synchronization", "@google_toolbox_for_mac//:GTM_Defines", ], + alwayslink = 1, ) objc_library( diff --git a/mediapipe/objc/MPPGraph.mm b/mediapipe/objc/DrishtiGraph.cc similarity index 74% rename from mediapipe/objc/MPPGraph.mm rename to mediapipe/objc/DrishtiGraph.cc index 3123eb863..c4c1096d3 100644 --- a/mediapipe/objc/MPPGraph.mm +++ b/mediapipe/objc/DrishtiGraph.cc @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -#import "mediapipe/objc/MPPGraph.h" - #import #import #include +#import "GTMDefines.h" #include "absl/memory/memory.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image.h" @@ -26,22 +25,23 @@ #include "mediapipe/framework/graph_service.h" #include "mediapipe/gpu/gl_base.h" #include "mediapipe/gpu/gpu_shared_data_internal.h" +#import "mediapipe/objc/MPPGraph.h" +#import "mediapipe/objc/NSError+util_status.h" #include "mediapipe/objc/util.h" -#import "mediapipe/objc/NSError+util_status.h" -#import "GTMDefines.h" - @implementation MPPGraph { - // Graph is wrapped in a unique_ptr because it was generating 39+KB of unnecessary ObjC runtime - // information. See https://medium.com/@dmaclach/objective-c-encoding-and-you-866624cc02de - // for details. + // Graph is wrapped in a unique_ptr because it was generating 39+KB of + // unnecessary ObjC runtime information. See + // https://medium.com/@dmaclach/objective-c-encoding-and-you-866624cc02de for + // details. std::unique_ptr _graph; /// Input side packets that will be added to the graph when it is started. std::map _inputSidePackets; /// Packet headers that will be added to the graph when it is started. std::map _streamHeaders; /// Service packets to be added to the graph when it is started. - std::map _servicePackets; + std::map + _servicePackets; /// Number of frames currently being processed by the graph. std::atomic _framesInFlight; @@ -56,7 +56,8 @@ BOOL _started; } -- (instancetype)initWithGraphConfig:(const mediapipe::CalculatorGraphConfig&)config { +- (instancetype)initWithGraphConfig: + (const mediapipe::CalculatorGraphConfig&)config { self = [super init]; if (self) { // Turn on Cocoa multithreading, since MediaPipe uses threads. @@ -76,40 +77,47 @@ return _graph->GetGraphInputStreamAddMode(); } -- (void)setPacketAddMode:(mediapipe::CalculatorGraph::GraphInputStreamAddMode)mode { +- (void)setPacketAddMode: + (mediapipe::CalculatorGraph::GraphInputStreamAddMode)mode { _graph->SetGraphInputStreamAddMode(mode); } - (void)addFrameOutputStream:(const std::string&)outputStreamName outputPacketType:(MPPPacketType)packetType { std::string callbackInputName; - mediapipe::tool::AddCallbackCalculator(outputStreamName, &_config, &callbackInputName, - /*use_std_function=*/true); - // No matter what ownership qualifiers are put on the pointer, NewPermanentCallback will - // still end up with a strong pointer to MPPGraph*. That is why we use void* instead. + mediapipe::tool::AddCallbackCalculator(outputStreamName, &_config, + &callbackInputName, + /*use_std_function=*/true); + // No matter what ownership qualifiers are put on the pointer, + // NewPermanentCallback will still end up with a strong pointer to MPPGraph*. + // That is why we use void* instead. void* wrapperVoid = (__bridge void*)self; _inputSidePackets[callbackInputName] = mediapipe::MakePacket>( - [wrapperVoid, outputStreamName, packetType](const mediapipe::Packet& packet) { - CallFrameDelegate(wrapperVoid, outputStreamName, packetType, packet); + [wrapperVoid, outputStreamName, + packetType](const mediapipe::Packet& packet) { + CallFrameDelegate(wrapperVoid, outputStreamName, packetType, + packet); }); } -- (NSString *)description { - return [NSString stringWithFormat:@"<%@: %p; framesInFlight = %d>", [self class], self, - _framesInFlight.load(std::memory_order_relaxed)]; +- (NSString*)description { + return [NSString + stringWithFormat:@"<%@: %p; framesInFlight = %d>", [self class], self, + _framesInFlight.load(std::memory_order_relaxed)]; } /// This is the function that gets called by the CallbackCalculator that /// receives the graph's output. void CallFrameDelegate(void* wrapperVoid, const std::string& streamName, - MPPPacketType packetType, const mediapipe::Packet& packet) { + MPPPacketType packetType, + const mediapipe::Packet& packet) { MPPGraph* wrapper = (__bridge MPPGraph*)wrapperVoid; @autoreleasepool { if (packetType == MPPPacketTypeRaw) { [wrapper.delegate mediapipeGraph:wrapper - didOutputPacket:packet - fromStream:streamName]; + didOutputPacket:packet + fromStream:streamName]; } else if (packetType == MPPPacketTypeImageFrame) { wrapper->_framesInFlight--; const auto& frame = packet.Get(); @@ -118,13 +126,16 @@ void CallFrameDelegate(void* wrapperVoid, const std::string& streamName, if (format == mediapipe::ImageFormat::SRGBA || format == mediapipe::ImageFormat::GRAY8) { CVPixelBufferRef pixelBuffer; - // If kCVPixelFormatType_32RGBA does not work, it returns kCVReturnInvalidPixelFormat. + // If kCVPixelFormatType_32RGBA does not work, it returns + // kCVReturnInvalidPixelFormat. CVReturn error = CVPixelBufferCreate( NULL, frame.Width(), frame.Height(), kCVPixelFormatType_32BGRA, GetCVPixelBufferAttributesForGlCompatibility(), &pixelBuffer); - _GTMDevAssert(error == kCVReturnSuccess, @"CVPixelBufferCreate failed: %d", error); + _GTMDevAssert(error == kCVReturnSuccess, + @"CVPixelBufferCreate failed: %d", error); error = CVPixelBufferLockBaseAddress(pixelBuffer, 0); - _GTMDevAssert(error == kCVReturnSuccess, @"CVPixelBufferLockBaseAddress failed: %d", error); + _GTMDevAssert(error == kCVReturnSuccess, + @"CVPixelBufferLockBaseAddress failed: %d", error); vImage_Buffer vDestination = vImageForCVPixelBuffer(pixelBuffer); // Note: we have to throw away const here, but we should not overwrite @@ -133,30 +144,35 @@ void CallFrameDelegate(void* wrapperVoid, const std::string& streamName, if (format == mediapipe::ImageFormat::SRGBA) { // Swap R and B channels. const uint8_t permuteMap[4] = {2, 1, 0, 3}; - vImage_Error __unused vError = - vImagePermuteChannels_ARGB8888(&vSource, &vDestination, permuteMap, kvImageNoFlags); - _GTMDevAssert(vError == kvImageNoError, @"vImagePermuteChannels failed: %zd", vError); + vImage_Error __unused vError = vImagePermuteChannels_ARGB8888( + &vSource, &vDestination, permuteMap, kvImageNoFlags); + _GTMDevAssert(vError == kvImageNoError, + @"vImagePermuteChannels failed: %zd", vError); } else { // Convert grayscale back to BGRA - vImage_Error __unused vError = vImageGrayToBGRA(&vSource, &vDestination); - _GTMDevAssert(vError == kvImageNoError, @"vImageGrayToBGRA failed: %zd", vError); + vImage_Error __unused vError = + vImageGrayToBGRA(&vSource, &vDestination); + _GTMDevAssert(vError == kvImageNoError, + @"vImageGrayToBGRA failed: %zd", vError); } error = CVPixelBufferUnlockBaseAddress(pixelBuffer, 0); _GTMDevAssert(error == kCVReturnSuccess, @"CVPixelBufferUnlockBaseAddress failed: %d", error); - if ([wrapper.delegate respondsToSelector:@selector - (mediapipeGraph:didOutputPixelBuffer:fromStream:timestamp:)]) { + if ([wrapper.delegate + respondsToSelector:@selector + (mediapipeGraph:didOutputPixelBuffer:fromStream:timestamp:)]) { [wrapper.delegate mediapipeGraph:wrapper - didOutputPixelBuffer:pixelBuffer - fromStream:streamName - timestamp:packet.Timestamp()]; - } else if ([wrapper.delegate respondsToSelector:@selector - (mediapipeGraph:didOutputPixelBuffer:fromStream:)]) { + didOutputPixelBuffer:pixelBuffer + fromStream:streamName + timestamp:packet.Timestamp()]; + } else if ([wrapper.delegate + respondsToSelector:@selector + (mediapipeGraph:didOutputPixelBuffer:fromStream:)]) { [wrapper.delegate mediapipeGraph:wrapper - didOutputPixelBuffer:pixelBuffer - fromStream:streamName]; + didOutputPixelBuffer:pixelBuffer + fromStream:streamName]; } CVPixelBufferRelease(pixelBuffer); } else { @@ -168,22 +184,23 @@ void CallFrameDelegate(void* wrapperVoid, const std::string& streamName, wrapper->_framesInFlight--; CVPixelBufferRef pixelBuffer; if (packetType == MPPPacketTypePixelBuffer) - pixelBuffer = mediapipe::GetCVPixelBufferRef(packet.Get()); + pixelBuffer = + mediapipe::GetCVPixelBufferRef(packet.Get()); else pixelBuffer = packet.Get().GetCVPixelBufferRef(); -if ([wrapper.delegate + if ([wrapper.delegate respondsToSelector:@selector (mediapipeGraph:didOutputPixelBuffer:fromStream:timestamp:)]) { [wrapper.delegate mediapipeGraph:wrapper - didOutputPixelBuffer:pixelBuffer - fromStream:streamName - timestamp:packet.Timestamp()]; + didOutputPixelBuffer:pixelBuffer + fromStream:streamName + timestamp:packet.Timestamp()]; } else if ([wrapper.delegate respondsToSelector:@selector (mediapipeGraph:didOutputPixelBuffer:fromStream:)]) { [wrapper.delegate mediapipeGraph:wrapper - didOutputPixelBuffer:pixelBuffer - fromStream:streamName]; + didOutputPixelBuffer:pixelBuffer + fromStream:streamName]; } #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER } else { @@ -192,13 +209,15 @@ if ([wrapper.delegate } } -- (void)setHeaderPacket:(const mediapipe::Packet&)packet forStream:(const std::string&)streamName { +- (void)setHeaderPacket:(const mediapipe::Packet&)packet + forStream:(const std::string&)streamName { _GTMDevAssert(!_started, @"%@ must be called before the graph is started", NSStringFromSelector(_cmd)); _streamHeaders[streamName] = packet; } -- (void)setSidePacket:(const mediapipe::Packet&)packet named:(const std::string&)name { +- (void)setSidePacket:(const mediapipe::Packet&)packet + named:(const std::string&)name { _GTMDevAssert(!_started, @"%@ must be called before the graph is started", NSStringFromSelector(_cmd)); _inputSidePackets[name] = packet; @@ -211,7 +230,8 @@ if ([wrapper.delegate _servicePackets[&service] = std::move(packet); } -- (void)addSidePackets:(const std::map&)extraSidePackets { +- (void)addSidePackets: + (const std::map&)extraSidePackets { _GTMDevAssert(!_started, @"%@ must be called before the graph is started", NSStringFromSelector(_cmd)); _inputSidePackets.insert(extraSidePackets.begin(), extraSidePackets.end()); @@ -232,7 +252,8 @@ if ([wrapper.delegate - (absl::Status)performStart { absl::Status status; for (const auto& service_packet : _servicePackets) { - status = _graph->SetServicePacket(*service_packet.first, service_packet.second); + status = + _graph->SetServicePacket(*service_packet.first, service_packet.second); if (!status.ok()) { return status; } @@ -269,11 +290,12 @@ if ([wrapper.delegate } - (BOOL)waitUntilDoneWithError:(NSError**)error { - // Since this method blocks with no timeout, it should not be called in the main thread in - // an app. However, it's fine to allow that in a test. + // Since this method blocks with no timeout, it should not be called in the + // main thread in an app. However, it's fine to allow that in a test. // TODO: is this too heavy-handed? Maybe a warning would be fine. - _GTMDevAssert(![NSThread isMainThread] || (NSClassFromString(@"XCTest")), - @"waitUntilDoneWithError: should not be called on the main thread"); + _GTMDevAssert( + ![NSThread isMainThread] || (NSClassFromString(@"XCTest")), + @"waitUntilDoneWithError: should not be called on the main thread"); absl::Status status = _graph->WaitUntilDone(); _started = NO; if (!status.ok() && error) *error = [NSError gus_errorWithStatus:status]; @@ -289,7 +311,8 @@ if ([wrapper.delegate - (BOOL)movePacket:(mediapipe::Packet&&)packet intoStream:(const std::string&)streamName error:(NSError**)error { - absl::Status status = _graph->AddPacketToInputStream(streamName, std::move(packet)); + absl::Status status = + _graph->AddPacketToInputStream(streamName, std::move(packet)); if (!status.ok() && error) *error = [NSError gus_errorWithStatus:status]; return status.ok(); } @@ -305,15 +328,17 @@ if ([wrapper.delegate - (BOOL)setMaxQueueSize:(int)maxQueueSize forStream:(const std::string&)streamName error:(NSError**)error { - absl::Status status = _graph->SetInputStreamMaxQueueSize(streamName, maxQueueSize); + absl::Status status = + _graph->SetInputStreamMaxQueueSize(streamName, maxQueueSize); if (!status.ok() && error) *error = [NSError gus_errorWithStatus:status]; return status.ok(); } - (mediapipe::Packet)packetWithPixelBuffer:(CVPixelBufferRef)imageBuffer - packetType:(MPPPacketType)packetType { + packetType:(MPPPacketType)packetType { mediapipe::Packet packet; - if (packetType == MPPPacketTypeImageFrame || packetType == MPPPacketTypeImageFrameBGRANoSwap) { + if (packetType == MPPPacketTypeImageFrame || + packetType == MPPPacketTypeImageFrameBGRANoSwap) { auto frame = CreateImageFrameForCVPixelBuffer( imageBuffer, /* canOverwrite = */ false, /* bgrAsRgb = */ packetType == MPPPacketTypeImageFrameBGRANoSwap); @@ -328,7 +353,8 @@ if ([wrapper.delegate packet = mediapipe::MakePacket(imageBuffer); #else // CPU - auto frame = CreateImageFrameForCVPixelBuffer(imageBuffer, /* canOverwrite = */ false, + auto frame = CreateImageFrameForCVPixelBuffer(imageBuffer, + /* canOverwrite = */ false, /* bgrAsRgb = */ false); packet = mediapipe::MakePacket(std::move(frame)); #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER @@ -339,7 +365,8 @@ if ([wrapper.delegate } - (mediapipe::Packet)imagePacketWithPixelBuffer:(CVPixelBufferRef)pixelBuffer { - return [self packetWithPixelBuffer:(pixelBuffer) packetType:(MPPPacketTypeImage)]; + return [self packetWithPixelBuffer:(pixelBuffer) + packetType:(MPPPacketTypeImage)]; } - (BOOL)sendPixelBuffer:(CVPixelBufferRef)imageBuffer @@ -367,13 +394,16 @@ if ([wrapper.delegate allowOverwrite:(BOOL)allowOverwrite error:(NSError**)error { if (_maxFramesInFlight && _framesInFlight >= _maxFramesInFlight) return NO; - mediapipe::Packet packet = [self packetWithPixelBuffer:imageBuffer packetType:packetType]; + mediapipe::Packet packet = + [self packetWithPixelBuffer:imageBuffer packetType:packetType]; BOOL success; if (allowOverwrite) { packet = std::move(packet).At(timestamp); - success = [self movePacket:std::move(packet) intoStream:inputName error:error]; + success = + [self movePacket:std::move(packet) intoStream:inputName error:error]; } else { - success = [self sendPacket:packet.At(timestamp) intoStream:inputName error:error]; + success = + [self sendPacket:packet.At(timestamp) intoStream:inputName error:error]; } if (success) _framesInFlight++; return success; @@ -407,22 +437,24 @@ if ([wrapper.delegate } - (void)debugPrintGlInfo { - std::shared_ptr gpu_resources = _graph->GetGpuResources(); + std::shared_ptr gpu_resources = + _graph->GetGpuResources(); if (!gpu_resources) { NSLog(@"GPU not set up."); return; } NSString* extensionString; - (void)gpu_resources->gl_context()->Run([&extensionString]{ - extensionString = [NSString stringWithUTF8String:(char*)glGetString(GL_EXTENSIONS)]; + (void)gpu_resources->gl_context()->Run([&extensionString] { + extensionString = + [NSString stringWithUTF8String:(char*)glGetString(GL_EXTENSIONS)]; return absl::OkStatus(); }); - NSArray* extensions = [extensionString componentsSeparatedByCharactersInSet: - [NSCharacterSet whitespaceCharacterSet]]; - for (NSString* oneExtension in extensions) - NSLog(@"%@", oneExtension); + NSArray* extensions = [extensionString + componentsSeparatedByCharactersInSet:[NSCharacterSet + whitespaceCharacterSet]]; + for (NSString* oneExtension in extensions) NSLog(@"%@", oneExtension); } @end diff --git a/mediapipe/objc/MPPTimestampConverter.mm b/mediapipe/objc/DrishtiTimestampConverter.cc similarity index 81% rename from mediapipe/objc/MPPTimestampConverter.mm rename to mediapipe/objc/DrishtiTimestampConverter.cc index e53758d71..44857c8e9 100644 --- a/mediapipe/objc/MPPTimestampConverter.mm +++ b/mediapipe/objc/DrishtiTimestampConverter.cc @@ -20,8 +20,7 @@ mediapipe::TimestampDiff _timestampOffset; } -- (instancetype)init -{ +- (instancetype)init { self = [super init]; if (self) { [self reset]; @@ -36,11 +35,14 @@ } - (mediapipe::Timestamp)timestampForMediaTime:(CMTime)mediaTime { - Float64 sampleSeconds = CMTIME_IS_VALID(mediaTime) ? CMTimeGetSeconds(mediaTime) : 0; - const int64 sampleUsec = sampleSeconds * mediapipe::Timestamp::kTimestampUnitsPerSecond; + Float64 sampleSeconds = + CMTIME_IS_VALID(mediaTime) ? CMTimeGetSeconds(mediaTime) : 0; + const int64 sampleUsec = + sampleSeconds * mediapipe::Timestamp::kTimestampUnitsPerSecond; _mediapipeTimestamp = mediapipe::Timestamp(sampleUsec) + _timestampOffset; if (_mediapipeTimestamp <= _lastTimestamp) { - _timestampOffset = _timestampOffset + _lastTimestamp + 1 - _mediapipeTimestamp; + _timestampOffset = + _timestampOffset + _lastTimestamp + 1 - _mediapipeTimestamp; _mediapipeTimestamp = _lastTimestamp + 1; } _lastTimestamp = _mediapipeTimestamp; diff --git a/mediapipe/objc/NSError+util_status.cc b/mediapipe/objc/NSError+util_status.cc new file mode 100644 index 000000000..144ec6ed4 --- /dev/null +++ b/mediapipe/objc/NSError+util_status.cc @@ -0,0 +1,72 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/objc/NSError+util_status.h" + +@implementation GUSUtilStatusWrapper + ++ (instancetype)wrapStatus:(const absl::Status &)status { + return [[self alloc] initWithStatus:status]; +} + +- (instancetype)initWithStatus:(const absl::Status &)status { + self = [super init]; + if (self) { + _status = status; + } + return self; +} + +- (NSString *)description { + return [NSString stringWithFormat:@"<%@: %p; status = %s>", [self class], + self, _status.message().data()]; +} + +@end + +@implementation NSError (GUSGoogleUtilStatus) + +NSString *const kGUSGoogleUtilStatusErrorDomain = + @"GoogleUtilStatusErrorDomain"; +NSString *const kGUSGoogleUtilStatusErrorKey = @"GUSGoogleUtilStatusErrorKey"; + ++ (NSError *)gus_errorWithStatus:(const absl::Status &)status { + NSDictionary *userInfo = @{ + NSLocalizedDescriptionKey : @(status.message().data()), + kGUSGoogleUtilStatusErrorKey : [GUSUtilStatusWrapper wrapStatus:status], + }; + NSError *error = + [NSError errorWithDomain:kGUSGoogleUtilStatusErrorDomain + code:static_cast(status.code()) + userInfo:userInfo]; + return error; +} + +- (absl::Status)gus_status { + NSString *domain = self.domain; + if ([domain isEqual:kGUSGoogleUtilStatusErrorDomain]) { + GUSUtilStatusWrapper *wrapper = self.userInfo[kGUSGoogleUtilStatusErrorKey]; + if (wrapper) return wrapper.status; +#if 0 + // Unfortunately, util/task/posixerrorspace.h is not in portable status yet. + // TODO: fix that. + } else if ([domain isEqual:NSPOSIXErrorDomain]) { + return ::util::PosixErrorToStatus(self.code, self.localizedDescription.UTF8String); +#endif + } + return absl::Status(absl::StatusCode::kUnknown, + self.localizedDescription.UTF8String); +} + +@end diff --git a/third_party/org_tensorflow_objc_build_fixes.diff b/third_party/org_tensorflow_objc_build_fixes.diff new file mode 100644 index 000000000..db7b827a9 --- /dev/null +++ b/third_party/org_tensorflow_objc_build_fixes.diff @@ -0,0 +1,86 @@ +diff --git a/tensorflow/lite/delegates/gpu/BUILD b/tensorflow/lite/delegates/gpu/BUILD +index 875c2a4f3da..e513db47388 100644 +--- a/tensorflow/lite/delegates/gpu/BUILD ++++ b/tensorflow/lite/delegates/gpu/BUILD +@@ -70,14 +70,17 @@ cc_library( + }) + tflite_extra_gles_deps(), + ) + +-objc_library( ++cc_library( + name = "metal_delegate", +- srcs = ["metal_delegate.mm"], ++ srcs = ["metal_delegate.cc"], + hdrs = ["metal_delegate.h"], +- copts = ["-std=c++17"], ++ copts = [ ++ "-ObjC++", ++ "-std=c++17", ++ "-fobjc-arc", ++ ], ++ linkopts = ["-framework Metal"], + features = ["-layering_check"], +- module_name = "TensorFlowLiteCMetal", +- sdk_frameworks = ["Metal"], + deps = [ + "//tensorflow/lite:kernel_api", + "//tensorflow/lite:minimal_logging", +@@ -98,14 +101,20 @@ objc_library( + "//tensorflow/lite/delegates/gpu/metal:metal_spatial_tensor", + "@com_google_absl//absl/types:span", + ], ++ alwayslink = 1, + ) + +-objc_library( ++cc_library( + name = "metal_delegate_internal", + hdrs = ["metal_delegate_internal.h"], +- copts = ["-std=c++17"], +- sdk_frameworks = ["Metal"], ++ copts = [ ++ "-ObjC++", ++ "-std=c++17", ++ "-fobjc-arc", ++ ], ++ linkopts = ["-framework Metal"], + deps = ["//tensorflow/lite/delegates/gpu:metal_delegate"], ++ alwayslink = 1, + ) + + # build -c opt --config android_arm64 --copt -Os --copt -DTFLITE_GPU_BINARY_RELEASE --linkopt -s --strip always :libtensorflowlite_gpu_gl.so +diff --git a/tensorflow/lite/delegates/gpu/metal/BUILD b/tensorflow/lite/delegates/gpu/metal/BUILD +index 8571ff7f041..82e6bb91d2d 100644 +--- a/tensorflow/lite/delegates/gpu/metal/BUILD ++++ b/tensorflow/lite/delegates/gpu/metal/BUILD +@@ -137,15 +137,16 @@ objc_library( + ], + ) + +-objc_library( ++cc_library( + name = "inference_context", + srcs = ["inference_context.cc"], + hdrs = ["inference_context.h"], + copts = DEFAULT_COPTS + [ + "-ObjC++", ++ "-fobjc-arc", + ], + features = ["-layering_check"], +- sdk_frameworks = ["Metal"], ++ linkopts = ["-framework Metal"], + deps = [ + ":compute_task", + ":inference_context_cc_fbs", +@@ -171,6 +172,7 @@ objc_library( + "@com_google_absl//absl/strings", + "@com_google_absl//absl/time", + ], ++ alwayslink = 1, + ) + + flatbuffer_cc_library( +diff --git a/tensorflow/lite/delegates/gpu/metal_delegate.mm b/tensorflow/lite/delegates/gpu/metal_delegate.cc +similarity index 100% +rename from tensorflow/lite/delegates/gpu/metal_delegate.mm +rename to tensorflow/lite/delegates/gpu/metal_delegate.cc From 05564cbe9a10bfc0c24dcc60d163af7117c554b5 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 23 Oct 2023 14:31:30 -0700 Subject: [PATCH 036/157] No public description PiperOrigin-RevId: 575930740 --- mediapipe/objc/{DrishtiGraph.cc => MPPGraph.cc} | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename mediapipe/objc/{DrishtiGraph.cc => MPPGraph.cc} (99%) diff --git a/mediapipe/objc/DrishtiGraph.cc b/mediapipe/objc/MPPGraph.cc similarity index 99% rename from mediapipe/objc/DrishtiGraph.cc rename to mediapipe/objc/MPPGraph.cc index c4c1096d3..df9a1ebd6 100644 --- a/mediapipe/objc/DrishtiGraph.cc +++ b/mediapipe/objc/MPPGraph.cc @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#import "mediapipe/objc/MPPGraph.h" + #import #import @@ -25,7 +27,6 @@ #include "mediapipe/framework/graph_service.h" #include "mediapipe/gpu/gl_base.h" #include "mediapipe/gpu/gpu_shared_data_internal.h" -#import "mediapipe/objc/MPPGraph.h" #import "mediapipe/objc/NSError+util_status.h" #include "mediapipe/objc/util.h" From 543b5959711abafd74d0b2eb661be15ff69a72a8 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 24 Oct 2023 04:23:18 -0700 Subject: [PATCH 037/157] Fix internal incensistency in parsing code PiperOrigin-RevId: 576094494 --- mediapipe/framework/tool/validate_name.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/framework/tool/validate_name.cc b/mediapipe/framework/tool/validate_name.cc index 8f9be7687..4415f76b5 100644 --- a/mediapipe/framework/tool/validate_name.cc +++ b/mediapipe/framework/tool/validate_name.cc @@ -134,7 +134,7 @@ absl::Status ParseTagAndName(absl::string_view tag_and_name, std::string* tag, RET_CHECK(name); absl::Status tag_status = absl::OkStatus(); absl::Status name_status = absl::UnknownError(""); - int name_index = 0; + int name_index = -1; std::vector v = absl::StrSplit(tag_and_name, ':'); if (v.size() == 1) { name_status = ValidateName(v[0]); @@ -143,7 +143,7 @@ absl::Status ParseTagAndName(absl::string_view tag_and_name, std::string* tag, tag_status = ValidateTag(v[0]); name_status = ValidateName(v[1]); name_index = 1; - } + } // else omitted, name_index == -1, triggering error. if (name_index == -1 || tag_status != absl::OkStatus() || name_status != absl::OkStatus()) { tag->clear(); From 5b0f1f9ac4dbb48250be68e2b815d611552e2494 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 24 Oct 2023 09:32:00 -0700 Subject: [PATCH 038/157] No public description PiperOrigin-RevId: 576166645 --- .../{DrishtiTimestampConverter.cc => MPPTimestampConverter.cc} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename mediapipe/objc/{DrishtiTimestampConverter.cc => MPPTimestampConverter.cc} (100%) diff --git a/mediapipe/objc/DrishtiTimestampConverter.cc b/mediapipe/objc/MPPTimestampConverter.cc similarity index 100% rename from mediapipe/objc/DrishtiTimestampConverter.cc rename to mediapipe/objc/MPPTimestampConverter.cc From 905a18c88c3b0b67882ab0ffd3930e0ce67fb440 Mon Sep 17 00:00:00 2001 From: Youchuan Hu Date: Tue, 24 Oct 2023 11:33:13 -0700 Subject: [PATCH 039/157] Add CPU tests for TensorsToSegmentationCalculator PiperOrigin-RevId: 576208735 --- ...tensors_to_segmentation_calculator_test.cc | 141 +++++++++++++++--- 1 file changed, 117 insertions(+), 24 deletions(-) diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc index 3db9145d2..f6f1c80ee 100644 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc @@ -62,12 +62,14 @@ struct FormattingTestCase { Options::Activation activation; int rows; int cols; + int rows_new; + int cols_new; int channels; + double max_abs_diff; }; using TensorsToSegmentationCalculatorTest = TestWithParam; -// Currently only useable for tests with no output resize. TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) { const FormattingTestCase& test_case = GetParam(); std::vector inputs = test_case.inputs; @@ -75,7 +77,10 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) { Options::Activation activation = test_case.activation; int rows = test_case.rows; int cols = test_case.cols; + int rows_new = test_case.rows_new; + int cols_new = test_case.cols_new; int channels = test_case.channels; + double max_abs_diff = test_case.max_abs_diff; std::string string_config = absl::Substitute( R"pb( @@ -119,28 +124,31 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) { MP_ASSERT_OK(graph.AddPacketToInputStream( "tensors", mediapipe::Adopt(tensors.release()).At(Timestamp(0)))); } + // The output size is defined as pair(new_width, new_height). MP_ASSERT_OK(graph.AddPacketToInputStream( - "size", - mediapipe::Adopt(new std::pair(rows, cols)).At(Timestamp(0)))); + "size", mediapipe::Adopt(new std::pair(cols_new, rows_new)) + .At(Timestamp(0)))); MP_ASSERT_OK(graph.WaitUntilIdle()); ASSERT_THAT(output_packets, SizeIs(1)); const Image& image_as_mask = output_packets[0].Get(); std::shared_ptr result_mat = formats::MatView(&image_as_mask); - EXPECT_EQ(result_mat->rows, rows); - EXPECT_EQ(result_mat->cols, cols); - EXPECT_EQ(result_mat->channels(), channels); + EXPECT_EQ(result_mat->rows, rows_new); + EXPECT_EQ(result_mat->cols, cols_new); + EXPECT_EQ(result_mat->channels(), 1); // Compare the real result with the expected result. - cv::Mat expected_result = cv::Mat( - rows, cols, CV_32FC1, const_cast(expected_outputs.data())); + cv::Mat expected_result = + cv::Mat(rows_new, cols_new, CV_32FC1, + const_cast(expected_outputs.data())); cv::Mat diff; cv::absdiff(*result_mat, expected_result, diff); double max_val; cv::minMaxLoc(diff, nullptr, &max_val); - // Expects the maximum absolute pixel-by-pixel difference is less than 1e-5. - // This delta is for passthorugh accuracy only. - EXPECT_LE(max_val, 1e-5); + + // The max allowable diff between output and expected output varies between + // tests. + EXPECT_LE(max_val, max_abs_diff); MP_ASSERT_OK(graph.CloseInputStream("tensors")); MP_ASSERT_OK(graph.CloseInputStream("size")); @@ -149,19 +157,104 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) { INSTANTIATE_TEST_SUITE_P( TensorsToSegmentationCalculatorTests, TensorsToSegmentationCalculatorTest, - testing::ValuesIn({ - {/*test_name=*/"NoActivationAndNoOutputResize", - /*inputs=*/ - {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, - 14.0, 15.0, 16.0}, - /*expected_outputs=*/ - {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, - 14.0, 15.0, 16.0}, - /*activation=*/Options::NONE, - /*rows=*/4, - /*cols=*/4, - /*channels=*/1}, - }), + testing::ValuesIn( + {{/*test_name=*/"NoActivationAndNoOutputResize", + /*inputs=*/ + {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, + 14.0, 15.0, 16.0}, + /*expected_outputs=*/ + {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, + 14.0, 15.0, 16.0}, + /*activation=*/Options::NONE, + /*rows=*/4, + /*cols=*/4, + /*rows_new=*/4, + /*cols_new=*/4, + /*channels=*/1, + /*max_abs_diff=*/1e-7}, + {/*test_name=*/"OutputResizeOnly", + /*inputs=*/ + {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, + 14.0, 15.0, 16.0}, + /*expected_outputs=*/ + {1, 1.5, 2.166667, 2.833333, 3.5, 4, + 3.8, 4.3, 4.966667, 5.633333, 6.3, 6.8, + 7, 7.5, 8.166667, 8.833333, 9.5, 10, + 10.2, 10.7, 11.366667, 12.033333, 12.7, 13.2, + 13, 13.5, 14.166667, 14.833333, 15.5, 16}, + /*activation=*/Options::NONE, + /*rows=*/4, + /*cols=*/4, + /*rows_new=*/5, + /*cols_new=*/6, + /*channels=*/1, + /*max_abs_diff=*/1e-6}, + {/*test_name=*/"SigmoidActivationWithNoOutputResize", + /*inputs=*/ + {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, + 14.0, 15.0, 16.0}, + /*expected_outputs=*/ + {0.731059, 0.880797, 0.952574, 0.982014, 0.993307, 0.997527, 0.999089, + 0.999665, 0.999877, 0.999955, 0.999983, 0.999994, 0.999998, 0.999999, + 1.0, 1.0}, + /*activation=*/Options::SIGMOID, + /*rows=*/4, + /*cols=*/4, + /*rows_new=*/4, + /*cols_new=*/4, + /*channels=*/1, + /*max_abs_diff=*/1e-6}, + {/*test_name=*/"SigmoidActivationWithOutputResize", + /*inputs=*/ + {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, + 14.0, 15.0, 16.0}, + /*expected_outputs=*/ + {0.731059, 0.805928, 0.89276, 0.940611, 0.967294, 0.982014, + 0.914633, 0.93857, 0.966279, 0.981363, 0.989752, 0.994369, + 0.996592, 0.997666, 0.998873, 0.999404, 0.999683, 0.999829, + 0.999913, 0.99994, 0.999971, 0.999985, 0.999992, 0.999996, + 0.999998, 0.999998, 0.999999, 1.0, 1.0, 1.0}, + /*activation=*/Options::SIGMOID, + /*rows=*/4, + /*cols=*/4, + /*rows_new=*/5, + /*cols_new=*/6, + /*channels=*/1, + /*max_abs_diff=*/1e-6}, + {/*test_name=*/"SoftmaxActivationWithNoOutputResize", + /*inputs=*/ + {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5, 7.0, 10.0, 11.0, + 4.0, 12.0, 15.0, 16.0, 18.5, 19.0, 20.0, 22.0, 23.0, 24.5, 23.4, + 25.6, 28.3, 29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3}, + /*expected_outputs=*/ + {0.731059, 0.119203, 0.880797, 0.0109869, 0.952574, 0.000911051, + 0.952574, 0.924142, 0.731059, 0.731059, 0.24974, 0.937027, 0.689974, + 0.990048, 0.0060598, 0.28905}, + /*activation=*/Options::SOFTMAX, + /*rows=*/4, + /*cols=*/4, + /*rows_new=*/4, + /*cols_new=*/4, + /*channels=*/2, + /*max_abs_diff=*/1e-6}, + {/*test_name=*/"SoftmaxActivationWithOutputResize", + /*inputs=*/ + {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5, 7.0, 10.0, 11.0, + 4.0, 12.0, 15.0, 16.0, 18.5, 19.0, 20.0, 22.0, 23.0, 24.5, 23.4, + 25.6, 28.3, 29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3}, + /*expected_outputs=*/ + {0.731059, 0.425131, 0.246135, 0.753865, 0.445892, 0.0109869, + 0.886119, 0.461259, 0.185506, 0.781934, 0.790618, 0.650195, + 0.841816, 0.603901, 0.40518, 0.561962, 0.765871, 0.930584, + 0.718733, 0.763744, 0.703402, 0.281989, 0.459635, 0.742634, + 0.689974, 0.840011, 0.82605, 0.170058, 0.147555, 0.28905}, + /*activation=*/Options::SOFTMAX, + /*rows=*/4, + /*cols=*/4, + /*rows_new=*/5, + /*cols_new=*/6, + /*channels=*/2, + /*max_abs_diff=*/1e-6}}), [](const testing::TestParamInfo< TensorsToSegmentationCalculatorTest::ParamType>& info) { return info.param.test_name; From c698414c718e41b6f24247e6b7c00e3359707697 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 24 Oct 2023 12:37:28 -0700 Subject: [PATCH 040/157] Use cc_library for DrishtiMetalHelper PiperOrigin-RevId: 576230898 --- mediapipe/gpu/BUILD | 6 +- .../{MPPMetalHelper.mm => MPPMetalHelper.cc} | 72 +++++++++++-------- 2 files changed, 47 insertions(+), 31 deletions(-) rename mediapipe/gpu/{MPPMetalHelper.mm => MPPMetalHelper.cc} (74%) diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 8a8a402f1..b75c824b3 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -863,12 +863,14 @@ cc_library( }), ) -objc_library( +cc_library( name = "MPPMetalHelper", - srcs = ["MPPMetalHelper.mm"], + srcs = ["MPPMetalHelper.cc"], hdrs = ["MPPMetalHelper.h"], copts = [ "-Wno-shorten-64-to-32", + "-x objective-c++", + "-fobjc-arc", ], features = ["-layering_check"], visibility = ["//visibility:public"], diff --git a/mediapipe/gpu/MPPMetalHelper.mm b/mediapipe/gpu/MPPMetalHelper.cc similarity index 74% rename from mediapipe/gpu/MPPMetalHelper.mm rename to mediapipe/gpu/MPPMetalHelper.cc index c66483698..e92d6aae7 100644 --- a/mediapipe/gpu/MPPMetalHelper.mm +++ b/mediapipe/gpu/MPPMetalHelper.cc @@ -14,15 +14,14 @@ #import "mediapipe/gpu/MPPMetalHelper.h" +#import "GTMDefines.h" #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" +#include "mediapipe/framework/port/ret_check.h" #import "mediapipe/gpu/gpu_buffer.h" #import "mediapipe/gpu/gpu_service.h" #import "mediapipe/gpu/graph_support.h" #import "mediapipe/gpu/metal_shared_resources.h" -#import "GTMDefines.h" - -#include "mediapipe/framework/port/ret_check.h" @interface MPPMetalHelper () { mediapipe::GpuResources* _gpuResources; @@ -31,7 +30,8 @@ namespace mediapipe { -// Using a C++ class so it can be declared as a friend of LegacyCalculatorSupport. +// Using a C++ class so it can be declared as a friend of +// LegacyCalculatorSupport. class MetalHelperLegacySupport { public: static CalculatorContract* GetCalculatorContract() { @@ -61,7 +61,8 @@ class MetalHelperLegacySupport { - (instancetype)initWithCalculatorContext:(mediapipe::CalculatorContext*)cc { if (!cc) return nil; - return [self initWithGpuResources:&cc->Service(mediapipe::kGpuService).GetObject()]; + return [self + initWithGpuResources:&cc->Service(mediapipe::kGpuService).GetObject()]; } + (absl::Status)updateContract:(mediapipe::CalculatorContract*)cc { @@ -77,7 +78,8 @@ class MetalHelperLegacySupport { } // Legacy support. -- (instancetype)initWithSidePackets:(const mediapipe::PacketSet&)inputSidePackets { +- (instancetype)initWithSidePackets: + (const mediapipe::PacketSet&)inputSidePackets { auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContext(); if (cc) { ABSL_CHECK_EQ(&inputSidePackets, &cc->InputSidePackets()); @@ -85,16 +87,19 @@ class MetalHelperLegacySupport { } // TODO: remove when we can. - ABSL_LOG(WARNING) << "CalculatorContext not available. If this calculator uses " - "CalculatorBase, call initWithCalculatorContext instead."; + ABSL_LOG(WARNING) + << "CalculatorContext not available. If this calculator uses " + "CalculatorBase, call initWithCalculatorContext instead."; mediapipe::GpuSharedData* gpu_shared = - inputSidePackets.Tag(mediapipe::kGpuSharedTagName).Get(); + inputSidePackets.Tag(mediapipe::kGpuSharedTagName) + .Get(); return [self initWithGpuResources:gpu_shared->gpu_resources.get()]; } // Legacy support. -+ (absl::Status)setupInputSidePackets:(mediapipe::PacketTypeSet*)inputSidePackets { ++ (absl::Status)setupInputSidePackets: + (mediapipe::PacketTypeSet*)inputSidePackets { auto cc = mediapipe::MetalHelperLegacySupport::GetCalculatorContract(); if (cc) { ABSL_CHECK_EQ(inputSidePackets, &cc->InputSidePackets()); @@ -102,12 +107,12 @@ class MetalHelperLegacySupport { } // TODO: remove when we can. - ABSL_LOG(WARNING) << "CalculatorContract not available. If you're calling this " - "from a GetContract method, call updateContract instead."; + ABSL_LOG(WARNING) + << "CalculatorContract not available. If you're calling this " + "from a GetContract method, call updateContract instead."; auto id = inputSidePackets->GetId(mediapipe::kGpuSharedTagName, 0); - RET_CHECK(id.IsValid()) - << "A " << mediapipe::kGpuSharedTagName - << " input side packet is required here."; + RET_CHECK(id.IsValid()) << "A " << mediapipe::kGpuSharedTagName + << " input side packet is required here."; inputSidePackets->Get(id).Set(); return absl::OkStatus(); } @@ -125,10 +130,12 @@ class MetalHelperLegacySupport { } - (id)commandBuffer { - return [_gpuResources->metal_shared().resources().mtlCommandQueue commandBuffer]; + return + [_gpuResources->metal_shared().resources().mtlCommandQueue commandBuffer]; } -- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer +- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer: + (const mediapipe::GpuBuffer&)gpuBuffer plane:(size_t)plane { CVPixelBufferRef pixel_buffer = mediapipe::GetCVPixelBufferRef(gpuBuffer); OSType pixel_format = CVPixelBufferGetPixelFormatType(pixel_buffer); @@ -178,41 +185,48 @@ class MetalHelperLegacySupport { CVMetalTextureRef texture; CVReturn err = CVMetalTextureCacheCreateTextureFromImage( NULL, _gpuResources->metal_shared().resources().mtlTextureCache, - mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width, height, plane, - &texture); + mediapipe::GetCVPixelBufferRef(gpuBuffer), NULL, metalPixelFormat, width, + height, plane, &texture); ABSL_CHECK_EQ(err, kCVReturnSuccess); return texture; } -- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer { +- (CVMetalTextureRef)copyCVMetalTextureWithGpuBuffer: + (const mediapipe::GpuBuffer&)gpuBuffer { return [self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:0]; } -- (id)metalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer { +- (id)metalTextureWithGpuBuffer: + (const mediapipe::GpuBuffer&)gpuBuffer { return [self metalTextureWithGpuBuffer:gpuBuffer plane:0]; } -- (id)metalTextureWithGpuBuffer:(const mediapipe::GpuBuffer&)gpuBuffer - plane:(size_t)plane { +- (id)metalTextureWithGpuBuffer: + (const mediapipe::GpuBuffer&)gpuBuffer + plane:(size_t)plane { CFHolder cvTexture; cvTexture.adopt([self copyCVMetalTextureWithGpuBuffer:gpuBuffer plane:plane]); return CVMetalTextureGetTexture(*cvTexture); } -- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width height:(int)height { +- (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width + height:(int)height { return _gpuResources->gpu_buffer_pool().GetBuffer(width, height); } - (mediapipe::GpuBuffer)mediapipeGpuBufferWithWidth:(int)width - height:(int)height - format:(mediapipe::GpuBufferFormat)format { + height:(int)height + format:(mediapipe::GpuBufferFormat) + format { return _gpuResources->gpu_buffer_pool().GetBuffer(width, height, format); } -- (id)newLibraryWithResourceName:(NSString*)name error:(NSError * _Nullable *)error { +- (id)newLibraryWithResourceName:(NSString*)name + error:(NSError* _Nullable*)error { return [_gpuResources->metal_shared().resources().mtlDevice - newLibraryWithFile:[[NSBundle bundleForClass:[self class]] pathForResource:name - ofType:@"metallib"] + newLibraryWithFile:[[NSBundle bundleForClass:[self class]] + pathForResource:name + ofType:@"metallib"] error:error]; } From 5f2b9fd7651e6049769a24be58c61b1d51f1e1eb Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 24 Oct 2023 13:39:00 -0700 Subject: [PATCH 041/157] Speed up Python build by only building binary graph PiperOrigin-RevId: 576260883 --- setup.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index b5b75b73c..aa6004b7e 100644 --- a/setup.py +++ b/setup.py @@ -272,13 +272,14 @@ class BuildModules(build_ext.build_ext): self._download_external_file(external_file) binary_graphs = [ - 'face_detection/face_detection_short_range_cpu', - 'face_detection/face_detection_full_range_cpu', - 'face_landmark/face_landmark_front_cpu', - 'hand_landmark/hand_landmark_tracking_cpu', - 'holistic_landmark/holistic_landmark_cpu', 'objectron/objectron_cpu', - 'pose_landmark/pose_landmark_cpu', - 'selfie_segmentation/selfie_segmentation_cpu' + 'face_detection/face_detection_short_range_cpu.binarypb', + 'face_detection/face_detection_full_range_cpu.binarypb', + 'face_landmark/face_landmark_front_cpu.binarypb', + 'hand_landmark/hand_landmark_tracking_cpu.binarypb', + 'holistic_landmark/holistic_landmark_cpu.binarypb', + 'objectron/objectron_cpu.binarypb', + 'pose_landmark/pose_landmark_cpu.binarypb', + 'selfie_segmentation/selfie_segmentation_cpu.binarypb' ] for elem in binary_graphs: binary_graph = os.path.join('mediapipe/modules/', elem) @@ -312,7 +313,7 @@ class BuildModules(build_ext.build_ext): bazel_command.append('--define=OPENCV=source') _invoke_shell_command(bazel_command) - _copy_to_build_lib_dir(self.build_lib, binary_graph_target + '.binarypb') + _copy_to_build_lib_dir(self.build_lib, binary_graph_target) class GenerateMetadataSchema(build_ext.build_ext): From 496a6ed80905d9fe077f5a4869887cf6f7d5cffe Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 24 Oct 2023 16:06:20 -0700 Subject: [PATCH 042/157] No public description PiperOrigin-RevId: 576314429 --- .../male_full_height_hands_result_cpu.pbtxt | 259 +++++++++++++++++- 1 file changed, 257 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/testdata/vision/male_full_height_hands_result_cpu.pbtxt b/mediapipe/tasks/testdata/vision/male_full_height_hands_result_cpu.pbtxt index 199dc6366..e50f777c5 100644 --- a/mediapipe/tasks/testdata/vision/male_full_height_hands_result_cpu.pbtxt +++ b/mediapipe/tasks/testdata/vision/male_full_height_hands_result_cpu.pbtxt @@ -2854,7 +2854,262 @@ auxiliary_landmarks { face_blendshapes { classification { index: 0 - score: 1.6770242e-05 - label: "tongueOut" + score: 8.47715e-07 + label: "_neutral" + } + classification { + index: 1 + score: 0.020850565 + label: "browDownLeft" + } + classification { + index: 2 + score: 0.007629181 + label: "browDownRight" + } + classification { + index: 3 + score: 0.26410568 + label: "browInnerUp" + } + classification { + index: 4 + score: 0.04212071 + label: "browOuterUpLeft" + } + classification { + index: 5 + score: 0.07319052 + label: "browOuterUpRight" + } + classification { + index: 6 + score: 9.39117e-06 + label: "cheekPuff" + } + classification { + index: 7 + score: 1.9243858e-07 + label: "cheekSquintLeft" + } + classification { + index: 8 + score: 4.066475e-08 + label: "cheekSquintRight" + } + classification { + index: 9 + score: 0.46092203 + label: "eyeBlinkLeft" + } + classification { + index: 10 + score: 0.40371567 + label: "eyeBlinkRight" + } + classification { + index: 11 + score: 0.65011656 + label: "eyeLookDownLeft" + } + classification { + index: 12 + score: 0.6423024 + label: "eyeLookDownRight" + } + classification { + index: 13 + score: 0.04721973 + label: "eyeLookInLeft" + } + classification { + index: 14 + score: 0.08176838 + label: "eyeLookInRight" + } + classification { + index: 15 + score: 0.09520102 + label: "eyeLookOutLeft" + } + classification { + index: 16 + score: 0.07271895 + label: "eyeLookOutRight" + } + classification { + index: 17 + score: 0.011193463 + label: "eyeLookUpLeft" + } + classification { + index: 18 + score: 0.007041815 + label: "eyeLookUpRight" + } + classification { + index: 19 + score: 0.27120194 + label: "eyeSquintLeft" + } + classification { + index: 20 + score: 0.21675573 + label: "eyeSquintRight" + } + classification { + index: 21 + score: 0.0018824162 + label: "eyeWideLeft" + } + classification { + index: 22 + score: 0.0011966582 + label: "eyeWideRight" + } + classification { + index: 23 + score: 1.9298719e-05 + label: "jawForward" + } + classification { + index: 24 + score: 9.670858e-06 + label: "jawLeft" + } + classification { + index: 25 + score: 0.000115385694 + label: "jawOpen" + } + classification { + index: 26 + score: 0.00023342477 + label: "jawRight" + } + classification { + index: 27 + score: 2.8894076e-05 + label: "mouthClose" + } + classification { + index: 28 + score: 0.003933548 + label: "mouthDimpleLeft" + } + classification { + index: 29 + score: 0.0051949574 + label: "mouthDimpleRight" + } + classification { + index: 30 + score: 0.00067943585 + label: "mouthFrownLeft" + } + classification { + index: 31 + score: 0.0006520291 + label: "mouthFrownRight" + } + classification { + index: 32 + score: 0.0006695333 + label: "mouthFunnel" + } + classification { + index: 33 + score: 8.578597e-05 + label: "mouthLeft" + } + classification { + index: 34 + score: 2.6707421e-05 + label: "mouthLowerDownLeft" + } + classification { + index: 35 + score: 2.153054e-05 + label: "mouthLowerDownRight" + } + classification { + index: 36 + score: 0.0132145975 + label: "mouthPressLeft" + } + classification { + index: 37 + score: 0.009528495 + label: "mouthPressRight" + } + classification { + index: 38 + score: 0.056963783 + label: "mouthPucker" + } + classification { + index: 39 + score: 0.027331185 + label: "mouthRight" + } + classification { + index: 40 + score: 0.00072388636 + label: "mouthRollLower" + } + classification { + index: 41 + score: 0.00021191382 + label: "mouthRollUpper" + } + classification { + index: 42 + score: 0.23938002 + label: "mouthShrugLower" + } + classification { + index: 43 + score: 0.052946873 + label: "mouthShrugUpper" + } + classification { + index: 44 + score: 0.68681276 + label: "mouthSmileLeft" + } + classification { + index: 45 + score: 0.68557316 + label: "mouthSmileRight" + } + classification { + index: 46 + score: 0.0030625665 + label: "mouthStretchLeft" + } + classification { + index: 47 + score: 0.003999545 + label: "mouthStretchRight" + } + classification { + index: 48 + score: 0.013184475 + label: "mouthUpperUpLeft" + } + classification { + index: 49 + score: 0.017995607 + label: "mouthUpperUpRight" + } + classification { + index: 50 + score: 2.0452394e-06 + label: "noseSneerLeft" + } + classification { + index: 51 + score: 3.7912793e-07 + label: "noseSneerRight" } } From 3017c02d3df6b5ce9ade14932a7f6871e8031104 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 25 Oct 2023 15:28:42 -0700 Subject: [PATCH 043/157] No public description PiperOrigin-RevId: 576663264 --- .../cc/components/processors/proto/BUILD | 6 +++ .../processors/proto/llm_params.proto | 41 +++++++++++++++++++ .../processors/proto/transformer_params.proto | 19 ++++++++- 3 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 mediapipe/tasks/cc/components/processors/proto/llm_params.proto diff --git a/mediapipe/tasks/cc/components/processors/proto/BUILD b/mediapipe/tasks/cc/components/processors/proto/BUILD index a45c91633..55cf3fca1 100644 --- a/mediapipe/tasks/cc/components/processors/proto/BUILD +++ b/mediapipe/tasks/cc/components/processors/proto/BUILD @@ -98,3 +98,9 @@ mediapipe_proto_library( name = "transformer_params_proto", srcs = ["transformer_params.proto"], ) + +mediapipe_proto_library( + name = "llm_params_proto", + srcs = ["llm_params.proto"], + deps = [":transformer_params_proto"], +) diff --git a/mediapipe/tasks/cc/components/processors/proto/llm_params.proto b/mediapipe/tasks/cc/components/processors/proto/llm_params.proto new file mode 100644 index 000000000..b0c253598 --- /dev/null +++ b/mediapipe/tasks/cc/components/processors/proto/llm_params.proto @@ -0,0 +1,41 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +syntax = "proto3"; + +package mediapipe.tasks.components.processors.proto; + +import "mediapipe/tasks/cc/components/processors/proto/transformer_params.proto"; + +option java_package = "com.google.mediapipe.tasks.components.processors.proto"; +option java_outer_classname = "LLMParametersProto"; + +// Parameters for Large Language Models (LLM). +message LLMParameters { + TransformerParameters transformer_parameters = 1; + + // Size of vocabulary. + int32 vocab_size = 2; + + // Whether or not to disable KV cache, which is also referred as state + // somewhere else. + bool disable_kv_cache = 3; + + // Id of the start token. + int32 start_token_id = 4; + + // Token to determine the end of output stream. + string stop_token = 5; +} diff --git a/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto b/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto index b2d13c3a2..a04aa9571 100644 --- a/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto +++ b/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto @@ -44,6 +44,21 @@ message TransformerParameters { // Number of stacked transformers, `N` in the paper. int32 num_stacks = 7; - // Whether to use Multi-Query-Attention (MQA). - bool use_mqa = 8; + // Deprecated: bool use_mqa. Use num_kv_heads below. + reserved 8; + + // Number of kv heads. 0 means Multi-Head-Attention (MHA), key and value have + // same number of heads as query; 1 means Multi-Query-Attention (MQA), key and + // value have one head; otherwise, this specifies the number of heads for key + // and value, and Grouped-Query-Attention (GQA) will be used. See + // https://arxiv.org/pdf/2305.13245.pdf for details. + int32 num_kv_heads = 9; + + // Different types of attention mask type. + enum AttentionMaskType { + UNSPECIFIED = 0; + CAUSAL = 1; + PREFIX = 2; + } + AttentionMaskType attention_mask_type = 10; } From a277d853ea23759d54cb335ef4fcd445e29f52d3 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 25 Oct 2023 15:44:27 -0700 Subject: [PATCH 044/157] Don't drop status message in ConvertFromImageFrame PiperOrigin-RevId: 576667666 --- mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc b/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc index da6c5a72d..5983758f9 100644 --- a/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc +++ b/mediapipe/gpu/gpu_buffer_storage_cv_pixel_buffer.cc @@ -151,7 +151,7 @@ static std::shared_ptr ConvertFromImageFrame( std::shared_ptr frame) { auto status_or_buffer = CreateCVPixelBufferForImageFrame(frame->image_frame()); - ABSL_CHECK(status_or_buffer.ok()); + ABSL_CHECK_OK(status_or_buffer); return std::make_shared( std::move(status_or_buffer).value()); } From e7121e4feb986bc067c879c954051f8c6cf1eaa2 Mon Sep 17 00:00:00 2001 From: Youchuan Hu Date: Wed, 25 Oct 2023 16:00:45 -0700 Subject: [PATCH 045/157] Use designated initializers for TensorsToSegmentationCalculator tests. PiperOrigin-RevId: 576671943 --- ...tensors_to_segmentation_calculator_test.cc | 187 +++++++++--------- 1 file changed, 90 insertions(+), 97 deletions(-) diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc index 14c132e77..e5c6b8ade 100644 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc @@ -152,103 +152,96 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) { INSTANTIATE_TEST_SUITE_P( TensorsToSegmentationCalculatorTests, TensorsToSegmentationCalculatorTest, testing::ValuesIn({ - {/*test_name=*/"NoActivationAndNoOutputResize", - /*inputs=*/ - {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, - 14.0, 15.0, 16.0}, - /*expected_outputs=*/ - {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, - 14.0, 15.0, 16.0}, - /*activation=*/Options::NONE, - /*rows=*/4, - /*cols=*/4, - /*rows_new=*/4, - /*cols_new=*/4, - /*channels=*/1, - /*max_abs_diff=*/1e-7}, - {/*test_name=*/"OutputResizeOnly", - /*inputs=*/ - {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, - 14.0, 15.0, 16.0}, - /*expected_outputs=*/ - {1, 1.5, 2.166667, 2.833333, 3.5, 4, - 3.8, 4.3, 4.966667, 5.633333, 6.3, 6.8, - 7, 7.5, 8.166667, 8.833333, 9.5, 10, - 10.2, 10.7, 11.366667, 12.033333, 12.7, 13.2, - 13, 13.5, 14.166667, 14.833333, 15.5, 16}, - /*activation=*/Options::NONE, - /*rows=*/4, - /*cols=*/4, - /*rows_new=*/5, - /*cols_new=*/6, - /*channels=*/1, - /*max_abs_diff=*/1e-6}, - {/*test_name=*/"SigmoidActivationWithNoOutputResize", - /*inputs=*/ - {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, - 14.0, 15.0, 16.0}, - /*expected_outputs=*/ - {0.731059, 0.880797, 0.952574, 0.982014, 0.993307, 0.997527, 0.999089, - 0.999665, 0.999877, 0.999955, 0.999983, 0.999994, 0.999998, 0.999999, - 1.0, 1.0}, - /*activation=*/Options::SIGMOID, - /*rows=*/4, - /*cols=*/4, - /*rows_new=*/4, - /*cols_new=*/4, - /*channels=*/1, - /*max_abs_diff=*/1e-6}, - {/*test_name=*/"SigmoidActivationWithOutputResize", - /*inputs=*/ - {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, - 14.0, 15.0, 16.0}, - /*expected_outputs=*/ - {0.731059, 0.805928, 0.89276, 0.940611, 0.967294, 0.982014, - 0.914633, 0.93857, 0.966279, 0.981363, 0.989752, 0.994369, - 0.996592, 0.997666, 0.998873, 0.999404, 0.999683, 0.999829, - 0.999913, 0.99994, 0.999971, 0.999985, 0.999992, 0.999996, - 0.999998, 0.999998, 0.999999, 1.0, 1.0, 1.0}, - /*activation=*/Options::SIGMOID, - /*rows=*/4, - /*cols=*/4, - /*rows_new=*/5, - /*cols_new=*/6, - /*channels=*/1, - /*max_abs_diff=*/1e-6}, - {/*test_name=*/"SoftmaxActivationWithNoOutputResize", - /*inputs=*/ - {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5, 7.0, 10.0, 11.0, - 4.0, 12.0, 15.0, 16.0, 18.5, 19.0, 20.0, 22.0, 23.0, 24.5, 23.4, - 25.6, 28.3, 29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3}, - /*expected_outputs=*/ - {0.731059, 0.119203, 0.880797, 0.0109869, 0.952574, 0.000911051, - 0.952574, 0.924142, 0.731059, 0.731059, 0.24974, 0.937027, 0.689974, - 0.990048, 0.0060598, 0.28905}, - /*activation=*/Options::SOFTMAX, - /*rows=*/4, - /*cols=*/4, - /*rows_new=*/4, - /*cols_new=*/4, - /*channels=*/2, - /*max_abs_diff=*/1e-6}, - {/*test_name=*/"SoftmaxActivationWithOutputResize", - /*inputs=*/ - {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5, 7.0, 10.0, 11.0, - 4.0, 12.0, 15.0, 16.0, 18.5, 19.0, 20.0, 22.0, 23.0, 24.5, 23.4, - 25.6, 28.3, 29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3}, - /*expected_outputs=*/ - {0.731059, 0.425131, 0.246135, 0.753865, 0.445892, 0.0109869, - 0.886119, 0.461259, 0.185506, 0.781934, 0.790618, 0.650195, - 0.841816, 0.603901, 0.40518, 0.561962, 0.765871, 0.930584, - 0.718733, 0.763744, 0.703402, 0.281989, 0.459635, 0.742634, - 0.689974, 0.840011, 0.82605, 0.170058, 0.147555, 0.28905}, - /*activation=*/Options::SOFTMAX, - /*rows=*/4, - /*cols=*/4, - /*rows_new=*/5, - /*cols_new=*/6, - /*channels=*/2, - /*max_abs_diff=*/1e-6}, + {.test_name = "NoActivationAndNoOutputResize", + .inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, + 12.0, 13.0, 14.0, 15.0, 16.0}, + .expected_outputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, + 11.0, 12.0, 13.0, 14.0, 15.0, 16.0}, + .activation = Options::NONE, + .rows = 4, + .cols = 4, + .rows_new = 4, + .cols_new = 4, + .channels = 1, + .max_abs_diff = 1e-7}, + {.test_name = "OutputResizeOnly", + .inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, + 12.0, 13.0, 14.0, 15.0, 16.0}, + .expected_outputs = {1, 1.5, 2.166667, 2.833333, 3.5, 4, + 3.8, 4.3, 4.966667, 5.633333, 6.3, 6.8, + 7, 7.5, 8.166667, 8.833333, 9.5, 10, + 10.2, 10.7, 11.366667, 12.033333, 12.7, 13.2, + 13, 13.5, 14.166667, 14.833333, 15.5, 16}, + .activation = Options::NONE, + .rows = 4, + .cols = 4, + .rows_new = 5, + .cols_new = 6, + .channels = 1, + .max_abs_diff = 1e-6}, + {.test_name = "SigmoidActivationWithNoOutputResize", + .inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, + 12.0, 13.0, 14.0, 15.0, 16.0}, + .expected_outputs = {0.731059, 0.880797, 0.952574, 0.982014, 0.993307, + 0.997527, 0.999089, 0.999665, 0.999877, 0.999955, + 0.999983, 0.999994, 0.999998, 0.999999, 1.0, 1.0}, + .activation = Options::SIGMOID, + .rows = 4, + .cols = 4, + .rows_new = 4, + .cols_new = 4, + .channels = 1, + .max_abs_diff = 1e-6}, + {.test_name = "SigmoidActivationWithOutputResize", + .inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, + 12.0, 13.0, 14.0, 15.0, 16.0}, + .expected_outputs = {0.731059, 0.805928, 0.89276, 0.940611, 0.967294, + 0.982014, 0.914633, 0.93857, 0.966279, 0.981363, + 0.989752, 0.994369, 0.996592, 0.997666, 0.998873, + 0.999404, 0.999683, 0.999829, 0.999913, 0.99994, + 0.999971, 0.999985, 0.999992, 0.999996, 0.999998, + 0.999998, 0.999999, 1.0, 1.0, 1.0}, + .activation = Options::SIGMOID, + .rows = 4, + .cols = 4, + .rows_new = 5, + .cols_new = 6, + .channels = 1, + .max_abs_diff = 1e-6}, + {.test_name = "SoftmaxActivationWithNoOutputResize", + .inputs = {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5, + 7.0, 10.0, 11.0, 4.0, 12.0, 15.0, 16.0, 18.5, + 19.0, 20.0, 22.0, 23.0, 24.5, 23.4, 25.6, 28.3, + 29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3}, + .expected_outputs = {0.731059, 0.119203, 0.880797, 0.0109869, 0.952574, + 0.000911051, 0.952574, 0.924142, 0.731059, + 0.731059, 0.24974, 0.937027, 0.689974, 0.990048, + 0.0060598, 0.28905}, + .activation = Options::SOFTMAX, + .rows = 4, + .cols = 4, + .rows_new = 4, + .cols_new = 4, + .channels = 2, + .max_abs_diff = 1e-6}, + {.test_name = "SoftmaxActivationWithOutputResize", + .inputs = {1.0, 2.0, 4.0, 2.0, 3.0, 5.0, 6.0, 1.5, + 7.0, 10.0, 11.0, 4.0, 12.0, 15.0, 16.0, 18.5, + 19.0, 20.0, 22.0, 23.0, 24.5, 23.4, 25.6, 28.3, + 29.2, 30.0, 24.6, 29.2, 30.0, 24.9, 31.2, 30.3}, + .expected_outputs = {0.731059, 0.425131, 0.246135, 0.753865, 0.445892, + 0.0109869, 0.886119, 0.461259, 0.185506, 0.781934, + 0.790618, 0.650195, 0.841816, 0.603901, 0.40518, + 0.561962, 0.765871, 0.930584, 0.718733, 0.763744, + 0.703402, 0.281989, 0.459635, 0.742634, 0.689974, + 0.840011, 0.82605, 0.170058, 0.147555, 0.28905}, + .activation = Options::SOFTMAX, + .rows = 4, + .cols = 4, + .rows_new = 5, + .cols_new = 6, + .channels = 2, + .max_abs_diff = 1e-6}, }), [](const testing::TestParamInfo< TensorsToSegmentationCalculatorTest::ParamType>& info) { From 5459705038098a280c36e55706f9b4901d2f3480 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 26 Oct 2023 15:22:08 -0700 Subject: [PATCH 046/157] Adding two new immutable texture GpuBufferFormat types PiperOrigin-RevId: 577002534 --- mediapipe/gpu/gl_texture_buffer.cc | 8 ++++++++ mediapipe/gpu/gpu_buffer_format.cc | 10 ++++++++++ mediapipe/gpu/gpu_buffer_format.h | 9 +++++++++ 3 files changed, 27 insertions(+) diff --git a/mediapipe/gpu/gl_texture_buffer.cc b/mediapipe/gpu/gl_texture_buffer.cc index 48afbd219..da2533f1c 100644 --- a/mediapipe/gpu/gl_texture_buffer.cc +++ b/mediapipe/gpu/gl_texture_buffer.cc @@ -14,6 +14,8 @@ #include "mediapipe/gpu/gl_texture_buffer.h" +#include + #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "mediapipe/framework/formats/image_frame.h" @@ -131,6 +133,12 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) { SymbolAvailable(&glTexStorage2D)) { ABSL_CHECK(data == nullptr) << "unimplemented"; glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_); + } else if (info.immutable) { + ABSL_CHECK(SymbolAvailable(&glTexStorage2D) && + context->GetGlVersion() != GlVersion::kGLES2) + << "Immutable GpuBuffer format requested is not supported in this " + << "GLContext. Format was " << static_cast(format_); + glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_); } else { glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_, height_, 0 /* border */, info.gl_format, info.gl_type, data); diff --git a/mediapipe/gpu/gpu_buffer_format.cc b/mediapipe/gpu/gpu_buffer_format.cc index 646fb383f..930bad735 100644 --- a/mediapipe/gpu/gpu_buffer_format.cc +++ b/mediapipe/gpu/gpu_buffer_format.cc @@ -163,6 +163,14 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, { {GL_RGBA32F, GL_RGBA, GL_FLOAT, 1}, }}, + {GpuBufferFormat::kImmutableRGBAFloat128, + { + {GL_RGBA32F, GL_RGBA, GL_FLOAT, 1, true /* immutable */}, + }}, + {GpuBufferFormat::kImmutableRGBA32, + { + {GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE, 1, true /* immutable */}, + }}, }}; static const auto* gles2_format_info = ([] { @@ -206,6 +214,7 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) { switch (format) { + case GpuBufferFormat::kImmutableRGBA32: case GpuBufferFormat::kBGRA32: // TODO: verify we are handling order of channels correctly. return ImageFormat::SRGBA; @@ -221,6 +230,7 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) { return ImageFormat::SRGB; case GpuBufferFormat::kTwoComponentFloat32: return ImageFormat::VEC32F2; + case GpuBufferFormat::kImmutableRGBAFloat128: case GpuBufferFormat::kRGBAFloat128: return ImageFormat::VEC32F4; case GpuBufferFormat::kRGBA32: diff --git a/mediapipe/gpu/gpu_buffer_format.h b/mediapipe/gpu/gpu_buffer_format.h index 06eabda77..0b6badfb5 100644 --- a/mediapipe/gpu/gpu_buffer_format.h +++ b/mediapipe/gpu/gpu_buffer_format.h @@ -53,6 +53,10 @@ enum class GpuBufferFormat : uint32_t { kRGB24 = 0x00000018, // Note: prefer BGRA32 whenever possible. kRGBAHalf64 = MEDIAPIPE_FOURCC('R', 'G', 'h', 'A'), kRGBAFloat128 = MEDIAPIPE_FOURCC('R', 'G', 'f', 'A'), + // Immutable version of kRGBA32 + kImmutableRGBA32 = MEDIAPIPE_FOURCC('4', 'C', 'I', '8'), + // Immutable version of kRGBAFloat128 + kImmutableRGBAFloat128 = MEDIAPIPE_FOURCC('4', 'C', 'I', 'f'), // 8-bit Y plane + interleaved 8-bit U/V plane with 2x2 subsampling. kNV12 = MEDIAPIPE_FOURCC('N', 'V', '1', '2'), // 8-bit Y plane + interleaved 8-bit V/U plane with 2x2 subsampling. @@ -78,6 +82,9 @@ struct GlTextureInfo { // For multiplane buffers, this represents how many times smaller than // the nominal image size a plane is. int downscale; + // For GLES3.1+ compute shaders, users may explicitly request immutable + // textures + bool immutable = false; }; const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, @@ -121,6 +128,8 @@ inline OSType CVPixelFormatForGpuBufferFormat(GpuBufferFormat format) { return kCVPixelFormatType_64RGBAHalf; case GpuBufferFormat::kRGBAFloat128: return kCVPixelFormatType_128RGBAFloat; + case GpuBufferFormat::kImmutableRGBA32: + case GpuBufferFormat::kImmutableRGBAFloat128: case GpuBufferFormat::kNV12: case GpuBufferFormat::kNV21: case GpuBufferFormat::kI420: From 2cb0100fe61a283ec9190cf755855eaf8305dafa Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 26 Oct 2023 15:24:15 -0700 Subject: [PATCH 047/157] Use mp.ImageFormat instead of just ImageFormat Fixes https://github.com/google/mediapipe/issues/4911 PiperOrigin-RevId: 577003083 --- mediapipe/python/pybind/image.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/python/pybind/image.cc b/mediapipe/python/pybind/image.cc index 800e883b4..62437e439 100644 --- a/mediapipe/python/pybind/image.cc +++ b/mediapipe/python/pybind/image.cc @@ -52,9 +52,9 @@ void ImageSubmodule(pybind11::module* module) { ```python import cv2 cv_mat = cv2.imread(input_file)[:, :, ::-1] - rgb_frame = mp.Image(image_format=ImageFormat.SRGB, data=cv_mat) + rgb_frame = mp.Image(image_format=mp.ImageFormat.SRGB, data=cv_mat) gray_frame = mp.Image( - image_format=ImageFormat.GRAY, + image_format=mp.ImageFormat.GRAY8, data=cv2.cvtColor(cv_mat, cv2.COLOR_RGB2GRAY)) from PIL import Image From 46cca0d4860b7a9b29246a3af04bc9cf658df204 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 27 Oct 2023 02:23:39 -0700 Subject: [PATCH 048/157] Rolling back. PiperOrigin-RevId: 577128565 --- mediapipe/gpu/gl_texture_buffer.cc | 8 -------- mediapipe/gpu/gpu_buffer_format.cc | 10 ---------- mediapipe/gpu/gpu_buffer_format.h | 9 --------- 3 files changed, 27 deletions(-) diff --git a/mediapipe/gpu/gl_texture_buffer.cc b/mediapipe/gpu/gl_texture_buffer.cc index da2533f1c..48afbd219 100644 --- a/mediapipe/gpu/gl_texture_buffer.cc +++ b/mediapipe/gpu/gl_texture_buffer.cc @@ -14,8 +14,6 @@ #include "mediapipe/gpu/gl_texture_buffer.h" -#include - #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "mediapipe/framework/formats/image_frame.h" @@ -133,12 +131,6 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) { SymbolAvailable(&glTexStorage2D)) { ABSL_CHECK(data == nullptr) << "unimplemented"; glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_); - } else if (info.immutable) { - ABSL_CHECK(SymbolAvailable(&glTexStorage2D) && - context->GetGlVersion() != GlVersion::kGLES2) - << "Immutable GpuBuffer format requested is not supported in this " - << "GLContext. Format was " << static_cast(format_); - glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_); } else { glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_, height_, 0 /* border */, info.gl_format, info.gl_type, data); diff --git a/mediapipe/gpu/gpu_buffer_format.cc b/mediapipe/gpu/gpu_buffer_format.cc index 930bad735..646fb383f 100644 --- a/mediapipe/gpu/gpu_buffer_format.cc +++ b/mediapipe/gpu/gpu_buffer_format.cc @@ -163,14 +163,6 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, { {GL_RGBA32F, GL_RGBA, GL_FLOAT, 1}, }}, - {GpuBufferFormat::kImmutableRGBAFloat128, - { - {GL_RGBA32F, GL_RGBA, GL_FLOAT, 1, true /* immutable */}, - }}, - {GpuBufferFormat::kImmutableRGBA32, - { - {GL_RGBA, GL_RGBA, GL_UNSIGNED_BYTE, 1, true /* immutable */}, - }}, }}; static const auto* gles2_format_info = ([] { @@ -214,7 +206,6 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) { switch (format) { - case GpuBufferFormat::kImmutableRGBA32: case GpuBufferFormat::kBGRA32: // TODO: verify we are handling order of channels correctly. return ImageFormat::SRGBA; @@ -230,7 +221,6 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) { return ImageFormat::SRGB; case GpuBufferFormat::kTwoComponentFloat32: return ImageFormat::VEC32F2; - case GpuBufferFormat::kImmutableRGBAFloat128: case GpuBufferFormat::kRGBAFloat128: return ImageFormat::VEC32F4; case GpuBufferFormat::kRGBA32: diff --git a/mediapipe/gpu/gpu_buffer_format.h b/mediapipe/gpu/gpu_buffer_format.h index 0b6badfb5..06eabda77 100644 --- a/mediapipe/gpu/gpu_buffer_format.h +++ b/mediapipe/gpu/gpu_buffer_format.h @@ -53,10 +53,6 @@ enum class GpuBufferFormat : uint32_t { kRGB24 = 0x00000018, // Note: prefer BGRA32 whenever possible. kRGBAHalf64 = MEDIAPIPE_FOURCC('R', 'G', 'h', 'A'), kRGBAFloat128 = MEDIAPIPE_FOURCC('R', 'G', 'f', 'A'), - // Immutable version of kRGBA32 - kImmutableRGBA32 = MEDIAPIPE_FOURCC('4', 'C', 'I', '8'), - // Immutable version of kRGBAFloat128 - kImmutableRGBAFloat128 = MEDIAPIPE_FOURCC('4', 'C', 'I', 'f'), // 8-bit Y plane + interleaved 8-bit U/V plane with 2x2 subsampling. kNV12 = MEDIAPIPE_FOURCC('N', 'V', '1', '2'), // 8-bit Y plane + interleaved 8-bit V/U plane with 2x2 subsampling. @@ -82,9 +78,6 @@ struct GlTextureInfo { // For multiplane buffers, this represents how many times smaller than // the nominal image size a plane is. int downscale; - // For GLES3.1+ compute shaders, users may explicitly request immutable - // textures - bool immutable = false; }; const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, @@ -128,8 +121,6 @@ inline OSType CVPixelFormatForGpuBufferFormat(GpuBufferFormat format) { return kCVPixelFormatType_64RGBAHalf; case GpuBufferFormat::kRGBAFloat128: return kCVPixelFormatType_128RGBAFloat; - case GpuBufferFormat::kImmutableRGBA32: - case GpuBufferFormat::kImmutableRGBAFloat128: case GpuBufferFormat::kNV12: case GpuBufferFormat::kNV21: case GpuBufferFormat::kI420: From eaf080784987a25aedcac8c7f77b735aba2a389f Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 27 Oct 2023 08:06:51 -0700 Subject: [PATCH 049/157] Fixes multiple typos in the calculator's internal files. PiperOrigin-RevId: 577202836 --- .../calculators/tensor/image_to_tensor_calculator.cc | 4 ++-- .../tensor/image_to_tensor_converter_gl_buffer.cc | 8 ++++---- .../tensor/image_to_tensor_converter_gl_texture.cc | 10 +++++----- .../tensor/image_to_tensor_converter_metal.cc | 10 +++++----- mediapipe/calculators/tensor/image_to_tensor_utils.cc | 8 ++++---- mediapipe/calculators/tensor/image_to_tensor_utils.h | 10 +++++----- .../tensor/tensor_converter_calculator.proto | 2 +- .../tensor/tensors_to_classification_calculator.proto | 2 +- .../tensor/tensors_to_detections_calculator.cc | 4 ++-- .../tensor/tensors_to_detections_calculator.proto | 2 +- .../tensor/tensors_to_segmentation_calculator.cc | 2 +- 11 files changed, 31 insertions(+), 31 deletions(-) diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc index 171b28eb4..924df6af3 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc @@ -82,7 +82,7 @@ namespace api2 { // // Outputs: // TENSORS - std::vector -// Vector containing a single Tensor populated with an extrated RGB image. +// Vector containing a single Tensor populated with an extracted RGB image. // MATRIX - std::array @Optional // An std::array representing a 4x4 row-major-order matrix that // maps a point on the input image to a point on the output tensor, and @@ -212,7 +212,7 @@ class ImageToTensorCalculator : public Node { std::array matrix; GetRotatedSubRectToRectTransformMatrix( roi, image->width(), image->height(), - /*flip_horizontaly=*/false, &matrix); + /*flip_horizontally=*/false, &matrix); kOutMatrix(cc).Send(std::move(matrix)); } diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc index b32b67869..04b791bd4 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.cc @@ -57,7 +57,7 @@ class SubRectExtractorGl { absl::Status ExtractSubRectToBuffer( const tflite::gpu::gl::GlTexture& texture, const tflite::gpu::HW& texture_size, const RotatedRect& sub_rect, - bool flip_horizontaly, float alpha, float beta, + bool flip_horizontally, float alpha, float beta, const tflite::gpu::HW& destination_size, tflite::gpu::gl::CommandQueue* command_queue, tflite::gpu::gl::GlBuffer* destination); @@ -154,13 +154,13 @@ void main() { absl::Status SubRectExtractorGl::ExtractSubRectToBuffer( const tflite::gpu::gl::GlTexture& texture, const tflite::gpu::HW& texture_size, const RotatedRect& texture_sub_rect, - bool flip_horizontaly, float alpha, float beta, + bool flip_horizontally, float alpha, float beta, const tflite::gpu::HW& destination_size, tflite::gpu::gl::CommandQueue* command_queue, tflite::gpu::gl::GlBuffer* destination) { std::array transform_mat; GetRotatedSubRectToRectTransformMatrix(texture_sub_rect, texture_size.w, - texture_size.h, flip_horizontaly, + texture_size.h, flip_horizontally, &transform_mat); MP_RETURN_IF_ERROR(texture.BindAsSampler2D(0)); @@ -308,7 +308,7 @@ class GlProcessor : public ImageToTensorConverter { input_texture, tflite::gpu::HW(source_texture.height(), source_texture.width()), roi, - /*flip_horizontaly=*/false, transform.scale, transform.offset, + /*flip_horizontally=*/false, transform.scale, transform.offset, tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]), command_queue_.get(), &output)); diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc index 2522cae85..930d9fe21 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc @@ -199,7 +199,7 @@ class GlProcessor : public ImageToTensorConverter { range_min, range_max)); auto tensor_view = output_tensor.GetOpenGlTexture2dWriteView(); MP_RETURN_IF_ERROR(ExtractSubRect(input_texture, roi, - /*flip_horizontaly=*/false, + /*flip_horizontally=*/false, transform.scale, transform.offset, output_shape, &tensor_view)); return absl::OkStatus(); @@ -210,7 +210,7 @@ class GlProcessor : public ImageToTensorConverter { absl::Status ExtractSubRect(const mediapipe::GlTexture& texture, const RotatedRect& sub_rect, - bool flip_horizontaly, float alpha, float beta, + bool flip_horizontally, float alpha, float beta, const Tensor::Shape& output_shape, Tensor::OpenGlTexture2dView* output) { const int output_height = output_shape.dims[1]; @@ -263,13 +263,13 @@ class GlProcessor : public ImageToTensorConverter { ABSL_LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread."; if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) { GetTransposedRotatedSubRectToRectTransformMatrix( - sub_rect, texture.width(), texture.height(), flip_horizontaly, + sub_rect, texture.width(), texture.height(), flip_horizontally, &transform_mat); glUniformMatrix4fv(matrix_id_, 1, GL_FALSE, transform_mat.data()); } else { GetRotatedSubRectToRectTransformMatrix(sub_rect, texture.width(), - texture.height(), flip_horizontaly, - &transform_mat); + texture.height(), + flip_horizontally, &transform_mat); glUniformMatrix4fv(matrix_id_, 1, GL_TRUE, transform_mat.data()); } diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc index cef2abcd7..f47d2da9a 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_converter_metal.cc @@ -179,13 +179,13 @@ class SubRectExtractorMetal { } absl::Status Execute(id input_texture, - const RotatedRect& sub_rect, bool flip_horizontaly, + const RotatedRect& sub_rect, bool flip_horizontally, float alpha, float beta, const tflite::gpu::HW& destination_size, id command_buffer, id destination) { auto output_texture = MTLTextureWithBuffer(destination_size, destination); - return InternalExecute(input_texture, sub_rect, flip_horizontaly, alpha, + return InternalExecute(input_texture, sub_rect, flip_horizontally, alpha, beta, destination_size, command_buffer, output_texture); } @@ -211,7 +211,7 @@ class SubRectExtractorMetal { absl::Status InternalExecute(id input_texture, const RotatedRect& sub_rect, - bool flip_horizontaly, float alpha, float beta, + bool flip_horizontally, float alpha, float beta, const tflite::gpu::HW& destination_size, id command_buffer, id output_texture) { @@ -223,7 +223,7 @@ class SubRectExtractorMetal { std::array transform_mat; GetRotatedSubRectToRectTransformMatrix(sub_rect, input_texture.width, input_texture.height, - flip_horizontaly, &transform_mat); + flip_horizontally, &transform_mat); id transform_mat_buffer = [device_ newBufferWithBytes:&transform_mat length:sizeof(transform_mat) @@ -383,7 +383,7 @@ class MetalProcessor : public ImageToTensorConverter { MtlBufferView::GetWriteView(output_tensor, command_buffer); MP_RETURN_IF_ERROR(extractor_->Execute( texture, roi, - /*flip_horizontaly=*/false, transform.scale, transform.offset, + /*flip_horizontally=*/false, transform.scale, transform.offset, tflite::gpu::HW(output_shape.dims[1], output_shape.dims[2]), command_buffer, buffer_view.buffer())); [command_buffer commit]; diff --git a/mediapipe/calculators/tensor/image_to_tensor_utils.cc b/mediapipe/calculators/tensor/image_to_tensor_utils.cc index 3f91f3dc2..b6ed5216c 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_utils.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_utils.cc @@ -92,7 +92,7 @@ absl::StatusOr GetValueRangeTransformation( void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect, int rect_width, int rect_height, - bool flip_horizontaly, + bool flip_horizontally, std::array* matrix_ptr) { std::array& matrix = *matrix_ptr; // The resulting matrix is multiplication of below commented out matrices: @@ -118,7 +118,7 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect, // {0.0f, 0.0f, a, 0.0f} // {0.0f, 0.0f, 0.0f, 1.0f} - const float flip = flip_horizontaly ? -1 : 1; + const float flip = flip_horizontally ? -1 : 1; // Matrix for optional horizontal flip around middle of output image. // { fl , 0.0f, 0.0f, 0.0f} // { 0.0f, 1.0f, 0.0f, 0.0f} @@ -177,13 +177,13 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect, void GetTransposedRotatedSubRectToRectTransformMatrix( const RotatedRect& sub_rect, int rect_width, int rect_height, - bool flip_horizontaly, std::array* matrix_ptr) { + bool flip_horizontally, std::array* matrix_ptr) { std::array& matrix = *matrix_ptr; // See comments in GetRotatedSubRectToRectTransformMatrix for detailed // calculations. const float a = sub_rect.width; const float b = sub_rect.height; - const float flip = flip_horizontaly ? -1 : 1; + const float flip = flip_horizontally ? -1 : 1; const float c = std::cos(sub_rect.rotation); const float d = std::sin(sub_rect.rotation); const float e = sub_rect.center_x; diff --git a/mediapipe/calculators/tensor/image_to_tensor_utils.h b/mediapipe/calculators/tensor/image_to_tensor_utils.h index a73529dce..63810923d 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_utils.h +++ b/mediapipe/calculators/tensor/image_to_tensor_utils.h @@ -74,7 +74,7 @@ absl::StatusOr> PadRoi(int input_tensor_width, // Represents a transformation of value which involves scaling and offsetting. // To apply transformation: // ValueTransformation transform = ... -// float transformed_value = transform.scale * value + transfrom.offset; +// float transformed_value = transform.scale * value + transform.offset; struct ValueTransformation { float scale; float offset; @@ -99,11 +99,11 @@ absl::StatusOr GetValueRangeTransformation( // @sub_rect - rotated sub rect in absolute coordinates // @rect_width - rect width // @rect_height - rect height -// @flip_horizontaly - we need to flip the output buffer. +// @flip_horizontally - we need to flip the output buffer. // @matrix - 4x4 matrix (array of 16 elements) to populate void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect, int rect_width, int rect_height, - bool flip_horizontaly, + bool flip_horizontally, std::array* matrix); // Returns the transpose of the matrix found with @@ -118,11 +118,11 @@ void GetRotatedSubRectToRectTransformMatrix(const RotatedRect& sub_rect, // @sub_rect - rotated sub rect in absolute coordinates // @rect_width - rect width // @rect_height - rect height -// @flip_horizontaly - we need to flip the output buffer. +// @flip_horizontally - we need to flip the output buffer. // @matrix - 4x4 matrix (array of 16 elements) to populate void GetTransposedRotatedSubRectToRectTransformMatrix( const RotatedRect& sub_rect, int rect_width, int rect_height, - bool flip_horizontaly, std::array* matrix); + bool flip_horizontally, std::array* matrix); // Validates the output dimensions set in the option proto. The input option // proto is expected to have to following fields: diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator.proto b/mediapipe/calculators/tensor/tensor_converter_calculator.proto index 2c5e0be56..b80d1e805 100644 --- a/mediapipe/calculators/tensor/tensor_converter_calculator.proto +++ b/mediapipe/calculators/tensor/tensor_converter_calculator.proto @@ -32,7 +32,7 @@ message TensorConverterCalculatorOptions { // Custom settings to override the internal scaling factors `div` and `sub`. // Both values must be set to non-negative values. Will only take effect on // CPU AND when |use_custom_normalization| is set to true. When these custom - // values take effect, the |zero_center| setting above will be overriden, and + // values take effect, the |zero_center| setting above will be overridden, and // the normalized_value will be calculated as: // normalized_value = input / custom_div - custom_sub. optional bool use_custom_normalization = 6 [default = false]; diff --git a/mediapipe/calculators/tensor/tensors_to_classification_calculator.proto b/mediapipe/calculators/tensor/tensors_to_classification_calculator.proto index 32bc4b63a..28012a455 100644 --- a/mediapipe/calculators/tensor/tensors_to_classification_calculator.proto +++ b/mediapipe/calculators/tensor/tensors_to_classification_calculator.proto @@ -34,7 +34,7 @@ message TensorsToClassificationCalculatorOptions { repeated Entry entries = 1; } - // Score threshold for perserving the class. + // Score threshold for preserving the class. optional float min_score_threshold = 1; // Number of highest scoring labels to output. If top_k is not positive then // all labels are used. diff --git a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc index 8e649c0a1..aa2cfe734 100644 --- a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc @@ -147,7 +147,7 @@ BoxFormat GetBoxFormat(const TensorsToDetectionsCalculatorOptions& options) { // TENSORS - Vector of Tensors of type kFloat32. The vector of tensors can have // 2 or 3 tensors. First tensor is the predicted raw boxes/keypoints. // The size of the values must be (num_boxes * num_predicted_values). -// Second tensor is the score tensor. The size of the valuse must be +// Second tensor is the score tensor. The size of the values must be // (num_boxes * num_classes). It's optional to pass in a third tensor // for anchors (e.g. for SSD models) depend on the outputs of the // detection model. The size of anchor tensor must be (num_boxes * @@ -267,7 +267,7 @@ absl::Status TensorsToDetectionsCalculator::UpdateContract( if (CanUseGpu()) { #ifndef MEDIAPIPE_DISABLE_GL_COMPUTE MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( - cc, /*requesst_gpu_as_optional=*/true)); + cc, /*request_gpu_as_optional=*/true)); #elif MEDIAPIPE_METAL_ENABLED MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); #endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) diff --git a/mediapipe/calculators/tensor/tensors_to_detections_calculator.proto b/mediapipe/calculators/tensor/tensors_to_detections_calculator.proto index 5cedff6c7..49db8e3e7 100644 --- a/mediapipe/calculators/tensor/tensors_to_detections_calculator.proto +++ b/mediapipe/calculators/tensor/tensors_to_detections_calculator.proto @@ -75,7 +75,7 @@ message TensorsToDetectionsCalculatorOptions { // representation has a bottom-left origin (e.g., in OpenGL). optional bool flip_vertically = 18 [default = false]; - // Score threshold for perserving decoded detections. + // Score threshold for preserving decoded detections. optional float min_score_thresh = 19; // The maximum number of the detection results to return. If < 0, all diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc index 6456126ae..24fd1bd52 100644 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc @@ -208,7 +208,7 @@ absl::Status TensorsToSegmentationCalculator::GetContract( if (CanUseGpu()) { #if !MEDIAPIPE_DISABLE_GPU MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( - cc, /*requesst_gpu_as_optional=*/true)); + cc, /*request_gpu_as_optional=*/true)); #if MEDIAPIPE_METAL_ENABLED MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); #endif // MEDIAPIPE_METAL_ENABLED From d73ef2440631d36e90bc3964129066e096b63127 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 27 Oct 2023 10:32:11 -0700 Subject: [PATCH 050/157] Support 3-channel RGB images for Mac Python PiperOrigin-RevId: 577240413 --- mediapipe/python/pybind/image.cc | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/mediapipe/python/pybind/image.cc b/mediapipe/python/pybind/image.cc index 62437e439..f3d89bada 100644 --- a/mediapipe/python/pybind/image.cc +++ b/mediapipe/python/pybind/image.cc @@ -244,12 +244,26 @@ void ImageSubmodule(pybind11::module* module) { image.def_static( "create_from_file", [](const std::string& file_name) { + unsigned char* image_data = nullptr; int width; int height; int channels; - auto* image_data = - stbi_load(file_name.c_str(), &width, &height, &channels, - /*desired_channels=*/0); + +#if TARGET_OS_OSX && !MEDIAPIPE_DISABLE_GPU + // Our ObjC layer does not support 3-channel images, so we read the + // number of channels first and request RGBA if needed. + if (stbi_info(file_name.c_str(), &width, &height, &channels)) { + if (channels == 3) { + channels = 4; + } + int unused; + image_data = + stbi_load(file_name.c_str(), &width, &height, &unused, channels); + } +#else + image_data = stbi_load(file_name.c_str(), &width, &height, &channels, + /*desired_channels=*/0); +#endif // TARGET_OS_OSX && !MEDIAPIPE_DISABLE_GPU if (image_data == nullptr) { throw RaisePyError(PyExc_RuntimeError, absl::StrFormat("Image decoding failed (%s): %s", @@ -263,11 +277,13 @@ void ImageSubmodule(pybind11::module* module) { ImageFormat::GRAY8, width, height, width, image_data, stbi_image_free); break; +#if !TARGET_OS_OSX || MEDIAPIPE_DISABLE_GPU case 3: image_frame = std::make_shared( ImageFormat::SRGB, width, height, 3 * width, image_data, stbi_image_free); break; +#endif // !TARGET_OS_OSX || MEDIAPIPE_DISABLE_GPU case 4: image_frame = std::make_shared( ImageFormat::SRGBA, width, height, 4 * width, image_data, From a96581e3b7685c9f12dfc206b885a362f9fb0396 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 27 Oct 2023 14:08:39 -0700 Subject: [PATCH 051/157] TensorsToDetectionsCalculator supports multi clasees for a bbox. PiperOrigin-RevId: 577300797 --- .../tensors_to_detections_calculator.cc | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc index aa2cfe734..95682d633 100644 --- a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc @@ -15,7 +15,6 @@ #include #include -#include "absl/log/absl_log.h" #include "absl/strings/str_format.h" #include "absl/types/span.h" #include "mediapipe/calculators/tensor/tensors_to_detections_calculator.pb.h" @@ -215,7 +214,8 @@ class TensorsToDetectionsCalculator : public Node { const int* detection_classes, std::vector* output_detections); Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax, - float box_xmax, float score, int class_id, + float box_xmax, absl::Span scores, + absl::Span class_ids, bool flip_vertically); bool IsClassIndexAllowed(int class_index); @@ -223,6 +223,7 @@ class TensorsToDetectionsCalculator : public Node { int num_boxes_ = 0; int num_coords_ = 0; int max_results_ = -1; + int classes_per_detection_ = 1; BoxFormat box_output_format_ = mediapipe::TensorsToDetectionsCalculatorOptions::YXHW; @@ -484,6 +485,16 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU( auto num_boxes_view = num_boxes_tensor->GetCpuReadView(); auto num_boxes = num_boxes_view.buffer(); num_boxes_ = num_boxes[0]; + // The detection model with Detection_PostProcess op may output duplicate + // boxes with different classes, in the following format: + // num_boxes_tensor = [num_boxes] + // detection_classes_tensor = [box_1_class_1, box_1_class_2, ...] + // detection_scores_tensor = [box_1_score_1, box_1_score_2, ... ] + // detection_boxes_tensor = [box_1, box1, ... ] + // Each box repeats classes_per_detection_ times. + // Note Detection_PostProcess op is only supported in CPU. + RET_CHECK_EQ(max_detections % num_boxes_, 0); + classes_per_detection_ = max_detections / num_boxes_; auto detection_boxes_view = detection_boxes_tensor->GetCpuReadView(); auto detection_boxes = detection_boxes_view.buffer(); @@ -493,8 +504,8 @@ absl::Status TensorsToDetectionsCalculator::ProcessCPU( auto detection_classes_view = detection_classes_tensor->GetCpuReadView(); auto detection_classes_ptr = detection_classes_view.buffer(); - std::vector detection_classes(num_boxes_); - for (int i = 0; i < num_boxes_; ++i) { + std::vector detection_classes(num_boxes_ * classes_per_detection_); + for (int i = 0; i < detection_classes.size(); ++i) { detection_classes[i] = static_cast(detection_classes_ptr[i]); } MP_RETURN_IF_ERROR(ConvertToDetections(detection_boxes, detection_scores, @@ -863,7 +874,8 @@ absl::Status TensorsToDetectionsCalculator::DecodeBoxes( absl::Status TensorsToDetectionsCalculator::ConvertToDetections( const float* detection_boxes, const float* detection_scores, const int* detection_classes, std::vector* output_detections) { - for (int i = 0; i < num_boxes_; ++i) { + for (int i = 0; i < num_boxes_ * classes_per_detection_; + i += classes_per_detection_) { if (max_results_ > 0 && output_detections->size() == max_results_) { break; } @@ -880,7 +892,9 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections( /*box_xmin=*/detection_boxes[box_offset + box_indices_[1]], /*box_ymax=*/detection_boxes[box_offset + box_indices_[2]], /*box_xmax=*/detection_boxes[box_offset + box_indices_[3]], - detection_scores[i], detection_classes[i], options_.flip_vertically()); + absl::MakeConstSpan(detection_scores + i, classes_per_detection_), + absl::MakeConstSpan(detection_classes + i, classes_per_detection_), + options_.flip_vertically()); const auto& bbox = detection.location_data().relative_bounding_box(); if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) || std::isnan(bbox.height())) { @@ -910,11 +924,17 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections( } Detection TensorsToDetectionsCalculator::ConvertToDetection( - float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score, - int class_id, bool flip_vertically) { + float box_ymin, float box_xmin, float box_ymax, float box_xmax, + absl::Span scores, absl::Span class_ids, + bool flip_vertically) { Detection detection; - detection.add_score(score); - detection.add_label_id(class_id); + for (int i = 0; i < scores.size(); ++i) { + if (!IsClassIndexAllowed(class_ids[i])) { + continue; + } + detection.add_score(scores[i]); + detection.add_label_id(class_ids[i]); + } LocationData* location_data = detection.mutable_location_data(); location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX); From 2f4d7b4079a32a99aa5e8e3593e7170dbfa757be Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Sat, 28 Oct 2023 01:07:56 -0700 Subject: [PATCH 052/157] No public description PiperOrigin-RevId: 577410310 --- mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc b/mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc index dbabf84b1..f958924f0 100644 --- a/mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc +++ b/mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc @@ -56,7 +56,7 @@ absl::Status RunMPPGraph() { for (const std::string& kv_pair : kv_pairs) { std::vector name_and_value = absl::StrSplit(kv_pair, '='); RET_CHECK(name_and_value.size() == 2); - RET_CHECK(!mediapipe::ContainsKey(input_side_packets, name_and_value[0])); + RET_CHECK(!input_side_packets.contains(name_and_value[0])); std::string input_side_packet_contents; MP_RETURN_IF_ERROR(mediapipe::file::GetContents( name_and_value[1], &input_side_packet_contents)); From 7256bd26386d08f1bf527af3d2f9cd6b6310aee8 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 30 Oct 2023 09:44:11 -0700 Subject: [PATCH 053/157] No public description PiperOrigin-RevId: 577873616 --- mediapipe/python/image_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mediapipe/python/image_test.py b/mediapipe/python/image_test.py index cd9124948..79056615e 100644 --- a/mediapipe/python/image_test.py +++ b/mediapipe/python/image_test.py @@ -207,7 +207,9 @@ class ImageTest(absltest.TestCase): loaded_image = Image.create_from_file(image_path) self.assertEqual(loaded_image.width, 720) self.assertEqual(loaded_image.height, 382) - self.assertEqual(loaded_image.channels, 3) + # On Mac w/ GPU support, images use 4 channels. Otherwise, all images use + # 3 channels. + self.assertIn(loaded_image.channels, [3, 4]) self.assertEqual(loaded_image.image_format, ImageFormat.SRGB) if __name__ == '__main__': From ec032fb018cbabf8d5442b30e14c72b517d8a8dc Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 30 Oct 2023 13:04:17 -0700 Subject: [PATCH 054/157] Use SRGBA for Mac on Python for image test PiperOrigin-RevId: 577931014 --- mediapipe/python/image_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mediapipe/python/image_test.py b/mediapipe/python/image_test.py index 79056615e..3181fb5f1 100644 --- a/mediapipe/python/image_test.py +++ b/mediapipe/python/image_test.py @@ -207,10 +207,12 @@ class ImageTest(absltest.TestCase): loaded_image = Image.create_from_file(image_path) self.assertEqual(loaded_image.width, 720) self.assertEqual(loaded_image.height, 382) - # On Mac w/ GPU support, images use 4 channels. Otherwise, all images use - # 3 channels. + # On Mac w/ GPU support, images use 4 channels (SRGBA). Otherwise, all + # images use 3 channels (SRGB). self.assertIn(loaded_image.channels, [3, 4]) - self.assertEqual(loaded_image.image_format, ImageFormat.SRGB) + self.assertIn( + loaded_image.image_format, [ImageFormat.SRGB, ImageFormat.SRGBA] + ) if __name__ == '__main__': absltest.main() From 95692c64a9fce02e2e55629222790f050a6e56ac Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 30 Oct 2023 16:05:12 -0600 Subject: [PATCH 055/157] Add GPU support --- mediapipe/tasks/ios/core/sources/MPPBaseOptions.h | 12 ++++++++++++ mediapipe/tasks/ios/core/sources/MPPBaseOptions.m | 2 ++ mediapipe/tasks/ios/core/utils/BUILD | 1 + .../ios/core/utils/sources/MPPBaseOptions+Helpers.mm | 6 ++++++ 4 files changed, 21 insertions(+) diff --git a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h index bef6bb9ee..eecb5e14e 100644 --- a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h +++ b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.h @@ -14,6 +14,15 @@ #import +/** + * The delegate to run MediaPipe. If the delegate is not set, the default + * delegate CPU is used. + */ +typedef NS_ENUM(NSUInteger, MPPDelegate) { + MPPDelegateCPU, + MPPDelegateGPU, +} NS_SWIFT_NAME(Delegate); + NS_ASSUME_NONNULL_BEGIN /** @@ -26,6 +35,9 @@ NS_SWIFT_NAME(BaseOptions) /** The path to the model asset to open and mmap in memory. */ @property(nonatomic, copy) NSString *modelAssetPath; +/** Overrides the default backend to use for the provided model. */ +@property(nonatomic) MPPDelegate delegate; + @end NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m index a43119ad8..fac1b94c0 100644 --- a/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m +++ b/mediapipe/tasks/ios/core/sources/MPPBaseOptions.m @@ -20,6 +20,7 @@ self = [super init]; if (self) { self.modelAssetPath = [[NSString alloc] init]; + self.delegate = MPPDelegateCPU; } return self; } @@ -28,6 +29,7 @@ MPPBaseOptions *baseOptions = [[MPPBaseOptions alloc] init]; baseOptions.modelAssetPath = self.modelAssetPath; + baseOptions.delegate = self.delegate; return baseOptions; } diff --git a/mediapipe/tasks/ios/core/utils/BUILD b/mediapipe/tasks/ios/core/utils/BUILD index 3cd8bf231..88c786642 100644 --- a/mediapipe/tasks/ios/core/utils/BUILD +++ b/mediapipe/tasks/ios/core/utils/BUILD @@ -24,6 +24,7 @@ objc_library( "//mediapipe/tasks/cc/core/proto:acceleration_cc_proto", "//mediapipe/tasks/cc/core/proto:base_options_cc_proto", "//mediapipe/tasks/cc/core/proto:external_file_cc_proto", + "//mediapipe/calculators/tensor:inference_calculator_cc_proto", "//mediapipe/tasks/ios/core:MPPBaseOptions", ], ) diff --git a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm index 73bcac49d..14ecbd708 100644 --- a/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm +++ b/mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm @@ -15,9 +15,11 @@ #include "mediapipe/tasks/cc/core/proto/acceleration.pb.h" #include "mediapipe/tasks/cc/core/proto/external_file.pb.h" #import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h" +#include "mediapipe/calculators/tensor/inference_calculator.pb.h" namespace { using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions; +using InferenceCalculatorOptionsProto = ::mediapipe::InferenceCalculatorOptions; } @implementation MPPBaseOptions (Helpers) @@ -33,6 +35,10 @@ using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions; if (self.modelAssetPath) { baseOptionsProto->mutable_model_asset()->set_file_name(self.modelAssetPath.UTF8String); } + + if (self.delegate == MPPDelegateGPU) { + baseOptionsProto->mutable_acceleration()->mutable_gpu()->MergeFrom(InferenceCalculatorOptionsProto::Delegate::Gpu()); + } } @end From 7da2810b83f751d23f5c340ad8228cc8a6301750 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 31 Oct 2023 08:16:58 -0700 Subject: [PATCH 056/157] Move filtering logic of score to ConvertToDetection. PiperOrigin-RevId: 578189518 --- .../tensor/tensors_to_detections_calculator.cc | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc index 95682d633..2b4a22fc6 100644 --- a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc @@ -879,13 +879,6 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections( if (max_results_ > 0 && output_detections->size() == max_results_) { break; } - if (options_.has_min_score_thresh() && - detection_scores[i] < options_.min_score_thresh()) { - continue; - } - if (!IsClassIndexAllowed(detection_classes[i])) { - continue; - } const int box_offset = i * num_coords_; Detection detection = ConvertToDetection( /*box_ymin=*/detection_boxes[box_offset + box_indices_[0]], @@ -895,6 +888,11 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections( absl::MakeConstSpan(detection_scores + i, classes_per_detection_), absl::MakeConstSpan(detection_classes + i, classes_per_detection_), options_.flip_vertically()); + // if all the scores and classes are filtered out, we skip the empty + // detection. + if (detection.score().empty()) { + continue; + } const auto& bbox = detection.location_data().relative_bounding_box(); if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) || std::isnan(bbox.height())) { @@ -932,6 +930,10 @@ Detection TensorsToDetectionsCalculator::ConvertToDetection( if (!IsClassIndexAllowed(class_ids[i])) { continue; } + if (options_.has_min_score_thresh() && + scores[i] < options_.min_score_thresh()) { + continue; + } detection.add_score(scores[i]); detection.add_label_id(class_ids[i]); } From a4048eee1124fd9690c918bd8811887b24b8e0e1 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 31 Oct 2023 11:11:21 -0700 Subject: [PATCH 057/157] Add video and live stream processing and tests for Image Classifier C API PiperOrigin-RevId: 578242391 --- .../tasks/c/vision/image_classifier/BUILD | 3 +- .../image_classifier/image_classifier.cc | 135 +++++++++++++++++- .../image_classifier/image_classifier.h | 26 +++- .../image_classifier/image_classifier_test.cc | 131 +++++++++++++++-- 4 files changed, 278 insertions(+), 17 deletions(-) diff --git a/mediapipe/tasks/c/vision/image_classifier/BUILD b/mediapipe/tasks/c/vision/image_classifier/BUILD index df0e636c5..e8ac090e9 100644 --- a/mediapipe/tasks/c/vision/image_classifier/BUILD +++ b/mediapipe/tasks/c/vision/image_classifier/BUILD @@ -30,13 +30,12 @@ cc_library( "//mediapipe/tasks/c/components/processors:classifier_options_converter", "//mediapipe/tasks/c/core:base_options", "//mediapipe/tasks/c/core:base_options_converter", + "//mediapipe/tasks/cc/vision/core:running_mode", "//mediapipe/tasks/cc/vision/image_classifier", "//mediapipe/tasks/cc/vision/utils:image_utils", "@com_google_absl//absl/log:absl_log", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/time", ], alwayslink = 1, ) diff --git a/mediapipe/tasks/c/vision/image_classifier/image_classifier.cc b/mediapipe/tasks/c/vision/image_classifier/image_classifier.cc index 4245ca4cd..ff6f5bdfc 100644 --- a/mediapipe/tasks/c/vision/image_classifier/image_classifier.cc +++ b/mediapipe/tasks/c/vision/image_classifier/image_classifier.cc @@ -15,6 +15,8 @@ limitations under the License. #include "mediapipe/tasks/c/vision/image_classifier/image_classifier.h" +#include +#include #include #include @@ -26,6 +28,7 @@ limitations under the License. #include "mediapipe/tasks/c/components/containers/classification_result_converter.h" #include "mediapipe/tasks/c/components/processors/classifier_options_converter.h" #include "mediapipe/tasks/c/core/base_options_converter.h" +#include "mediapipe/tasks/cc/vision/core/running_mode.h" #include "mediapipe/tasks/cc/vision/image_classifier/image_classifier.h" #include "mediapipe/tasks/cc/vision/utils/image_utils.h" @@ -41,7 +44,10 @@ using ::mediapipe::tasks::c::components::processors:: CppConvertToClassifierOptions; using ::mediapipe::tasks::c::core::CppConvertToBaseOptions; using ::mediapipe::tasks::vision::CreateImageFromBuffer; +using ::mediapipe::tasks::vision::core::RunningMode; using ::mediapipe::tasks::vision::image_classifier::ImageClassifier; +typedef ::mediapipe::tasks::vision::image_classifier::ImageClassifierResult + CppImageClassifierResult; int CppProcessError(absl::Status status, char** error_msg) { if (error_msg) { @@ -60,6 +66,53 @@ ImageClassifier* CppImageClassifierCreate(const ImageClassifierOptions& options, CppConvertToBaseOptions(options.base_options, &cpp_options->base_options); CppConvertToClassifierOptions(options.classifier_options, &cpp_options->classifier_options); + cpp_options->running_mode = static_cast(options.running_mode); + + // Enable callback for processing live stream data when the running mode is + // set to RunningMode::LIVE_STREAM. + if (cpp_options->running_mode == RunningMode::LIVE_STREAM) { + if (options.result_callback == nullptr) { + const absl::Status status = absl::InvalidArgumentError( + "Provided null pointer to callback function."); + ABSL_LOG(ERROR) << "Failed to create ImageClassifier: " << status; + CppProcessError(status, error_msg); + return nullptr; + } + + ImageClassifierOptions::result_callback_fn result_callback = + options.result_callback; + cpp_options->result_callback = + [result_callback](absl::StatusOr cpp_result, + const Image& image, int64_t timestamp) { + char* error_msg = nullptr; + + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Classification failed: " << cpp_result.status(); + CppProcessError(cpp_result.status(), &error_msg); + result_callback(nullptr, MpImage(), timestamp, error_msg); + free(error_msg); + return; + } + + // Result is valid for the lifetime of the callback function. + ImageClassifierResult result; + CppConvertToClassificationResult(*cpp_result, &result); + + const auto& image_frame = image.GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = { + .format = static_cast<::ImageFormat>(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + result_callback(&result, mp_image, timestamp, + /* error_msg= */ nullptr); + + CppCloseClassificationResult(&result); + }; + } auto classifier = ImageClassifier::Create(std::move(cpp_options)); if (!classifier.ok()) { @@ -75,8 +128,8 @@ int CppImageClassifierClassify(void* classifier, const MpImage* image, ImageClassifierResult* result, char** error_msg) { if (image->type == MpImage::GPU_BUFFER) { - absl::Status status = - absl::InvalidArgumentError("gpu buffer not supported yet"); + const absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet."); ABSL_LOG(ERROR) << "Classification failed: " << status.message(); return CppProcessError(status, error_msg); @@ -102,6 +155,68 @@ int CppImageClassifierClassify(void* classifier, const MpImage* image, return 0; } +int CppImageClassifierClassifyForVideo(void* classifier, const MpImage* image, + int64_t timestamp_ms, + ImageClassifierResult* result, + char** error_msg) { + if (image->type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Classification failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image->image_frame.format), + image->image_frame.image_buffer, image->image_frame.width, + image->image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_classifier = static_cast(classifier); + auto cpp_result = cpp_classifier->ClassifyForVideo(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Classification failed: " << cpp_result.status(); + return CppProcessError(cpp_result.status(), error_msg); + } + CppConvertToClassificationResult(*cpp_result, result); + return 0; +} + +int CppImageClassifierClassifyAsync(void* classifier, const MpImage* image, + int64_t timestamp_ms, char** error_msg) { + if (image->type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Classification failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image->image_frame.format), + image->image_frame.image_buffer, image->image_frame.width, + image->image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_classifier = static_cast(classifier); + auto cpp_result = cpp_classifier->ClassifyAsync(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Data preparation for the image classification failed: " + << cpp_result; + return CppProcessError(cpp_result, error_msg); + } + return 0; +} + void CppImageClassifierCloseResult(ImageClassifierResult* result) { CppCloseClassificationResult(result); } @@ -134,6 +249,22 @@ int image_classifier_classify_image(void* classifier, const MpImage* image, CppImageClassifierClassify(classifier, image, result, error_msg); } +int image_classifier_classify_for_video(void* classifier, const MpImage* image, + int64_t timestamp_ms, + ImageClassifierResult* result, + char** error_msg) { + return mediapipe::tasks::c::vision::image_classifier:: + CppImageClassifierClassifyForVideo(classifier, image, timestamp_ms, + result, error_msg); +} + +int image_classifier_classify_async(void* classifier, const MpImage* image, + int64_t timestamp_ms, char** error_msg) { + return mediapipe::tasks::c::vision::image_classifier:: + CppImageClassifierClassifyAsync(classifier, image, timestamp_ms, + error_msg); +} + void image_classifier_close_result(ImageClassifierResult* result) { mediapipe::tasks::c::vision::image_classifier::CppImageClassifierCloseResult( result); diff --git a/mediapipe/tasks/c/vision/image_classifier/image_classifier.h b/mediapipe/tasks/c/vision/image_classifier/image_classifier.h index 60dc4a2c4..549c3f300 100644 --- a/mediapipe/tasks/c/vision/image_classifier/image_classifier.h +++ b/mediapipe/tasks/c/vision/image_classifier/image_classifier.h @@ -92,9 +92,16 @@ struct ImageClassifierOptions { // The user-defined result callback for processing live stream data. // The result callback should only be specified when the running mode is set - // to RunningMode::LIVE_STREAM. - typedef void (*result_callback_fn)(ImageClassifierResult*, const MpImage*, - int64_t); + // to RunningMode::LIVE_STREAM. Arguments of the callback function include: + // the pointer to classification result, the image that result was obtained + // on, the timestamp relevant to classification results and pointer to error + // message in case of any failure. The validity of the passed arguments is + // true for the lifetime of the callback function. + // + // A caller is responsible for closing image classifier result. + typedef void (*result_callback_fn)(ImageClassifierResult* result, + const MpImage image, int64_t timestamp_ms, + char* error_msg); result_callback_fn result_callback; }; @@ -110,13 +117,22 @@ MP_EXPORT void* image_classifier_create(struct ImageClassifierOptions* options, // If an error occurs, returns an error code and sets the error parameter to an // an error message (if `error_msg` is not nullptr). You must free the memory // allocated for the error message. -// -// TODO: Add API for video and live stream processing. MP_EXPORT int image_classifier_classify_image(void* classifier, const MpImage* image, ImageClassifierResult* result, char** error_msg = nullptr); +MP_EXPORT int image_classifier_classify_for_video(void* classifier, + const MpImage* image, + int64_t timestamp_ms, + ImageClassifierResult* result, + char** error_msg = nullptr); + +MP_EXPORT int image_classifier_classify_async(void* classifier, + const MpImage* image, + int64_t timestamp_ms, + char** error_msg = nullptr); + // Frees the memory allocated inside a ImageClassifierResult result. // Does not free the result pointer itself. MP_EXPORT void image_classifier_close_result(ImageClassifierResult* result); diff --git a/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc b/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc index e8e84d864..790f5ce36 100644 --- a/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc +++ b/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc @@ -15,6 +15,7 @@ limitations under the License. #include "mediapipe/tasks/c/vision/image_classifier/image_classifier.h" +#include #include #include @@ -36,12 +37,13 @@ using testing::HasSubstr; constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; constexpr char kModelName[] = "mobilenet_v2_1.0_224.tflite"; constexpr float kPrecision = 1e-4; +constexpr int kIterations = 100; std::string GetFullPath(absl::string_view file_name) { return JoinPath("./", kTestDataDirectory, file_name); } -TEST(ImageClassifierTest, SmokeTest) { +TEST(ImageClassifierTest, ImageModeTest) { const auto image = DecodeImageFromFile(GetFullPath("burger.jpg")); ASSERT_TRUE(image.ok()); @@ -63,14 +65,13 @@ TEST(ImageClassifierTest, SmokeTest) { void* classifier = image_classifier_create(&options); EXPECT_NE(classifier, nullptr); + const auto& image_frame = image->GetImageFrameSharedPtr(); const MpImage mp_image = { .type = MpImage::IMAGE_FRAME, - .image_frame = { - .format = static_cast( - image->GetImageFrameSharedPtr()->Format()), - .image_buffer = image->GetImageFrameSharedPtr()->PixelData(), - .width = image->GetImageFrameSharedPtr()->Width(), - .height = image->GetImageFrameSharedPtr()->Height()}}; + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; ImageClassifierResult result; image_classifier_classify_image(classifier, &mp_image, &result); @@ -84,6 +85,120 @@ TEST(ImageClassifierTest, SmokeTest) { image_classifier_close(classifier); } +TEST(ImageClassifierTest, VideoModeTest) { + const auto image = DecodeImageFromFile(GetFullPath("burger.jpg")); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + ImageClassifierOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::VIDEO, + /* classifier_options= */ + {/* display_names_locale= */ nullptr, + /* max_results= */ 3, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + /* result_callback= */ nullptr, + }; + + void* classifier = image_classifier_create(&options); + EXPECT_NE(classifier, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + ImageClassifierResult result; + image_classifier_classify_for_video(classifier, &mp_image, i, &result); + EXPECT_EQ(result.classifications_count, 1); + EXPECT_EQ(result.classifications[0].categories_count, 3); + EXPECT_EQ( + std::string{result.classifications[0].categories[0].category_name}, + "cheeseburger"); + EXPECT_NEAR(result.classifications[0].categories[0].score, 0.7939f, + kPrecision); + image_classifier_close_result(&result); + } + image_classifier_close(classifier); +} + +// A structure to support LiveStreamModeTest below. This structure holds a +// static method `Fn` for a callback function of C API. A `static` qualifier +// allows to take an address of the method to follow API style. Another static +// struct member is `last_timestamp` that is used to verify that current +// timestamp is greater than the previous one. +struct LiveStreamModeCallback { + static int64_t last_timestamp; + static void Fn(ImageClassifierResult* classifier_result, const MpImage image, + int64_t timestamp, char* error_msg) { + ASSERT_NE(classifier_result, nullptr); + ASSERT_EQ(error_msg, nullptr); + EXPECT_EQ( + std::string{ + classifier_result->classifications[0].categories[0].category_name}, + "cheeseburger"); + EXPECT_NEAR(classifier_result->classifications[0].categories[0].score, + 0.7939f, kPrecision); + EXPECT_GT(image.image_frame.width, 0); + EXPECT_GT(image.image_frame.height, 0); + EXPECT_GT(timestamp, last_timestamp); + last_timestamp++; + } +}; +int64_t LiveStreamModeCallback::last_timestamp = -1; + +TEST(ImageClassifierTest, LiveStreamModeTest) { + const auto image = DecodeImageFromFile(GetFullPath("burger.jpg")); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + + ImageClassifierOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::LIVE_STREAM, + /* classifier_options= */ + {/* display_names_locale= */ nullptr, + /* max_results= */ 3, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + /* result_callback= */ LiveStreamModeCallback::Fn, + }; + + void* classifier = image_classifier_create(&options); + EXPECT_NE(classifier, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + EXPECT_GE(image_classifier_classify_async(classifier, &mp_image, i), 0); + } + image_classifier_close(classifier); + + // Due to the flow limiter, the total of outputs might be smaller than the + // number of iterations. + EXPECT_LE(LiveStreamModeCallback::last_timestamp, kIterations); + EXPECT_GT(LiveStreamModeCallback::last_timestamp, 0); +} + TEST(ImageClassifierTest, InvalidArgumentHandling) { // It is an error to set neither the asset buffer nor the path. ImageClassifierOptions options = { @@ -124,7 +239,7 @@ TEST(ImageClassifierTest, FailedClassificationHandling) { ImageClassifierResult result; char* error_msg; image_classifier_classify_image(classifier, &mp_image, &result, &error_msg); - EXPECT_THAT(error_msg, HasSubstr("gpu buffer not supported yet")); + EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet")); free(error_msg); image_classifier_close(classifier); } From b9ff9708e312bb63b16075ecffa2cdb8f457bd73 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 31 Oct 2023 12:04:55 -0700 Subject: [PATCH 058/157] Upgrade to use Gradle 8.4 PiperOrigin-RevId: 578259506 --- .../gradle/wrapper/gradle-wrapper.jar | Bin 61624 -> 63721 bytes .../gradle/wrapper/gradle-wrapper.properties | 3 +- mediapipe/examples/android/solutions/gradlew | 29 ++++++++++-------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar b/mediapipe/examples/android/solutions/gradle/wrapper/gradle-wrapper.jar index afba109285af78dbd2a1d187e33ac4f87c76e392..7f93135c49b765f8051ef9d0a6055ff8e46073d8 100644 GIT binary patch delta 41204 zcmZ5{b95%((seSiZQHhO+qRP@<}68$AGx`lh~)fFsaj;MiX97^(}w;JdeIg_EK=f{qUxfgB1;VggKi&SA9v8P52JERa$mc|$lmVvfx{qi69Qt` z*fZ)^M2W{1fVoSt`s`K{cHR_Ov(<|-{;q0kYpPXc^iz@|EJ@}9Rkl^$`6gZKo4=jl zD2P(#uTcD*y0DWmpF`Dtw^s9E&tgueEt9jYdg)|4^CH zb}H5tkozJO-=TYiQxbjRT%6PCU;JFRQPXAQ>rlRozg33`4{|4|Q{K zP#?n!$RS}A3u8%zoT~YzxuAqLNKTa(NWlJ4XU{EHZHY-(GB_2uuL{a8BMNcj)?=dUUT2HO%1h(mqG)ntl zN?SX{UvgL}$DJsYuN~%ASKh2fJrBPH#2??t43t8?^fOWdaMj%wk$O`DK4(tRuA(&E zog=Ry-f5N`!=ByQeZ>yqokNEb4OV)~d*KM!+n@>L3iD=%hcWl5GV|Tcwvb**xo{vG!%lw${AnQ~eWceyLLtO0ikN#30gs-w0?6D+m(Pg;;(saJJH6dgz zPtR9$S)TMwL6Y4dX7dYUtY^k@&mLj>shqlfVB>uT4%k z-sD&k5#S$1G!f+SeqV-O07FX!@mC%6H?4gT42hV?{rCiRc9Cr9B1@ZjfX@!wme?JN zAJ(4)af-zesN2Gr=Jn#7mg9j8%5Jvi=KRdf+^w(o&rhoFI@|08W-G$DW;^7um(;k@ zrb`3p^aRVime{Nq^@gWKx`2>bX7zjX*(w=Bcc4S{A@7F|ytuV3;DP(RjMa4RmukeWjWwVyaGM*D6m`mn7ZGF34w6Gb!;w3^St z3XgDy{pdd{y~uiAiiTGa2wO@_XU;qFfTIXAZ1RMapg5FqfM@t-DJO(?zaynola?z< zq^^3=9HQZI#n>+*T*@Eef3h6)^xyrwTYa3S(|cxi6h6LV6~ufKNVoA3J4aC#kWj^9 zU$rtnC%FQ(!JWlPz7l4OHcH%})DUBe?Ui1bJ3TXHGHytpNOUMTkK>O63oL1f0R~Vl z5Q&~zYZ~evszs-g;%QS1E$G6>(o=zr5zmFhgtr72k-dOTT1h;>q0$c5&3}By18rDv za{zTEQI@fiS&|kEha>S}so7LsIpivt5vVHE)F!Z@B(20{Xj&vc)i}Ts+cmgXxl^-r zfwK2C3bRXuS7T6w6&q}%IY$i(=8BOF%1u8n14+J?DQ}GLQC#%nt^E3w&nfRL0C{8o z1kK9$eh2k$?D=o(;&vtePX11A3JhkVfk_W3;pVPI(@owrFYG{UOG!MTtY45i(<@=S zN=P|BG2-(N7nJ4OA>%O~hs;#8d_UEH{2?oTcEBiMv{=Wp@WM@3;N z0{Wcat;=K&itA=*;~J(LXVs)n9~I^r%a4*|S$W0d?v?Q`{73J$KUQ_4q5`Sm3BFa? zV(eIUY#=VrrlfmrNrXeQej79wf^Or9m^X&%-g`DZ;Yi7Q4g&5ll^1FyD)W@D)KrVh zl|juxjkCEccS-qdV|;zxQr12&?HgSGVY)!2YK($@QY2deG%8D@JL$e1pta(|J1t#f zUVXCDHNM8*jvht|y!My(A~`L3>E<-DLux~cq>XNop^vVrBZxb}{nb5fA()O1T|^S(dc&uFNXX z1sOhL*eHNaulR)m@PE4EooRR|dslppGSL#gzaw^n>yc)xGtbqT1u>8fY9^T3tzg#i zEiiQ#AQn{r#eEUXY8>4+Mrqk_>>(;GWqA*KMTOm5@A`nZy=B$H-C?( zlF23*1jz}fkjVi@l)zFo10^&Obb)3fg9^P$h^iuJQ;Z*^a6Q;+qAQe?3Q<`pw}KAg zydA$doAnNj?u2d+;V1>M^FI~FysSLcf+g$@#ZKq8d0w`C6|MR|Uw=akFnT;VD^Hq9 zGI0`KoFoRj=k7WyyDED&OeSn42gbbM%*0-x1h2w>1ew$^fC2Ap-F{$PBzXEyrTj37 z|B%v`@t;f5{H-YoPp3c_sfip(oYcsVma9E!ya2Bhu7Ag^4(~_@9b)^=9|exlV;ye0 zkAQyjF?9L1MD~jY>AmX`ua2~kR*f1DUXh7s5&IJF3N2tvARh{hNrID-Z54AhWLP_F zFm8P-mQDjHL1nr5DLdug=(vCHfzCO_8J zEp}ADT4%<15ZL|G=AWR}ikGH-x5cS%WlZt>I`kA<%!uoz)|GAv_cah!+Y7h`@it zu8EEaceVwi{ki&o3?rRBTObvKrW@02mSpQ zD^OwQTHZFUnC}|{%QT@iPD87>s#7FYY?`8-wKN+V%kaw0n!Qwe`tejf7W4-uxMOEn z1(Vn|#vOg*_XsP}L^5eoi@55}u8>pdAD)HjFyCfJ{bvuXw4_?hqm*=Twu)n|8EDp} z+eYBWOMCAP%?4JsQxu}o<$vt%1F1M%3xE{do_y6?#u;H-jLua7VHFrsRou*Me7~Cn z1C`5D3zl~V=mjbZ4Z8f}+vsFcsxmOvll-$BZaszJJA)xbGmlU*&%|6w=(r4LQP#B8qZL4?#bw;rFwrUZe%Cl!17wp@m_D+-Nz5%B% zt?XT&zh679<@}P7y-eRZC5R&rPv$LF|5=J}QFnrP@w`@`o?^n3b3RXV&PMS=L{r+! za7Y9Oi2VBUB6J$%+V116)8xd1RLkFT*ANt*svo|*ApHGBmxixy&&1idy8M>im6v+s zJJG|fRAA@k#dU*&gHSrR)2GZ}tv0Z_0F(hXtyrUQ8WJo|a{ ziMWa`=&FqTs``tea&48@-F9e+V6{bmb@ln8%ChSG*rBYNCc8(^^J?+vcr*X*TlY&* zP`b0Uu~>NJh1l@*S4;+P70*S3Lo7CoK;d@+>MMofl3$_1TyK>=!U7BZw?Q zhMTsbi~7e-*pV{h+ncBqVqK6o#QCoWmO$-zcY?U8;xN7nN;fJ;JVN(eDNSd5D)gSH zWK>0(S2{&H?qD%k!{JNVuQ3tZ8$}9(;3+1vRTU|By?jbx*xdmg2%wem?uec@=pCKR zAWz(=cMrn>0cfkX0ZqqumOvoKy0-yri$e3lL`0*=|UF=Xo;nNB$R++7AqF1I$2 zZvCA#{pqrH;X7TditMr4QzKe8tYz#nZ6q5vj(Ua#bB__qb=;@iZ@p%Qg^;6D$z44A zh(2vy6$HmZxDhoJsL3VEmK6%>Dlcc!l(mcUV1(oOxE z`#HlgHd3=fPzjTlsAlQngunQjxoK8k2*4DNg|*W#mnx?YRFxRP7EwrMqU2oRtpMtL zxPS_E(=%Mu@}{MByzHWw_iUI8VR?cgJ+c;FlPE8-W=Lmm)AL@i#FS3`Nlhz; z;VGl5;Zpo)d6Bv+f3=idnJQ{bP%GC!Xo6b_hE|$O4RtQ1jx#G)d`K!=l&<^(h*HRs zB7rxC9LE_a62%!yf|{!|a{rJQm%q?~j7l6=D332v${pu@2oAIzsn#Y}EYi#wau{h| zT8}|=`?eb;fUz6p&v2*t%b~K}6e!jLKTlnsUkmQxi4+N7N)oF?5 zTBO*$S5&^zn+N6;KzZ&f-aD|>ic#ydB&->9(A|2Jp9(0;J(sv_dtpCp-h&4Q4`;Z; ztbWGaBh8)gv1G?HF&5~?h?M$s*1Sf28_pImEJRH-6bfwR%*v!e)#l5NRLD3#{KS_? z6i(c44`X6_Mr|W0@*mtFl(dVhZY50MO0@|5h4iNR^AspN6*hcRG`6RM{DAR0fzaxx zV^i-Je6DP8BQrssMCdS^h?JD)5qQ7|;!N=(g7m^(s~Gl)4Og&J6psP1 zS#-)hqqC_6Ngq1M9Xll{ONfA$!bY2bGt!YanL@r}o`q{yizFpUBV|F>HmDP_qul~O z^GDy4;p&fuVEdH)0HvGx#b1NSl4Sbqi(j-^TH$Q3^GH~D=O2WcMmU9To-E(KfTN=~ zV)vf^#83N&^X~F*ARv#BARzqz#m{|KLg1d#vNHO3zbD?#8U_w+git1$g-`;5xUe$X zC=rVy`=D`<64u&G)^*ypN5$PJmeHfpod_yHB+?h?m&%yy7VEe+t9F{(>2}@&kI>fF z+s7r>cOCjmY2tdaNYSK{BGq!Y5@&;41-eS~ae4e^_^jqDow$f2$l+U8qkzFuJKz*Y zyf0<&oj9~w-Nv@NLb^%a;e>7_rS~eQoL-xTuFHixV-<#6M&RBE$pouJrvuD%_jE(T zB~*fuRYWbf#*~#bP95h(cA4KKmjf@4j!Rod?_6o=<1O>g-bU)3M60w*-6Blln@L^m zRnr^#E+18zbKXAh@xiqtiJ8OP~2gfQxKj?}%vZ1~_f?(Z9C zL1*8t>OW|tYIseB)Gr47Zak-ZJu`wqm*TA1ys4|yeHr@+$MJ_R`zUv}tj6}DCO(?F zW|-yiM?*!7TyX)@e&&@7I0EKuwOJ;{&^GR!Hp1gDU=P@A4c{fePF)5E7=SV3jU&u^ zd#Eh(1m*qk&ErnQVr`y$*p%_iIR%R_(>q4QJNeIT4pq!2+l}oKFM5j=*+XU%j{wQA zT|*h4`o-juGLfXPh%Dq>_~hq|7U<7%SbL;I(L1W5?h?rfd~-sIc}U0&M7L+e@-usO z=5d>(eL2|aGfzk}5;vLEq`Jc_ zqs5Y&%;Ky-dQqvn2U=F zOw&jV-Ln)N*Q3%LSYyx$ROZ(q>N#foT1`&!0S8&Ft3nwRVG%D&0Jdd;1%i%e5t7B} zGzV`_Y(mT6gTebw*5Ia#73$3pH~W?>3^o3vR7h{gD8*E+DShpw=iDgybRfx-`BpQ& zuSGbVQSGry{Na0-L$+z_pusx%wy1Gh=C1r>-6SPYk7W{dS1`&b5t zEzQ7aO27)*RV5w96QYtLuJ>RA3jB_2a`#38E9Y*J{li9W$7iPf6V`^JoTBPy7DcT3 z`Z-Ny1zBK8+;Yb9V!Rr!BhXZPv?oqaTe-2q3}zbMH88WL=T^Sa5@wyQl{O&5q#<3e zYR58zpYSxGw<6G|oQB8x`KQu`T395ovNoz4-fYk{FVkk;Wd(rLm4k@ucY#~KO4xG; z&h#KT{d?LxwY#!wrShOhm&1UIjZ2~3>k<1!Rq@vvu(N->cGf5T2XL}eGRQ?XOE8_+ zUs?=17A7jj4id&@jsb&G#$gk46NK}NPIPR9aVY)DON&6w*03?B`6Z$+U_HmM@5@aIrq<(LqW+FVb*+6#Ws3C^mh`AE%ol zMtB;`f@JdQ{w+;T5l%f~wn!IcE#@Jt@qJRJNR0}6D}rr%nzpt{ff&dZ%>-ScVmm?g zzNyc&qQ&v*w1&`RS0(ia__dIwDCMU*6yhvkTIg?oSeRuZuxppe0mTN^i z*2P*uOiV8$ww9$R>NwH7toID>_M3muuJbZpxtb-f{!1V>F9X4xEUkhW$}*1c{PpI| z@9XyJy+2=1Tp)5CR7YSKXdMNI9oS=y^hZTe@S5r=jkHH(5O%cae)MKQgHaSv)drF< zIa3`KXPpT;*1D<*m}dOC=?d8T{E1g}Nrf?lutJUL#vqQc7Dsy)X^eIYR{UC>IGvm8 z>x^B7KC@&>E!XVr-NQVJwMJFgYdt|Rk7hLOIyDzhzU`yHOBQR5ahBewS^quqkrsi~mA1?%c`k2!r9qq@hyJktY{Wzu2NN8NPL*fCZqund&7Vh;v9i$|t(GgzQQ{bCshm$73o|_+C17X%@WIp!y1!r7O zBnLC~&VgBG`mY498zb+QaY>+(#d|TbvLv z4K`g8D{8%IE&1}RO^fq3Qv*mHSg-B1`+^e7A5Jwc;e>))Za=;WBN6q&4F`CR$Mdk% zX~mY<36Ag)EZeZ4C;L5Dn!YC#qIUebb&ywbGmyT`KrW5~LiqH@2b(1Ip`ofC-Y;Ab z=%z}uCS`64lmN?QU~iVV$y0ve>=t)of;xX&CeJL8H=jWVq>=0tcM_IRP1rXJz<+*w zqe*%9%eR%F{Tb$~3>>MX;%@4NV$hF&ywyaKa@E20;eJLdG4)rCN?c}PRecm;OJ0JN zn2e+@9+jyChBuCiFagZwS*E^(13zVyMvYzrd!g-kiB1UC$t|sJ1GG=c(5=&k zsQbbZgZ(g!P17t0l&&CA(jB0 zr!^m>UO@LhJ~xZ3O#f#x_i3j0mxF?U6#q?V?tdh6-U&)F96vJ{x<6L4AiozV*8dWx z&;@fH|AM482oMmq|8o-t@JbO)1zEuJqw%UKMHO6J1tq#gZd!Fj2|0#?QW6{uJ@^mf zRT;gbXZFtMS@A3QSExAgf0!OuCoJ$E;MPl>Y_3*wIhmbT`Tasbu#C}Wb{`XK@-lXF zWmHR^P#Xj;ld6oxf+BKpiHJ?~>&Oir3?y}a^9Yyz!34eCpTPXibLV!Gy3oEW&Yw6* zhIB0g=_wFft-}&wlcHAo<>ei7$n`$%(S=T%9<`~tKg&+~kFw(^XuUbEYh3p*J&t3- z^*ja&>ZgJnuvv{d>rS)?2*ELTvZnL)nDGSb`Oj#3ZO1-|ga_totxMKL2pn5a)Q6Q?=n`#eE)1NynuJ&48uPJwQ zj({)t^IEKt*H+@=0Bpa8VTQ0l#%zXw5~_JUhUr?g9Y;`tx(X}Y^9^rtW=Wxb_)j+s zT&mfmmKIC*)=fN9rpVY>?%A=g>8Xr@>F5)m5O4JoPd63sWXG0?}Iqw62ipHvB`4ka~b zzdcfVWZjRxZD!CCh;H&F*G;$;69D*;F^(36)rGec!BBJw(IPC=I-(~LEmzKUET?EJ zk5F|*M5ah3j|0$kpZxVhCHV@KIG>RInF(?AFZXH5{lc=q|Fn16y038c@n?A1Yna6l~&_zku`JHXY z^SIu0ma~1FY<fY(69) zfHMlon>SWBSih%>gZ>une91BIl%-(wi!W`{**>{JYG@~YHV;Oq;rI4a>J+>#SyaWj zeV+%lV#qmkyUo}Hz_U!bl+94RmVDC_l(9Ws8W^%a@E|tSm@f2fs_dfBy2pn!j&Iq} zrl;A{nq%l+?yC*xfxd`(v~rVgml9#y{RKGhmmbR)CUO@IcT}9U3m!B1hYoJ@eKMtw zUF-LFYi4?JY3H`>#7H>I39Jn<7zpv$Z9agSh=HoEz!OLh>wN7UF4HC`QI8x}<}i4m z*t_R0yxkw~z&iBkQB&+*MTW5R-9O+4sN;a_1CdAjvoeJz-e3tIPhM=uv(&16RqZH9 z8rDhHrpQjXThC)>_{FRg8Nu)DZ<6o>2r)?(s(Et<^UBmn_E__U6;1_}PKC7D5B-wW zw$JGA89rg)PFC|JzEL4RzCfNJx`m5?(r4z`QPe6&Fz@!?%b6#j5>Lc34s4F!m}ULI zr~T+jTTr>Kkdt;5wxgsSk2t+^CK$}{Ju56RdA2E(D3IKEZR{KCnYDW^&K+>AirD6G z4=|c4I}fR>A0GLnj6RsMN=#vGaFpDmYLZU~L^hv-EKLU}{(xVbWIbbS$KUeM^GE!G zmz;=G>m!1IPv)XII2X(+`%XI65BAT1^_pga15d6OlLdZf-T7@g!HJExx`Pu#iK-w! zVHvbJdTm%>VlcIJ;$ZbC3- zy4aR1*cMaCVWq*sWLz{i!TVf>$wxwZ6>l8884ccG$aX#X`e3xP!GwxJ^>9<_rZJ9Xa&|wh$)!wYRWh>RnHzY+lWWY638>&B zcWF! zqi2Ug7G}JzyXDW=u$EsPx>$DTh%qLF2lk+USORzX-uEz)0@L+e!6dLIB_trKWvCC4hfn#Wo~-40JnH9 z@;iegoEv`sfLR?KHn2;>hnQ!^T-;X&_NGd}TZRH_;U6sq7wA9^Fr^UnOtW`CL67kG zAc6J)mTpN?6OH*wAog0I$*Bd{XsH{NkPPZTM}6AUX_Avol7}^f2yfEJ6)I?FW2#cO&=1Nu!!TQ9s=`_n zs13DPWMs0+P0O1sfr_TRUHd1|Z^CkZ7JA-vzvQ^i3ru^?`}-ViJV1fSr+<1|50_uu z0W*Dm*GKx`v)*c}ee95H* z00=Ib8;p+iUh?o*!^riKDoLjpYQ3k`;ic4&nP(}3hB!r%VO8VI9IE@N2P+biUZo2O ziIkK_$XaPCVKJO4A!3Gijj72iN9Lm}KuLiyTF|4E?YoNuEg#!Hwfnm!d1WLP^P))E zO!EnIG`16Uu`+B`bJI5T?205*3HH?lRkQ~i2pTq(T1(o}?F3k~=+pru^B}FVMOr{s zR_1l;@+z&BmqnN@odS=VWz{adSk--_zNN249Eq!jXP&NmpB5(dYISduaxW7r(7Z8n zHHAS=d?UH}45GDp70^Ut<$HqP+KP*oKG9j_nWvRT3$~i2N;NI7!+7d@@{Lw3UAmWs z?pU`;bO`81!Y*CmWm7_$2IY5Qj zay`V%gBvpjimBNb4JM3t``oV@2vN)g-yTYqJv?`OFU)Q!b$Jxn;V^+)=d}y-a}74k z0>_Ywx&~p)RR2e2ZCu+K&2?!PkE2xXVg?Booa9+zbI_u8PoEjs(+_lLE|me%ZXVQD z-&njRL5uGfvw;=4jn@=GzheSHu;km*j9;5h=xozgYwN4)44MMGpQ142fQ%JruW63( z`j{1G5l~_nD&3>Is*U`}<(dbZ>gFyA{!c@ta%)uF=|&C5HMl;d0|zh=Lkt*kZm3AQ zBMzfr2d`7Y6~oU{kb?;k&6}aQG6Z2IwVHU=RCKu$hiLJLH=5Yfch)Kh?C_S}aJc{F+2Bm(sz z%4SR1@Rsjo=IHC@v#ITvcu?9)fAWk#%*>;|mG9D@&6sS8ER7n(qJ}X}$!<%IkivGi zfnQ+v>#uqC#94pGUMMsPt|wzcfGqSe*lbjQ@~2(3FT(b#GlJ(mLf%g9mrFfi0R z3gRQQDce@ITv&7Rcv!87&rJ47#OyI}wpD%)X%~bt^|{5Xx3{x-{5THbI zZZ4Ypa2LyVK3DA27$o51WR-0%8Oxt%Z&`YxA~ULiOsye~1B);GBbPBPi;l> zTri?_ILsaI=r>D91k^@!CQX*$=ajRiI>lOGwzd@l8G<5vuQE82sGfc-z)VsVcq|UUJ9d&4ioxIk^p4Wlou|i3+8re>zwMK2?38r zTqHX$rqplA7l`mqZdqQ1Iv*DI)AA%|(H~Q{!fFLDTZK8nU*iitN28LKneQ)U3;3S> zK1P!}CC|37VLfh|Y$=~jnW4WT1dIN_KDWtQ8}3?m$nQ^_<4+q=5}_j-;)%GK6TDRu z?@4^Si>(7{xJ6jndq$Ej zk9{N_fednv33xE^K86a_I(|@|jm9!DskL-WCfntpT207$sIihoY1<_!gsWT(L4}m5 zEJOY~^zGV!>TS!(t10BYw5zmj5>H(~`nL7tPRDxE&4fVOcD4A^vtGV-+i~~3z?!m- zvFhm9JlpRsQroS^>irf_O<@5pB#R#e7mdqFz>njvV@k9Q9LMHt)WGo?{DRxv7XmrL z1SqyPRf1SZUJOER`YiIwqvP#Y)^I_sNtiS^^CoGZ2GrG z*`JARasbZ+}NrixnS z7~_RuSD?$D>h!GW5N*}J4^2H6ctOZQ`w^SN<^V<={oW7#4D1w8sMlQ3f=kmFOqrFk zv}Zbpa~H5f7`gbx;PN>c@{~(gV9EPd}v^P^Xy>%Xqb4 zxDri#4uiV#$g;54=O0||Z&rP8mdCq^ilV2jp4@6qSQ5nR09V~?f8f*!q_p}pXXS3p zU~T$xV|4&|c{a?ak1_6;+%00fWpu2R>};m|Sd&K(;YEOeYwrA+U6-5BoosqqX;(*8 zz?O2BsPPOFB^Y%#MpilPp`y78)xP1frX+BYt$2megrVy#>d`^5TQ!^lIT0UnNqSHM z7H&Ir%$Y|03D84t$;n~>J~u&lFwR>+D#@`gE%wn?tkFo%`#f2Qj&7!WCG(;r_6n1E zKcDj;VO1lB_Ce}o5K?j_bZsI{OBHQRfCm%MG+%*9QF zdXZm)Okvrk-L>|p|LQ8V>Cq1;efT?&T!%(5rxl$S?b8+h0ebuhWWqCRqHt8A@5~k> zXk0rUT%7jehl0Y+inM#LaVoL1oUw!u^4VA!HJOPlY0(oMYkx<#mw2M+l3F0e^n2>! zb!uU=KW4_TZzTUo!VpFF4k!@iH+ zQb_n6imLKQdT+8b5X~`cAM@GOnGV=f{P4N;{)zbIWw8F)t-(++JyhI`kt=tQBhkQz8xi|X!@p~s8XrLY=>be(%_lACSA!#Va zH4~O3FtHi$H)yEs%Tm#X7mOz>Yrzz@qOcQR*rV0qSNDlcxk2JoWf-_m0bdVDf7>>| zZWPOV?1Y2uW(Giod6=mOn-Unb>k;lZ)WcC-4ak%{cFOnxu*THA{qh)NC^1HWmilwU0CeB(!;XA~g9= z*s&NH5pXK8soYrAuO2Ahtc{y$B&pW+ZEGxSYs?0WVx$@feVt(pDM#xYLHotX1jW$= z!()$t)@dTvi7M8qjnGjg=zs| zbjKX_sqHSYzhSnH)%2OdorC>CvyZVICskmoO#i^Z7aJ1(1`#@bAdD&3jBk2@4m)EB z&eEBqcXv61uj2%Lgp-dn5eyKIU->4LrS3o zRk11RP80O*)tS({f~WQ}!$c)b@kC>$X+_^7{xa~_v!teY&nEq3qTAcugN?1QPS2)3 zS1VmjV5xQ6vYv(A3$4+W`e%8DCo~E_kI#9`1<0VP>s;Bc|0s76v4Oi=AVQm{>wF8Q z#b8bKIGEWP7h*h*Jb`&cwKif>amK#_vc)=3+}-&3IAi#CWGxmx3Iu1xi-`lS9h}C=0E|umAtpQJFQ%9-?NMI*-GR3}R!l$FQcJ&k3;kTg+{sV^ZZT8< zQf*w-QGTOBdaFI5V%~9^R;6Zyq8F2-C^u5mn6re^laOP_rSKaWicZO{${BZ8E|d>F zF;l+wY!286%@sB5VDGURYHT}-;RF( zV0^OYR~IxGS3XRgDVHnM{^(xiQT3e~XUoBj|iLno{kdnZBhn-PCI$aGLh_lG+{Yz9-yG ze!0UVxNJ9LK4BW8gMI3_F!M@XLa6iQ`aAj*^D~|$(%O@)^*6ZL9h^dYP8R=M&aE?n z99h|*2ICl>m)c1)zWh%ZUts@`9{d&m8^g^POa|^Ws~=ik8&`+63yabu+&%spZ?n4CTS%I%T;FfzZ zV5wby0x5ZeI!9K9xig`C(9PKU5a$IH$vS--zvW=%4$K^j{FPUA{{}K!Qa^I6QeBC6 zdPLNQUv=(i&o=0&U1o7TxqHFR8R#|<*)3mE%PX#^Xk0h)C2CD2=(bm>=gwkP8(#P9 zj02~!b@zg8PGIS81wY-C>v=M&5sz&2qJWfr{^EWgTJ!_`CL~`IpTL}A=T7qKd6ZqyAd&IHl)*CNy>Y*!87YW(I6|zKP)lN>=|#^5HI@ z8z1=?zq^6yL0a;hLWL?@OC!uk*E^?mi`@yGEe)+Z*KkC*zZT~?xg4WFfCv%bf_`zC4p}RkirLY0OqX!jiT6jl2vO5zFna6Es%ZE zXkqRduoi$qkwB~24CqbK>Zabp-rBT3vgg|{lyu47S2#r3PQw*OPrd+4q+w-V+C$PksRZ-3e=6)`N%sEBD(sBLn*@_ft8; zX(q7h`j-H3!+PoO@-Mw%@_;Sw|5&%SWdElVMl?q%-~E>ci~Kui!Ipeyi=QlMhY!@# z!qY(i3In%<7GqLtgrJr536$T$@ez?LY{^iqEu-NSE|o9{ch1W3U`3nH`Og1#m`@BG zNEfU?qLE8~zm5lV&Xbqg#3E7J8bYlum?yGI%jaa?v zSHognXH(;1Ya;4?%dVBxhgFludT3Zs@Ir|LA!l`Oo(G1pd+ za_(c1>fF&KIY4bYQKP()s_Tc*-)d!4)ycLETg(LcrKab*@;D8|ZK~C|O8_;=gig!h z<~I-@z%Vs`Cnb^A%$$BPkPa|6ditD)$wj5+?^!(|cAAkI!tIUVoQFPa40!RtVxXD$ zNl6=OCT`O;5vZ8RiWdfopa`KJc(mYA>5zWN)7Vb1_S3P?S=VC=#ENx=$0|qS8eugO ztIC;RSB}ti;@-^RR>SpUhXO|>k?B`Qdt#Tp=Ev9pkd#JheOTZa;QQUGMI%%sWR+;o#qloiKrQ6ezta?;MI=Le8O%FdVgKMqOTOy%2HE%=1 z0<4Qeq9Nh;#;SfA+y8J*9!9{k({6VWx%}Y2Px&wcS1J(*}(Y zMss+gYjCAk|4=eYPw42m>!!9BjZu$-UFFU7${F$0CSTS*b{PmR*1Qn-mM3jftkVur z9SK**wjo*;DYK3vi&z}u#C6ay=L{$H%;NXG_=9;_h&0#7B|=bA1(a=KpH*?e(-U5c zCe7|K|J;;yYt;O^E}wfgukgHR_zY6Gd5#(QnmF(x^b+4`&_?t_@p}_b=ut6tYjg>mJ|l$^HB4 zaK&@jg|_U))j8g*Jt-+0#a->CYy(37h5Oan zjp3pNWo4kV=d9ei_!>+dHhOkC!n{}C!jUJvu8#br1Dn@%4ed=2DY#tO=4aTN*?ffp z{68U$5tINECcC06;XrWA80{A4y)!7DlDmb7q|l~Nim<{{RhiEq5*^xg9929KHRhPL z5>&n1;joC^hcKl)KZe^5qtirokNM1{kEYq%8ccD9a*Vq&rkuF6KC)EI#KXP^l0y^f zqV%R(vQ1)oWxm9IlnS!H;)GO_sY~2-QEW}QROOKiR<~%~G3NA_X{-b*xMLKSZeR#fdtJi99%71Rd`iJYTn~&6GmQ zRJVp$Hyv|qys$Cqc&;q~jR@!q{V7sl3nbolHf9|=n#2CapI2;Lku@7ZVD|Fm&R@u3 zN)Ft`OeS`s1q$m;6^FDfkSFLiAef41kSD-J7ce4cM+zvE0mc@+z6p(nAxP7|f(XL8 z{1X2rie)gfJ)Yz0e%<@L{>l4mM3!0K3g=<%JepaLC{Xxy3flsSO$R;dpqRUT*xK46 ztMV*4H94|rF`C~q)XbT_^_6KUBfd|ifjzN7VRH081PR&TQm()h_*R2fO6BuQ=&{$5AmgTr%K*?2azdM zH;G>jEYN;V_reljo0QnJ6FdjSS{vVsjC8A9;D0a%aRE%}Bgm-?LECuku52RFErNJv z(oSc{u}vOx4XmeN+k!K5V2Z2j<7Nc~(i6Zrr(f1StbM2C|Ds}y&$o*Ve=u>2G!&<^ zPUqVr?o*!!y-B6r9JJUyfEVtTl$))XpOg|`mz@H{agXDv_P)P`)75T0#NzG{GEw`hXny)_@7UT4*X9EiPhB8 zLeoP3LX^p1hBqXI6$#WXAV-#7{GnN(9!tGYkVeh9^aF8{Bu3G4DHBH2>DiHK-v72% zXs{1lSMy4*TN9|JrS+02Lk@HI%`P`{(>K4j|MRN(Wm4$R=NI+>&2MtD?bA z5pt-E(;34(@I;v8Jq$n+3bGgp94%*!F^(RxGzuJ%5)1utghqNO7d7c!1X8ktv=A1Y zT0SES3D*k~BsAFO<8661vP@xC_tH#QI+8nz9V=iBIPRjF&1@tC$5X5= z_ma!*X|WS*4`~OIs74T)uwADaP|_hDvT@h=FGFlmUutZ){TEyo6o1&pr?6aHvb3eN zHKtanL`^}7UPj{}tqcL45@gM@%&u0OldPac9BqGvV;$<<)3G~k0T;btI-lG)$8ssl zv7XSj@ht=O7)&-FPqvAecuTyN?;dnhS(XQ?w!g>f+$I$2nAiIB$kC&FtQ4OK^#3>& z)EX`_3^?fOVSIB)uVKKJHLcVk9Swtkg1}K{iQ|e2>DEt|4KD^TpU_tke#-nTIVKA~ z6?Z2a+(os@m162Vj8bouz?x>3U@xYdw!u)-V}wOc+9mb#&e@$cOsuZeO$W!14@uUBZXn&w3sJDAeZt8{>2z-y?Jj0F9Ok_W z#`sS2gpohTgLw=DS2)mAhRnDa4Pe9d3~8bDXx<<~S3A?ashy*~lYv^{dIlk&w)ekq zEW!U>-1#0td_z8X1MgVixF)8fRw@=pC^={{)eJ}HasKDz6NSit)>LR-Cc9{t(Pw= zv|Qx;p-J}ZtbzA2N7dKQbOz66Q)yu?YAzP>p2>-uHEsVk2VD(yuCKO5N|npuDz0Bx zvx^OZ9ibCAvZ_vVntyB9)VstI>$F^nZRN~$H~rbxakhzTD-tWP~! zHTKSR!y8c@JpKz79>G?+F7Uvg&+mk>lwM4~jY|VaGv*ZGN@L?42r@y}JI1RGz4N_z?kbMpbw#2i!r~O zo{VCUNvlrNc8LoyLt9%T4Uo=~74jgvQ(>!|o4B!~4+fVfx-D<;%#=&X5TC zMDGIZhLoEQ?z;$vo`rhy+x z$}^nt<&qQJw$7dBs7ppG|Rj7Y|XQ|B*t=R^Pf zH|L(xbB3D|;Q9OiJ&091JTiVRXse$TgUEl4(L_(!6n`&PV1nvDF|o?;A=XPe$?FZ5O)=SsAg7a1rGV2IL7sryQZ_PmE0BWKixf|uA8fGVmvXaj z?T<%!e$$Tu$2r}vFZtUaU^jw2OG$mBj3<4~IF>_+OEzs-BOZiYH!}3eZM2?_r)_ci z^p>pUr1iUM03oVBR!2MZ^PEaw7tWQn_UxyJlT2*Pvs&Yd6@+0>pH=B=IehekaAsp| z&#;UpPAA+sY`x3&C1-bB=Bw2v;i%`-xWG)UZ_)u{q3e%ifri%eld=+!#*sZXmXWCj5g{)USL*MG;}`s)fjIi&u_&U?w;}CI z$~hSNK<=PwriP*GJr%6F+E?w|!;a&m!^qlcLG=b54)n zdXC?i(mI#C0u}^IaQTY7+e$Mwc%Kg#`@gy3Sc!t+oeT>=7&6h?2ZKOndV3 zxJvc$1ZD1~^E2X+ze0B>U+XBQ;W@20pZnjEY6xm@UwJzR<)BeRk;hkVRB{`PRx&zy z16T3vFgA1KgCVq8qJP-)BZ=rR#F1&UvC?XfV{=Rr9S0#JnK6bWMrR+eu@%JJgh7Z? zB2AA)2_u0ot%|R3Q&AuyZC`urV9*m&YI9S}j@&7Suo<;_5EQC8ovt>FN?BoO7%Va< zJ@$o<%x?3Hs0ipu-BHU7pG$Q4Vv;;3Cvz#rd_GEn z--ppDjTWA0fz2`+UYc4h&Ln%lrY?!8<6Ez=C!m6JkbQTjijhK3Dw|X^pj_r?$D%8P zy@`ToAXg&mO>K&4adO(H#ueNmKDvEf96)@VpIwR^Wqol+TiUZGEVm;Tg+*d9a8ItE z>SWmAuS^m4()<7KWmxeVA$gx`cHrkgVgJ7a1<@Z5cy&UTLizY>@A}t5Ar@a+4Np1- zz9nT9sT3K_u%9+{*(Xs;2AwARew@d*+im7J{7vlrhvrVRMAcK2?7J2n>5Fi!I;_Xv zJ@FQF9(GqxE6?5^xzCoK*DD!9?+7~#ve5A+?6QM85CPs=mB&Ti$k+6z5D>(BUBRre z=WWI$KoyvZe(X+i7PHqr9F@C%TmeddNqhnVXeTQevX0(~_`I!fA3dB2*sP+uM}Ux6 zp+6qaqxti#bvH^@!na+T_twA%s@OkAcv{+IG?@#FXY-58N-SIZhcP zIkGj;Bvvr~Cf_iWa$yb9L9S&#Y`kWpa{vYaZ;u7s_5;on;%Jg#Kh{zeg?!J#-0yBA znyZf{kkxPh!3S9dzw~pK9NE&C$007VV`-sQp-BEAy7BAj7Fn`}#xS+%sn@*2R9Fns z#OtiODHOz;7!8@*(W3y9Kpsjukvd*Mi?)p)z45s5&>-8#ds`d}ZXd9${!A~sJlc9d zC;D`CT2s@a?-n}Vtt&o|5|gKlrPwTepX!MFsIkB-$qY9Cv~9RCb_VxljjdPr=S4Dz zTeCqn!oo}KxUCl9+Sxz26XDiVEPU_}r=iF*fifaeFb8JsZx=Y3@QZ)vU&GBhshb@T z+|83oQ&ks7E3=OdRycGhi^Ha($$lvSN*r7o6QsYfC@`s?>A1tie!8)>rsFF1R&AHj zM4b({lQD8gNzi}ao8lE~?V#yY*ys#(a=70o)Gm5~Ur1fDb!AsV%o5niUG+4&XXKN5 z6zgVz+fB?)THGprlw7bD@Obq1Dz7wURIId3#Xw!TaJIb6V}Hkrh@;%=^MXGI2C{GX z8)cBy=(<%4%C@O<(nNh>LqSN7b9z&IE*>$`a3X3M(Y((J^V3OXbN1%pL?RRRq#!~Y zJZS3+GK3p^#n~~4KYcP?3U1mSo`Zu%C1Eyo7H2R(g4p%|S!Fd+ccu@_;pJlT+Xhz5`9FjF`NUpI>#Jyf`Qah`b|38_uy)e2%O6wskSP_E9lA@05r>1Jv=Y_}IJG+8c8J02AdIHVSE0{@dl_Ye#TK7Q_dAkn0PLDUG{y#3-^evfIyd$RT z=l-NJ65?HB)P5}maFa)pEw!^yZ6r%!ZT-WhJHEk- zJ8qkXMSc(UI`5L&@Ba2n>6qRlFjsPW>FI%}C%S(58INZl5znal7T+T*%?A}tHYTNw(;JyXptLTrmXJkO1;zdChEz?a(tz& zDc$-;=jq}&&Vo6D;s866eY4;Kr3R|SGnh^mNH~)=jPR~k-$P$wc zvv?hOB2bI=oHh%ZWfe|-nRh^so8hz~cRN`N@;hq{5X3@+ah|>|DKivWIO6C=AU+(t zBfT0+U4*g|vaz(Zz{nj?24$5RR{NAE?XG3yJQ+=~ryStHLJ-@^@=J53;I)#V*Bw5k zn(>=?)J1>8QVPzxjl}C@E~%V80WQ0k-lVni$aD$4Al~;fc*}2C=8mu1vdgqZxtut# z*WoS@zyd>nqDML_8~l`9xlaKMOZ?TKRp&bob%tqD7Idx^|M^qZTlpXe?8|@@jce>r zCw_~9BmZNoSKozc`kH?4dl2rehYg7%($#>VN7l&o{L?00Z$#zmB8H%avVlNx=tr(J zH)gzA7g^8y4xk*Q)|?#aGX=~m(!pH4|8t@ND3ARi&z+qiD;;g2RB&4xbR);jG`%YZ zzeV`gu29wBfLG3c;Y`*y1DwiplbtbICTD8ghxYg1yA3LsO}@h+L4N-uE9J&AzuI0I za9!}}0>Y#!5;33NC{{r$ox(8Mc5oNV62hO_5>pe zSRkuG4rie3wM0UUI#ihsFED4hSS?`4z~G?pu(jfLlo%|7NsrW-`$e9vCZ6Q#NfjPZ zDvW9GG(5LEq+L&GnmR=8!nSA;Cbd_rL#-oLNxKE+SAM3iM?uPvd2~dRXEw{mCKo)m z!$g0jo9GESXWpN_|@@Q0qVzA-wUARtQx#pYU0>IMe- zc`IX%Q!e?OS@*0f4@Y9Qa!GX7K3PKwW@kuX{|Xn|S+=f+1!t7VDO$QPVSlUx4j18e z#E+O(F&wyx3k#H|eix0m#F6PINt|f1h#WW95T1pvmSXKx@}XwXIE6+f;z&CWE zNEc}5FF2Q0p@&;^ClL7M6QgoYsLj9WGKJJ3G_&cqR{jq20HiX4iRg_j_j+s zC#KNrBR>RQd19^;Ltp8v2tu37ecs7cn)uW(Da1e_KzJV+&z~6>+Yx&+2^AV3w#Z65 zJ~O@=5lJ%dhN0_BS$tLpxH#U-Nfo z%BIzC&Lu*`d*mXJ_RQ!2PCfMg#ONHj-J^sxj!EK#C&Ui|z%Xkx(d2!Qs^Ht=fl?Yq zFJx1%?lN%%gPg=ngzD9Ft${Rz-1(ae-|b6vlJlr(^angNkd)8zBqt0X5R)hFBk$m( z0X2X(K0S;=-nnef{M*JT{fHhK(3rgC2%O)kzqdmCfCkj|op6CShmZLF?L%a?XVt5_ zYgV(5g2hiG6u%=PhxMC4Xzz>=%TvS~Io15rD6|mN>%9^rnN^o4qlkM+e7ME?;M`ay zu??9+Cj&apADPje{Af0SYHCK}sJ!$smVKruSa*h$-;xHq;njCo?2Xe=a+ioF{X>R% zH?4rg{^tG-z6^NRTlxFf{jdJTbwX4mVL(9kd)4@p+eEBTh*b18!XQ7Z9SSxHP?ijn zqHgooEajgmCgKO_kQRFz8!0>D#}w&=It%P*2w0|2k|mC{@HnbxlJ$*jc}2SeDZ&e5 zxAhJ4vljHftS`@Sa~&e{QexF~I?(^Zjf82;;ZQ@|Wv2nIr`EU6ckZG)W}a{SLUa+k zF|=dLY3-3+pLk-Em_PmClBl`CzXOI6kJ`B$jdQc4Ue6kI9 zG^J`P@FNN(ZJ(ROF=ja$K8^p7X#!fAsB`4aFXVO(_YEud`9HzXbR6P@ps7S+x7Wn; zcl;LjLEy@e&y9oo$AqGdMte$jE*y2^yH)7Ffs53d@J+=pY&l74nd6tc+Ln?rLwFay zry|T&YwDC2py9@mPm{yLp7zg|R`zPU8>45ZF(@B2hvN=Vnn%C9&l7(Mg?jsZ&K|@i z+rDC3-v^nsXOEP%<9E-}aK$qjhb&i}@09o3!$8x$DR&G%Ja0hp+!)boy8!hyCKa~d z**XMGB+D3_{_xp0E^|K7CeXZUJr$s#nl`s$kqvVLEw__JT#4IAa61W)X*aR05Q`vD zZv3SCve^UvY`VS$pXsuzE4RCD)V_smQjeP!=6i!>dO$-f>JsT3lo4EK`|z@ylN<`k zM#lfmNe8M;QCOA%H{AK592Deo#XF^T0^C1&(7pRhvk$sXO`esR>DknnF2@eIdD6-7 z(cg!ACS2mdNt*p|LTfVG>zUM_meW`sPi31dY^I&)W1v3FNgrl>rM2t5+=2Zr%>S_A zDY5jktHRf>Ij{yB2N}Th_jh~se|Kgk>L8`5BV7As|%K&He z&%;z5wJk?{&I*!Mbcri)e-e*!@Z@WYd+;EPXcA;h#BefJAIE;^;C>Z3x{UBZDe<0} zq%BKzwQ{Ry8wxjr<0P1JsvE942RmtX!Cv{-wDb2D;8 z9Dxbk;l2D@+&LR4b z)e6Lt=l#@k&+3xcvr@CwjlWNeL97pY6Rwod)p;riA*X-Fe*Pg?qU#6~2*d&rwk1Dj0UY!#25u zxZJets$>7aB-4e5mr_02;fn`V+$1f=xO@()rEQ5VF!crcNo}%I*4(bZqf)>XRMLt zVGtbrB31S#ia>1%HMp^&tT~(`8}s0Ez#fc)Z_K^j=yFLpw8Z$8{{L4zQu7qT|sT=)|@`ribR$0-MMKqu7j_bkjh*3{NYp8NYe|IW#Mxbq|?~HBW z8OupwvtO(38={Hm-APsPYOe7+XYc(eI=M>qK}T1EH`{k48hJ|uI-UO&c}wzsG3Qqi z#>LUHXO#MFHh6k&tvnFRN61}LEV`ZBI0o6HodVI=Inn=%vh%V>vfsT^@&JyAch zsKJkbH4SWuNEgP5p=<@>yA~;^zRa=%+U6cnntg>#bvqK=kI=ZCxDXH^@qxH3;|c}# z7@uA}Q}6Iqf*rw6kVt|*Gkl34|LL8+?tn1R<`$&6XUI{zgR1*tKt$56^eeLt8`7=! z&+cFAOZa>^Hm9&jabZKY;7g?UUO1;0`glaP3$MY=R3OJ6ojqA8VD~|xV80|fPric0 z=z`UYLF6FW!clC}!Gz#O>`K)B+f<-L$}mNg!WW5b?kB2FA0r6;g%(p3+Vd}#3)_2a zAQF7q?KezuM@(KN)!!7_X}0Q>bDVK`%O|*Tc}pie-&5$cTuiYCT+j+x&@t(&epXYF zcBN+^wdTv5Wd4+%Mzd>%`ttThQ6(xN2!4z#Ju5HzJ7YfyGoj;6w@>CytA83$7-8*UqQ#Ncdv z>Mf$iX=imf$4@gT^}Q?ydgr!Di?!R*;i&q1l>2eMfRLkt>8XQ>w0vb#;L0k;@05Ih zi`)g^M6sj~1A@4;j2| z|Cff4!hC_7@_h?ECD9Wmg&?K#lN^ozKTpOkNPbeJc6|0q{M^nO{H&fxDb78`_@9)N z-C{2ru}>PL20o=Aj|NE7JpE4qp4*SZOgVqudONZ3rFcL+NyUWL zx~nYPp)7ZCDn`&Udk)IF?wWA>Kzd{2UP{K14xck`;|-rlHnlR3ELW z1e=gJCTUjE5|CS4#w5>mb3am>6*T8NwO*T3D~p}x_2s0`Juz?io6(f)U(4mt=CBFp zsfiY?Q&*l%bx+8~FvwTf2U|a)O{7LybMgZ5x++*mvglM!V}U3ZPATR3kLCJ;hn8IR5jkSfzvbZF_ zDY@(t576}0_FeFl9gYD<(ho**hvN^Cg8It}Q*1yI@(ylrBdt)08B9Q=)!^`FiJPM( zP4EZXWU4!4or&fSuOm&xRg)E@p=lR9GkjM_HhBFv?>L+KeZ*j{+w{J5Z>ix<@ax@x zVC0*^=mbc?5o+-KZ;zjJ@Gv&G(H=c!D`b!5Dd2Qd+vn9*Y5dwmZ^kph_RW>{!9KH@ ze}w2-zt*NgNAt+AzfL2i-gq0L1uibg;!v(9Y5y^!T$GBi($6*MC_H2)UR$u)7rFT&u4XP_?_VZpHGb zpPd@h&NhmwBM!&m*H;ovj%~sDhzC|i1HYznsHHrdve1gDQ|3!$Ruq0Rbsyns5{=+7 z`Ssf)XJ2Yh*|$?jsig|^HKp0<=y+N%%4M+!4nCNN-%%3!nzb;v#H*SU4O2hJN3DGQ zb9c+3$K({HfN8HxZGp>Hncu2@hC%1-APZ#8r%Y@Sm^v+gyfFxHA6nV@1bV6)6E#>d zE-Y6rX=wG5jS`0DWjeS2{+m|+i)Q%L*T>VSO9+eJGmv$Dkig~`kL=;v*7-r|fFWg8 z)QF^3c{V(2L^5w4to9X-e##aob-Ao(A3qJP$_g}yOjhT*oGym@)!ZWfcSbDAeG(|9 zo99V5lXkNvDwKAc;J1uf3p1tsDOFv^MXQr<3WWiHYl zi?@BW$){XS)>}xI(7{)cuN#DAe13b@F6oL{0AqAtja@OxSd)%S$IFjJx()$G6RxTSw z2tb0%TO{n;cbPX?f|#}gE9MZC-+t#sXfQv|2 zRneo9{nOXtW>O_#Ukv^~t5P}u-B^c_QJtCsMM8v=7DFBk4xD7hBFe@6vr^Hv(O2hj zA^Z|$;EkZHJX<%_Ho}}rYeqsyY#M1AOCgz3h|MN5!5U`wNtI)o!5q7VjGQ;vFRK|{z|w-NSStWS>-RU5W+RiT)7Evr{XLb5uZ+VIyS z9+~$bex2*#`bO=2Uy_fj$~GT+ZG^*m*>{st_pU*aDl1~XJaj<7B=aH|WK*SRS?mx& zhosu?JkSA?!zg9{twe6Y3HR<-DI%kw)SYwSPb;+BedSl@hC&|0Z-B9gUcqNnwwisF^I+8O_FP1j&no3I<3VOUJG$r#lu>Ua&Z$s-P_XG$_6)px}4dG>hK z=fbL}8Xtk1Lvr^ixIv;MTe7@!MN#k1BSw1E#RVf3u{Q3@sd`8k#P z&2@9Fzt03HO=ybF*Zbf4J#M;O{rJc87=66;JZ~o6tj;$fH>^*c;OoQgU75>u;@=Va zbO=y)9qh&VoH(YAQu*YNCQva8%uE?N?oJu{rnZjl%jfEKv;SB(pVQ23{hf2kdwrvd zdX2GmdAE94>U8U9Q<=M6{F!?!D0S@Sf#hq$|Gc2( zGC$l{h#qtk1?Fei)_hFdpJMw<2)-xRr;ch61@^idUGCbMUj-r?WzVeKKHRy4@;icV zzIK6#dt{E0eJdrNyyKkG^YltEGnON|{ON&JIn*A7p_j-Dm%hQ)bN>kUz(qOYLm)xs zIG3nLZp+K>nW$%FkT@er2?ioRH7ft^MuZv25+cV3G>>1q76y4KhyrY2Ays$tczXo4pUq5~A6J#o1 zyMDk(N}Fb8=9jVD5dlLMU~+YifIKATH;Gu-ptJtRW@LY=Ut0NAGdD(bPz5_E$4`L< zO}55@w-bcpN~o9YW#u>LNo8fwJYk--h!LI12oXSNX>W%TYMN(l#nL5!1OK~q^Hd%b zZoI6+9_*)uh|x@=AF|m?0BJf}A4;U`T+GsFt8Mjh>!# zDU%pgfJ5^zdDX5k6Ygmp9*{?8yhCME>8|~Sss=YeW*RWUp9_4yDr?-7pV-#Oh99XN zK4Uc^q5?he9jBw8Cp{r?$`T>`+z;)ly$k~=)!8S)P6o#O9A~seg8kRyv z)C>Mzp4Iv;>0{#aH}vn5+@x$3IWBL8DU2^g`W1T$poeny%WyYO8Er+ST799}O^1=m zI!z9;0So2d7;7ivuIc$9AYsUHl@_b4>;*~M<(6`o;%XPbjcV7%z;abSGkSX28c1m$ z{mwi*ge^O&=HebDhZ|KBmBMzh1j#AlVe5&2{Fi!vRmk!x>q208`Q(i)7eKo0C~j1A z3)Mce-lH^7#>pVRV{bt6IeKF0F&)@XXvL{5&hK{&20 zzjpO7D_#$@BfpahtF@BVJ@h|V>c#9^9t$(1;?y`ow}I=NiwTs?#}w-+GYn!SeFQ^# zvW9+UNAW^`G3*ylq(wfc-AtMoacK@D!N3CMp0RoVsg?_g?)yz@SNO3kRO=PbeRn+} z=$OD@rf&OF&_TjBLFe^*wO}jOYee)NOP;$kmCYMq^EcTb5n{(cB%9FWPCQ-6{`?C<} z4^KgKT*evGh7I^^yPt~gXtu7EP)#8ilGCSwGZL{(nT?)tEV8Nz#ZWUAU)hwd3^FoV z-+uZe1Cmx zx7@jpegtmgD!M6H^meG^omo-kaxtDHa)zo0=(G-O2E(q*N@^=pxn;$FO0lzl8Pi7r zG#>^9w?_T8$Xpiee-WZ$>#g!c6AN86D28-OLD;1G4r)7QuRjFpWy^5wXA=-!-~yHk zxK0#8i4L}dK43Aon@(Dx1LBETPDldQ3T9@l5IYjCxSqV;{Br-1Tnpj7QGfjG2X=*u z6?wyD(qYl#%^SuXnge{d_&b3+cfga4&?22Sw-`){{^w z#XN)H8*C^)t^BdTiz4&=WpWYW`;1BxZj;qFl}v+=N&bCo{lEBc#DNM#_2f10g>6D; z57)5aVg`ZEg=OlvZ{ZCupTD(k^vFsLF|~ed?X*45);yi|k@TV^LHmR}eOw{G>NnabvAF%%>tbweq1#@kfumar?aA5QIfNG3pNXRI z`)Z~UF17mCF!XzRpp~^;bLnLFdg&sZ2i@YmRHs|*ogQ_G4`ouaG+zV5aZb+;$(u*Q z={3t#q!8-GdB)hyH6o=iP%TRar*9VUA@0Ny9C2!tdC%T5yG@7oax)I3pV=Qqg_nTR{K{j-%i13I}HQcxhL`TcyY`PsifZ$NtLWf%hhhS&c zxKoe{gnLFqPT4wC_6xdrI7!i%hBBUI7B4yKd}J|v802;bs9>1JYtT1X^bOSEiRl&R z+FIKUhXwFdLE`^x_E@v|*2@_)qpktFib5!@yd%zbygFQ1w}h+Py=-ia5BWSkFZ_KR zYb&3@;r;z|O_PNg2vSu{7^xjjGss?UcE$cTOqFXA4XNB&_UIDRp2*%H)AJRtA9M9faR>cp z0^M)(a$Y=i)keuf&3l~V)ya)iod&E}w+jX~ON=3THCK(sGvQ#-35~VY3Zw__Nvm^H z*VCd#FPjgq1=#eTLj1FYpTiqv`wA-k~fvbW5$m%H{ z<^qPWMTpLQz&*X94uXD<>+=w-$&@B`iPOkdr%{%A)Ftf_v6p^>jyAW}(lIhZG2I?! z#N(oQ)P{$nL>I6zmF48-S@WDH&fae2Qde6*PG}l6wWMLnkGEx3Rv+cskoI>N*0zvl z8ch{OM`}N1>m3UkAG_4*4_CNEhM0NtsOz7jgMespbBC|$-x66E5q(5yCo^@ke2|le(0J^MKO9&ezi{WE zK8$DFQ3^+x3<*Q@jx< zG8QKLo28-a#ZbO1b{pRFF;uV-QhiYu_<aqX3cHw} zvO_uoj(=}s_Bfe^WpIc?31k$m#$~-c=A3ap+dGW%XqOxB9c<#ONBP?;HjE8~u=>14 zq7#XuSEv&(XDu4w1G#bWYT+e|P}Sc}=<&+akJ=Q}XgQc(&_~#o@_Xg995}8-~{&U*9Wyx<5`+us+*u6VtJxIHfVs9k8CVvVm4+kxTT^2$Xg!7p}f2 z$KM6vAzN0Z(92R1=vPXf8WE^YjhIYZf0YKxPUBRleA&X-j5)x^s8GC0;-Nk!$~%Zc zzsdWpW#GiJ3_Z#gtT+BU$K+6In3n5z**-9FSAT&&B_q6dWIbsQlN~Ti*)onT-Xip~ zzGl(2bV{0~#_@@v*!@*>Q1}DOHZ+p(o)=>l&Du zgYd>Sxe9fMnHhZYWo{pfdJJhYaZQg)tnQvQ5S~Fw`9Sd*A15yAYKo2+)6;xLIV9HP z^Cz0YJs5Gr)T}F@b%*oTR&b*crdB}=_(E=hrSL)~AYP(7fL@*MTw0lab0~a`f>YC<^#OotFa64{B`$s53CtnHU_K59z($cjM7Mh#JO|lHB;A2Ve(q8ok*^6rka@iVRNS40x4|lv0(cQG?kwV;) zoL&S?w9f~W3hLGXv5I(wJeTC`tpS;DVfW$If-QWN&R_7v)W6A73mIPY1b~_bcRauy zO4zKiD9|pOMVX37^s7U=7J;Za$2&FTG-c+rh38o@;@V;w4rQTbgUB>&Wwv2!>fuJ) zpbl$f%j;5>%i{A>vKJDA8uoQW+d%Gv1rUNygDIhaoRJv3u8hsAWvPTFqulyD8X=#$ zoc%7O)ykZkvnw`@p_tKE0j{>L=$H?XnwYZAm-T^O@bTpRe1z%vb+^ln`%s^R+JY8hF__?;xXXU zsC2gVI7Q$5;Ia;J)xs;m1?lo&;S-c85!9EM%4&(qhQGAV2#CvpcXXw{fW*EeEZy-F zIS(}6Cok(GnWjluhXFXB9{X#JA8d*u^=;Fi)gr_p+EyvB=cpe=XO~6y#+4S<*BVX} z5a2g1^dRPl2&3S!w&<)A+nTL}hZU2lz?!yrw1#WLj$r6h$&OG#u=R}+hE2HY>Eg@r z5XBlpN#@E`ZhYYz2VdM}Jn0?jBZ}Cv9Ye^ z$$vgEosoVxb{jt|Dd)%FJbk)xrkw#Y5RyKwu< zbgHUnnW$+w8~~|MNtVzonQOZ>lw2~nY&BXV2Fpe`W{aH5>6N88|j% z?_0C5xE)f;Nh(V4Wb&>(YLJ@q8yg6(GB1C9Pl4jn0f^c*Isr{^rJ8>(!1fS5_T?o= zQO$SmC28wPEu@$>$vBGJNwnLNjTM;Ov}V5a1+kJ53|o-wOrPaJ-g%9m%^33BM!Sn5 zT|$VUe+=H;37q5t4pQ?QaqhU{1M<`@@u^*eAiH&MLe9J?sp;pq0k)(CvA(iR=Ou*I z+cAX}AeZY(-CD|B%wz%8))B8T-IHQ(YAXI0RLUu;QXh|AXN*6lX%nNJ)i%#df@}T) zt6$_aKlh8Cx8u3UH*hAKC;bGmk`^pI)Gr5kT8-6rjeeCKL6Y<_1`|vIqI!&AYe!w# zcLCGCV1MCF_rg6S{wrlZK-wR;4Fj!y<^@1h^ z2t@?&O}fG0Xu{yqNj>U9@F8ym5wCfl9wj3Is`*1lib+F?NO-W*P-&!VvF#;i3zbWq zn`iKf>qju3uNBiJLbJq$d4@sb`8$!0YjA78NJZN_A_$vCXYa=Lh5j8eI#)bne*2Ud z*a;w4T@h+r4i;FM?LFBpAREyZeW8Kr#BhC+DqjgAtVjQtS_QKf5&aSClzTK_4xh+l zC^h|C9?NMq+w&qW1j?jHZ#WNf6A8A1HjFPfLDjFbxabn*l2^17+2nv>vBAsaM8GV9 z0D0k07G|N~F)Hsdt~{^TBn2)Ek?imx@S9Ge8?4aidmA(9dJjUv8`9n}CNCja8}5!P z6N>0!`^Fy2VdbW$x$HS(g&IVf+o{7)1%%!4JRe>9!W^BMP;0k08qXG)P?1OYygVva z!nZ%WFKC=-Z57r2;W-hg{)9A3LkYadW3A$PCK+`HbWV0&a9b_EI_cKeHU-Pa09gHg43P3Q&+=Y2?ICSx(dSra(ZTAU;{2&N{KaKrdVTO3}4rKQr6TCufWJsCZ# z&%T0%*<-L;wbkHs^y=;lb&qhw_PSPUG=_W|&*vX=NSgA>EOT_yu6(CbWBlJOz1)V` zmPcV8NS=K4=!QxXYE$VNRt+vXfQ+N(oZ+j|vDX)rFr-A4(*@$Pd-gS#Qhb%il=0@M z>OP`5JBqv?$Pw8HW(vWyWnx%D-*pj5ZTml{np=Z@mQ-qyds2|Or{$&CTK|d@#MKe8 zLVkK&P+dQfQPUHUQOiiACtgaO_9^COJ@FJ6_5okYiBPbawmUFwj30S^0@#vqWqu}; z&8Z1D+zxCZ#JF|pldHB^yTq@mJ%mT+na}j~whs|V-UzS;67rFN(V}?UK3Zl86 zE(dz;&~Jw2QsdVG%w?$O*V;218Na@Jc7^-XFl=4c{rhIbIf*f|`sie4M;_^)3{B^= z3@{{#OoT-fCHvisX-X+VY=B)Cv%yzwoqd)PT1gdmL{Yh|*3T8h>}p^)gro;)SAah_ z+M5jX-duMC3S&p~q#eeXC2abz(p^xgcw$vbt1-WB_lG;k)gXoP#OV2y1ONYuAztS5^1h-6c7If&%i5Uhfs| z@1Jk}IM068e&2o8*|X0)XV%$ky$5+a$t-O+vp%JXK>xD(-krsO6w5NMxh*E&%v1b$ zt-Q)&gT5*Cwkqo?uM8r=x$!EAj!d26n<|s9>H<_snaSfOyGL6tOTahq@4jj%-d(~JMyr)g6GR>wiCojoOH6>bOMw`qY3Wug zmbS@h0p%-ZgaauYHS)*Ly}lk$7YyB9yfPgIe~y=SyG1#2l;>;Kw#hi?b$wKPz)+et z)gr0XOpeIn{53xBsuQ>L!XrGaFhyXR4RIj`nNr-{BpR>b*SGK*3RCUT4 zpZFzhX|Tk7oTRT}cvKxeFt^DyC2`;d4>;YgOW>d6a42wakd(3OkE{qqI$asi-=lSq z4XK-ivV$+(+W_vJ%=vs`sA}s{0JRZ@=f&g(W9#5`P9b&mG-dCEv-1NDrS~}_hxDGJ zz*JW84~48vaQ!Nq+4VKURj4+KKz%OY`|(-FX%?Ynl=oE7eCmAa;JFB_)p*xI+oVe* z>oU8+iVxf28p7=U7=9bY42Y@(F6`SAM=4*E$Azs-WZx8$aJdb1O^SDs>cfg4Plc8| zZudrR_9iAgb;#(mguncdkLc?lr^l(N2sD7c>_K}`pCbJvac;M4cqow8hpajPq?brm z)Gq6=MMx1UICf!%UA?8{VF!4XoTcjK?%`78sJUbVdfXSZn$i9C6Wd+{w(`CZa#0P; z5SgLr!8%*3_F0hGn=C+Gk6@`@khg|rJ4>12^LhugHYdPAap#Ebtcv7gHPa97zJLsN zxz2v7#u>{jA8~Y>rPAA_ul{69x-N^aTQt&p0^tp~EB0ma>VNnUVBL^x-xB}Ylv0cN zzWnEYleOcYds7I&(W@S*9h65#3{E}#+lM?%ry6kHQsT2tXy zf0-YW(V!BvIxmt8VB+--O0Lzs`J{NNK-T+dDw+or)B4uE(M@HcNF|eRIrF$EhJmRL za`{8W-oM?`B0eC3@!Q!>2fwG_W!v?!&z=9}kta-6*DaT-X(gp|l9w*Fbl(>)MSd^l z^XItvkeYFDSp?9sXRnhCS|vH1M)#(^2W6d+b9KX5y7!3dH6)!&4D=xs;_VxF^q!bA?|5sv|;=RxXhZ=W~D*%B|eba9H7F@V?NJIsL{C&I>!e; z(I{6&*Elbqc5N=54ek4Ocqj{-kbZEcBB+&8uWl1aVfDrtHky9&1SDDFxf-5&SC&!q#iHBua}%fkF$vgm@#Bnusuh;T%zZOYsw zJKueBI9jd+CYZzih&H6JUVc(mJi3tFX2Lf7t)|;a@T2+sbMIsJ@0IS!i4+-=G3vvz zDJ|xMvt=el93ykz5fYx$s02H?rxMumYn8v#_ITD}L~EkLJ|8yPvPpMBzP{!2HE5sh z@Pan)O=b!#V#rxd>FDsNbBTh(%Xd8!?!|N2kvG8>IY2krO0+akyQitROb&aLR9(zj zom~}JhN5Mvlt@jF%u}0SI!SrBNKgH#a%Wz-*<-}T=aq}4OXI6Wj|Nv8nw%D^iV;zg zHoGVd-}0d;&2BPT3DGb^Yp@YJY;Be45G;qOPdYxx!dFY5n+aUp-q=PhHy>CFOEj zIt`hv_1TFDrMo$GcT=wjxxtC2wmC7I{CNH&QzyXFjDRPPI?mB>E!WCzXJvIatTkY%*rM`lmF&rMA0>}~AEpstR- z^@0;%s&IX4JEnHGIE1r|q}o+I-$f-0q`nL-fMa9J&wHC9Jt=IqEF+e>Th(l9-Bs>n z6s^B`iY(H*Max&ysDypE?{EF~^tCB_xnDKU+o#@6%$3ZqF5t8Ms(YKjFTQ&)GU*w@ zv{JCQB1gVX2Tf)+k;K-my-M6;JK%xESMgQ>zTnY>mDhjtneoxoxZoEY>;2o(y_d(dxpL zV&D7{qD}d}xu(fsudxXrwB#`{63|JU9McT&n1>UFOgb!lJNogOImMnb)e*j$H_6hz z8EAJ5+10+d8cg=vVA~WBO;Mf3ABz(ciLS*t<<6+5kvxw(NsFWHUXI&B;g|RxbrF|= z5fe9M7wYDFaTNQJrWfCHlIYn==n3Y)RgmeJsrg2l)!OIkmj`Nf&GIi|wSfPP56fpAJ(S;SmHSQ!HWTHsrm9*C$2 zt|E;?2V0EpJ7vLqKUsf{8Ghj9)Q}EB2hN19hL9I?p5h>i9(96R9mr0@D4B!kxSxkz zQd>}X+53ij!5A_LH&B2smZgEQa8&6k>UDwa7*$uOGm}EGU$QD_TGAK0CzQ4jj%J6h zV|O<%Lc~;mWL${KZ6b)CPVeQ<0K=KjP@~4Ks2I_LNIEGvG;Xj8_B7I%7V+shUri2K zR0R5CC}Vv$PYlL6qwTZaWL=Q<%VulddX1kDH{!xts|Wtr`VAUzM?6j4s#;$rTAVE6Ty4zwx$mW4A+NDIVR2Yj=-@}^k4#! z=5&e+qjKXS(pwyJjA3qxZ;DCA*xLm86dA3vs}d7&Zy^|cC~MT(*(Qm5PTWMf!EY$L zfw;KNG3yihy}{=q&Bhz!Otc8h-_WEW{F4po5z%}W9V#n;>MUAQO*vJSHw|$;(wCPo z9ZFp}1#8cyiA1HzJ;q>t2kBy_VSK4V`;lIYkBwp5g;zlI%66C2$5Vx#Am;tO8TehJ*9i5hDxM z;8EbV^VK`^1D@<*;9Ic;CUPl0e&Ww)v=>cj>AQ}6DlK@-C^_ux>|5*8ynJ`e*_e9s z{dSWH=4q!K#s(K_j{$&sJIpQ9Fk+} zB8N)VkCFm!h6K)8Cx<`S_r&g|_|@2Febyc`??l6B5x3DQtZ07Skv|dpvTu(nIEA!_ zcj*NZe+^wLMv-iJG{Yh&tkiKv72C}Rs4radx@j(7dwJF_I44+TNR)>HiB5K;s)SIV zRmc6y(%ksCo4(({wd}~vD@~gQxt9>kMb?(g_b$4SLlmIF5ms)}py3$JoN84PknAP# zc2$nuTRJ6Gi|XAdrR#x_kEfo~1`a*ZQ>s?XVP)+g)c&k~WbqARY}y~vaCGELzz}CU zfz1$CQCf5Ki^rnO3z)62CH=w^n>Tc_!^djrxMJJg|u>%tz9c zP_!?@bv5tH$Z7%miUlyZ>09-cm>C>|&4ORe!=VnVGF5@w(GvyGV;Gv0OQ@Mgm`w>7 zr=4-fi9{Ykk#ymD442>)uQn4M8ZdEM0!hP>*^$(w8HDGK7!4MY8w2CSo5ApKAXslo z=AcPxax0Ky^TcGN?$Nul6vOKL-{ebj1H zW|)9JoOm~wGXzFNYJPE7>P)G4779}ADUi_g%(3k4K1%HGJeW;CuDqb=F+Cmj`N(fBWLG>lDB)of8 zGh`R0NKKxYNb)%aNr8_>av1ZCic~y%^Qc1*s!i6RYGUBnxQ9my%$%NnY5n6bRWq+= z;p58k7|1~c<@K-Ht6vkJTzAARd|w&;D}5Ow;{6TJ>Sg<7$DMG;<;vxe=`+zQoB)7S zX@pKh({!AU5_5OCOq(813N=;%$B9_OkZz;izfHCGfPtc)oIYti29IbhB7$F0H-gTb z-kTq|GMZc12BG@V7V5dmx?br;KGMD%=XrXiy0bh`>-i1hK(*SFd->ftdf<}#`c~vR z;L)Qki2o1Y!8q*mE8HIxmzxN2kId)eAcwHxs%HW{No*f1WPO)_ame#a8vedPenMlT z;a9GDy`-p_r;3c?Y{q!G8=|=>rzm1{>uOo%MjsTf*r+z=cjMY zL;_;JDx{**1!8*Oq_|CN^*RA{4l&{CHB){i3NW4z}6$EV$2XpRO!Crf+nDf zyvVYe>`9|7lArr#UO;`!x3yHML!rEWp1PP$-|W{}Z2F%n8D>{TXVPqBlDQiI6O{u= zz40biZ5|PajfJTu@*V;Kw}`PR@=qBWs^t|0uaz2Y6vvhrUzrCcujJd3;ZaXC*ywP$ zQ%01sFu{qr0@VvU`3-VF+hJwBjkea)6UZ?k0evwlhJtzt6Ukoh4-9Vg64T0flAnZB zU@>WsPBS{yZZfuKZ!$tQA8>y_P{CG+ce+5*h)T8jofJu%U(*y0;g%d^Z642Y8;zP>S?GQTN~cqPwr|`9Vv1^nuXU4tJJh} z)%7!7>NNAkUp28`EzWUE2QbkEl|+kOjCf%o0nJM6Y%!ckSGzoAeV_G1-$F*8U%^^O zw~|SOCu*DD#4})TFp`R!)O+LIMhUWP4zNM+Ewo$H6BI)^niVXR`k&cH>zm(1_iah* zJ(iL$SRnHsUozU_wp%me+0+>=$;0BUB)={ocqS_Cd#~wnf z#+-tf`%?t7fc--n4KYGsWGRGqDuXW^Kr1z*eO!N(5liZQ~a^cI)GNL zSw6c21{LVjp^c8_rX3j98d^^BuCTa(BSrT3?TidgTa8qL!5y1a-F;9E{Q{uijh(yt zLi=BWd`&&?h?9Hw$5<^`aratEpMKcABI`NyptL7Dt8y|a=Ipuv#BxdOO*Lw9Y*$k2 z5OpYi6YTN}44T~gR%8;Hs4Hdp@c_v2BQNiHufhu7w9?gxAOF~hUiIDUm|U zx4W*kIfVtYM&J9Ji@S5`wFTwTHAp7qKToEA?@7YZ zco9O;rFG%z1bd5P7@+57j9i(cQchb@_(k<&OUjS#sxRt#t8045*6kzp4tegtM%(tp zIngTjh`IJEcUCyx_9>B0{tCcUwz4VWZ3tyAj^c5#7L$@S(o9YemoJ)ZPrigRQHKUI z>39JzI;08l(%}0-kP1n0wVh*k1mepuQNd2s2%>5RUidpvT^BU-9@rsIGW0Nzbi#|l z7fHFs=J~#qeJQbMWPw^ClTojM6(0LG&P!f746HNL64(^_rB1~5r?}g`%pf#4LxoO| zY5~uT3tr&!b45)67$6UEavfnK3a6r0C7P8fPwyjqqHCfCS1&S}#{se&&08sEuea9G ztFv6F%TsBMf~&Vf(pl_9%4}M~GccakUZ|4;&6!P0@l^_fQb;RmtSfh zGhg)7TKi-5p2*v45!rsoqV^2-La=<-ANdi}t)dgKb%Vy%tH;)WXD_N~-|J%6jbvBM zyAY&i7aD2njNGtpXb`^Hl;b3XT1vS)5y-^+LU4{oN+_Hx5t#3ex10Y8<1Y&eU{{8- z3E>MSwcvucs6}bx1~Li22t2BfILVJ(Rpl_IqAq6>QUg0%NGrtASqOAf`@)vz{(5(M z<2jWct|puEAsd)&8tM8&EB#ZTzq4nGwp+ut0x}xx1&Y{JTsAcNbMIv^8OJUxyrXuj@kr4FlVas}Ow&yq4PcWAjGc*XaCl&3vg_ zd~cl!@yiheHM?)~`sO<9YaO**D;<$OW)ITL4{)h+=SVOJ_C(koy=)Fa>6BQQ!poZ( zo>0v>8sI%j9z2960CE{|z}QASb5t7$tdwD`ftz5vDv3h#`($Zd@y^_Z?A9w`cAaFO z5!o7IS^Cb&&Ahg*;Uv4GXi6Lr7N(wW+3t)DL*IhYa-z$D7^errl9aB)BENc&P$X65f ztOJX)RyA})N(ur-i4ZVQms4-t9eYsVupg?1yuy6$Me~UZBF4I5*qkk<+LXSVi9*|ckezycy=zh0cR#>5}oE07D zF#(+vjJ?YLtA`Z)YE|?%rgoJH+FD#SgdT;~tf0rjwV!QeLQ*jK`cG6j5h>VZT?pEO zP>_QA*F~`YL|=O&X-vU58~XQ5^L*$+VDAl$dnU4$gd6a}hQU44PLYHXSZ34WUL@D@ zf<7Mu`q1kIZr{{p{BsVw^ZnMh(4?0OXwr)a%%4wD;35xrotX}7xJ7dh-sOQC_fxaI z7YY~sgN7*zTmpc*1X#f1TNL-;U+pvTGogQf6;WVd_#eR61)y+2w%?Qfuk`E#A^8V{ zNfGd*un-t~3+dkY4Lj~4uS7X!L1)=WvkID2StjOI`yFVvv6M+c{51tzx?LH>u-3^h^NfKRN1 z|26$BQG&qm_Ze|ujwB(rFffiTmYkL@#%6ZroKTGdJI;Rw_=!HXLBRNfrm+LF+ERdX z_Yv>S-ve!v2iKza6Al31?ce`)9>^p-fLl00;pA%nFV%$qE|K3Q@BglN|6o*jS1_(K z3okwQgDL%YZ9S-~{(y+)4kdE_J9izQ>EH}sLrsLxd|jOXS)-5c7Z!dnj-=0T($6}> z4F5xXP(btnaSZq`5!09Ce~5p|6~e&KJRr6Q|0bd&1|a==0i1_)aC~Y1^gt7iiT>@1 zg?2Hf|LPh%WP~oK6R3eb@0URp775IG^i$SG>4**vQUI2GO9YEKX@ei;HzFu_bGRQLS(_pKHdRIK{J366$;Q%Ip>WSr2!$xQt+%$(5O v<39-MJ>YA8{009z#z%UO@^{eTcg*1d1!3_wg%V2uMgS%YANotV{S)zjc9=jx delta 39055 zcmZ6yV{~OPhb)HS zzXM2kh6Ci(M*<>p!vPJ?;G51dAtC>%@9@tNVL?Ekq5pk?Fv#hvbszAIKnbABRZp)} zN%R|qR)jG*7+OmS^jL=qI%%)3ME^o$gM3Te4+2pv=w~V!8;ORZXk949Fxl|JiMZgkGrZF_RwgkF{t!MAmjs#D zoliD7MXvrj5Y*)I1<@Io6KOc03~+RrX|gJ+!xl+%}`Qk~+FEQ8NZyJH9;A zM!+on+=b(NYY^2k>JoLQ3O*NxzlezqW?*o3wbP_}{DM0PJjq9Awq`P%<{6?ua^Ae1 z>v>TvMtWsDYl`;*LJD#>rj&YAOkwr}qY>Bb2%8Tk;^46~o?fv%^By3Djq7H=k&M9F zVqe)g$~-V{S7)zT%m#BIuIVBgKtJZvm>Nr;6<%qMcAVs3`Nw>Hcz_;H+u7ON>+#ra zDpv|4ER|+l;qP;6EObJ5G3Uv#{e`L%pYb|h8l6)dyMuYug?t_Mfyt>Uw1#V(u+FMy zOfySG)T1?h(;15Au_yop%kQ=*91T!Ju9d03lkDcVR7Ei z`K;5mxsQN_TC3x5SvSUc#YBwQdxG=#%3|PtM{QyN@EFCL5bRLdqTn;Ek*}@dF znqpF4y+MqA9c92SG0u?ty+u)P5mso#pM`m-c7Xx3w-{Wk-U1KecY+|W+ZHD4a}m3r zPrgtItB8C0aFSLR(S$U(&vcE!5<-Cu;%rUi&GnYhjTq4~;~p+z=ICwn!sx^295s6A zPT<1Z#Muf-t3jV42}P@nOkJ>Ms>K*w+flg$l7_VT>vB4Yv;jv^MOpnBEwI{m(dy~j zk$+=#u=(5N9=JN}i!C+qq#fq{mPq$&18>(BBDo-me``9E5fSP-7cv-FILG+vr5ACZ zC4T)MfBql$YIirv1I7{y1MC&igptR-ed&8#=v&K)1T?{eXq^J1NJhnOiODeJ=OvU4 z>8_L&Yke(zsRF?Jq8Z7QrP=PlUJGKH)|#@LkNVRox>%c<`Z6|hw!S`|uAl;Owwdxx zhlOy9cF1DSsBooB2K&kA2vm7Waab_w3{DDlTta7GjDy0qF0rph~JgcM(cpmP~ zYnHjh?n@|ffM{bcw%`8M(+PTymEd#KL-!W`GpX@!nX;)xax@ zwl#lU0l!1uT&43QNM%~dRZBn}yIMB!iR3%{H`!zN8XhR@Q(0>|dZ%)aB) zDe)PdyAo2H2h%Ee01SgIO-HMEIH95#GLHqePnHWS4gI^?kPVbegw~ySbHmD;ukja- zBAxGnB6~(3i`J={u3xtm89^FODx%Za-TGNRQgaP8yV4p0ff-y#!-4%|l>eNv|HAGc z)(VZ>GSqfz=-=avmAaUD9$L7>oYuS zgd@Vc-=SiPaq*SJOIMvp_RpaRh1jp3xD=Za-3#Z|D_RxHiVh_3{!ac6LVS3<1Oh6(Ew>O&~F@kGMBc}3AZfdd94lC58* zy!Tv?;7NllfcJkmgU%_@_O_0Ur`L$GGvXVZ*MX0vqx>-nq7V@S)f`et`a&>aDgMBU zN;6ce=o@$eT0ie*59&&+ByB0RC?yd6K=xsMNOx1@iS0RZPx_-O z{d8k_fIkBPFZ)aE_u3lt^ zbcB^4yF zv+%tEalY`M&ubF=JVQ*WRocgD8!DbOJ~CC<6)id~m6^QUL(4Z68v1RG#XV-};XTt% zwu`!-?g*0IOND7Unv9CuCw6a4aYaE*?~SET*2 zTwFX3)G@^ZD3x9@hbwE)mO{KO1hTRm?)P}-DGUp@WR_H*Tj=8XY1pm7SyJVh} z3)VHL{e%6MT3s9xJW{JtisL&MLzW(aFm}PJ$4#YTcKnpG(YimW_c6g>HPE-$lGOx| zjxmHU+MMPK?^3W@&o+sZm@q`ZvlQ$Lal%|^gnhzw%}S;?T+#j^G=M3v|Eea8vIURA zD1gB?YzZHon@gD9WpP!pLC!k&)4h1_$0&FuBhG;&uU5lkY^B5I#T;8t1{78%1?ddF zw1yPp0fmv7Nw)y=3^I@0s^3{d?FK%;kXlI@m5@CgVlIIr4%({yZ&1DLh{;zUMuY)K z2ZLsAW$JIl)AEoGV4sJN+3zzSyMuNacszdfVuPP4)i#+1b&^Z0kL?Ukx4C6{h6r2b z#pT$)2IndAD>YW_!=r1McI3R>+tl5D87UKC-#`p-K;7ZknZ9f}RCfn0F6q<*e2>{4 zc~EIik~7s)th6tKANKF$2$HW07(B!J1iPhezu~;n{(eVy4og>)i1aKWx9HKKqmE>0 z1iXsTLk(>;cggz-&dAVxk@_`wnIL`LSld3H!BRMr#KQTHzE>vHO(u|@3?2!Dk8h-Bi27L|7I{gd~B_fTL>N8!%$a_cJ_a7hO zvJYWi)Xcpx&Bqp?|BPW32Q=RPDBy?x>Wp_{<|gS`^ttFjO9*@?>;Ts#N!MhaOl2Oe zB_60HozkeF*YnIb^bjTx(qs;j4yMykZN4X#60E>RHFHJhOc-WkFQKLcNbT;9yGY>& zH4!5RHO(kJ6w@wD6I`zodwvcwbsccUbi-0Q zTl-s7*&iT~AUFTYRSBRm1_3Zo1`-HH%?S9^D%Y)ms4Rpgk?AulU>t3>UZ$XsKbKS) z{M$@$zSp=l?GOnV`JTrzWV#!8y>uiw&DoJhz^sWx%HefA*>=6*&iM?uJEjf9wZTbW zpEVL@q~=?mB1Pln(PvZUP-a#(m*Om@4WSS%)Z#IdYV8g((mI#Y>?X{64CUG5j{vZ| z#jTyp5^JjD9h$LtQTm6+V!nuPJ%wFN3FSsP{Z%wrMiH(5ae6H<{g=37Cdep+9@C)zG%u&$+GEI zCgS%ryvP8RtFtY(BZ`m1#$GY&0tMXk0Y)SnPGxs~KgNj*md{zp*rUaZ-s)BSb-(Jq zGmba@Yd8-dtFoN-plGDY; zq7|W}{zJRMN|t%?Gf^k+)Zay9Lu@>G4ca4!?Qc+wuw6JxV`|Z4y84ZVGBoe8GtdKa z#%LeY0c}o)ZGoFE)mP#3X4v2}Z(cvxq*d=NTrDNQQcCfQgaof;DhOXIUOHEW%>-(F z1reR3-)$BB+BoDIZ5{sZr2%6UsvTw4F=2=NyfM5BH&^!+=0Ed|Zjmq65+n!+1qyIq zm=f52fe)C`I`c$RNB{2Om?>TYl})4(cNRNMatjLP)vy&WZx*k?q-B7gNI)!Rb+=dy z{@cq~{%aLM30zrCNw>N^r-+b~?+YXMaCdLN{AE^dkvyo`?|#bH@w)5pdy?Vz+dUr0 z%&VFNy+fLE1TWb(brCk6EKK2DV`Lf^BaDHY#14Rt7)Lk{Ty&JgE>StW#!p6k-VhQE zlN4dRd{qC#TGq*2&4ekK7>C&o*g)X`dGsj@N1KqB%txtfQI#Y2a#2d_ zQ@&@SS4b2k0U@tdX@rxBK{qB2TMHP{30j|6nIIdew0GwJb$-9sNi6BwYF{0q&R9zrY7?V7t_AjYzYdHete)EZR!;b#5(D7vO@sof z#>l{c>*52``KnPtQW?V0v@Mh$_W6JS;>IK`*xyTWARtam){^HKSIty=HIAov>%BSS zXa8SV#KxzedqdI&FH_zjV0`b z^8+Z@>6Q38(M?fDQkr3;%Mv%&~zt4t%EhDBe#*16J+1*SH_ z3r-WPM1PL6M5g&lvZUMQfGZ08fcvSyejO6#Fdouy3t@=ZU#n`9xRVMnxD`qJ)d%nV z<~na4uy*Mld4f5sx}_tBoriP=ttQk@d#xF>$=$ixHfjpt^QO_+sS^O;zxuiNwt5n0 zl@c%GF~K>eOPeCBkkad0eR=h`7o@7OVUjHkhB+F01HU#~EPKMTUIwc>YRIp2h9ybx znNBSX5zob=ew+N@feUc^tzQU&6|w{mD!eSF zp^-E+Z1Eow!9QUIdgjO5h=U0%cM*#$f-8GT=VmXYFtNS{n_`MJ++WS$PWjoRT)T%x z=X+z|W@o_7jlc3^64w^A1tmOy>)vi$hI_np=TYo22f+Za@2}$~Z9WNZ_Bc-u$hyvG z43g!{e@f}GMdvaO;4$*9N$o2l^BuqV6Vx_!z;MDV4DZMsHfG;E?HQ*ZdE6n?9d7_C z!0Rh2Nzp^)w}(o0^c{wGZ8S^D!-Ln}5GG%|d%M7RFa-3N!Y%M39E~c)>2SB42Xurt zTNjb08ITq1WKU6=Y?n~f@E-nFP=qX53T~HHCKMf?BF~M(2_;TiL`j8k`emYw zGy%cD^|$Hy)&;z*H$u%G=%=6s5}e*(!v%%weNms|r*TCVc?`U1ex z2a@8@VR;s*O0{XI-BlzZIafB4lP0C*D49j)v*~wT%uw5)Lm0u{Qk?jg^Z`#{j;Jw~ zDw)@f^n7!V&`s)NHE9>MORbYr3Pr?bH8#?$k9dMVv=68}>8;;hU>@EKK*HEX zOg1W<1;8eqN;cM+cGgUW{s5QMEmhZT2V0T}2`D~1G0S)#2k-f0S4EG&+6dKK=J})J za`j|diCa14R`mqC$CU3^UH}InR9hmeQ&qiF4zgkHB6{uaDZ7bTp#hSFLe-u^Zh_CM zpE0DG_#HoKT#b5>c#nVcPY-}-OSWe4rLx!Z8*0g#q~5lcMS8wuAhwb`Hxx1~h29;u zBR{H_ojGJG(zTEeN!pdFvBJX>RWY<7oN}2PA(BG=X)xL&VZdn*Spbk>!npR(qZr?l zeUeXScL#+wTwf8%;hS<97pbPU^$GbfUD0o;8PmL>IYtvU7j7_>D2}_kO2q9NSO?VI z0gFoqlETw7k@MgipOag#zb&JhBeWVG(e4N|OzjXk;+x7M1nvt|(JWu>xY)+DQ<0@$ zWj}Z$5}`Y>gg^d^tq!i!dv$EH6sg7gjhci9i@l8XYaVd907{nZ2>!JLt#U;4&SK(}z}|*?*p*M7jDs z^!azXlrc&ztbtgDh7&Wu6;>e#3C zT8xi#UG_6uRs_?@!+*Ujx-7jBo+9e~%}+QrpK9&qg-4U;@Lb__D+r#?l zmU=Qy^?~wVttLt2-Tv@H^I~@xfEiZ=hc#DnMJ*41>w9j|^yp5xc3LTV=Sze!0is_& zrJn;BJNIJZOE(m=&R0!)uQa7-x#^YJ*XXQ^I~C@n#0g-?+_Ois;{vi5odIY=G=JQ; z2ZN);31qb_(&47fj>9aY*k#)dD%9_W;6r-0RjxRO9`CqbF2oU*dF3aBn>Yc3xh!&@ zvi9oR=GaKN507@faYd=V^T&z#17)>$I#&aDinlC|&C`8y8XZ}cvN^c3Rq9LbORnCX zBN?XKvfK^2-^X8sZ5$W+HUP>KT!c|-(#MYFGf~{Z{A@>}jAS&$TqbnfLQtk`;wq_M zOf}M^U2qolbf;K0tQ0qpP*pfCJ;?_ZsAyt@b zlKEu%&hZ3=2D~|ed_q1xPxy58g%^&p_TtaDChtDvHs-72B_I9E-9f0 zT(2%D^WfLP_6EZ5z=p60AxA+oEYNB9n}rX@o3k|)+e&Yt0Kx@B83RB`Blny7#>(?z zPFfKZIH(!)=cfE;U9b619s@p~-rxeolgZ7PNDK1%-kE4k6oNt22Y<+rV2aa8OS$sK zQj%zJ`Kb@9qS%tG;`ngL0k$4Fagrd+R{f2eBGRG^^b+ga!ClN<$24>Gn|Tzr>H_M4GOZWnxJUY8ZG^}5Y# zq4IlEeu|U+NwI|U0IorLS!Kq$!~L*if>&@eF(6lUYTQI<6f!q5KSQU-cx_0eb#Bh; z1!6W2?M`nLv(@^x7VsQwOv_=e{Sp&(h0|*J{-4OQzKN<|8#2(%zyiVUYH6hHnn7_|_Ay=f6Y(^lZN9n5d zxBnhni7QK;AOAwx%9%2aTZ8#&XOV0agAgc;fQuPIQc02-896Z`2WYXxk-xh2m*PV7uz2Qw`jSlxvr#gB(|wn@dK` zPic~`IO1n8?`PKWPJ9+g?G6vDm*AZ0JppSzmI$cy{B=G8yCgQ^F8MBB>??;9jbB(9 z85?kq7-s<3H=UzQQj0+|?XM$}B=)4pa)D}Je+kk4V+GapQBobn1LwfGB4jCuHfaUr z85Sy)Wv;WwsnQA6VZIIPq$%ugg#Uzj23_A_87K%y5f}&v*Z;sAiC+o~*&nA#hu;en z>3=CcZaT`#3gEtg5a3(k9|`gG>~ocJXl09nEG|yBNJuY<0tmLMW@aWy&&U37v}QcM z^;z2{koY-5d21*Ijwyaf9*eYykF1M(ODMxWndK2 z4daVm^LfL6cI&hZF=^)Jl|0IP1uva$w)g(I0ayQ9LYm679mGVc0A`Sh^!egss3X?`ey*XH` zpk!P{2%4HW;a>-`GJQGW)SfDhyKnm2*}CU;}s-^R;2k&Vd~eUt4ybJSnv=q z&3WU0b1@z8_WVT}C|-RkWh@cFfQ0~o(8QxQJ#kzR?%g53x?G=M@Q)m8)92kbQi2^J zfNVLx0%*ezxo@@Pwjz1be|N0AQ?aVNWDahi^wJ+Xs35#t1rm4pqi^PJLHlpntM?3% z?nlo*;KXEJ=me~w2pgd|)Lg&OHyuawkFjLM*`q!ywfi=1up=Jj=|gbCY9ps?pq zFCz^%FcIvxPGf7I&7j-5jYe0dkpCs@A9*H;1R!7f21s!|-%AQ@xRIjIXetgXdv40c zws|5|TWI~4IW8ut)2{Ip1~oUJr$6Bvi%qoBxJ?hGwFL13V6ud0KlVZW+vnyIPffWW8cbH}pDP$QF^*M2S7+jBIT%GeH>~)QQoIwHjY)%Dn-^CFjB&HqH82wemg7LW~m(DqtDHO00g7 z0O?4x()pnPl89sfDvXRHX|r)o6OdZtWIEuz8xkN~#M`LK4N}}6Ox_<(eIxxZyNrYlT&pZFc9Ht147; zywp-QFMnp?1|BS@&nhX7+RTh^gT7HB)f2*86D~6r6zc=6J%V~u(@AjfxWZdYAjnN z2TQrd-LM3LBK@USIT{Ni|Cx9lbcN4hsF6o-fvnb*&mw;}_KW_L@T321a_4~1E2W4x zPc9i?=5!hkro*&AMDvLR#hK;XKt|Ku4<($T&(0l;-9q$nxyxB*+-IMhjC@(DDtiD$;|KSM zMS(CC#TWZdnQP{>b(dE*j>BpK+>tR6<47+cgL7`t7xoDKs0LqWo&0hc>w(YlqKHF4 zkaYPAO1|EAAPl$thR^@}f;BxgmjVe+J6Ih+!11gr4>!a2tkhhuxUOr#L3p+65%+#_ zO+Ivw79gPEk zX651J6tlQoo}htjsrQ?aQ;mv|x57wgRl&>e(3SKKefWe}kd|sI62Zt{rZ__lhuXH| zq3sS)oWQ))Ewdk{-(*d;mL@BUXwN4jQHC+D;{qEjqNY@R7Bzhqy+Ui+t5FYs9&&8S z7r)MMXvk&Jwl)kr2Zdub0zm%@692zZo2``ZARv)sC`=8WAsnZ}_9<3Vjbz>r^#9G& zh3~!BhNvJQXVgGueT;vDJ177KQ#)f<*BotcBej)|Z~uwu?THx(5u$#osd18x5Gs^G zTugSEAGqK{BnnAcX5&m~78D}e<-zV1BdayR?$kA<@tpYW@8irdh zezKZ1_88jly#Ko57D>^OpQhK`FWEPKC%S+WUQYvouSa7L*5eBN=0kwu`JLs{Yjm8C zU;e%*frDTEVco0L$?Y!NhyIYD%N>QlYQZ5I0XJ9SzAC|xGaJ^J6Cc)Jr(mQYibB%} zGjDR;F)C;4gH>h?K7)njE=_(H^sw3@S(hG@@(@Zu zDJywcH}nIrx)@_x(-nYew$8q4yefK#D#q^JIkWJDbxl@!^bkuV68_q7U6G2yl38Qi z28%ebB2#XOSyN$6nS2SuynsTS2XhftrMgIGMolJ?qJ$@vCfsp?tS)(hMy1w*rkh3d z$|YRWqVicze>O?GQnO-WqeF|jsfR@pNVAet<&kre@+R{LVF1uB|BYp>Qe#T*J!OJ# zshBzsC)3ozq5v#0&@NTul^ip7sXhN^jBB3%;NBsWdZ9w=ZlEn}e2S`FF%{;LsCm9} zL?h3DiOs%WkE4$YJ8WVh$7CTc$%hTIHdZv?bdfavCr*eBIbMuin?Ajn44Kt-keL-9 zZZsq<>oXGE8}NMo=xz&s$bi@uL8?3Sd^9PP4bGU=uiHh6%>*1~Rx9G< z``8lMSX_1qCuA`@QFF?*r5Y_qrBvByEZbLC)#|r81I(W))u$LL(Oy&k^`!~RPh+4& zF_fEDwlx}9kjvEeJAzSAbk|4pr3pzF%Wc63iYF?NTEoHaW~tcqQn`8O0uUeuYcM!M zI;Kb$R-;y}^1014*usJ_V0KcWhDzx#w;_(WS&(7HTUR05GG<_0VUo=ywGVN=4yMa1 zXm|=&0b7ekbo1CO)?#zrMqJp2MB@x}CT{JiVwv*ff}WKZ+uK$Wn$#k~U-wd~M9nZbU@xwf_y;z^E+Ef!m-9kes<~s4xQ{rVdQ=FYIDYtVv{JC* z@Zdo-!7Y{BoHCNX&bPAC1)-&4WV4qisX`5N13J8>omfq?;wp@h#gYDeL5gWgvQTByT_YdR1V$uk ztMaP$>r_sd2Oo--EtnBxOX6SGQCaS)kV-{gd_oWt$)}v)5%73!H z0h*R56R#@D^ZVbxw1!zg%YMhW4rFAIK7{|E#=Tx3%O?rJ9bt@v&^c2y?3AqJ<-wl! zKt*AQk%DxKixbP5v0|~jR3_JxT()b_OTeFyMRV|GDm|KU;OzQ^ng%73v%PE|eyIG^ zJM93+LhM?}UacX{b4Y_pKyEg87r1V>4{)Q1c1;>8#>pjS3v-RILTlZ~0^0&tecxJO zPAYp=JtZ@9%~^ID1dOdTK?-u>uUapbO9Lsueb-8*54Z)Y=OO8gc9Nbh78r7{SLrk1kNF0<{Rc=W9n%O3RSfpdI8fWO$Y zv>K_}r`3s>_=PHYyd|m`6;PR((^_`t)i|u+OR!LqK_r9=+FKTnHQ6;9x!lPqV3Sc~ z(cDpCo`h)|WgP}T&tfSs$$~XHsn^>dky8fY6?_0@!G@Vnby= z@*#g*53)vqXRnGuIe!xG7)Zc7rVw?|2!Bno<2=HAYb5S6Dr zg&b}qZkr;YtHw7{85woO3DC!2mw#7x8bzKe?7awOxym_rP9ru1b^&E5>RGoPI4FO_ zYjcbJ`DA1czf9%FsZ^6J0t7LHO~Xg3^aU9n*o$5#W`Q#uUAWZcuT;*9G6TA29v`6Z zBy~Pwn?1Veu%9Uvc3r+ZQkep(+@ju0b7GlYWhs$p*-_W+7B!dwM1SH#MfJbDh*oDE zmd}f|uwC*l7mfnJ>L$R#{fhGPD@+FxdT8qPmrRxhaqjF*YlwG~0qMlMNl+CFY_!;f zCf%bS0WJLr4k^>7*~;obN80xYZQo)HU9p)}p=>1ofduKDUAktx#ZFKXSAw zipZpcS|Hz^vf)r*{GMK}ACd$5hhJ(2$1v*k-BvykXlg`^znr^Q7;KCpjEINx5i2g1 zhe*&z{}m5GMetG1W2K4)?rcOPY0q-{3{1}fkZ@tQ^Nwu$0o4Ts&w=q^qT+F1dC=e` zc_$=n$)@BF4wOW0vgL7!dFfenf4<2%gf2(o+YM8}xn{7raAj18C<$7&-hPQ3|2vVuf zhb^@epb+gT1~4vKR{{yBQ~{dS|zB&##nUZ!dV=ziTR;DtXna z8zQ#4-E*BwCOm?}*g#^R8fqFIa_=seexw)8Hj7nbqOWpcP7SGXl~vTQ3WouXgD+np z%)Lxl&CM=kCu4}W*Z=xg&j;^YxMp>Bc9pBdx>&|G0c<)U1T=7AF?l=wJY45wamAWo zmBZCi8kti$4<)`xPx3e3jgnT_dK46$4(ZNt(Uu9^xp$qzX26#55wmXNIyMa^LypIW zUXbh;gFVnq7UNDO`)$;~>L_?d@t5__{+XEGvyPSE;avq^5qLY(uIrU9=gJcBQlTVi z{Aop|2;jG!9h#{WEtnI+ER9frQ1N_E#5uDC5Gz0H*o3@=un_?K6?spqH?eIg+aJKvqrtjhOeT@ zJ!Q*TS5x`8-f`k7_mD+dorUurI?QHbv?Xtg01@t0bTJ!=$sU5^uaCLGHRbO9*QU*( zB?dj}X`C529F1vfo6&!GoNDFo@e)VnKWjwBtvE~v$~Uq*%PPQcjzU+5Pn#Jw^AeX3 z0r-1t%|(QPcxC04X+3oAcoBQD8sC^-Ud(OpZIv93*`1>I{@B)Dx+f1Fz4SsHzS#@{ z01!=Q#%%RKFatvl-zmN$ZoDGpnG~I;T!O zcIS01{yQAs*^tmE)!82T<{*bB2Vf8{)@8_2Ui0^p&_y5ECe$${Y(2w{Ay*5?WE*y7QT1{748ucm3l;q;=BuKdTpddHMS)bMe#XN|+M3rGY zr-GTKaOeQMllOz`9Ij7j_DvM6z^&0{e3_AcjJic7B*~G$xok18c_!}U{v+JN7wgz` zqOp-YE%5xjXKb;}^#;}(0=Qw}EE{*EIw_j&8%sKKQ1*}KKMwL9=Z!-};Kuc^TK7b6 zVvD)YWB0#QMsM_lXZN!?<&lYuGa5jVIHcsFzkueuEH&>yQ2V<92o7l2x5AV}SFswY zElbe<@DfDd6yJB07bT`I2zOfkH;85A%&Wb<-+!RP7e<&B)?U#y3TSwObwo6>u$R`~ z*q71tDCz3g-`kv0V{n?QzE7rby`QGTAJo>?YFe6U2edsRTfm%8+940FH-bZ|)0oj* z4Q%XC{${+6KAw0)wZdYwXGv`546!Ho_artoMey%Q?bW=`6`!Ca5!|1Fd*clZITGep z)#}U*rlNadLCck31h6Z-&l7rwliHk6JLMQr`LH$HdT9(93xxf7#a4~4T=17<%8l+- zte^*)3C>+X58Zn(T5u-g_v0TpVBC2*fbF~f2A!^>l?DCR9?7ToX-H_*RXe6>npPIS zmBl{7SdrBlHf^fVu)_q#LKK8XkxNs^PMF~k3paHfy}F?40W_)%er&mi31HMhF>Rc< z|0omXikR+Xu5j!$+Ix(Q@caGYigdttErMc$zJ--wd-ofR{@V|+-H%(~6PaOL` z_9OV?_?fmyG>4-_20&gRDIQLKCbl9gRf*A#F|Lj;reBiPP?%bc-Cj)EVph^PPON7| z-v-Xz!uCzF252gG9M?dr0)OIfPoJEjmOpRJw822;UOO^0p*@`?a&#i++JsZw9!$4e zPm3C}{Urli;qnVqU{n!>6+lsBLH>-w{6aeIw=?P16^-L6==z7L=$2q~wGK!B=1}`1 zkyH|!XY3Aa;ZF46{NZ3InuU;Yg5*Oh_0U{9C=Yqc0FX%VTCp9XdF>k11 zT#v9mhNYv5#d36>Yok8p4b3uqSPn>geI~rS;?xS=HDL~r{QRa3b1bPw+#LXvvqqh~ z)wTc7xY2)CFWRh-xLMTCo# zA1R(0ob~6Zdjk+4sFU2H<2&C75XkSs?h3mQ3< zG^L(7T=-IbPPNq-KpyS4r-&UZ_~7;i2Y`D)vhL_>qA1Lsx$+G4{-Rzu zp(=XTnuIG_u*hL?m_=;5m&##@P%WF@AEs`K?p1qB#w3&uO6pLF&j|4t8c8aiE}mjU7pkN3b+FxMmN11vSnj^1NshU zSweGgy!#aOGcdJT0~z~zpoj_KYaQAf^ZJuNW}r|4e$a#6D0sU~9Ab9SnI<~jB4JMa zTo`zUT@64PNg&Z^a@iTA)lasMxzcNOZq2b{r0lA(YiqbN@ePw5OR=jv_^Z{b+0+w$ z<>5j&`wVASXYQLr3&*>DP4@X@hty|9e4OPq0en#W?*?ZRp*}qHza_Hn8=NLFb6Bu{ z+h&SZOaHZP7I2Hx#Anm-U*f-YHz1rH?*El|?MTu7x5b5P=)WERkF^p7I@|F9WPP{9 z(D~6e676XH%R?!wVMzSRG$NNVRmMY-4oa1rmI8sv&Bn~mrL8*JETr7Tv>YFiuSoT0 zKeXE4_Q0M$s06OZ9nLYEgrdE-eNS>-4c{+r48K8~2zHMW!_vmM^;Vo``rQd`j_(Bi z+|N6S<_Iv|7Hpm%&{1`XgUOx)z7jjP*=`$x%-#FU_t3z|!P*e2kDQ_M{@mhR)2y68 z_RcF?U>n?v3U*B7&!BVSu#{M7GG){KEzq)O_XyQH;RThhc2PSXph)ZOdee`b%Kb?p63%zK>ZM|s=BGHG4pq`wE`Z^ZI1uw@THsa%=oSuOm`l#P z<-dmDtM*P2dpEXkcy&Yp4X@~7m*P~B_If(w|9zRrdpw4>U&U1v;zY=jT_s^W_{r3J zC95n?;$1)a@`0=PL1i}nb_XW79s7L9CTo^gCq}Q0;prE~DnKQnxu4Dz-zGPmKhE@o z`s#rE3m3^@LZ^y&(CoMiP-*3nxP2yj*L*eRflbcQhT)wxWwW!3#-Tk4#+&2;+_Ou8 zN;C00d-|G1E37@tja1*(oVTUu9qxjkX1z-}=x>$hIx`*+gSeomrTR>e&26?{V{a&E z&b)9~IxjILVrFjYz8t1VuyW_4j>4qDU~p8hVl$pA*35r5s+}~F^x8+((=19b zi?wh0E-8KzDTsv>D4|Vc#$<}ngw_gYhPio^tA}&YjLmG75e+q?P-37skEmCwxg=Jwe&UasCWV+ptLu?# zWQFO&w2MY<&UWHv)a*Rc;6iY;mF`k4;)6};*MuA@n#~uQ`-XyZwZZ%j zE)FqGf`nyBbqk|cqE)-V=+GKJq4yJLY-4Yq!6P&y5&2VzAB;N`$wycw|+Aj z>OzhN&%K&cCGt$Gh`MBLLQVx0T?>&@sh@AwBuI)5bvo%?{~L;!AY(9$fi!cBu!w|N zW!L&*hBW(=5yF%pMi(u#a$d`jJIy*rUiR_}>_3T&6!N9gj(iUl8hvLJV{&?m_Evm&6&9IFg4jALw`-L=AgjuT;YgXfV+F zZp^MfU1wK1US{VEcL5+?Q9U3J?M=VBK(F_E^1g;0+f%bya$n>zZ_9I|5~7I2>)1x>lH*2^+1DbgYuU5E(@IBoz4U^i1-8N9w z0|czkn1H50_+);*O@LzQQv-nsRM(FyT22O~@1jTh8xivC{>@$a6ods-`g+fiPP_dO z4r@M)R~?onZr0W3%hU5FQncHM#Rd@1j?GRx1gpVAz`^}%#~&}QpAu`Q;?2Px0pEDT zY3JN~Jxb(APC1@jpz{n1Dy-+K_b>}B;*HzFrq{GKd6uWj;*JVLsy=5sB1Jwg4KXQ| z!!{#&Lp(JRTkHJ@KA_45BLn}>n2a?jUDZzr5D+pTu{#G~OBGEM`;GfiZ0 zgNQT;8e@w_y+$n6JOn+cee)*61aY;4$C*XZyvQkq980C@Pw z+a)b+(%nD3(X{*YqR;s)@cGFcSoA6_0fNS2Fa!aEC7PbhL~e?EGkGhV!bEDUApxIe zrZ?)#@+Sd+G1eV}jA^DfpzhrLQ$#_jHq1$X5DHa=g`}O>6rYpaZ#~k$k4zZOSb2Ri zx`+so!B~1U0F)@aF!bNZ*Cbm<4s)|hMjB3VnVYJo0_!X41TFkmjQSq@mio?7Q_3Pq z+ewRf2$k{5r0poi%<715?5Ss}dt;&+eOEferp^Fh@2sn}wQ=z3k$cKz60v96sx!l# zZEOFTOEjOZ79K3};+*~}E|r6=DSeiw)d6#iKgywm@f7l>=iR^X$#-c-I!)xuKv{!o zeCD8GaeCXnlqrAuka~8jK8B#%()@$cWO<3H189)0il-&gAy%%BIur-F%OKs3`GIpy zrwaiH*9r-oNrTVzTqKMq^W>y-urJB&ma5uN9V(OVjdn@t<1eadH$ioC;b_P_p^bCM zFo$U?=aKEyR%(gLF#OYsX9y@HcdwaHilKLgwf_5L3Gq1c&?!Gp?;ERUQ4^JO5-c5Q zqdAq#ly3wQ0;_OHFgd8f<%+SiAo~GWhere8RGFY3lpYTwOhyd-D>vc|-DuJ4OrsU^ za-uIRa`fgW%6hd>Fb_mSjtkI5=Td>P(QVK*p6B>O()#sB_Md*uyoZ@uUbI} zlNCkz!Mw27QKleOsH^CH!HO8$L;jNtEuNDXrD_DWlVH@Sg?p@i3NJR|K zJy{6DecSV5pGDSi^*@EZu)PCD1bQ{NY|(1)phvn8yZb@GH+O-hmlzRtd%uEz-jWW@ z)6y_K3e*9t?_0GHjxViqBy`ARQ}q|E8WkyWb5=(0p3XHK7yd;4UXjqO4hl4-pO#PD z6>D_1DjUtZ;86#v^O3wj4(6aJQVDbFmUJ_7V{(J`5$X-k8WBX@ zAoWeqVR$eUL8RfAfX3iOWa5*T;AHPm*YN^_tsA1>`JQOKdh0jeU_sfAY=5B#5TpPO zB*~;IlCz%WupLnfCRFNf_W1zc>2K(*qAwbs!mrynnnBYbE23^-F)6p9mu73!j_H6^ zf;Xn*ssG$c`8VTZt!UgkKr~r6@VP5r^|aWPQFtc;@v$H;fGdVH4qm)Y<%821nLMgT zWMP*S0cE=wONgG!nODps-BfUp1RreR@e^>X~d$0A|k5^?&`{wy?r=C8@b(;E>5x62ERAK_f>IHF~6{N3u3-{1hAfPeMj-yi%=~?5it}vvQ)|_maJk0 zN+6#ZL8yeH6E+Fj9cjSLDS4buz@g&ti-F*gF@SkMpFu|;IL1siyE){~ewpmwxGM+! z{5PrL<{mS_$7>Az|4CYc36vzJe+jwO-~WIL|9+8p30~ezfDBm)VWGTP5G^Z+#9TF$ z`Q>H-7M4UQicqyI(l*&+Uhcm{JCuhDT{*WG5NQs-r}eB|OOU-hMpSU9G}%V)|7`)P}J zliGR2+yJbf-<~rW5VvRb4gpdF%30fskm*1@*wkbJ;QZ7wHcj(>IIfqa4$IeOy8>Hi z82mXlnyt}v4bGPIo3L5^HcZ2e3=lpDv|YZ)1fjzjx_AVb6ck9Rrgz8Nzel}`Ou3YI z_Rl8MZL_URRt|h!Mzsmqd1+b9PfCfB7=(76nuR9ohQz;%UaNdzWnHrT?M~1!i9`~5 zn~rP)tgB=q>11UorK;*VJ&-}FENp089R;7nogAfy?b>=wOre>TTqnPk+qc2ho#rOc zvfG@;r}UfXQMnJg`z~zfVeS;PHWa38f)5?i_A#|5m}>xGoesPHy5g|Q{gnkSmpjp` z34>f=2#b%IDm*5@KaKIY*;!?{-3v=&p@EVOh&j`M&z*7*s$O?_Tx$zH(5CiTp+rbw z@iE#}{#SEg+F8A3629=4%jLh^nx)u3Fl9~eQa7^yjYmU0=Xtcu3$f~GhasL4i?>+R z(+dNv?aHN{fIoB6O>E^eI?>Nkzhxh(l-(`Mbo80qD;_Nbefe)-Q&4y+Fzu&ibD)#} zjQN}1(od?TLt@c3oF`NBDHBm2s!#KT1RrDTjTG0~qdAx#=WSC#?}7pXPchoRk@77q zU_!g|7$v^#GD*C{ziAiRg^ZaRAd@jS1qhfMfk_K&2wo+`o?>l7q@LBozhG%!aTz}a zB>+fxr{o87Aq)5IvQj!^<{+E`&gWAAvh5a#IdqI0+qNIpXw&YHd+0|Qdv6D1sri2N(qQ0@L6}-Gu2C3h{SrTE!<7l5d&`PJGqoUhiykNGn(CrRhol{9PDE z=;B3t2PFr&GVR$gx6U6%B48RbC914_;l}6+e!}*}GCQK@$2d^@A&ki4$Q1-|4GhNR z0J~!2Kv9NqElD91PP74(2x-cbP@Tw8bIHHR`GHX7bZQ$_vDq*RGw(&&W zvqTC?3Jkt=_6l15S5tCccV70LZFP_EZup&0qMuz87YI^Y>!e4H?dUZrT4(I+QkBA#3>S8s4b&n>kn9GW3H}*|)bmgodd5I#J4vQT6k2h)m2RDc?EsAi zOHxG_Swnir86_`c#!8{wF3p*)7EXZT!%ESCcJZxljK97s&KatWVwF_?AvbQ;=Ea}= zYkFxw@+7-X&0?c@8{EK{O1p=;53N)5qw~k#QQ^w`&ug0s5)f={j-$=a8%?=10Ys3u z>a%s2IiMbmBou(nHu^ETaz4^w`GImUbk^HN>Yjx_jT?PUuwpRs?_A7C&lbRO+I@IY zYV?(jsVX>5%^t@H<0Ez1C}R*NZUat^VYD$GG#46YvV-mjY_#d7$*J|sr_xlt^!$NF z9~Ex$mTcC6v#CeK0oygh;07!vocV?a*pagUD+lPZN+a2bBnf8yPYpgs^mX=ZGvjQW zWUzviB7TWodV>CvNm3*l8#N$&k&Sn0Q-wB!L_1a`^KfDp18!=1YE#3E+BcG%k z{Up3`5eKyjA)NgJX6rovlFji3uDfX}#@!(7oL)v*tzvABht=ZwU(9U7cdF$QXAAzR zpCVA$PE5%=W*b|!;AwH|VJoXRyosZzTv16xr(qzisbl_88;319^cP@`dsEz=HDY9mmWUEmR+sAha7B_^I9;qE4)YyY%M3ctUUGp! zz!=@|&xprgF}zW5xn0`yf$|1yyl+xk8E0B$@mFq~cW^XGUDwm0$(LC?f_sFyBU%h$ zR~K>)Sl2>zz9Hks6nZO8hc--5i5_V?g^`Fe91A6s z0vEB$iqC!^8c!HpA8mfVf5rUez_DUrkOU_jQa7`Jw)y98Zb1}+Es?@1wyG7t5@ahC zqN|-ekfpdGEZ@XfvqYi$8X`u~!!M?P?)m?qFCn!m?_2+DTz@EmfC&F12>#o}HKhgV z^B)&i_oS0O89N~+0YfMQmGm6||jd#nN(B z_0s%uk%cLU*brTQQ`P>wtoFxq_sX;8&iA{X?K}U;_F6pS{LA}C)8}N{iO&i5$wo@T z>-tTOcU)-l{mgw?Ai(b=FXr?GZ*rKCAEE!hnomuF8?-{-RBiFVxK4EDzt< zfb*LPJno67Q7ZlHrxC}Hu&qyde+gx0uW*ECUI^M95q#G`1YpQ54P#uj`gT@ZhccLo zQ6(Zz45kUsD?|R}7DET{_&yC!YkESLZwaJc zYY#kV=2?N5TZK>DXC*XG@rdkBx5BC66)hB78%2+NRhK?kjnM{SK#Z|5EKe5(7s{Qg zQ}W1}cX5DL2=Iq4=@%Z>v0ak4%68>F8ZfWCMdOO_SEuTcE#5n6PR%_Sls<*PhHAHj zscv2h%J$wK-?jQ0f;$WaPw8#!xL}fdd8hPI?D#;yNNp~jqNm7Fglcp-j`ErcXZ5|E zqeD!;H|q9CVN>vcSR-ZOh>=*N<+qlxKupMFQylM}0zl+u2PFzdEJ~x27K0gSkrge(-n-1Fgo!W_@uylt&DNde!{nSFxa?vd$?ka+B=9$fnmhHUKR~_u;ERx zF6qL85?=Qz;~+D#Kl;;JVAc)qM0;Bbb_Vgq=qaWY%w3B5d}APVt&S-EyH?HJ%ngia z5&(u<9%9Uul;dvP@(^SKA5gJmr{^dVRF#~IZy9a(>s8Cgw5J5Z{^cNG<(rKl=X#b7NhZV)q#9zg@8is9FQk=Zt>;mWvOE(6-p7P`;-NEL zfGYtsz-Z(J1+s)y&lHYS+|}Hu9iWL+7!gtO@G;u+3*IVzcky7;k31%93x$8~E!k6;YSjVl@fMv5T0GfuLMqyIBkY zZX~1wrq9G5lkrI*0}+oUgvpNSJ%M2R&te)4nQUAufjg06?=@2p)4Xy*1p+*nU$fjkxW^QXUFCRDqeEP(CG3lk28 zDT)ctzv8`Y*n56+aA+FP9Vt|j_C*HQ5|iic_r!HFyXh7?u@*z`?+A3z$(ep!%Cgg+ z$>*eG>W`gW97m8M`*VE4gWTn2NF8FkRxaeea(iD``XejWj|{x|bG$AJFf*&0S}Rp6 z#=@;iHaPRI;N6OQ87$u{JAleNTAzyHoV|_I;uDG!texo=%TLXBg#+%vOSx|xtU1cT z*;BRmNU@f=Caf(v2GcK;QE0wA$=?H?o>6?#?iMcWTe))$E)aaICh5ei#6&Mp&Q?C? zJ5ie?xpgcU@7dw^hsXK|9pZc5tlcp^xp!Zme}>_OxF12HtR98m;{e0(cLNeXV>pWx z%w=;qX=Slc%*-yw$*uYrp*^K{1)r&6H}l?DU%`BVQD6g0^YpCWNcgj_6Q8J2JygLX zZ;(+rhb1iESokWhmQgu-HP)XrJLxQG5+PY-xyJXTL60|c#^1RTi*v7`mX9kC=OHL^ z7Chr2tbF}5jB)+OSOESRhe-55w-bO+x!9pxWcsiN1Br1vi6o}Lj{pnP<;trlA60M) zLeMubF4tMjL&kE}H&fjM?BU#=M%(UH#;9-_WA$B2;EAs`3_Ua|4swI~;4Kw+<||yM zV>nJ!VTxHZG%IuJm8h}n{MHo*1DZbi+lOa z+eoPXX=^Lvlj-22LAjehfNZ-aB=0eGE(E+UXZk!qqQ3myakNLytGEZh8xQt66gm^@ zr#RN%hdz2JK%tC3n+T&+@DiHi{tEZqFI!N0Et00xfTPPeR09ucmBZz_c$4+L?KI@%w2@16k#7@d!{9OJc`f_t z?|B#p8ato|WyLO1gUD}UuXDwL8#9|ZEW;j|CwMz6_%VViS|>vv`%vWWiwHlB?+2z+ zf=#>32lWs)X)@0~oyj)So+9ssvgr=?IrRQE|Gk#}aXL)^yr>2oJo}dIRC7)oIyXyf z11euic135!)jp@j6(pVqq&t}$XmNUFx{>!5t6I6K^pmRVZKI4dGg0fEDiEF&PkGv= zp8?QRL^K%IvhEBJ4?6G!5{hr)tyFfBYWqOv(mcy7fg6KY3*?=QY&Hn)9nT_&E1^0P z3skPhAFHYUAVP?Pr4{rjVPP-Lt1t)WfOBN8wzVpiFVIgML6(1^f=bRIx1q)Ac&S3A zv^NIpT>DXkB8fg9UlKE|23d-jXBqTmw6fF9R>E)x;9^=uUk8ouYB9M{R_9Ny#~(Kt zhbu9U)4Wwse93aVilNF%*#6nH@%Ub^iHC;p5PPQdFPWhDz@OC>+E35}GDwIa09nrJ z$stFB?R?s#HW}}_Ibahn5fSDl`ZC~adOR=ER=GosiXS?N^_uyJKHOQ7!Y_!WrP7kB z2U#!9Z=mBh(VXE{X0IyT9{=**#Tg1!9s#4SAm05Ps|)S%M;Jv+^}xGVJ7H6I-0`d| z=7qemuk-Lb+I{#6)v?~4LDJ3!;EH}b^F9Kd?UtYZ?xr7WMqx4ns3YWh$OB((V^mho zr$kl7N%hHaoLeSZ(TR^1-WRyww|+jegAL|ZjAl@f9Q^Fx`x?&$5^NgtfH)pbCT0pXTKBu$fCu|%rz|8*H9ATbl>AOl)>WMT~9Mb`Xr z`>u-_onE5j=hfl}o*<5xo78a$V|ML1$-rdjQ4Sc=teyLklCn}8l6 zP`NE8{O{DM;P>T94%<}jHRP$P@`D=F*_3{{&aTA+_)F;vMIS}IWSm;Vx0=OUqU9uTG~Z`umJ?6Rv_BYd>#AOGqV^6S;xUvl9?;_l zalG>&??QH*TmN3@B!yTC2B0msjaVq&V9;KJ?bRz%-2iUekW~~*s)sG+2wk6LQ~ePOd$GXW=Gg@zLiKhxfp(SexM zP=a=DqyU2=F!9oD*(JUuMrMA&3b@FfkgBuoB}l8=w&=KXXVTNlbK}M;F2NI=+-ojT z1e^k~!&I`Hbx>AlNbSxGh6WRj8`~isL2RFK3R@m6cG{Pq@YY%OtJ8&903vP)yhYUO zy^qDmFFsK}TK86TKRFk|M?XX#+23mTQw6{#@c_bdp(sr$?zkj3LNDRl7sK=5qUJe> ztK5jaAVq^A3-&Y@(ji6b=l+HlLOA*&@-rrni=#)|ca+nclAK(ou9)%1mfm+Ts`=k8 ztiBQ}e-gfQ1Xmns{{`|-#q=cE@2J|-X2eX9@Lbq=LGa+kXJz44iUc2L!663SmOFP`K)+vO(KQr?x@cacRz> zt6>uO;uJDLLr52m^NtKPhl9d|cp;IR82}UeIw5N@l=nUkBo#+G&<}hChDhPjn1VRO znDSTOnk;*|kZe)73cP}5y~S>~;aUG2xlfz^nYPz6tfJn&X2$cgC%bB#B3g0N+>g0R z7xSa+4xqLSxuA?q`;;nW!l#Ny<1l?#bxtF5LiVs9`C@F0e|pXh&kIQmp{yYf1i&(V zQ5x)-{rs#JPy%Y(mBNCFuz`S}y8Vz4=i(olbFSDv8;(m7;=-3nuKe;V_laQf1A)Uc z&@3T5WPv=^9l6o_L);J7w`>=iL1H|T!-gkv~_U{2L@E>tj7HO7jj+m1gQxv zqfbw~$Xz`^NMQ912^N2FER=&Y-go}wP-oRu&ayf}^gFQY-r!NT6xQ%u$l8;%J2{#v z7gOj>Q4!vZSaL<+yCK)#8IegB!4abDLn`o86+$kr<{{62t@oAs-Fj0&7*H~|V{i;= z!iLnl(-$B-TL;zwwRCcGhR<*zl>Y{1)-P@+dG?N0rvA7mLG&Z>1Bw?EOigKzbBH@1 zg(%R2!iazu^1`7^07QZ!?kC#Q%=-pe8OWGnQWZI_1u5G&VM^0>S;{5$5GO8Q&kMm? z)7DBSXx4Nj_NJs`a$C9XWWFGo*M$~Q>_N%BKZSt^_CYzJo3^l_&CEi;7V zNpRd*U8t? z>griftx`(7Z#Yt*rZ)%kG^hj2Yl}}kSN*Q-SC^G(Zp*4}kYE`c4xo4fzBnC-I$bZs z#ooXV=h*{U@exgGWWxm*Nv2)1ePDT%MdF)&zW*V1^GDUH5DV;dr%+67!R+0Aji#vp z^#guVr`4SEi`?wmDCs-mdxOn}wws8kNW?OD6RDMAqx~ zFafco$iue4rfilb&=hJzf`f(}vVdR#7RnVR>liSLUrTd!OBUcQZMM?3zermC=%=C@ zGD8k$+y1R};Ty4BVAXYj|ExUp{xNk)){#LElgmgJ6+uEv-w~S_I3~x$8b++2xkq$r-B&mtowX4?+^9k~;-u6R!m>eJ!^BFPZPyYk zJD_wh*qT$DtOjp=RivSeL%mdnI|tMMZwh`TS>jB*F4;)Z+RWiHqV#sVlrn2a12^bAN`nVqE4e5tmObPpFgJN3F8B~q}h%`*;?NrQKX+VuUhztmENWNb; zfu($fscAWAfWjLN44ApQ_1yjd=`P)seytCv*{wk8F5bQ1DA~n+Ee;e%@qvz^dSxRc z@e%KHzvJ~6P?l=K1=|^cA1&npl5ppx{>%%+r+j5}{sQGsx@89AAE8Id9HB@3go>g3 zS6cfO?6y??xJw|It+NU{r)J%XBeq-`lbkw!>hl63}OtX-Ilgnt@hlBC>$2=o*CqjuW!2br+i>Cj8T<7sWyf6c6ezeG+vdpz>AR z9g70jQ>)6NwL3s)O}TB?@HngZ--tFzDv;6}q)ltOxMD4~ipZMgsWRJPHJT-D3mKJ) zujDqy`!X?vqmOJMb&x4no~1wBi?Z!zeoe8aysIh5d%H~&bGJN}&=Mb5bf}mE{(uw= z8)>D}l_$BVlA8H~p>{AvOlK-@Oj@Z=7B;6Qbd5b!8*SgW(adP4Pk@-b;7D_0=^pV- zHAB;f>K>*Ee_LAB_nu#9J``(O(*+NHGI2pjQtDv_o`4<23PHjRx8*4?p(iQwyXs5& zs9)8&wnq^U!3UZa*%x$bSLCn)a*v!hElH8+G$MMlItT8vnvFH}3tuzv|F&e|(qR^Df<$WXIJ5T&yK$(Dfw? z4eCoxOi)`Gf?}}y5xUR$v(pl`{Ow2sIM?C{$itM?;IR~{ z;J=&+Lpo?>64!D@MPxB^WJNSpyusxQ84CF_uN@4j-AQnhwcp-%q_K6C&!L{hg7`hX ztL2YqvtM%o%1bX?)rUlisDuO;pOMYVDZ01Ik^GeHVMavG87nG`Hld0CwXheyDHv4J zdTizmPt-PkMepeYHgb+(gAZe5;t zzv56+=373zS6sQU%jSw`^8Iuj!dR9S2>nY$sq|Q|ePeWCN0a9WO%%9;CS*^(*Al|e z6r->qZy_~1&EU4nv3KDnv?{q7xNtgLAo7Iawt(D}ItpV@z(p~pWsbul-ke)k0!^I~) zXk1Cb?TQWpiV(Wo-9q#8=H-|Vp;yTXN%B4{C#%{8b^Z{OJnHv?Kr%w!mCUhG1un%8}+tVyZy1Z4hBFtWgLICF(*e?WkFp@B@e z2U?ufgYN9HTfg%E{Qn?#{+Rraklz1jW0?PFW3>NQmFPSp1CkPT|HG*X-mz@aZQl~A z+IdwilF)|IhDt{m!jKFl6Q#FG-`%Q(F*URtv#a*5uRt+YORyk{YsyK7DUq9RoYrUI;WL6tuN@L# zSl>Rg<2+#r(8P|W&Ylg@y>7*ow}IL@fz^tGRBP>AtF^M7v1GGI!mBo7&}aQ(bKiR6 zHN-D%r0jG}`RKmIT)sD73bLB8*6@eC_&X949cUhuxJeLoZgEkCOU0z>n!fLP;-Kwe z|JGcg(`0kK+;%d`(^-n|CVSyG(oAWnF_m-eka>OwD3M#Rk0H!_IwxoM@|I)WR>qQd*-Y$0=8`WcpM{wuwCTG z{@;EQE~9b%9TXs-P+A}$g8zO`VF6W|kUl!9OZ?;wonw!#(izRd=-?LXm9j6UO5v92 z^NB39vgixtxAC1MzZwGB?bidTtVF>C71vA@5r(LUlyW7d{{#?lhqnIxLzGXHL7gkm z^V4Q;#w4j{d2wT%@Uh)7q51O<^=^;n`>z=hYQfioT&T;Vm~LcACdsUsP6-II5Sqs! z&r$xXGQ7Z&eNFiN)+Gpa4};0Am+lZNzl&VNNjb{gMk1m9Kx1iUt%Fqa+8LKSradd5)=H;>Ds?sYu2;p3^Dd)H48lez#Asbcgh&Lp8-@NmOu3umbAQ zHcUQo8&2SxDIYft57w%dO%3>~Gg1MfLr+{G`mSucpH3iq91K*GbYh9c4sHoqv(QQ@ zG0_W`by2iNFm)zmUO;0DcT0A*R(XdO)dK!H0Bjc*Z7ZlXwo&vZ5u1GwWA(`xH50qS z{MQ8wTTpAF3abUovP(gK;tH7Us6aS9bT~_JeNq=o{I3sM4y`PWgf^gv9Ix<-hqf2e zldLEVv@K%>vKI9aWg{r_7&m`@8t7%TK^>G!{Xkdn9I3+XZbW=e`xDY~df_!eF#Mq; z;L~!O*1lxoa>l}^dpHMHNeWHllT-7_bM7^Opo_NqH2&DOTj7A-!Ur*7J62*%j7Dw6 zv^lGa54l(somq*Jc?f`_rP|Z(=6q1htj5Y>zrI@Q^iUl9Bx`O{y44t5R(eNc9HNNJ zjFh!B)r>0Zvdj=wEe+Kb*fkVdGzFwTc^E^=%`F>YX=OZrUU}UZ`JAfRwZPQgAzRU| z2y0}T%iG3`D2;U`!@NnxRyiIConkj|T^<};8TAAb@6E=5J%`X2aRc`*s!M!vxy;_ zDRXUGX=T8`g#V7W6|;#j6g}czIBV+2zuja}pN_5}^sSv@g5A(Tm0gpW&|zIY=h>T$ zwlH63y0l5J=hoeiM2eBLsfSxAx~5lsmT{}r`wIXY1e6Y zYDpc8+$QQ1hxG%r8CH{{`#L>Aax}OSrP8z?83vEQ4FDCaTXTf9G^^e5V+*Fwx%^N& zZWO^_j-X>MAp+wz(%On0Enc{@kQUY9Zt0ZRjXby@fA?`#f<@fd!O0gwr5l@``#*wh z8BhZ{iGddq5tXV`;zJ8EYzwX~UBz1Z(}q(lK24*ZEk03;CAnp&*XJ_q`=Z83VqMo6 zs>G?T!T~8Ta^}Y;WT6b4_%IcdDjBF(Za$Z`F$Rm?ub7A)N3EywNTi?49>P%oAw zsqI#q(|4l?-~8p=sgXRh8QC=?B`2sC=2C?rxv69^I0eNf3HNIG(3@B}e=1O6A!{4w zw*U-Y--&Z@vFGu3t#JaMXQ_#eOo}CUMGTfKO|0p?Y~Z@QMW!3MYD`Ne5VuepQ>m0^ z%T9IhWL9O3>v{yF-5|EQ3L{LwmyjD<_kq71R!;w|8_VKghl+}Ek81FNf1M1peLBv} zg^4U$8Z~-i%6Bs`fg>+Ea}cA`mDn^ns{+hiq>xam6oAx^dAeL6pS7k<+~bkvsfQZ( zwXIchdKMC-J#y`2E#8L@gwgGJ{yWcK=eA?l+nq8aEatOJZXgulxfI>i0wkde_r zVUN<&oKL1r&t0~t8a~c{%J$ncc4`}xTAjw0G&O}H0%z}y$pMM_ zA*PnJd`wluW)(D5q(T+v8Wy(LI-M$LIesV+F|5kyk2DyE^)juCILt`*1CA=ei=AyF zm2_&qGnU}u!j>jGIhkZj$WJA+zJ+Q|UFf$;7H-Z&v+xQ-T;6GX;{-O)8GZy7$kRX;^%h5C7dQc&?=jd64n4y!Lf@>k4QUP$pH4c<| z*b=8hV2A0+G}Q8vlm4EDTrqz>ug!vSHz0?|;9Xrd?%MNg^Xzg~D1w1%>3*Vdx#-M+ zKlLI1ccbqNKh6z6#qDK*M3``V_ekbGZ280t|90(8xiQxAHi;T?4JH+FZV%B_^S_UI z^~bEqG%wQL>u`1!T9v=~%$NX?n>p zP*t1;u_4x}9Ugi+UAo0uLXszkuEt~Ij>(3_%(L-y$-up7OIkO34J#a@w|6AaNcC%u zGtCi$g$7lHBRya}Sp%NVK84h9AiyGx2f~HYE$wIzjv6z>*M>R^2_Vlgw27lglc`S- zqPQd4Pg)i)!j>0Y#Fa!?!hJH)ER%()s0Wa2s5kqj=qGbBa&!gB5u2To3ehZvoW_9$BA*}5p##PSy0xYp zR8vKrEi!hfS#J?H4d557^dDE|`bo@LelrJomApNk!ugNQPr7dvrO3+_}$(b15lb!#a4#p}5jFh`uS9{IPXP+*a@vnrg< zVf7Y48R^|+oxgu{YGIq!6{%13B;TVZC2N+SCW{^X0lH`ggTS-Z)qfp8stM7>r9|{zn^79JftTM zwX8m8ZOW<@#n;AI|F%dD6~R_x+#l#HS-Ml&K+D%<;gppmFGpqObo?pj>2$MO_~ph# z9fBsi1Q_YC&Gh@iwK7*AFGU<|j%lo|cx_wv&0f4F?+AB&JmImY@}jG}@Fy47pgA#b zsnt)DS%&~F@WzFzH`}PSx$D)S{J@HG0U7ddS{9grdD+P7qZm75=67z!Vf#9Dy~{Ys zrHjK+KR1$YLvXwn%)C_DNS7L(Z>GHCvyt@c1f=rH4bvx1)GSRWEqe$!1Y*rCmfJ)@ zdt+S*EJ})Z13)7j5sIfJNRUevjiXr-q;%=@R9FfJ=ZWmP3&8Eu8neU}#lHNf1cuD? z$Z|4Ma)Y;rTY=hdg`O7GWE_DvTG6LT-PSuuxUcsmdlEDm19ay zZWe1AhjNb2&et)Tp1)YBGFZvniK?*Uf9iSePg?IGOyDi!=kCcCT`|>|i3UaL)#i4i z%82qfejn$#2o=4_VSAOaHBU%XNo$$;0wyB|xD8mT44+riX7=pHj}T$iob5;+m<`$ zK(6wci0&0b`6P3QiscU~{xrNjd=Yi?0l=X2 zc+8Vulnvj2z_i1~qDn$OBzMFMCa9m|)b!e7K95vCEGG4a_M|Ohi22t&H8f?BE%{6| z1eh%dvqU0Jks=ehSp3C9txosopIvxmXU; z$8T~OJuva_f;)XxQ0~Tzg$8530ds^uK(aQJ@NdH1!>fbLl=Dtj&ahKm ziDU~dZjVbY>eeHYb_TICLO!mtVtq?9{@1@{Ea@-6ylss=h47@MnlUoq?afsXtC%L*4)M(|)-K z4W4LT(m#iEzHACOk1XWHwygH-c7C}Si9lQWy2QVDVWHkfWo(Euw*>pltJXjH@ZWGg z7S?9CQU~HdA7p|6%jao50UjwDZ1ap7xpSB0CQrd!rh_8f=k6XI(`9pVrMTcP+y+u} z`pUOdVotC)njx#Vpjw*y>$d)A(*wO&3S0?2>gInVyY?1}vP5*tviEZk&VIsO`BQe> zed}Uq71#pJy)aN!ckrB~Pn?lqU>Sl@?UZCBUFHUf`ne_Q(AM<{=Y> zbI*|9CiVpNm6w3ir+Nnl^ObJ4RF?P4Y_{c(KFaphB2|bSbwl#X_k^>5v6TD)QWSjf zEBAv)IEKq0`SoC@3SbJQH&Y~Bm(F`JoqctJ`jve2#{^dQS5-~X)1Un`pgmG4eNw|| zGhC?R%5AvbX0Ti*vMtG9wQFUypSl=N^II4zNv5Wkc#&aO!|^A&P}VNDt?rg&mEuG% zzG)o3D3OD_uBz}#ww|W^Q!ySDBn;>}vJZdeg0v&hETyW#j{#`(TxjequjT35e^eEj z_px(HX$kfDX?a`OL%75j>;?0&Em0I05%ExECIk4%Rt$} zlV`s?5(<@57P_PH6v7m@NhF{bH)2#7c8ZnAf6zf6w!R2>6X$LbvXR=Xe_LQ0(Grc7 z9#d&WkKB`4bOCaubUbE^$U!4~*=He<{2sDqx!1_BDY{~|iPJ0PXeT&bBsar* ziWo#a>I_jPJrO)07=@SOJ>23J zNYJJV=UbF-x@fihqt^U-Q{0XwKS$-!Rf@+Ka-q)tvQQ@XtoQdaYI4B=wF)n{iT0O^ zR)i1x^Fq6yRX9OLO-L1cZ}2xCR~IRgM=V)LV}6ueU?9kp?_wvarlL$|nI*ie^Qc`P z^>fJA;~s#|>R$!FpU2<^Vf&}~@4k3A-?}(HGRFR#xcTEhIDMWhqtY^1qb4uC=(&SI zYZO_YFUGPT=(9J>$I(IR9SOlJa`>!r?WsKDizVp9%$dzQnA?uN;nzd%BX(hj`EJsQ z;FV_Jxjkb346r&q&}`2y6@3-FD{!5bc^n0AsUUz3h@Gkb~I|jci zR@LGuXVkKl1o0!?lt2zpUj~1e$-J4&8W$H#{fsa!IdOI{eOMW!=&+wvSJe7@X;)qT z=p~>aI>VXUK|%J7*u+oMl7CDyK2AGiyGL12ivSQjhR}`PE=5#PrVz)1A2aQm1h154 zKJ+Mc=}MkRy{Rgf1Hb!}p#D5nF?Q1S<^xZ31tz2K@lacN4~6lCdZ}0B_=pv7<%%M2 z-wSC!uj4~s+~U-y7MupiO)=OeN~EZV_XN~d<4|>0iqwpybq0SI%FrF-%4WZmN)+%v zJh}@M3Ve4amH}T6xFaaj;pNh;eHF;APRt-3;^j7&LNgxG4l8ieg*iG&7lDcz@ssB1 z4btOpjgliFp8$Quz{Lj5N;U<4ewn-|bLA&hyc>#3T$E%XVe@)hxRB&-3;F-`h6CtA zMt3QRp^QtHB1o=4XSyqZIFbI5@I*8EyAyR{r()IN+-61W1UCm@e8+P_byFsZK%&bf zz06Zym43P`V8zL`#VLea&YQE4+D#j72pO5B!xPUrylcdsfxhhwjYuqG2RR{KgemES z`k)bfyGt)GXwHlF!L6R_r3HWalm+a4?q|-?7x$PqH%6S{L!AUs@O?}E1Z31}&S<{b zSE8&e#)Hox>SJ|1;n6Np!0F(FJbWs|3Yv=)v_P^7?QE#L;!`Pz3fV=h!nplYolM$; zD|6%ww3s)dlq)L08*(=pw$vTL2orHZzCfWWet$(kFeJ5*$7`bw8u&a+M)Q+h^~#eIi}N zfUn_**m`$CR)8WS4noaqs7X&8P(~G@{=_SnfEwtQ$UzeLkhq#>y~^HV4j4201asB( zW=;L(R$M(K3>+b3(~a!AZiF|6p%QR0=9BLQ7L2{_^EX6?gAq&;J%rm?;z8~(+WT)) zLW9w8#ecj9h%cB8zKF*HfYM25rEZd_S074nC*@3^gr8@wkvB1{EE+1lOPt zBzRLqEcy$cvQQ5IPb6FAZFpZYX26glr= zOaCihHpBsvU+;1Pk=^Y>8N>S>3;ax|huWYsArJmAkRL{W%<3B$p5CDVzu5&jksj2Z zDVsjz-V%Zy^q%W8KzJ|%#4cn0!O2pGl~#CaExFtpq%acjZd*z?eMbo$POsX+0M%I7E+Wpc$&y2dvw3-cQNAXngI}d&C*Rc*3CDDFOou)rJ@_LYwj+m&WO=I$21z#pUV^= z>A{vz6aTqLo(=D?bLK1g!MoX{L8i340n^t;f&5Jp-1_O9@fsloIPDgCZ|O(Hzmmp ztFl}SB)@6Co?d#%w8TLDI~$aqfgklW{>{0BCy9$SD!oUPmnbtAmpmASu;KeutLHLN@k%TDr@yhPWy_4K4jHp>oR zZl82Z$X2S+SuDvqe&oO;MT)LERa&Jx_h=o>^^?B^d(6yfm%5RB+`0ce)JhHK*LRa5 zG#KoLcAAYFMSr!Z<6dc*cjI!LNC_W24+Rq3Gi!dMiq6g`y)${}u&~`jJDdLjtBCE) z>4ls!fGf*_LCz`NE;A^itfzP>9H1u zc2sxpPUmME)TOmhhpHFd1F4I!l;hI4oeijq?5Jl?w)En;Gn?wFW9qRwU1^2p(rTU~ z{WFT5*p!9}w-(5KwDW(;x(=u&nyyW6g7iR;5(vF_kS0}1M5=TY=~Y02bOT6HDT0fD zf&r-(nl$NMq$ov@D$5Ca+_~3vtooBljYzUf ze0SWW0O7p1rSMbvnQD3kIpj@-bPeV#R8YTIP27I}y;^TPytNnR=V&Aw{knr0sc#|e zHLHgEjdJ`3^_=T691(M~*-lP%MC|CQ2g&|Fxr5i`)AZ!W4<%VMF)#~i4Pq5koJrQ2 zv_hyvsGV5slo|P;>K!IK){prlRH9yR^vjkTYOo7>LsaSUlsv9fdcS0-EM`m#QBn~y zeU--d)XR4G%Qq*U~gySg+GB%3`5&c1P!lLZ*yqP z!shAWLuw;>S@DSxZd!5|uhyimT>RD~%gvi@QiE2~;&|65e8H*nXa6_x{S4Ui_6~V9 zG6LTC2p@aGPT+_a4dMP|Hqb0fjoO-QY2e=(_Bueq8B&f)5RED_SM;WY@e~*OVrp+L zZxONZhueu6eKOi6X{gUo$oZ~Shc9k9R&3U*K zcn2RJk=Tkf(Nyy0V9Of+$^LLSVI7yLx6fKBPFN=-G=Y**N}Ke*dfa`~HK2Zz{02=#A}b zVjWETl}RfX<*K4d9LRo@HyRX8_?F)aZdIY}v?FC8>1_Q?IrhD>%@mh%mvpRCEJ!=w zOuAFkXA5~LjAt15eT2A1E`{pbHE7Rd#!t#nhu$%`O5OFs+oIl=sNSFM5%p!9ptq}Z zfI1t459Xqk1^XcbHFZkrd24wExn3hi$eJ;SYQ$qtcDiF6p_K<6ULgyXi$5PS!A!#Pgeo z@6)XFZ?@AFN1%K6#IujK%=hyw!I^(hhOiWK2pB<1A9UgS|Aq2~Jz^ksm>y+qG@wiu z)vM|cY6w zWKP_2s=Me^wph2WG&-==J9(oK*M@LMl`4t{Pl2MT3>MoGg_DLmlGVK^Y40)unIilo zFS+_TffgpB92ROeAUHnwS_ow+fyB$s^_ef@Sa~hP_9P~afaCeL^?O6owy)@@tl@dG zp)I-b-K%eVNjT--@jG>OjRod@;e-YE3tREj+z!&jv>{p#kuRmLWjoNqH1Ljubca_3 z!N5tHa5DuzL5R`HMlHf^dfZc{B(`$Vr>!paZd>YP31_Qb4_h@X**P4H-y&o&V(Ps zJO59eNKcX8v3lj9)(IA;K>0k;TiZgB`rFg4m)YNJHWyYcTz?;O-_{}HDSJFcK4V3o zKm#(dLFc7y`9~vPy=?q6)ReBn)9Ya*fT)H|vV?ZvUcAFA-Sp}Qn}O!vI#bAaG@q#M z&wXhg4<`}3Pn$ZY6;tCN5Jwc2_=e?IP=&gdvQ-2fg`Khd`~HPEMEy-(DeCr{su3jW z8d{$h-X9-UurGh|qI|u${EEVFqn3PDQ>Y2Yx9tl~uHWsJZf%kLItA2%Lt7Iwk!ms7 ztfbO4?+lv;f!_oi?}^5z0NW#Jk{-FKf_sKYvCmP{>+#(}vB1=aR2()mwdyg6T)vkd zQ;1UuM-AHRf}-Oww(A#K81P^l+M zur11ataS^{Lx0{%|K?(UXOi1y) zzR@IOOPYVM#Clz%lPATrhk{a5#=Gc^^Q@DdbHo96WF!1AdsL`zEG1bV{!w(OOyiA0d8BvM1+UtRP`d3%SMN7V@ zS?OQCg7TOUt^4`<4Y)ixL9?)~wL_6l6nG#nK&_ahQ-$sc`%(W}<~${5t$YwMu>vQT z)6WGL+1huM2SyKwjtA&6OSwqMJH~=(m>eu-f6OLJ zKkFagHM5BCsqpk`*A36NajCrN_+5WC)g>`KZ5vfyKU%WnW^urYRPS!iIRDnC7%ki< zhIh_gTvNNO+WNA|I^UJ}rJ+#imYS&AkLAgudFDjL6S|e-PY_4!idIbxWNIshmC4_} z6wZJ3ky?o@xLKT1Qm%?X4GWEPt7O^fqJzJGO&jtlBF5W^P>J`HzeJk`x~dEahLk1d zr(ZzviFRZ#K2j%q`LWY{#7WHQ2`NDG8FM`}G=_=H8rf&%x@F^iW-b>;!t7kL zL(t_UQ1dQt>ombJmCHP3)!{$fi=SXzpc#o6^V`Q&w}QUFC!F&IH)v-1k9iRG*8ZVa zMxhfIYu64F9$Q5}jTQ08b{WgbNM0El5&ncK^+mMtWtz|0dt^T%H*E2Oag{4QS0>%O z9K6wM0BLxwAIe-=xWGB85dC$L=$`9r>5}_=^8WWXHT1Q`ns!NFloop!T#VTU$exzz z7LF40RSRmnTy1pgL3})a<03nvF<05zCMold7Rj1pfVh&Twx+qK#hrJFy|iwfP#v8d zRF%=oP5v$i!kC1TITQor=~AIRX+POrr|PfPMZDXp^zxVhnjfaWt{mW@ZW&@Uf*&y5 z?OZrg9g^nWwq?=Ls*=;tnMxEh;!o0=G;tLdnMM z{rRJ`{0*O&CvVyUDXGu;mWUAa+}DDKn@5Ugfv9l-pRh%cX)f832 zgDSF355b&+93l>N-WLb#T!__AxgfOC1cji|9@ct~kJ{fJ7n}IrXEIlhCC) zG}rT!JUV*w2(q+yF0X#S5G@u>+9#^Fvv3`1S zDT>UZ1p2&uO!W;vvhNoZy&Y`fxn01}FZbR2hStgjg%z&yhW@gKq#P zZp~OEOTU>OYTb|#)}vzKp-S;7G#RR;Ch?N#VdR^9>QogkXC|qEghymm9;hu}#uQum zkZX%nXs%2*F=GPeXhG)(BXp{}cFp9j(JeYtVjrQe<8!YnKfgQyZs`?BF%YsJ93W{U zg@>6GY;@>UIj&OG846G!cjZ-cnOR8sdR~o4E|dBTdu--_gXD1_|Ort2jkKJ>{(3OGA)%@5KQDKLpd$_a7|Y2UZxZL{Rd_M~ExmNzT9 zu3{A-d0AUzh@0R$d2)w6s_g0;oFDPjyZ#11S{tjnbIWu*pVc-&l3A4qNpEkH2p4JU{i^slq<%m#PPmDNu@kGwMFiibr7Qt z<)dy5eK&0rBZIZbq|!`tp~ynCeFWe0G-oRL7JjFvJahLtIrvVr5ED z1fOE4YC5o1u(eNt;i7h1oioqyqI;_F9+0D`OUfmNwNh zRk9wgkD_aw!*x{Wycit5PpCJ3pv|nz686xr9 zFW@Swjk&p;{cFmoFbi>utj78_ZxexMYdKP#}Ju7o5L53BLVtp7~GdXdeEjB(op@mh2Wx!DHe%C@{Dn zb6NuhxE2-vVEh+(z*6Sob?|t+=nNj6mrlwEdMGfpbb@02ZUi1N%upcayTX~=c1wFM zjlh@hmS;p&zBUpz_GIMV!=!J!fHnx{04&$ZPlkszl>DFCoUk5CV0xVcC_b&K zTZiL~+kpZ}5U1x11w=N;@Qxh;`3+T2aCZZaH{cA&xyl2gKTmc8o($t(8|N?Nx#Hk> z-gXjkx_#5$-9f|`>EaFs%R7AzPBF5W^LRt9zy{`m;Qu}ze?!fKosWI5Dcw#8z-tWb zjEPv0uKzItJm-Y1-vVqt)1L&smgE%s|AdVHd#%3#mBHZ(I$8hy7jqFodN%we=5$_OH?1pr)xVU}IPI yj0UWUF)R%my@1!S$bUKvYaRfLU@!o{#`z)o2tshY /dev/null && pwd -P ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -133,10 +131,13 @@ location of your Java installation." fi else JAVACMD=java - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. Please set the JAVA_HOME variable in your environment to match the location of your Java installation." + fi fi # Increase the maximum file descriptors if we can. @@ -144,7 +145,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac @@ -152,7 +153,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then '' | soft) :;; #( *) # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC3045 + # shellcheck disable=SC2039,SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -197,11 +198,15 @@ if "$cygwin" || "$msys" ; then done fi -# Collect all arguments for the java command; -# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of -# shell script including quotes and variable substitutions, so put them in -# double quotes to make sure that they get re-expanded; and -# * put everything else in single quotes, so that it's not re-expanded. + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ From 35f2f98a1cfb874890947d2e4ac775e709015721 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 31 Oct 2023 12:27:05 -0700 Subject: [PATCH 059/157] No public description PiperOrigin-RevId: 578266141 --- mediapipe/python/pybind/image_frame.cc | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/mediapipe/python/pybind/image_frame.cc b/mediapipe/python/pybind/image_frame.cc index 7348133eb..9ec60dbec 100644 --- a/mediapipe/python/pybind/image_frame.cc +++ b/mediapipe/python/pybind/image_frame.cc @@ -81,17 +81,20 @@ void ImageFrameSubmodule(pybind11::module* module) { become immutable after creation. Creation examples: - import cv2 - cv_mat = cv2.imread(input_file)[:, :, ::-1] - rgb_frame = mp.ImageFrame(image_format=ImageFormat.SRGB, data=cv_mat) - gray_frame = mp.ImageFrame( - image_format=ImageFormat.GRAY, - data=cv2.cvtColor(cv_mat, cv2.COLOR_RGB2GRAY)) - from PIL import Image - pil_img = Image.new('RGB', (60, 30), color = 'red') - image_frame = mp.ImageFrame( - image_format=mp.ImageFormat.SRGB, data=np.asarray(pil_img)) + ```python + import cv2 + cv_mat = cv2.imread(input_file)[:, :, ::-1] + rgb_frame = mp.ImageFrame(image_format=ImageFormat.SRGB, data=cv_mat) + gray_frame = mp.ImageFrame( + image_format=ImageFormat.GRAY, + data=cv2.cvtColor(cv_mat, cv2.COLOR_RGB2GRAY)) + + from PIL import Image + pil_img = Image.new('RGB', (60, 30), color = 'red') + image_frame = mp.ImageFrame( + image_format=mp.ImageFormat.SRGB, data=np.asarray(pil_img)) + ``` The pixel data in an ImageFrame can be retrieved as a numpy ndarray by calling `ImageFrame.numpy_view()`. The returned numpy ndarray is a reference to the From c6aa9cbaef0d5bb5b9f9e5f3c9d72c17b24a5a01 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 31 Oct 2023 14:27:06 -0700 Subject: [PATCH 060/157] No public description PiperOrigin-RevId: 578303180 --- mediapipe/framework/tool/BUILD | 1 + mediapipe/framework/tool/sink_test.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/mediapipe/framework/tool/BUILD b/mediapipe/framework/tool/BUILD index 77e3ab16d..65c8a15c8 100644 --- a/mediapipe/framework/tool/BUILD +++ b/mediapipe/framework/tool/BUILD @@ -616,6 +616,7 @@ cc_test( "//mediapipe/framework:calculator_runner", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", + "@com_google_absl//absl/functional:bind_front", "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/framework/tool/sink_test.cc b/mediapipe/framework/tool/sink_test.cc index c5316af4d..9769aeeee 100644 --- a/mediapipe/framework/tool/sink_test.cc +++ b/mediapipe/framework/tool/sink_test.cc @@ -17,6 +17,7 @@ #include #include +#include "absl/functional/bind_front.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_runner.h" From 3a55f1156aad1ace7a752acdfdba57389d8c9a54 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 1 Nov 2023 06:19:05 -0700 Subject: [PATCH 061/157] No public description PiperOrigin-RevId: 578496866 --- mediapipe/gpu/gl_texture_buffer.cc | 9 +++++++++ mediapipe/gpu/gpu_buffer_format.cc | 14 ++++++++++++++ mediapipe/gpu/gpu_buffer_format.h | 9 +++++++++ 3 files changed, 32 insertions(+) diff --git a/mediapipe/gpu/gl_texture_buffer.cc b/mediapipe/gpu/gl_texture_buffer.cc index 48afbd219..7e4694a0e 100644 --- a/mediapipe/gpu/gl_texture_buffer.cc +++ b/mediapipe/gpu/gl_texture_buffer.cc @@ -14,6 +14,8 @@ #include "mediapipe/gpu/gl_texture_buffer.h" +#include + #include "absl/log/absl_check.h" #include "absl/log/absl_log.h" #include "mediapipe/framework/formats/image_frame.h" @@ -131,6 +133,13 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) { SymbolAvailable(&glTexStorage2D)) { ABSL_CHECK(data == nullptr) << "unimplemented"; glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_); + } else if (info.immutable) { + ABSL_CHECK(SymbolAvailable(&glTexStorage2D) && + context->GetGlVersion() != GlVersion::kGLES2) + << "Immutable GpuBuffer format requested is not supported in this " + << "GlContext. Format was " << static_cast(format_); + ABSL_CHECK(data == nullptr) << "unimplemented"; + glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_); } else { glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_, height_, 0 /* border */, info.gl_format, info.gl_type, data); diff --git a/mediapipe/gpu/gpu_buffer_format.cc b/mediapipe/gpu/gpu_buffer_format.cc index 646fb383f..b099f4a25 100644 --- a/mediapipe/gpu/gpu_buffer_format.cc +++ b/mediapipe/gpu/gpu_buffer_format.cc @@ -35,6 +35,10 @@ namespace mediapipe { #endif // GL_HALF_FLOAT_OES #endif // __EMSCRIPTEN__ +#ifndef GL_RGBA8 +#define GL_RGBA8 0x8058 +#endif // GL_RGBA8 + #if !MEDIAPIPE_DISABLE_GPU #ifdef GL_ES_VERSION_2_0 static void AdaptGlTextureInfoForGLES2(GlTextureInfo* info) { @@ -163,6 +167,14 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, { {GL_RGBA32F, GL_RGBA, GL_FLOAT, 1}, }}, + {GpuBufferFormat::kImmutableRGBAFloat128, + { + {GL_RGBA32F, GL_RGBA, GL_FLOAT, 1, true /* immutable */}, + }}, + {GpuBufferFormat::kImmutableRGBA32, + { + {GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE, 1, true /* immutable */}, + }}, }}; static const auto* gles2_format_info = ([] { @@ -206,6 +218,7 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) { switch (format) { + case GpuBufferFormat::kImmutableRGBA32: case GpuBufferFormat::kBGRA32: // TODO: verify we are handling order of channels correctly. return ImageFormat::SRGBA; @@ -221,6 +234,7 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) { return ImageFormat::SRGB; case GpuBufferFormat::kTwoComponentFloat32: return ImageFormat::VEC32F2; + case GpuBufferFormat::kImmutableRGBAFloat128: case GpuBufferFormat::kRGBAFloat128: return ImageFormat::VEC32F4; case GpuBufferFormat::kRGBA32: diff --git a/mediapipe/gpu/gpu_buffer_format.h b/mediapipe/gpu/gpu_buffer_format.h index 06eabda77..223780939 100644 --- a/mediapipe/gpu/gpu_buffer_format.h +++ b/mediapipe/gpu/gpu_buffer_format.h @@ -53,6 +53,10 @@ enum class GpuBufferFormat : uint32_t { kRGB24 = 0x00000018, // Note: prefer BGRA32 whenever possible. kRGBAHalf64 = MEDIAPIPE_FOURCC('R', 'G', 'h', 'A'), kRGBAFloat128 = MEDIAPIPE_FOURCC('R', 'G', 'f', 'A'), + // Immutable version of kRGBA32 + kImmutableRGBA32 = MEDIAPIPE_FOURCC('4', 'C', 'I', '8'), + // Immutable version of kRGBAFloat128 + kImmutableRGBAFloat128 = MEDIAPIPE_FOURCC('4', 'C', 'I', 'f'), // 8-bit Y plane + interleaved 8-bit U/V plane with 2x2 subsampling. kNV12 = MEDIAPIPE_FOURCC('N', 'V', '1', '2'), // 8-bit Y plane + interleaved 8-bit V/U plane with 2x2 subsampling. @@ -78,6 +82,9 @@ struct GlTextureInfo { // For multiplane buffers, this represents how many times smaller than // the nominal image size a plane is. int downscale; + // For GLES3.1+ compute shaders, users may explicitly request immutable + // textures. + bool immutable = false; }; const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format, @@ -121,6 +128,8 @@ inline OSType CVPixelFormatForGpuBufferFormat(GpuBufferFormat format) { return kCVPixelFormatType_64RGBAHalf; case GpuBufferFormat::kRGBAFloat128: return kCVPixelFormatType_128RGBAFloat; + case GpuBufferFormat::kImmutableRGBA32: + case GpuBufferFormat::kImmutableRGBAFloat128: case GpuBufferFormat::kNV12: case GpuBufferFormat::kNV21: case GpuBufferFormat::kI420: From 9474394768ea77ef58280f7114f0fd0e934a62ef Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 1 Nov 2023 08:31:11 -0700 Subject: [PATCH 062/157] Add drawCategoryMask() to our public API PiperOrigin-RevId: 578526413 --- mediapipe/tasks/web/vision/core/BUILD | 45 +++- .../web/vision/core/drawing_utils.test.ts | 103 ++++++++ .../tasks/web/vision/core/drawing_utils.ts | 220 +++++++++++++++++- .../core/drawing_utils_category_mask.ts | 189 +++++++++++++++ .../web/vision/core/image_shader_context.ts | 56 ++++- .../tasks/web/vision/core/render_utils.ts | 38 --- 6 files changed, 594 insertions(+), 57 deletions(-) create mode 100644 mediapipe/tasks/web/vision/core/drawing_utils.test.ts create mode 100644 mediapipe/tasks/web/vision/core/drawing_utils_category_mask.ts diff --git a/mediapipe/tasks/web/vision/core/BUILD b/mediapipe/tasks/web/vision/core/BUILD index dfbbb9f91..31bad937d 100644 --- a/mediapipe/tasks/web/vision/core/BUILD +++ b/mediapipe/tasks/web/vision/core/BUILD @@ -31,27 +31,57 @@ mediapipe_ts_library( mediapipe_ts_library( name = "drawing_utils", - srcs = ["drawing_utils.ts"], + srcs = [ + "drawing_utils.ts", + "drawing_utils_category_mask.ts", + ], deps = [ + ":image", + ":image_shader_context", + ":mask", ":types", "//mediapipe/tasks/web/components/containers:bounding_box", "//mediapipe/tasks/web/components/containers:landmark", + "//mediapipe/web/graph_runner:graph_runner_ts", ], ) mediapipe_ts_library( - name = "image", - srcs = [ - "image.ts", - "image_shader_context.ts", + name = "drawing_utils_test_lib", + testonly = True, + srcs = ["drawing_utils.test.ts"], + deps = [ + ":drawing_utils", + ":image", + ":image_shader_context", + ":mask", ], ) +jasmine_node_test( + name = "drawing_utils_test", + deps = [":drawing_utils_test_lib"], +) + +mediapipe_ts_library( + name = "image", + srcs = ["image.ts"], + deps = ["image_shader_context"], +) + +mediapipe_ts_library( + name = "image_shader_context", + srcs = ["image_shader_context.ts"], +) + mediapipe_ts_library( name = "image_test_lib", testonly = True, srcs = ["image.test.ts"], - deps = [":image"], + deps = [ + ":image", + ":image_shader_context", + ], ) jasmine_node_test( @@ -64,6 +94,7 @@ mediapipe_ts_library( srcs = ["mask.ts"], deps = [ ":image", + ":image_shader_context", "//mediapipe/web/graph_runner:platform_utils", ], ) @@ -74,6 +105,7 @@ mediapipe_ts_library( srcs = ["mask.test.ts"], deps = [ ":image", + ":image_shader_context", ":mask", ], ) @@ -89,6 +121,7 @@ mediapipe_ts_library( deps = [ ":image", ":image_processing_options", + ":image_shader_context", ":mask", ":vision_task_options", "//mediapipe/framework/formats:rect_jspb_proto", diff --git a/mediapipe/tasks/web/vision/core/drawing_utils.test.ts b/mediapipe/tasks/web/vision/core/drawing_utils.test.ts new file mode 100644 index 000000000..b5ba8e9a4 --- /dev/null +++ b/mediapipe/tasks/web/vision/core/drawing_utils.test.ts @@ -0,0 +1,103 @@ +/** + * Copyright 2023 The MediaPipe Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import 'jasmine'; + +import {DrawingUtils} from './drawing_utils'; +import {MPImageShaderContext} from './image_shader_context'; +import {MPMask} from './mask'; + +const WIDTH = 2; +const HEIGHT = 2; + +const skip = typeof document === 'undefined'; +if (skip) { + console.log('These tests must be run in a browser.'); +} + +(skip ? xdescribe : describe)('DrawingUtils', () => { + let shaderContext = new MPImageShaderContext(); + let canvas2D: HTMLCanvasElement; + let context2D: CanvasRenderingContext2D; + let drawingUtils2D: DrawingUtils; + let canvasWebGL: HTMLCanvasElement; + let contextWebGL: WebGL2RenderingContext; + let drawingUtilsWebGL: DrawingUtils; + + beforeEach(() => { + shaderContext = new MPImageShaderContext(); + + canvasWebGL = document.createElement('canvas'); + canvasWebGL.width = WIDTH; + canvasWebGL.height = HEIGHT; + contextWebGL = canvasWebGL.getContext('webgl2')!; + drawingUtilsWebGL = new DrawingUtils(contextWebGL); + + canvas2D = document.createElement('canvas'); + canvas2D.width = WIDTH; + canvas2D.height = HEIGHT; + context2D = canvas2D.getContext('2d')!; + drawingUtils2D = new DrawingUtils(context2D, contextWebGL); + }); + + afterEach(() => { + shaderContext.close(); + drawingUtils2D.close(); + drawingUtilsWebGL.close(); + }); + + describe('drawCategoryMask() ', () => { + const colors = [ + [0, 0, 0, 255], + [0, 255, 0, 255], + [0, 0, 255, 255], + [255, 255, 255, 255], + ]; + const expectedResult = new Uint8Array( + [0, 0, 0, 255, 0, 255, 0, 255, 0, 0, 255, 255, 255, 255, 255, 255], + ); + + it('on 2D canvas', () => { + const categoryMask = new MPMask( + [new Uint8Array([0, 1, 2, 3])], + /* ownsWebGLTexture= */ false, canvas2D, shaderContext, WIDTH, + HEIGHT); + + drawingUtils2D.drawCategoryMask(categoryMask, colors); + + const actualResult = context2D.getImageData(0, 0, WIDTH, HEIGHT).data; + expect(actualResult) + .toEqual(new Uint8ClampedArray(expectedResult.buffer)); + }); + + it('on WebGL canvas', () => { + const categoryMask = new MPMask( + [new Uint8Array([2, 3, 0, 1])], // Note: Vertically flipped + /* ownsWebGLTexture= */ false, canvasWebGL, shaderContext, WIDTH, + HEIGHT); + + drawingUtilsWebGL.drawCategoryMask(categoryMask, colors); + + const actualResult = new Uint8Array(WIDTH * WIDTH * 4); + contextWebGL.readPixels( + 0, 0, WIDTH, HEIGHT, contextWebGL.RGBA, contextWebGL.UNSIGNED_BYTE, + actualResult); + expect(actualResult).toEqual(expectedResult); + }); + }); + + // TODO: Add tests for drawConnectors/drawLandmarks/drawBoundingBox +}); diff --git a/mediapipe/tasks/web/vision/core/drawing_utils.ts b/mediapipe/tasks/web/vision/core/drawing_utils.ts index c1e84fa11..95e376fb2 100644 --- a/mediapipe/tasks/web/vision/core/drawing_utils.ts +++ b/mediapipe/tasks/web/vision/core/drawing_utils.ts @@ -16,7 +16,11 @@ import {BoundingBox} from '../../../../tasks/web/components/containers/bounding_box'; import {NormalizedLandmark} from '../../../../tasks/web/components/containers/landmark'; +import {CategoryMaskShaderContext, CategoryToColorMap, RGBAColor} from '../../../../tasks/web/vision/core/drawing_utils_category_mask'; +import {MPImageShaderContext} from '../../../../tasks/web/vision/core/image_shader_context'; +import {MPMask} from '../../../../tasks/web/vision/core/mask'; import {Connection} from '../../../../tasks/web/vision/core/types'; +import {ImageSource} from '../../../../web/graph_runner/graph_runner'; /** * A user-defined callback to take input data and map it to a custom output @@ -24,6 +28,9 @@ import {Connection} from '../../../../tasks/web/vision/core/types'; */ export type Callback = (input: I) => O; +// Used in public API +export {ImageSource}; + /** Data that a user can use to specialize drawing options. */ export declare interface LandmarkData { index?: number; @@ -31,6 +38,32 @@ export declare interface LandmarkData { to?: NormalizedLandmark; } +/** A color map with 22 classes. Used in our demos. */ +export const DEFAULT_CATEGORY_TO_COLOR_MAP = [ + [0, 0, 0, 0], // class 0 is BG = transparent + [255, 0, 0, 255], // class 1 is red + [0, 255, 0, 255], // class 2 is light green + [0, 0, 255, 255], // class 3 is blue + [255, 255, 0, 255], // class 4 is yellow + [255, 0, 255, 255], // class 5 is light purple / magenta + [0, 255, 255, 255], // class 6 is light blue / aqua + [128, 128, 128, 255], // class 7 is gray + [255, 100, 0, 255], // class 8 is dark orange + [128, 0, 255, 255], // class 9 is dark purple + [0, 150, 0, 255], // class 10 is green + [255, 255, 255, 255], // class 11 is white + [255, 105, 180, 255], // class 12 is pink + [255, 150, 0, 255], // class 13 is orange + [255, 250, 224, 255], // class 14 is light yellow + [148, 0, 211, 255], // class 15 is dark violet + [0, 100, 0, 255], // class 16 is dark green + [0, 0, 128, 255], // class 17 is navy blue + [165, 42, 42, 255], // class 18 is brown + [64, 224, 208, 255], // class 19 is turquoise + [255, 218, 185, 255], // class 20 is peach + [192, 192, 192, 255], // class 21 is silver +]; + /** * Options for customizing the drawing routines */ @@ -77,14 +110,47 @@ function resolve(value: O|Callback, data: I): O { return value instanceof Function ? value(data) : value; } +export {RGBAColor, CategoryToColorMap}; + /** Helper class to visualize the result of a MediaPipe Vision task. */ export class DrawingUtils { + private categoryMaskShaderContext?: CategoryMaskShaderContext; + private convertToWebGLTextureShaderContext?: MPImageShaderContext; + private readonly context2d?: CanvasRenderingContext2D; + private readonly contextWebGL?: WebGL2RenderingContext; + /** * Creates a new DrawingUtils class. * - * @param ctx The canvas to render onto. + * @param gpuContext The WebGL canvas rendering context to render into. If + * your Task is using a GPU delegate, the context must be obtained from + * its canvas (provided via `setOptions({ canvas: .. })`). */ - constructor(private readonly ctx: CanvasRenderingContext2D) {} + constructor(gpuContext: WebGL2RenderingContext); + /** + * Creates a new DrawingUtils class. + * + * @param cpuContext The 2D canvas rendering context to render into. If + * you are rendering GPU data you must also provide `gpuContext` to allow + * for data conversion. + * @param gpuContext A WebGL canvas that is used for GPU rendering and for + * converting GPU to CPU data. If your Task is using a GPU delegate, the + * context must be obtained from its canvas (provided via + * `setOptions({ canvas: .. })`). + */ + constructor( + cpuContext: CanvasRenderingContext2D, + gpuContext?: WebGL2RenderingContext); + constructor( + cpuOrGpuGontext: CanvasRenderingContext2D|WebGL2RenderingContext, + gpuContext?: WebGL2RenderingContext) { + if (cpuOrGpuGontext instanceof CanvasRenderingContext2D) { + this.context2d = cpuOrGpuGontext; + this.contextWebGL = gpuContext; + } else { + this.contextWebGL = cpuOrGpuGontext; + } + } /** * Restricts a number between two endpoints (order doesn't matter). @@ -120,9 +186,35 @@ export class DrawingUtils { return DrawingUtils.clamp(out, y0, y1); } + private getCanvasRenderingContext(): CanvasRenderingContext2D { + if (!this.context2d) { + throw new Error( + 'CPU rendering requested but CanvasRenderingContext2D not provided.'); + } + return this.context2d; + } + + private getWebGLRenderingContext(): WebGL2RenderingContext { + if (!this.contextWebGL) { + throw new Error( + 'GPU rendering requested but WebGL2RenderingContext not provided.'); + } + return this.contextWebGL; + } + + private getCategoryMaskShaderContext(): CategoryMaskShaderContext { + if (!this.categoryMaskShaderContext) { + this.categoryMaskShaderContext = new CategoryMaskShaderContext(); + } + return this.categoryMaskShaderContext; + } + /** * Draws circles onto the provided landmarks. * + * This method can only be used when `DrawingUtils` is initialized with a + * `CanvasRenderingContext2D`. + * * @export * @param landmarks The landmarks to draw. * @param style The style to visualize the landmarks. @@ -132,7 +224,7 @@ export class DrawingUtils { if (!landmarks) { return; } - const ctx = this.ctx; + const ctx = this.getCanvasRenderingContext(); const options = addDefaultOptions(style); ctx.save(); const canvas = ctx.canvas; @@ -159,6 +251,9 @@ export class DrawingUtils { /** * Draws lines between landmarks (given a connection graph). * + * This method can only be used when `DrawingUtils` is initialized with a + * `CanvasRenderingContext2D`. + * * @export * @param landmarks The landmarks to draw. * @param connections The connections array that contains the start and the @@ -171,7 +266,7 @@ export class DrawingUtils { if (!landmarks || !connections) { return; } - const ctx = this.ctx; + const ctx = this.getCanvasRenderingContext(); const options = addDefaultOptions(style); ctx.save(); const canvas = ctx.canvas; @@ -195,12 +290,15 @@ export class DrawingUtils { /** * Draws a bounding box. * + * This method can only be used when `DrawingUtils` is initialized with a + * `CanvasRenderingContext2D`. + * * @export * @param boundingBox The bounding box to draw. * @param style The style to visualize the boundin box. */ drawBoundingBox(boundingBox: BoundingBox, style?: DrawingOptions): void { - const ctx = this.ctx; + const ctx = this.getCanvasRenderingContext(); const options = addDefaultOptions(style); ctx.save(); ctx.beginPath(); @@ -218,6 +316,118 @@ export class DrawingUtils { ctx.fill(); ctx.restore(); } + + /** Draws a category mask on a CanvasRenderingContext2D. */ + private drawCategoryMask2D( + mask: MPMask, background: RGBAColor|ImageSource, + categoryToColorMap: Map|RGBAColor[]): void { + // Use the WebGL renderer to draw result on our internal canvas. + const gl = this.getWebGLRenderingContext(); + this.runWithWebGLTexture(mask, texture => { + this.drawCategoryMaskWebGL(texture, background, categoryToColorMap); + // Draw the result on the user canvas. + const ctx = this.getCanvasRenderingContext(); + ctx.drawImage(gl.canvas, 0, 0, ctx.canvas.width, ctx.canvas.height); + }); + } + + /** Draws a category mask on a WebGL2RenderingContext2D. */ + private drawCategoryMaskWebGL( + categoryTexture: WebGLTexture, background: RGBAColor|ImageSource, + categoryToColorMap: Map|RGBAColor[]): void { + const shaderContext = this.getCategoryMaskShaderContext(); + const gl = this.getWebGLRenderingContext(); + const backgroundImage = Array.isArray(background) ? + new ImageData(new Uint8ClampedArray(background), 1, 1) : + background; + + shaderContext.run(gl, /* flipTexturesVertically= */ true, () => { + shaderContext.bindAndUploadTextures( + categoryTexture, backgroundImage, categoryToColorMap); + gl.clearColor(0, 0, 0, 0); + gl.clear(gl.COLOR_BUFFER_BIT); + gl.drawArrays(gl.TRIANGLE_FAN, 0, 4); + shaderContext.unbindTextures(); + }); + } + + /** + * Draws a category mask using the provided category-to-color mapping. + * + * @export + * @param mask A category mask that was returned from a segmentation task. + * @param categoryToColorMap A map that maps category indices to RGBA + * values. You must specify a map entry for each category. + * @param background A color or image to use as the background. Defaults to + * black. + */ + drawCategoryMask( + mask: MPMask, categoryToColorMap: Map, + background?: RGBAColor|ImageSource): void; + /** + * Draws a category mask using the provided color array. + * + * @export + * @param mask A category mask that was returned from a segmentation task. + * @param categoryToColorMap An array that maps indices to RGBA values. The + * array's indices must correspond to the category indices of the model + * and an entry must be provided for each category. + * @param background A color or image to use as the background. Defaults to + * black. + */ + drawCategoryMask( + mask: MPMask, categoryToColorMap: RGBAColor[], + background?: RGBAColor|ImageSource): void; + drawCategoryMask( + mask: MPMask, categoryToColorMap: CategoryToColorMap, + background: RGBAColor|ImageSource = [0, 0, 0, 255]): void { + if (this.context2d) { + this.drawCategoryMask2D(mask, background, categoryToColorMap); + } else { + this.drawCategoryMaskWebGL( + mask.getAsWebGLTexture(), background, categoryToColorMap); + } + } + + /** + * Converts the given mask to a WebGLTexture and runs the callback. Cleans + * up any new resources after the callback finished executing. + */ + private runWithWebGLTexture( + mask: MPMask, callback: (texture: WebGLTexture) => void): void { + if (!mask.hasWebGLTexture()) { + // Re-create the MPMask but use our the WebGL canvas so we can draw the + // texture directly. + const data = mask.hasFloat32Array() ? mask.getAsFloat32Array() : + mask.getAsUint8Array(); + this.convertToWebGLTextureShaderContext = + this.convertToWebGLTextureShaderContext ?? new MPImageShaderContext(); + const gl = this.getWebGLRenderingContext(); + + const convertedMask = new MPMask( + [data], + /* ownsWebGlTexture= */ false, + gl.canvas, + this.convertToWebGLTextureShaderContext, + mask.width, + mask.height, + ); + callback(convertedMask.getAsWebGLTexture()); + convertedMask.close(); + } else { + callback(mask.getAsWebGLTexture()); + } + } + /** + * Frees all WebGL resources held by this class. + * @export + */ + close(): void { + this.categoryMaskShaderContext?.close(); + this.categoryMaskShaderContext = undefined; + this.convertToWebGLTextureShaderContext?.close(); + this.convertToWebGLTextureShaderContext = undefined; + } } diff --git a/mediapipe/tasks/web/vision/core/drawing_utils_category_mask.ts b/mediapipe/tasks/web/vision/core/drawing_utils_category_mask.ts new file mode 100644 index 000000000..d7706075f --- /dev/null +++ b/mediapipe/tasks/web/vision/core/drawing_utils_category_mask.ts @@ -0,0 +1,189 @@ +/** + * Copyright 2023 The MediaPipe Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {assertNotNull, MPImageShaderContext} from '../../../../tasks/web/vision/core/image_shader_context'; +import {ImageSource} from '../../../../web/graph_runner/graph_runner'; + +/** + * A fragment shader that maps categories to colors based on a background + * texture, a mask texture and a 256x1 "color mapping texture" that contains one + * color for each pixel. + */ +const FRAGMENT_SHADER = ` + precision mediump float; + uniform sampler2D backgroundTexture; + uniform sampler2D maskTexture; + uniform sampler2D colorMappingTexture; + varying vec2 vTex; + void main() { + vec4 backgroundColor = texture2D(backgroundTexture, vTex); + float category = texture2D(maskTexture, vTex).r; + vec4 categoryColor = texture2D(colorMappingTexture, vec2(category, 0.0)); + gl_FragColor = mix(backgroundColor, categoryColor, categoryColor.a); + } + `; + +/** + * A four channel color with values for red, green, blue and alpha + * respectively. + */ +export type RGBAColor = [number, number, number, number]|number[]; + +/** + * A category to color mapping that uses either a map or an array to assign + * category indexes to RGBA colors. + */ +export type CategoryToColorMap = Map|RGBAColor[]; + + +/** Checks CategoryToColorMap maps for deep equality. */ +function isEqualColorMap( + a: CategoryToColorMap, b: CategoryToColorMap): boolean { + if (a !== b) { + return false; + } + + const aEntries = a.entries(); + const bEntries = b.entries(); + for (const [aKey, aValue] of aEntries) { + const bNext = bEntries.next(); + if (bNext.done) { + return false; + } + + const [bKey, bValue] = bNext.value; + if (aKey !== bKey) { + return false; + } + + if (aValue[0] !== bValue[0] || aValue[1] !== bValue[1] || + aValue[2] !== bValue[2] || aValue[3] !== bValue[3]) { + return false; + } + } + return !!bEntries.next().done; +} + + +/** A drawing util class for category masks. */ +export class CategoryMaskShaderContext extends MPImageShaderContext { + backgroundTexture?: WebGLTexture; + colorMappingTexture?: WebGLTexture; + colorMappingTextureUniform?: WebGLUniformLocation; + backgroundTextureUniform?: WebGLUniformLocation; + maskTextureUniform?: WebGLUniformLocation; + currentColorMap?: CategoryToColorMap; + + bindAndUploadTextures( + categoryMask: WebGLTexture, background: ImageSource, + colorMap: Map|number[][]) { + const gl = this.gl!; + + // TODO: We should avoid uploading textures from CPU to GPU + // if the textures haven't changed. This can lead to drastic performance + // slowdowns (~50ms per frame). Users can reduce the penalty by passing a + // canvas object instead of ImageData/HTMLImageElement. + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this.backgroundTexture!); + gl.texImage2D( + gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, background); + + // Bind color mapping texture if changed. + if (!this.currentColorMap || + !isEqualColorMap(this.currentColorMap, colorMap)) { + this.currentColorMap = colorMap; + + const pixels = new Array(256 * 4).fill(0); + colorMap.forEach((rgba, index) => { + if (rgba.length !== 4) { + throw new Error( + `Color at index ${index} is not a four-channel value.`); + } + pixels[index * 4] = rgba[0]; + pixels[index * 4 + 1] = rgba[1]; + pixels[index * 4 + 2] = rgba[2]; + pixels[index * 4 + 3] = rgba[3]; + }); + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, this.colorMappingTexture!); + gl.texImage2D( + gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, + new Uint8Array(pixels)); + } else { + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, this.colorMappingTexture!); + } + + // Bind category mask + gl.activeTexture(gl.TEXTURE2); + gl.bindTexture(gl.TEXTURE_2D, categoryMask); + } + + unbindTextures() { + const gl = this.gl!; + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, null); + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, null); + gl.activeTexture(gl.TEXTURE2); + gl.bindTexture(gl.TEXTURE_2D, null); + } + + protected override getFragmentShader(): string { + return FRAGMENT_SHADER; + } + + protected override setupTextures(): void { + const gl = this.gl!; + gl.activeTexture(gl.TEXTURE0); + this.backgroundTexture = this.createTexture(gl, gl.LINEAR); + // Use `gl.NEAREST` to prevent interpolating values in our category to + // color map. + this.colorMappingTexture = this.createTexture(gl, gl.NEAREST); + } + + protected override setupShaders(): void { + super.setupShaders(); + const gl = this.gl!; + this.backgroundTextureUniform = assertNotNull( + gl.getUniformLocation(this.program!, 'backgroundTexture'), + 'Uniform location'); + this.colorMappingTextureUniform = assertNotNull( + gl.getUniformLocation(this.program!, 'colorMappingTexture'), + 'Uniform location'); + this.maskTextureUniform = assertNotNull( + gl.getUniformLocation(this.program!, 'maskTexture'), + 'Uniform location'); + } + + protected override configureUniforms(): void { + super.configureUniforms(); + const gl = this.gl!; + gl.uniform1i(this.backgroundTextureUniform!, 0); + gl.uniform1i(this.colorMappingTextureUniform!, 1); + gl.uniform1i(this.maskTextureUniform!, 2); + } + + override close(): void { + if (this.backgroundTexture) { + this.gl!.deleteTexture(this.backgroundTexture); + } + if (this.colorMappingTexture) { + this.gl!.deleteTexture(this.colorMappingTexture); + } + super.close(); + } +} diff --git a/mediapipe/tasks/web/vision/core/image_shader_context.ts b/mediapipe/tasks/web/vision/core/image_shader_context.ts index eb17d001a..3dec9da95 100644 --- a/mediapipe/tasks/web/vision/core/image_shader_context.ts +++ b/mediapipe/tasks/web/vision/core/image_shader_context.ts @@ -27,9 +27,9 @@ const FRAGMENT_SHADER = ` precision mediump float; varying vec2 vTex; uniform sampler2D inputTexture; - void main() { - gl_FragColor = texture2D(inputTexture, vTex); - } + void main() { + gl_FragColor = texture2D(inputTexture, vTex); + } `; /** Helper to assert that `value` is not null. */ @@ -73,9 +73,9 @@ class MPImageShaderBuffers { * For internal use only. */ export class MPImageShaderContext { - private gl?: WebGL2RenderingContext; + protected gl?: WebGL2RenderingContext; private framebuffer?: WebGLFramebuffer; - private program?: WebGLProgram; + protected program?: WebGLProgram; private vertexShader?: WebGLShader; private fragmentShader?: WebGLShader; private aVertex?: GLint; @@ -94,6 +94,14 @@ export class MPImageShaderContext { */ private shaderBuffersFlipVertically?: MPImageShaderBuffers; + protected getFragmentShader(): string { + return FRAGMENT_SHADER; + } + + protected getVertexShader(): string { + return VERTEX_SHADER; + } + private compileShader(source: string, type: number): WebGLShader { const gl = this.gl!; const shader = @@ -108,14 +116,15 @@ export class MPImageShaderContext { return shader; } - private setupShaders(): void { + protected setupShaders(): void { const gl = this.gl!; this.program = assertNotNull(gl.createProgram()!, 'Failed to create WebGL program'); - this.vertexShader = this.compileShader(VERTEX_SHADER, gl.VERTEX_SHADER); + this.vertexShader = + this.compileShader(this.getVertexShader(), gl.VERTEX_SHADER); this.fragmentShader = - this.compileShader(FRAGMENT_SHADER, gl.FRAGMENT_SHADER); + this.compileShader(this.getFragmentShader(), gl.FRAGMENT_SHADER); gl.linkProgram(this.program); const linked = gl.getProgramParameter(this.program, gl.LINK_STATUS); @@ -128,6 +137,10 @@ export class MPImageShaderContext { this.aTex = gl.getAttribLocation(this.program, 'aTex'); } + protected setupTextures(): void {} + + protected configureUniforms(): void {} + private createBuffers(flipVertically: boolean): MPImageShaderBuffers { const gl = this.gl!; const vertexArrayObject = @@ -193,17 +206,44 @@ export class MPImageShaderContext { if (!this.program) { this.setupShaders(); + this.setupTextures(); } const shaderBuffers = this.getShaderBuffers(flipVertically); gl.useProgram(this.program!); shaderBuffers.bind(); + this.configureUniforms(); const result = callback(); shaderBuffers.unbind(); return result; } + /** + * Creates and configures a texture. + * + * @param gl The rendering context. + * @param filter The setting to use for `gl.TEXTURE_MIN_FILTER` and + * `gl.TEXTURE_MAG_FILTER`. Defaults to `gl.LINEAR`. + * @param wrapping The setting to use for `gl.TEXTURE_WRAP_S` and + * `gl.TEXTURE_WRAP_T`. Defaults to `gl.CLAMP_TO_EDGE`. + */ + createTexture(gl: WebGL2RenderingContext, filter?: GLenum, wrapping?: GLenum): + WebGLTexture { + this.maybeInitGL(gl); + const texture = + assertNotNull(gl.createTexture(), 'Failed to create texture'); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texParameteri( + gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, wrapping ?? gl.CLAMP_TO_EDGE); + gl.texParameteri( + gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, wrapping ?? gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, filter ?? gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, filter ?? gl.LINEAR); + gl.bindTexture(gl.TEXTURE_2D, null); + return texture; + } + /** * Binds a framebuffer to the canvas. If the framebuffer does not yet exist, * creates it first. Binds the provided texture to the framebuffer. diff --git a/mediapipe/tasks/web/vision/core/render_utils.ts b/mediapipe/tasks/web/vision/core/render_utils.ts index ebb3be16a..3ee981bab 100644 --- a/mediapipe/tasks/web/vision/core/render_utils.ts +++ b/mediapipe/tasks/web/vision/core/render_utils.ts @@ -16,24 +16,6 @@ * limitations under the License. */ -// Pre-baked color table for a maximum of 12 classes. -const CM_ALPHA = 128; -const COLOR_MAP: Array<[number, number, number, number]> = [ - [0, 0, 0, CM_ALPHA], // class 0 is BG = transparent - [255, 0, 0, CM_ALPHA], // class 1 is red - [0, 255, 0, CM_ALPHA], // class 2 is light green - [0, 0, 255, CM_ALPHA], // class 3 is blue - [255, 255, 0, CM_ALPHA], // class 4 is yellow - [255, 0, 255, CM_ALPHA], // class 5 is light purple / magenta - [0, 255, 255, CM_ALPHA], // class 6 is light blue / aqua - [128, 128, 128, CM_ALPHA], // class 7 is gray - [255, 128, 0, CM_ALPHA], // class 8 is orange - [128, 0, 255, CM_ALPHA], // class 9 is dark purple - [0, 128, 0, CM_ALPHA], // class 10 is dark green - [255, 255, 255, CM_ALPHA] // class 11 is white; could do black instead? -]; - - /** Helper function to draw a confidence mask */ export function drawConfidenceMask( ctx: CanvasRenderingContext2D, image: Float32Array, width: number, @@ -47,23 +29,3 @@ export function drawConfidenceMask( } ctx.putImageData(new ImageData(uint8Array, width, height), 0, 0); } - -/** - * Helper function to draw a category mask. For GPU, we only have F32Arrays - * for now. - */ -export function drawCategoryMask( - ctx: CanvasRenderingContext2D, image: Uint8Array|Float32Array, - width: number, height: number): void { - const rgbaArray = new Uint8ClampedArray(width * height * 4); - const isFloatArray = image instanceof Float32Array; - for (let i = 0; i < image.length; i++) { - const colorIndex = isFloatArray ? Math.round(image[i] * 255) : image[i]; - const color = COLOR_MAP[colorIndex % COLOR_MAP.length]; - rgbaArray[4 * i] = color[0]; - rgbaArray[4 * i + 1] = color[1]; - rgbaArray[4 * i + 2] = color[2]; - rgbaArray[4 * i + 3] = color[3]; - } - ctx.putImageData(new ImageData(rgbaArray, width, height), 0, 0); -} From e81fc5d0aa55f5349473c0cb8b87f5b989547a27 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 1 Nov 2023 14:06:03 -0700 Subject: [PATCH 063/157] Access document via self.document PiperOrigin-RevId: 578635298 --- mediapipe/web/graph_runner/platform_utils.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mediapipe/web/graph_runner/platform_utils.ts b/mediapipe/web/graph_runner/platform_utils.ts index d86e002de..a9a62a884 100644 --- a/mediapipe/web/graph_runner/platform_utils.ts +++ b/mediapipe/web/graph_runner/platform_utils.ts @@ -32,6 +32,5 @@ export function isIOS() { // tslint:disable-next-line:deprecation ].includes(navigator.platform) // iPad on iOS 13 detection - || (navigator.userAgent.includes('Mac') && - (typeof document !== undefined && 'ontouchend' in document)); + || (navigator.userAgent.includes('Mac') && 'ontouchend' in self.document); } From f8197651e8827c8efa99063eb14f3abfc5151b58 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 2 Nov 2023 05:56:09 -0700 Subject: [PATCH 064/157] Add AT_FIRST_TICK processing to SidePacketToStreamCalculator. PiperOrigin-RevId: 578824863 --- mediapipe/calculators/core/BUILD | 2 + .../core/side_packet_to_stream_calculator.cc | 35 +++- .../side_packet_to_stream_calculator_test.cc | 166 +++++++++++++++--- 3 files changed, 174 insertions(+), 29 deletions(-) diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index aacf694c1..729e91492 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -727,6 +727,7 @@ cc_library( "//mediapipe/framework/port:logging", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/status", ], alwayslink = 1, ) @@ -742,6 +743,7 @@ cc_test( "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/port:status", "//mediapipe/framework/tool:options_util", + "//mediapipe/util:packet_test_util", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", ], diff --git a/mediapipe/calculators/core/side_packet_to_stream_calculator.cc b/mediapipe/calculators/core/side_packet_to_stream_calculator.cc index 311f7d815..686d705dd 100644 --- a/mediapipe/calculators/core/side_packet_to_stream_calculator.cc +++ b/mediapipe/calculators/core/side_packet_to_stream_calculator.cc @@ -17,6 +17,7 @@ #include #include +#include "absl/status/status.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/logging.h" #include "mediapipe/framework/port/ret_check.h" @@ -32,6 +33,7 @@ namespace { constexpr char kTagAtPreStream[] = "AT_PRESTREAM"; constexpr char kTagAtPostStream[] = "AT_POSTSTREAM"; constexpr char kTagAtZero[] = "AT_ZERO"; +constexpr char kTagAtFirstTick[] = "AT_FIRST_TICK"; constexpr char kTagAtTick[] = "AT_TICK"; constexpr char kTagTick[] = "TICK"; constexpr char kTagAtTimestamp[] = "AT_TIMESTAMP"; @@ -43,6 +45,7 @@ static std::map* kTimestampMap = []() { res->emplace(kTagAtPostStream, Timestamp::PostStream()); res->emplace(kTagAtZero, Timestamp(0)); res->emplace(kTagAtTick, Timestamp::Unset()); + res->emplace(kTagAtFirstTick, Timestamp::Unset()); res->emplace(kTagAtTimestamp, Timestamp::Unset()); return res; }(); @@ -59,8 +62,8 @@ std::string GetOutputTag(const CC& cc) { // timestamp, depending on the tag used to define output stream(s). (One tag can // be used only.) // -// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_TIMESTAMP -// and corresponding timestamps are Timestamp::PreStream(), +// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_FIRST_TICK, +// AT_TIMESTAMP and corresponding timestamps are Timestamp::PreStream(), // Timestamp::PostStream(), Timestamp(0), timestamp of a packet received in TICK // input, and timestamp received from a side input. // @@ -96,6 +99,7 @@ class SidePacketToStreamCalculator : public CalculatorBase { private: bool is_tick_processing_ = false; + bool close_on_first_tick_ = false; std::string output_tag_; }; REGISTER_CALCULATOR(SidePacketToStreamCalculator); @@ -103,13 +107,16 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator); absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) { const auto& tags = cc->Outputs().GetTags(); RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1) - << "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and " - "AT_TIMESTAMP tags is allowed and required to specify output " - "stream(s)."; - RET_CHECK( - (cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) || - (!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick))) - << "Either both of TICK and AT_TICK should be used or none of them."; + << "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, " + "AT_FIRST_TICK and AT_TIMESTAMP tags is allowed and required to " + "specify output stream(s)."; + const bool has_tick_output = + cc->Outputs().HasTag(kTagAtTick) || cc->Outputs().HasTag(kTagAtFirstTick); + const bool has_tick_input = cc->Inputs().HasTag(kTagTick); + RET_CHECK((has_tick_output && has_tick_input) || + (!has_tick_output && !has_tick_input)) + << "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output " + "should be used or none of them."; RET_CHECK((cc->Outputs().HasTag(kTagAtTimestamp) && cc->InputSidePackets().HasTag(kTagSideInputTimestamp)) || (!cc->Outputs().HasTag(kTagAtTimestamp) && @@ -148,11 +155,17 @@ absl::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) { // timestamp bound update. cc->SetOffset(TimestampDiff(0)); } + if (output_tag_ == kTagAtFirstTick) { + close_on_first_tick_ = true; + } return absl::OkStatus(); } absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) { if (is_tick_processing_) { + if (cc->Outputs().Get(output_tag_, 0).IsClosed()) { + return absl::OkStatus(); + } // TICK input is guaranteed to be non-empty, as it's the only input stream // for this calculator. const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp(); @@ -160,6 +173,9 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) { cc->Outputs() .Get(output_tag_, i) .AddPacket(cc->InputSidePackets().Index(i).At(timestamp)); + if (close_on_first_tick_) { + cc->Outputs().Get(output_tag_, i).Close(); + } } return absl::OkStatus(); @@ -170,6 +186,7 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) { absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) { if (!cc->Outputs().HasTag(kTagAtTick) && + !cc->Outputs().HasTag(kTagAtFirstTick) && !cc->Outputs().HasTag(kTagAtTimestamp)) { const auto& timestamp = kTimestampMap->at(output_tag_); for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) { diff --git a/mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc b/mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc index 086b73fcd..6c0941b44 100644 --- a/mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc +++ b/mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc @@ -27,13 +27,17 @@ #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status_matchers.h" #include "mediapipe/framework/tool/options_util.h" +#include "mediapipe/util/packet_test_util.h" namespace mediapipe { namespace { -using testing::HasSubstr; +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::IsEmpty; -TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) { +TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTick) { CalculatorGraphConfig graph_config = ParseTextProtoOrDie( R"pb( @@ -52,10 +56,35 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) { EXPECT_THAT( status.message(), HasSubstr( - "Either both of TICK and AT_TICK should be used or none of them.")); + "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output " + "should be used or none of them.")); } -TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) { +TEST(SidePacketToStreamCalculator, + WrongConfigWithMissingTickForFirstTickProcessing) { + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie( + R"pb( + input_stream: "tick" + input_side_packet: "side_packet" + output_stream: "packet" + node { + calculator: "SidePacketToStreamCalculator" + input_side_packet: "side_packet" + output_stream: "AT_FIRST_TICK:packet" + } + )pb"); + CalculatorGraph graph; + auto status = graph.Initialize(graph_config); + EXPECT_FALSE(status.ok()); + EXPECT_THAT( + status.message(), + HasSubstr( + "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output " + "should be used or none of them.")); +} + +TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTimestampSideInput) { CalculatorGraphConfig graph_config = ParseTextProtoOrDie( R"pb( @@ -76,7 +105,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) { "or none of them.")); } -TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) { +TEST(SidePacketToStreamCalculator, WrongConfigWithNonExistentTag) { CalculatorGraphConfig graph_config = ParseTextProtoOrDie( R"pb( @@ -92,14 +121,13 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) { CalculatorGraph graph; auto status = graph.Initialize(graph_config); EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and " - "AT_TIMESTAMP tags is allowed and required to specify output " - "stream(s).")); + EXPECT_THAT(status.message(), + HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, " + "AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is " + "allowed and required to specify output stream(s).")); } -TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) { +TEST(SidePacketToStreamCalculator, WrongConfigWithMixedTags) { CalculatorGraphConfig graph_config = ParseTextProtoOrDie( R"pb( @@ -117,14 +145,13 @@ TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) { CalculatorGraph graph; auto status = graph.Initialize(graph_config); EXPECT_FALSE(status.ok()); - EXPECT_THAT( - status.message(), - HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and " - "AT_TIMESTAMP tags is allowed and required to specify output " - "stream(s).")); + EXPECT_THAT(status.message(), + HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, " + "AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is " + "allowed and required to specify output stream(s).")); } -TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) { +TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughSidePackets) { CalculatorGraphConfig graph_config = ParseTextProtoOrDie( R"pb( @@ -146,7 +173,7 @@ TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) { "Same number of input side packets and output streams is required.")); } -TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) { +TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughOutputStreams) { CalculatorGraphConfig graph_config = ParseTextProtoOrDie( R"pb( @@ -248,7 +275,50 @@ TEST(SidePacketToStreamCalculator, AtTick) { tick_and_verify(/*at_timestamp=*/1025); } -TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) { +TEST(SidePacketToStreamCalculator, AtFirstTick) { + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie( + R"pb( + input_stream: "tick" + input_side_packet: "side_packet" + output_stream: "packet" + node { + calculator: "SidePacketToStreamCalculator" + input_stream: "TICK:tick" + input_side_packet: "side_packet" + output_stream: "AT_FIRST_TICK:packet" + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("packet", &graph_config, &output_packets); + CalculatorGraph graph; + + MP_ASSERT_OK(graph.Initialize(graph_config)); + const int expected_value = 20; + const Timestamp kTestTimestamp(1234); + MP_ASSERT_OK( + graph.StartRun({{"side_packet", MakePacket(expected_value)}})); + + auto insert_tick = [&graph](Timestamp at_timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "tick", MakePacket(/*doesn't matter*/ 1).At(at_timestamp))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + }; + + insert_tick(kTestTimestamp); + + EXPECT_THAT(output_packets, + ElementsAre(PacketContainsTimestampAndPayload( + Eq(kTestTimestamp), Eq(expected_value)))); + + output_packets.clear(); + + // Should not result in an additional output. + insert_tick(kTestTimestamp + 1); + EXPECT_THAT(output_packets, IsEmpty()); +} + +TEST(SidePacketToStreamCalculator, AtTickWithMultipleSidePackets) { CalculatorGraphConfig graph_config = ParseTextProtoOrDie( R"pb( @@ -302,6 +372,62 @@ TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) { tick_and_verify(/*at_timestamp=*/1025); } +TEST(SidePacketToStreamCalculator, AtFirstTickWithMultipleSidePackets) { + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie( + R"pb( + input_stream: "tick" + input_side_packet: "side_packet0" + input_side_packet: "side_packet1" + output_stream: "packet0" + output_stream: "packet1" + node { + calculator: "SidePacketToStreamCalculator" + input_stream: "TICK:tick" + input_side_packet: "side_packet0" + input_side_packet: "side_packet1" + output_stream: "AT_FIRST_TICK:0:packet0" + output_stream: "AT_FIRST_TICK:1:packet1" + } + )pb"); + std::vector output_packets0; + tool::AddVectorSink("packet0", &graph_config, &output_packets0); + std::vector output_packets1; + tool::AddVectorSink("packet1", &graph_config, &output_packets1); + CalculatorGraph graph; + + MP_ASSERT_OK(graph.Initialize(graph_config)); + const int expected_value0 = 20; + const int expected_value1 = 128; + const Timestamp kTestTimestamp(1234); + MP_ASSERT_OK( + graph.StartRun({{"side_packet0", MakePacket(expected_value0)}, + {"side_packet1", MakePacket(expected_value1)}})); + + auto insert_tick = [&graph](Timestamp at_timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "tick", MakePacket(/*doesn't matter*/ 1).At(at_timestamp))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + }; + + insert_tick(kTestTimestamp); + + EXPECT_THAT(output_packets0, + ElementsAre(PacketContainsTimestampAndPayload( + Eq(kTestTimestamp), Eq(expected_value0)))); + EXPECT_THAT(output_packets1, + ElementsAre(PacketContainsTimestampAndPayload( + Eq(kTestTimestamp), Eq(expected_value1)))); + + output_packets0.clear(); + output_packets1.clear(); + + // Should not result in an additional output. + insert_tick(kTestTimestamp + 1); + EXPECT_THAT(output_packets0, IsEmpty()); + EXPECT_THAT(output_packets1, IsEmpty()); +} + TEST(SidePacketToStreamCalculator, AtTimestamp) { CalculatorGraphConfig graph_config = ParseTextProtoOrDie( @@ -334,7 +460,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) { EXPECT_EQ(expected_value, output_packets.back().Get()); } -TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) { +TEST(SidePacketToStreamCalculator, AtTimestampWithMultipleOutputs) { CalculatorGraphConfig graph_config = ParseTextProtoOrDie( R"pb( From 9018ca699b7a3391c5a3d70b698634e01d234899 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 2 Nov 2023 07:44:40 -0700 Subject: [PATCH 065/157] Creates GpuBuffers around pre-allocated AHardware_Buffer objects. PiperOrigin-RevId: 578850184 --- mediapipe/gpu/BUILD | 5 +++++ mediapipe/gpu/gpu_buffer_format.cc | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index b75c824b3..f39b8d3f7 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -516,6 +516,7 @@ cc_library( ":gpu_buffer_storage", ":image_frame_view", "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/port:ret_check", "@com_google_absl//absl/strings:str_format", ], ) @@ -1223,9 +1224,13 @@ mediapipe_cc_test( ], requires_full_emulation = True, deps = [ + ":gl_texture_buffer", + ":gl_texture_util", ":gpu_buffer_format", ":gpu_buffer_storage_ahwb", + ":gpu_test_base", "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/tool:test_util", ], ) diff --git a/mediapipe/gpu/gpu_buffer_format.cc b/mediapipe/gpu/gpu_buffer_format.cc index b099f4a25..510a9cd48 100644 --- a/mediapipe/gpu/gpu_buffer_format.cc +++ b/mediapipe/gpu/gpu_buffer_format.cc @@ -238,7 +238,7 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) { case GpuBufferFormat::kRGBAFloat128: return ImageFormat::VEC32F4; case GpuBufferFormat::kRGBA32: - // TODO: this likely maps to ImageFormat::SRGBA + return ImageFormat::SRGBA; case GpuBufferFormat::kGrayHalf16: case GpuBufferFormat::kOneComponent8Alpha: case GpuBufferFormat::kOneComponent8Red: From 8f564c4b7bf7c63d1f72ce6435d5c938f78d0d56 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 2 Nov 2023 17:00:30 -0700 Subject: [PATCH 066/157] Allow OffscreenCanvas to be used by DrawingUtils PiperOrigin-RevId: 579021013 --- mediapipe/tasks/web/vision/core/drawing_utils.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/web/vision/core/drawing_utils.ts b/mediapipe/tasks/web/vision/core/drawing_utils.ts index 95e376fb2..796d7dcb6 100644 --- a/mediapipe/tasks/web/vision/core/drawing_utils.ts +++ b/mediapipe/tasks/web/vision/core/drawing_utils.ts @@ -116,7 +116,8 @@ export {RGBAColor, CategoryToColorMap}; export class DrawingUtils { private categoryMaskShaderContext?: CategoryMaskShaderContext; private convertToWebGLTextureShaderContext?: MPImageShaderContext; - private readonly context2d?: CanvasRenderingContext2D; + private readonly context2d?: CanvasRenderingContext2D| + OffscreenCanvasRenderingContext2D; private readonly contextWebGL?: WebGL2RenderingContext; /** @@ -139,12 +140,14 @@ export class DrawingUtils { * `setOptions({ canvas: .. })`). */ constructor( - cpuContext: CanvasRenderingContext2D, + cpuContext: CanvasRenderingContext2D|OffscreenCanvasRenderingContext2D, gpuContext?: WebGL2RenderingContext); constructor( - cpuOrGpuGontext: CanvasRenderingContext2D|WebGL2RenderingContext, + cpuOrGpuGontext: CanvasRenderingContext2D| + OffscreenCanvasRenderingContext2D|WebGL2RenderingContext, gpuContext?: WebGL2RenderingContext) { - if (cpuOrGpuGontext instanceof CanvasRenderingContext2D) { + if (cpuOrGpuGontext instanceof CanvasRenderingContext2D || + cpuOrGpuGontext instanceof OffscreenCanvasRenderingContext2D) { this.context2d = cpuOrGpuGontext; this.contextWebGL = gpuContext; } else { @@ -186,7 +189,8 @@ export class DrawingUtils { return DrawingUtils.clamp(out, y0, y1); } - private getCanvasRenderingContext(): CanvasRenderingContext2D { + private getCanvasRenderingContext(): CanvasRenderingContext2D + |OffscreenCanvasRenderingContext2D { if (!this.context2d) { throw new Error( 'CPU rendering requested but CanvasRenderingContext2D not provided.'); From 1c46e430883c56faefb8d829ad9f51cc94c25f5b Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 2 Nov 2023 17:58:44 -0700 Subject: [PATCH 067/157] Update WASM files for 0.10.8 relese PiperOrigin-RevId: 579032432 --- third_party/wasm_files.bzl | 48 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/third_party/wasm_files.bzl b/third_party/wasm_files.bzl index f6256b91b..58c0570e9 100644 --- a/third_party/wasm_files.bzl +++ b/third_party/wasm_files.bzl @@ -12,72 +12,72 @@ def wasm_files(): http_file( name = "com_google_mediapipe_wasm_audio_wasm_internal_js", - sha256 = "bb10ba65b0135f3d22c380bd87712f6a859ecdebdf1e3243407bc2f3ac5ccf71", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.js?generation=1696624044559284"], + sha256 = "8722e1047a54dcd08206d018a4bc348dd820f479cb10218c5cbcd411dd9e1c0c", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.js?generation=1698954798232640"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_internal_wasm", - sha256 = "01079a05e1ce4963e3a78cd5fee8f33be9fda10f1c6c450d00cf71251d817e0c", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.wasm?generation=1696624047350232"], + sha256 = "bcd230238dbabdf09eab58dbbe7e36deacf7e3fc57c2d67af679188d37731883", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.wasm?generation=1698954800502145"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_nosimd_internal_js", - sha256 = "c9484189261601052357359edd4a8575b0d51d0ce12cf3343d12a933307303c6", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.js?generation=1696624049287593"], + sha256 = "a5d80eefde268611ed385b90fab9defc37df50a124a15282961dbaa30b62c14d", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.js?generation=1698954802474619"], ) http_file( name = "com_google_mediapipe_wasm_audio_wasm_nosimd_internal_wasm", - sha256 = "c00c3455b9ca1f477879383dde7a5853e1bac06cad8c48088c53c6bd1bcd3890", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.wasm?generation=1696624051752560"], + sha256 = "2fc431cc62330332c0c1e730d44b933a79e4572be0dc5c5a82635bd5dc330b94", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.wasm?generation=1698954804758365"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_internal_js", - sha256 = "b5a8b98ac3927c28b462f8a0f9523fb7b0b6ac720a0009ba2c4f211516cc5a5e", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.js?generation=1696624053716124"], + sha256 = "b100d299cb06c0fd7cf40099653e8d4a3ac953937402a5d7c3a3a02fa59d8105", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.js?generation=1698954806886809"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_internal_wasm", - sha256 = "a5f28b7aa458f7e34c78cb90aac0fc3a228a5d0f3ae675fb7a5e2dca4299363c", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.wasm?generation=1696624056400203"], + sha256 = "ae1b8f9684b9afa989b1144f25a2ae1bda809c811367475567c823d65d4fef0a", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.wasm?generation=1698954809121561"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_nosimd_internal_js", - sha256 = "0eb6417d62f860161b504a72c88659a6a866b36e477da600630bebaca01bf7c6", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.js?generation=1696624058379631"], + sha256 = "d8db720214acfa1b758099daeb07c02e04b7221805523e9b6926a1f11ec00183", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.js?generation=1698954811167986"], ) http_file( name = "com_google_mediapipe_wasm_text_wasm_nosimd_internal_wasm", - sha256 = "af9e62a8e63ea38e3f984f2937cd856cb6a599cdda169bb1c29169a1a08f60f9", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.wasm?generation=1696624060772680"], + sha256 = "44b8e5be980e6fe79fa9a8b02551ef50e1d74682dd8f3e6cf92435cf43e8ef91", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.wasm?generation=1698954813498288"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_internal_js", - sha256 = "addc64f89585eaa7322f649158d57ac70fd0e9ae0d37f4aeb4016b04d0e18d2a", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.js?generation=1696624062874405"], + sha256 = "f5ba7b1d0adad63c581a80113567913a7106b20f8d26982f82c56998c7d44465", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.js?generation=1698954815469471"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_internal_wasm", - sha256 = "357a65e76313ceb65ffec453c47275abfc387b7b6e77823b6c4c016ab7a40cf5", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.wasm?generation=1696624065934602"], + sha256 = "d502a753b40626a36734806599bf0e765cf3a611653d980b39a5474998f1d6fe", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.wasm?generation=1698954817976682"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_nosimd_internal_js", - sha256 = "c41e7c4b00d2ab701d6e805917a26da7b563737fd12671f2f3496c036c94d633", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.js?generation=1696624067936821"], + sha256 = "731786df74b19150eecc8fe69ddf16040bbbba8cf2d22c964ef38ecef25d1e1f", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.js?generation=1698954819912485"], ) http_file( name = "com_google_mediapipe_wasm_vision_wasm_nosimd_internal_wasm", - sha256 = "620986d18baa69934579bbd701d72532cedc2a6b052932cb19e302bd748afcba", - urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.wasm?generation=1696624070556427"], + sha256 = "2c16ecc52398857c5ce45d58c98fe16e795b6a6eda6a2a8aa00f519a4bd15f2a", + urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.wasm?generation=1698954822497945"], ) From e22b7d5dd46ec5efc75b5d71f0ae7bcad5c03490 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 3 Nov 2023 12:55:50 -0700 Subject: [PATCH 068/157] Example updated for mp.Image in documentation PiperOrigin-RevId: 579277510 --- mediapipe/python/pybind/image.cc | 2 +- mediapipe/python/pybind/image_frame.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/python/pybind/image.cc b/mediapipe/python/pybind/image.cc index f3d89bada..98f162342 100644 --- a/mediapipe/python/pybind/image.cc +++ b/mediapipe/python/pybind/image.cc @@ -51,7 +51,7 @@ void ImageSubmodule(pybind11::module* module) { ```python import cv2 - cv_mat = cv2.imread(input_file)[:, :, ::-1] + cv_mat = cv2.imread(input_file) rgb_frame = mp.Image(image_format=mp.ImageFormat.SRGB, data=cv_mat) gray_frame = mp.Image( image_format=mp.ImageFormat.GRAY8, diff --git a/mediapipe/python/pybind/image_frame.cc b/mediapipe/python/pybind/image_frame.cc index 9ec60dbec..90db05066 100644 --- a/mediapipe/python/pybind/image_frame.cc +++ b/mediapipe/python/pybind/image_frame.cc @@ -84,7 +84,7 @@ void ImageFrameSubmodule(pybind11::module* module) { ```python import cv2 - cv_mat = cv2.imread(input_file)[:, :, ::-1] + cv_mat = cv2.imread(input_file) rgb_frame = mp.ImageFrame(image_format=ImageFormat.SRGB, data=cv_mat) gray_frame = mp.ImageFrame( image_format=ImageFormat.GRAY, From 0b53c9752ff635bbe3c1835f15e4f2877abef45a Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Sun, 5 Nov 2023 20:45:18 -0800 Subject: [PATCH 069/157] Fixes multiple typos in the calculator's internal files. PiperOrigin-RevId: 579718764 --- mediapipe/calculators/tensor/audio_to_tensor_calculator.cc | 4 ++-- .../calculators/tensor/audio_to_tensor_calculator.proto | 2 +- .../calculators/tensor/image_to_tensor_calculator_test.cc | 2 +- .../calculators/tensor/tensors_to_landmarks_calculator.cc | 2 +- .../tensorflow/pack_media_sequence_calculator.cc | 2 +- .../calculators/tensorflow/tensor_to_matrix_calculator.cc | 4 ++-- .../tensorflow/tensorflow_inference_calculator.cc | 6 +++--- .../tensorflow/tensorflow_inference_calculator.proto | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc b/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc index c8d38a653..7230e178d 100644 --- a/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc +++ b/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc @@ -109,7 +109,7 @@ bool IsValidFftSize(int size) { // Non-streaming mode: when "stream_mode" is set to false in the calculator // options, the calculators treats the packets in the input audio stream as // a batch of unrelated audio buffers. In each Process() call, the input -// buffer will be frist resampled, and framed as fixed-sized, possibly +// buffer will be first resampled, and framed as fixed-sized, possibly // overlapping tensors. The last tensor produced by a Process() invocation // will be zero-padding if the remaining samples are insufficient. As the // calculator treats the input packets as unrelated, all samples will be @@ -159,7 +159,7 @@ class AudioToTensorCalculator : public Node { public: static constexpr Input kAudioIn{"AUDIO"}; // TODO: Removes this optional input stream when the "AUDIO" stream - // uses the new mediapipe audio data containers that carry audio metatdata, + // uses the new mediapipe audio data containers that carry audio metadata, // such as sample rate. static constexpr Input::Optional kAudioSampleRateIn{"SAMPLE_RATE"}; static constexpr Output> kTensorsOut{"TENSORS"}; diff --git a/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto b/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto index 948c82a36..a49825586 100644 --- a/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto +++ b/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto @@ -37,7 +37,7 @@ message AudioToTensorCalculatorOptions { // will be converted into tensors. optional double target_sample_rate = 4; - // Whether to treat the input audio stream as a continous stream or a batch + // Whether to treat the input audio stream as a continuous stream or a batch // of unrelated audio buffers. optional bool stream_mode = 5 [default = true]; diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc index 7017c1e3a..51150a1ca 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc @@ -206,7 +206,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) { } else if (image_channels == 1) { return ImageFormat::GRAY8; } - ABSL_CHECK(false) << "Unsupported input image channles: " << image_channels; + ABSL_CHECK(false) << "Unsupported input image channels: " << image_channels; } Packet MakeImageFramePacket(cv::Mat input) { diff --git a/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc b/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc index 5942f234d..77488443f 100644 --- a/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc @@ -124,7 +124,7 @@ absl::Status TensorsToLandmarksCalculator::Open(CalculatorContext* cc) { kFlipVertically(cc).IsConnected())) { RET_CHECK(options_.has_input_image_height() && options_.has_input_image_width()) - << "Must provide input width/height for using flipping when outputing " + << "Must provide input width/height for using flipping when outputting " "landmarks in absolute coordinates."; } return absl::OkStatus(); diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc index 4972b202d..95962c261 100644 --- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc +++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc @@ -79,7 +79,7 @@ namespace mpms = mediapipe::mediasequence; // and label and label_id are optional but at least one of them should be set. // "IMAGE_${NAME}", "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store // prefixed versions of each stream, which allows for multiple image streams to -// be included. However, the default names are suppored by more tools. +// be included. However, the default names are supported by more tools. // // Example config: // node { diff --git a/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc b/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc index dc3d97844..ed234b3fa 100644 --- a/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensor_to_matrix_calculator.cc @@ -67,8 +67,8 @@ absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet, // -- 1-D or 2-D Tensor // Output: // -- Matrix with the same values as the Tensor -// If input tensor is 1 dimensional, the ouput Matrix is of (1xn) shape. -// If input tensor is 2 dimensional (batched), the ouput Matrix is (mxn) shape. +// If input tensor is 1 dimensional, the output Matrix is of (1xn) shape. +// If input tensor is 2 dimensional (batched), the output Matrix is (mxn) shape. // // Example Config // node: { diff --git a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc index 84c32fed6..39993ada0 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc @@ -111,8 +111,8 @@ class InferenceState { // input_side_packet. // // The input and output streams are TensorFlow tensors labeled by tags. The tags -// for the streams are matched to feeds and fetchs in a TensorFlow session using -// a named_signature.generic_signature in the ModelManifest. The +// for the streams are matched to feeds and fetches in a TensorFlow session +// using a named_signature.generic_signature in the ModelManifest. The // generic_signature is used as key-value pairs between the MediaPipe tag and // the TensorFlow tensor. The signature_name in the options proto determines // which named_signature is used. The keys in the generic_signature must be @@ -128,7 +128,7 @@ class InferenceState { // addition. Once batch_size inputs have been provided, the batch will be run // and the output tensors sent out on the output streams with timestamps // corresponding to the input stream packets. Setting the batch_size to 1 -// completely disables batching, but is indepdent of add_batch_dim_to_tensors. +// completely disables batching, but is independent of add_batch_dim_to_tensors. // // The TensorFlowInferenceCalculator also support feeding states recurrently for // RNNs and LSTMs. Simply set the recurrent_tag_pair options to define the diff --git a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.proto b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.proto index a243412c0..f09664592 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.proto +++ b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.proto @@ -42,7 +42,7 @@ message TensorFlowInferenceCalculatorOptions { // If the 0th dimension is the batch dimension, then the tensors are // concatenated on that dimension. If the 0th is a data dimension, then a 0th // dimension is added before concatenating. If added, the extra dimension is - // removed before outputing the tensor. Examples of each case: If you want + // removed before outputting the tensor. Examples of each case: If you want // to batch spectra of audio over time for an LSTM, a time-frequency // representation has a 0th dimension as the batch dimension. If you want to // batch frames of video that are [width, height, channels], the batch From 5f0d24d7413c2f8e8fce8c3c6851c0710b43b8a6 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 6 Nov 2023 07:05:54 -0800 Subject: [PATCH 070/157] Fixes typo in GlCalculatorHelper::UpdateContract argument name PiperOrigin-RevId: 579832146 --- mediapipe/calculators/image/image_clone_calculator.cc | 2 +- .../calculators/image/segmentation_smoothing_calculator.cc | 2 +- mediapipe/calculators/image/warp_affine_calculator.cc | 2 +- mediapipe/gpu/gl_calculator_helper.cc | 4 ++-- mediapipe/gpu/gl_calculator_helper.h | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mediapipe/calculators/image/image_clone_calculator.cc b/mediapipe/calculators/image/image_clone_calculator.cc index 563b4a4ad..0929e81e5 100644 --- a/mediapipe/calculators/image/image_clone_calculator.cc +++ b/mediapipe/calculators/image/image_clone_calculator.cc @@ -65,7 +65,7 @@ class ImageCloneCalculator : public Node { } #else MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( - cc, /*requesst_gpu_as_optional=*/true)); + cc, /*request_gpu_as_optional=*/true)); #endif // MEDIAPIPE_DISABLE_GPU return absl::OkStatus(); } diff --git a/mediapipe/calculators/image/segmentation_smoothing_calculator.cc b/mediapipe/calculators/image/segmentation_smoothing_calculator.cc index d238975c6..ab2148f36 100644 --- a/mediapipe/calculators/image/segmentation_smoothing_calculator.cc +++ b/mediapipe/calculators/image/segmentation_smoothing_calculator.cc @@ -118,7 +118,7 @@ absl::Status SegmentationSmoothingCalculator::GetContract( #if !MEDIAPIPE_DISABLE_GPU MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( - cc, /*requesst_gpu_as_optional=*/true)); + cc, /*request_gpu_as_optional=*/true)); #endif // !MEDIAPIPE_DISABLE_GPU return absl::OkStatus(); diff --git a/mediapipe/calculators/image/warp_affine_calculator.cc b/mediapipe/calculators/image/warp_affine_calculator.cc index dba500dfa..0bbf6c72d 100644 --- a/mediapipe/calculators/image/warp_affine_calculator.cc +++ b/mediapipe/calculators/image/warp_affine_calculator.cc @@ -206,7 +206,7 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl { if constexpr (std::is_same_v || std::is_same_v) { MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( - cc, /*requesst_gpu_as_optional=*/true)); + cc, /*request_gpu_as_optional=*/true)); } return absl::OkStatus(); } diff --git a/mediapipe/gpu/gl_calculator_helper.cc b/mediapipe/gpu/gl_calculator_helper.cc index 1b113f8ac..20f155e15 100644 --- a/mediapipe/gpu/gl_calculator_helper.cc +++ b/mediapipe/gpu/gl_calculator_helper.cc @@ -57,8 +57,8 @@ void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) { // static absl::Status GlCalculatorHelper::UpdateContract(CalculatorContract* cc, - bool requesst_gpu_as_optional) { - if (requesst_gpu_as_optional) { + bool request_gpu_as_optional) { + if (request_gpu_as_optional) { cc->UseService(kGpuService).Optional(); } else { cc->UseService(kGpuService); diff --git a/mediapipe/gpu/gl_calculator_helper.h b/mediapipe/gpu/gl_calculator_helper.h index f5d98ebfe..45b25f67e 100644 --- a/mediapipe/gpu/gl_calculator_helper.h +++ b/mediapipe/gpu/gl_calculator_helper.h @@ -68,7 +68,7 @@ class GlCalculatorHelper { // This method can be called from GetContract to set up the needed GPU // resources. static absl::Status UpdateContract(CalculatorContract* cc, - bool requesst_gpu_as_optional = false); + bool request_gpu_as_optional = false); // This method can be called from FillExpectations to set the correct types // for the shared GL input side packet(s). From 077b52250d704286f9db1a5b57ba980262434cf9 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 6 Nov 2023 13:39:48 -0800 Subject: [PATCH 071/157] Pass Model Asset Buffer as byte array + length PiperOrigin-RevId: 579944283 --- mediapipe/tasks/c/core/base_options.h | 5 ++++- mediapipe/tasks/c/core/base_options_converter.cc | 4 +++- mediapipe/tasks/c/core/base_options_converter_test.cc | 4 ++++ .../tasks/c/text/language_detector/language_detector_test.cc | 2 ++ .../tasks/c/text/text_classifier/text_classifier_test.cc | 2 ++ mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc | 2 ++ mediapipe/tasks/c/vision/image_classifier/BUILD | 4 ---- .../tasks/c/vision/image_classifier/image_classifier_test.cc | 5 +++++ 8 files changed, 22 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/c/core/base_options.h b/mediapipe/tasks/c/core/base_options.h index 78d89ce8c..bbfd3ef7e 100644 --- a/mediapipe/tasks/c/core/base_options.h +++ b/mediapipe/tasks/c/core/base_options.h @@ -22,9 +22,12 @@ extern "C" { // Base options for MediaPipe C Tasks. struct BaseOptions { - // The model asset file contents as a string. + // The model asset file contents as bytes. const char* model_asset_buffer; + // The size of the model assets buffer (or `0` if not set). + const unsigned int model_asset_buffer_count; + // The path to the model asset to open and mmap in memory. const char* model_asset_path; }; diff --git a/mediapipe/tasks/c/core/base_options_converter.cc b/mediapipe/tasks/c/core/base_options_converter.cc index 3f126168b..07a9e81d0 100644 --- a/mediapipe/tasks/c/core/base_options_converter.cc +++ b/mediapipe/tasks/c/core/base_options_converter.cc @@ -27,7 +27,9 @@ void CppConvertToBaseOptions(const BaseOptions& in, mediapipe::tasks::core::BaseOptions* out) { out->model_asset_buffer = in.model_asset_buffer - ? std::make_unique(in.model_asset_buffer) + ? std::make_unique( + in.model_asset_buffer, + in.model_asset_buffer + in.model_asset_buffer_count) : nullptr; out->model_asset_path = in.model_asset_path ? std::string(in.model_asset_path) : ""; diff --git a/mediapipe/tasks/c/core/base_options_converter_test.cc b/mediapipe/tasks/c/core/base_options_converter_test.cc index 27c7fb3ec..37ab90f94 100644 --- a/mediapipe/tasks/c/core/base_options_converter_test.cc +++ b/mediapipe/tasks/c/core/base_options_converter_test.cc @@ -15,6 +15,7 @@ limitations under the License. #include "mediapipe/tasks/c/core/base_options_converter.h" +#include #include #include "mediapipe/framework/port/gtest.h" @@ -28,6 +29,8 @@ constexpr char kModelAssetPath[] = "abc.tflite"; TEST(BaseOptionsConverterTest, ConvertsBaseOptionsAssetBuffer) { BaseOptions c_base_options = {/* model_asset_buffer= */ kAssetBuffer, + /* model_asset_buffer_count= */ + static_cast(strlen(kAssetBuffer)), /* model_asset_path= */ nullptr}; mediapipe::tasks::core::BaseOptions cpp_base_options = {}; @@ -39,6 +42,7 @@ TEST(BaseOptionsConverterTest, ConvertsBaseOptionsAssetBuffer) { TEST(BaseOptionsConverterTest, ConvertsBaseOptionsAssetPath) { BaseOptions c_base_options = {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ kModelAssetPath}; mediapipe::tasks::core::BaseOptions cpp_base_options = {}; diff --git a/mediapipe/tasks/c/text/language_detector/language_detector_test.cc b/mediapipe/tasks/c/text/language_detector/language_detector_test.cc index b8653e616..1c55b2ab5 100644 --- a/mediapipe/tasks/c/text/language_detector/language_detector_test.cc +++ b/mediapipe/tasks/c/text/language_detector/language_detector_test.cc @@ -44,6 +44,7 @@ TEST(LanguageDetectorTest, SmokeTest) { std::string model_path = GetFullPath(kTestLanguageDetectorModelPath); LanguageDetectorOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* classifier_options= */ {/* display_names_locale= */ nullptr, @@ -71,6 +72,7 @@ TEST(LanguageDetectorTest, ErrorHandling) { // It is an error to set neither the asset buffer nor the path. LanguageDetectorOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ nullptr}, /* classifier_options= */ {}, }; diff --git a/mediapipe/tasks/c/text/text_classifier/text_classifier_test.cc b/mediapipe/tasks/c/text/text_classifier/text_classifier_test.cc index 51232d63a..d261be1c8 100644 --- a/mediapipe/tasks/c/text/text_classifier/text_classifier_test.cc +++ b/mediapipe/tasks/c/text/text_classifier/text_classifier_test.cc @@ -43,6 +43,7 @@ TEST(TextClassifierTest, SmokeTest) { std::string model_path = GetFullPath(kTestBertModelPath); TextClassifierOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* classifier_options= */ {/* display_names_locale= */ nullptr, @@ -74,6 +75,7 @@ TEST(TextClassifierTest, ErrorHandling) { // It is an error to set neither the asset buffer nor the path. TextClassifierOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ nullptr}, /* classifier_options= */ {}, }; diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc index c823e01b4..40c449b8a 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc @@ -42,6 +42,7 @@ TEST(TextEmbedderTest, SmokeTest) { std::string model_path = GetFullPath(kTestBertModelPath); TextEmbedderOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* embedder_options= */ {/* l2_normalize= */ false, /* quantize= */ true}, @@ -63,6 +64,7 @@ TEST(TextEmbedderTest, ErrorHandling) { // It is an error to set neither the asset buffer nor the path. TextEmbedderOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ nullptr}, /* embedder_options= */ {}, }; diff --git a/mediapipe/tasks/c/vision/image_classifier/BUILD b/mediapipe/tasks/c/vision/image_classifier/BUILD index e8ac090e9..a2c7ca290 100644 --- a/mediapipe/tasks/c/vision/image_classifier/BUILD +++ b/mediapipe/tasks/c/vision/image_classifier/BUILD @@ -55,11 +55,7 @@ cc_test( ":image_classifier_lib", "//mediapipe/framework/deps:file_path", "//mediapipe/framework/formats:image", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_frame_opencv", "//mediapipe/framework/port:gtest", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/tasks/c/components/containers:category", "//mediapipe/tasks/cc/vision/utils:image_utils", "@com_google_absl//absl/flags:flag", diff --git a/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc b/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc index 790f5ce36..a66e42832 100644 --- a/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc +++ b/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc @@ -50,6 +50,7 @@ TEST(ImageClassifierTest, ImageModeTest) { const std::string model_path = GetFullPath(kModelName); ImageClassifierOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* running_mode= */ RunningMode::IMAGE, /* classifier_options= */ @@ -92,6 +93,7 @@ TEST(ImageClassifierTest, VideoModeTest) { const std::string model_path = GetFullPath(kModelName); ImageClassifierOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* running_mode= */ RunningMode::VIDEO, /* classifier_options= */ @@ -164,6 +166,7 @@ TEST(ImageClassifierTest, LiveStreamModeTest) { ImageClassifierOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* running_mode= */ RunningMode::LIVE_STREAM, /* classifier_options= */ @@ -203,6 +206,7 @@ TEST(ImageClassifierTest, InvalidArgumentHandling) { // It is an error to set neither the asset buffer nor the path. ImageClassifierOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ nullptr}, /* classifier_options= */ {}, }; @@ -220,6 +224,7 @@ TEST(ImageClassifierTest, FailedClassificationHandling) { const std::string model_path = GetFullPath(kModelName); ImageClassifierOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* running_mode= */ RunningMode::IMAGE, /* classifier_options= */ From a8d88bf7cf398a382ddfa049b11fd79655abe37b Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 6 Nov 2023 14:39:10 -0800 Subject: [PATCH 072/157] Creates GpuBuffers around pre-allocated AHardware_Buffer objects. PiperOrigin-RevId: 579961642 --- mediapipe/gpu/BUILD | 5 ----- mediapipe/gpu/gpu_buffer_format.cc | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index f39b8d3f7..b75c824b3 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -516,7 +516,6 @@ cc_library( ":gpu_buffer_storage", ":image_frame_view", "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/port:ret_check", "@com_google_absl//absl/strings:str_format", ], ) @@ -1224,13 +1223,9 @@ mediapipe_cc_test( ], requires_full_emulation = True, deps = [ - ":gl_texture_buffer", - ":gl_texture_util", ":gpu_buffer_format", ":gpu_buffer_storage_ahwb", - ":gpu_test_base", "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/tool:test_util", ], ) diff --git a/mediapipe/gpu/gpu_buffer_format.cc b/mediapipe/gpu/gpu_buffer_format.cc index 510a9cd48..b099f4a25 100644 --- a/mediapipe/gpu/gpu_buffer_format.cc +++ b/mediapipe/gpu/gpu_buffer_format.cc @@ -238,7 +238,7 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) { case GpuBufferFormat::kRGBAFloat128: return ImageFormat::VEC32F4; case GpuBufferFormat::kRGBA32: - return ImageFormat::SRGBA; + // TODO: this likely maps to ImageFormat::SRGBA case GpuBufferFormat::kGrayHalf16: case GpuBufferFormat::kOneComponent8Alpha: case GpuBufferFormat::kOneComponent8Red: From 2abaabce0e851521bf05540318062f4fee79b50b Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 6 Nov 2023 14:53:42 -0800 Subject: [PATCH 073/157] Drop default arguments in C API PiperOrigin-RevId: 579965820 --- .../language_detector/language_detector.h | 13 +++++----- .../language_detector_test.cc | 7 ++--- .../c/text/text_classifier/text_classifier.h | 13 +++++----- .../text_classifier/text_classifier_test.cc | 7 ++--- .../c/text/text_embedder/text_embedder.h | 7 +++-- .../text/text_embedder/text_embedder_test.cc | 6 ++--- .../image_classifier/image_classifier.h | 17 ++++++------ .../image_classifier/image_classifier_test.cc | 26 +++++++++++-------- 8 files changed, 49 insertions(+), 47 deletions(-) diff --git a/mediapipe/tasks/c/text/language_detector/language_detector.h b/mediapipe/tasks/c/text/language_detector/language_detector.h index f1c85069f..a7f2159b6 100644 --- a/mediapipe/tasks/c/text/language_detector/language_detector.h +++ b/mediapipe/tasks/c/text/language_detector/language_detector.h @@ -60,18 +60,18 @@ struct LanguageDetectorOptions { // Creates a LanguageDetector from the provided `options`. // Returns a pointer to the language detector on success. // If an error occurs, returns `nullptr` and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT void* language_detector_create( - struct LanguageDetectorOptions* options, char** error_msg = nullptr); + struct LanguageDetectorOptions* options, char** error_msg); // Performs language detection on the input `text`. Returns `0` on success. // If an error occurs, returns an error code and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT int language_detector_detect(void* detector, const char* utf8_str, LanguageDetectorResult* result, - char** error_msg = nullptr); + char** error_msg); // Frees the memory allocated inside a LanguageDetectorResult result. Does not // free the result pointer itself. @@ -79,10 +79,9 @@ MP_EXPORT void language_detector_close_result(LanguageDetectorResult* result); // Shuts down the LanguageDetector when all the work is done. Frees all memory. // If an error occurs, returns an error code and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. -MP_EXPORT int language_detector_close(void* detector, - char** error_msg = nullptr); +MP_EXPORT int language_detector_close(void* detector, char** error_msg); #ifdef __cplusplus } // extern C diff --git a/mediapipe/tasks/c/text/language_detector/language_detector_test.cc b/mediapipe/tasks/c/text/language_detector/language_detector_test.cc index 1c55b2ab5..47770986f 100644 --- a/mediapipe/tasks/c/text/language_detector/language_detector_test.cc +++ b/mediapipe/tasks/c/text/language_detector/language_detector_test.cc @@ -56,16 +56,17 @@ TEST(LanguageDetectorTest, SmokeTest) { /* category_denylist_count= */ 0}, }; - void* detector = language_detector_create(&options); + void* detector = language_detector_create(&options, /* error_msg */ nullptr); EXPECT_NE(detector, nullptr); LanguageDetectorResult result; - language_detector_detect(detector, kTestString, &result); + language_detector_detect(detector, kTestString, &result, + /* error_msg */ nullptr); EXPECT_EQ(std::string(result.predictions[0].language_code), "fr"); EXPECT_NEAR(result.predictions[0].probability, 0.999781, kPrecision); language_detector_close_result(&result); - language_detector_close(detector); + language_detector_close(detector, /* error_msg */ nullptr); } TEST(LanguageDetectorTest, ErrorHandling) { diff --git a/mediapipe/tasks/c/text/text_classifier/text_classifier.h b/mediapipe/tasks/c/text/text_classifier/text_classifier.h index 057b00f99..55b4bc710 100644 --- a/mediapipe/tasks/c/text/text_classifier/text_classifier.h +++ b/mediapipe/tasks/c/text/text_classifier/text_classifier.h @@ -44,18 +44,18 @@ struct TextClassifierOptions { // Creates a TextClassifier from the provided `options`. // Returns a pointer to the text classifier on success. // If an error occurs, returns `nullptr` and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT void* text_classifier_create(struct TextClassifierOptions* options, - char** error_msg = nullptr); + char** error_msg); // Performs classification on the input `text`. Returns `0` on success. // If an error occurs, returns an error code and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT int text_classifier_classify(void* classifier, const char* utf8_str, TextClassifierResult* result, - char** error_msg = nullptr); + char** error_msg); // Frees the memory allocated inside a TextClassifierResult result. Does not // free the result pointer itself. @@ -63,10 +63,9 @@ MP_EXPORT void text_classifier_close_result(TextClassifierResult* result); // Shuts down the TextClassifier when all the work is done. Frees all memory. // If an error occurs, returns an error code and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. -MP_EXPORT int text_classifier_close(void* classifier, - char** error_msg = nullptr); +MP_EXPORT int text_classifier_close(void* classifier, char** error_msg); #ifdef __cplusplus } // extern C diff --git a/mediapipe/tasks/c/text/text_classifier/text_classifier_test.cc b/mediapipe/tasks/c/text/text_classifier/text_classifier_test.cc index d261be1c8..1dd3f6910 100644 --- a/mediapipe/tasks/c/text/text_classifier/text_classifier_test.cc +++ b/mediapipe/tasks/c/text/text_classifier/text_classifier_test.cc @@ -55,11 +55,12 @@ TEST(TextClassifierTest, SmokeTest) { /* category_denylist_count= */ 0}, }; - void* classifier = text_classifier_create(&options); + void* classifier = text_classifier_create(&options, /* error_msg */ nullptr); EXPECT_NE(classifier, nullptr); TextClassifierResult result; - text_classifier_classify(classifier, kTestString, &result); + text_classifier_classify(classifier, kTestString, &result, + /* error_msg */ nullptr); EXPECT_EQ(result.classifications_count, 1); EXPECT_EQ(result.classifications[0].categories_count, 2); EXPECT_EQ(std::string{result.classifications[0].categories[0].category_name}, @@ -68,7 +69,7 @@ TEST(TextClassifierTest, SmokeTest) { kPrecision); text_classifier_close_result(&result); - text_classifier_close(classifier); + text_classifier_close(classifier, /* error_msg */ nullptr); } TEST(TextClassifierTest, ErrorHandling) { diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder.h b/mediapipe/tasks/c/text/text_embedder/text_embedder.h index c9ccf816b..c0aab5c76 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder.h +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder.h @@ -47,15 +47,14 @@ struct TextEmbedderOptions { // an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT void* text_embedder_create(struct TextEmbedderOptions* options, - char** error_msg = nullptr); + char** error_msg); // Performs embedding extraction on the input `text`. Returns `0` on success. // If an error occurs, returns an error code and sets the error parameter to an // an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT int text_embedder_embed(void* embedder, const char* utf8_str, - TextEmbedderResult* result, - char** error_msg = nullptr); + TextEmbedderResult* result, char** error_msg); // Frees the memory allocated inside a TextEmbedderResult result. Does not // free the result pointer itself. @@ -65,7 +64,7 @@ MP_EXPORT void text_embedder_close_result(TextEmbedderResult* result); // If an error occurs, returns an error code and sets the error parameter to an // an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. -MP_EXPORT int text_embedder_close(void* embedder, char** error_msg = nullptr); +MP_EXPORT int text_embedder_close(void* embedder, char** error_msg); #ifdef __cplusplus } // extern C diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc index 40c449b8a..1c38a1703 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc @@ -48,16 +48,16 @@ TEST(TextEmbedderTest, SmokeTest) { {/* l2_normalize= */ false, /* quantize= */ true}, }; - void* embedder = text_embedder_create(&options); + void* embedder = text_embedder_create(&options, /* error_msg */ nullptr); EXPECT_NE(embedder, nullptr); TextEmbedderResult result; - text_embedder_embed(embedder, kTestString, &result); + text_embedder_embed(embedder, kTestString, &result, /* error_msg */ nullptr); EXPECT_EQ(result.embeddings_count, 1); EXPECT_EQ(result.embeddings[0].values_count, 512); text_embedder_close_result(&result); - text_embedder_close(embedder); + text_embedder_close(embedder, /* error_msg */ nullptr); } TEST(TextEmbedderTest, ErrorHandling) { diff --git a/mediapipe/tasks/c/vision/image_classifier/image_classifier.h b/mediapipe/tasks/c/vision/image_classifier/image_classifier.h index 549c3f300..51316bcbe 100644 --- a/mediapipe/tasks/c/vision/image_classifier/image_classifier.h +++ b/mediapipe/tasks/c/vision/image_classifier/image_classifier.h @@ -108,30 +108,30 @@ struct ImageClassifierOptions { // Creates an ImageClassifier from provided `options`. // Returns a pointer to the image classifier on success. // If an error occurs, returns `nullptr` and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT void* image_classifier_create(struct ImageClassifierOptions* options, - char** error_msg = nullptr); + char** error_msg); // Performs image classification on the input `image`. Returns `0` on success. // If an error occurs, returns an error code and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT int image_classifier_classify_image(void* classifier, const MpImage* image, ImageClassifierResult* result, - char** error_msg = nullptr); + char** error_msg); MP_EXPORT int image_classifier_classify_for_video(void* classifier, const MpImage* image, int64_t timestamp_ms, ImageClassifierResult* result, - char** error_msg = nullptr); + char** error_msg); MP_EXPORT int image_classifier_classify_async(void* classifier, const MpImage* image, int64_t timestamp_ms, - char** error_msg = nullptr); + char** error_msg); // Frees the memory allocated inside a ImageClassifierResult result. // Does not free the result pointer itself. @@ -139,10 +139,9 @@ MP_EXPORT void image_classifier_close_result(ImageClassifierResult* result); // Frees image classifier. // If an error occurs, returns an error code and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. -MP_EXPORT int image_classifier_close(void* classifier, - char** error_msg = nullptr); +MP_EXPORT int image_classifier_close(void* classifier, char** error_msg); #ifdef __cplusplus } // extern C diff --git a/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc b/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc index a66e42832..22a716dfd 100644 --- a/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc +++ b/mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc @@ -63,7 +63,7 @@ TEST(ImageClassifierTest, ImageModeTest) { /* category_denylist_count= */ 0}, }; - void* classifier = image_classifier_create(&options); + void* classifier = image_classifier_create(&options, /* error_msg */ nullptr); EXPECT_NE(classifier, nullptr); const auto& image_frame = image->GetImageFrameSharedPtr(); @@ -75,7 +75,8 @@ TEST(ImageClassifierTest, ImageModeTest) { .height = image_frame->Height()}}; ImageClassifierResult result; - image_classifier_classify_image(classifier, &mp_image, &result); + image_classifier_classify_image(classifier, &mp_image, &result, + /* error_msg */ nullptr); EXPECT_EQ(result.classifications_count, 1); EXPECT_EQ(result.classifications[0].categories_count, 1001); EXPECT_EQ(std::string{result.classifications[0].categories[0].category_name}, @@ -83,7 +84,7 @@ TEST(ImageClassifierTest, ImageModeTest) { EXPECT_NEAR(result.classifications[0].categories[0].score, 0.7939f, kPrecision); image_classifier_close_result(&result); - image_classifier_close(classifier); + image_classifier_close(classifier, /* error_msg */ nullptr); } TEST(ImageClassifierTest, VideoModeTest) { @@ -107,7 +108,7 @@ TEST(ImageClassifierTest, VideoModeTest) { /* result_callback= */ nullptr, }; - void* classifier = image_classifier_create(&options); + void* classifier = image_classifier_create(&options, /* error_msg */ nullptr); EXPECT_NE(classifier, nullptr); const auto& image_frame = image->GetImageFrameSharedPtr(); @@ -120,7 +121,8 @@ TEST(ImageClassifierTest, VideoModeTest) { for (int i = 0; i < kIterations; ++i) { ImageClassifierResult result; - image_classifier_classify_for_video(classifier, &mp_image, i, &result); + image_classifier_classify_for_video(classifier, &mp_image, i, &result, + /* error_msg */ nullptr); EXPECT_EQ(result.classifications_count, 1); EXPECT_EQ(result.classifications[0].categories_count, 3); EXPECT_EQ( @@ -130,7 +132,7 @@ TEST(ImageClassifierTest, VideoModeTest) { kPrecision); image_classifier_close_result(&result); } - image_classifier_close(classifier); + image_classifier_close(classifier, /* error_msg */ nullptr); } // A structure to support LiveStreamModeTest below. This structure holds a @@ -180,7 +182,7 @@ TEST(ImageClassifierTest, LiveStreamModeTest) { /* result_callback= */ LiveStreamModeCallback::Fn, }; - void* classifier = image_classifier_create(&options); + void* classifier = image_classifier_create(&options, /* error_msg */ nullptr); EXPECT_NE(classifier, nullptr); const auto& image_frame = image->GetImageFrameSharedPtr(); @@ -192,9 +194,11 @@ TEST(ImageClassifierTest, LiveStreamModeTest) { .height = image_frame->Height()}}; for (int i = 0; i < kIterations; ++i) { - EXPECT_GE(image_classifier_classify_async(classifier, &mp_image, i), 0); + EXPECT_GE(image_classifier_classify_async(classifier, &mp_image, i, + /* error_msg */ nullptr), + 0); } - image_classifier_close(classifier); + image_classifier_close(classifier, /* error_msg */ nullptr); // Due to the flow limiter, the total of outputs might be smaller than the // number of iterations. @@ -237,7 +241,7 @@ TEST(ImageClassifierTest, FailedClassificationHandling) { /* category_denylist_count= */ 0}, }; - void* classifier = image_classifier_create(&options); + void* classifier = image_classifier_create(&options, /* error_msg */ nullptr); EXPECT_NE(classifier, nullptr); const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}}; @@ -246,7 +250,7 @@ TEST(ImageClassifierTest, FailedClassificationHandling) { image_classifier_classify_image(classifier, &mp_image, &result, &error_msg); EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet")); free(error_msg); - image_classifier_close(classifier); + image_classifier_close(classifier, /* error_msg */ nullptr); } } // namespace From 1d0f3734b4a0c45c623a72c8f7cbce680ad3ccfa Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 7 Nov 2023 09:46:39 +0530 Subject: [PATCH 074/157] Added iOS MPPPoseLandmarker.mm --- .../tasks/ios/vision/pose_landmarker/BUILD | 14 ++ .../sources/MPPPoseLandmarker.mm | 222 ++++++++++++++++++ 2 files changed, 236 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarker.mm diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/BUILD b/mediapipe/tasks/ios/vision/pose_landmarker/BUILD index a7b612bce..97cb278d9 100644 --- a/mediapipe/tasks/ios/vision/pose_landmarker/BUILD +++ b/mediapipe/tasks/ios/vision/pose_landmarker/BUILD @@ -47,13 +47,27 @@ objc_library( objc_library( name = "MPPPoseLandmarker", + srcs = ["sources/MPPPoseLandmarker.mm"], hdrs = ["sources/MPPPoseLandmarker.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], module_name = "MPPPoseLandmarker", deps = [ ":MPPPoseLandmarkerOptions", ":MPPPoseLandmarkerResult", ":MPPPoseLandmarksConnections", + "//mediapipe/tasks/cc/vision/pose_landmarker:pose_landmarker_graph", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", "//mediapipe/tasks/ios/components/containers:MPPConnection", + "//mediapipe/tasks/ios/core:MPPTaskInfo", "//mediapipe/tasks/ios/vision/core:MPPImage", + "//mediapipe/tasks/ios/vision/core:MPPVisionPacketCreator", + "//mediapipe/tasks/ios/vision/core:MPPVisionTaskRunner", + "//mediapipe/tasks/ios/vision/pose_landmarker/utils:MPPPoseLandmarkerOptionsHelpers", + "//mediapipe/tasks/ios/vision/pose_landmarker/utils:MPPPoseLandmarkerResultHelpers", ], ) diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarker.mm b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarker.mm new file mode 100644 index 000000000..815578500 --- /dev/null +++ b/mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarker.mm @@ -0,0 +1,222 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarker.h" + +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskInfo.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h" +#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarksConnections.h" +#import "mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerOptions+Helpers.h" +#import "mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h" + +namespace { +using ::mediapipe::NormalizedRect; +using ::mediapipe::Packet; +using ::mediapipe::Timestamp; +using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; +} // namespace + +static NSString *const kImageTag = @"IMAGE"; +static NSString *const kImageInStreamName = @"image_in"; +static NSString *const kNormRectTag = @"NORM_RECT"; +static NSString *const kNormRectInStreamName = @"norm_rect_in"; +static NSString *const kImageOutStreamName = @"image_out"; +static NSString *const kPoseLandmarksTag = @"NORM_LANDMARKS"; +static NSString *const kPoseLandmarksOutStreamName = @"pose_landmarks"; +static NSString *const kWorldLandmarksTag = @"WORLD_LANDMARKS"; +static NSString *const kWorldLandmarksOutStreamName = @"world_landmarks"; +static NSString *const kSegmentationMasksTag = @"SEGMENTATION_MASK"; +static NSString *const kSegmentationMasksOutStreamName = @"segmentation_masks"; +static NSString *const kTaskGraphName = + @"mediapipe.tasks.vision.pose_landmarker.PoseLandmarkerGraph"; +static NSString *const kTaskName = @"poseLandmarker"; + +#define InputPacketMap(imagePacket, normalizedRectPacket) \ + { \ + {kImageInStreamName.cppString, imagePacket}, { \ + kNormRectInStreamName.cppString, normalizedRectPacket \ + } \ + } + +#define PoseLandmarkerResultWithOutputPacketMap(outputPacketMap) \ + ([MPPPoseLandmarkerResult \ + poseLandmarkerResultWithLandmarksPacket:outputPacketMap[kPoseLandmarksOutStreamName \ + .cppString] \ + worldLandmarksPacket:outputPacketMap[kWorldLandmarksOutStreamName \ + .cppString] \ + segmentationMasksPacket:&(outputPacketMap[kSegmentationMasksOutStreamName \ + .cppString])]) + +@interface MPPPoseLandmarker () { + /** iOS Vision Task Runner */ + MPPVisionTaskRunner *_visionTaskRunner; + dispatch_queue_t _callbackQueue; +} +@property(nonatomic, weak) id poseLandmarkerLiveStreamDelegate; +@end + +@implementation MPPPoseLandmarker + +#pragma mark - Public + +- (instancetype)initWithOptions:(MPPPoseLandmarkerOptions *)options error:(NSError **)error { + self = [super init]; + if (self) { + MPPTaskInfo *taskInfo = [[MPPTaskInfo alloc] + initWithTaskGraphName:kTaskGraphName + inputStreams:@[ + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageInStreamName], + [NSString stringWithFormat:@"%@:%@", kNormRectTag, kNormRectInStreamName] + ] + outputStreams:@[ + [NSString + stringWithFormat:@"%@:%@", kPoseLandmarksTag, kPoseLandmarksOutStreamName], + [NSString + stringWithFormat:@"%@:%@", kWorldLandmarksTag, kWorldLandmarksOutStreamName], + [NSString stringWithFormat:@"%@:%@", kSegmentationMasksTag, + kSegmentationMasksOutStreamName], + [NSString stringWithFormat:@"%@:%@", kImageTag, kImageOutStreamName] + ] + taskOptions:options + enableFlowLimiting:options.runningMode == MPPRunningModeLiveStream + error:error]; + + if (!taskInfo) { + return nil; + } + + PacketsCallback packetsCallback = nullptr; + + if (options.poseLandmarkerLiveStreamDelegate) { + _poseLandmarkerLiveStreamDelegate = options.poseLandmarkerLiveStreamDelegate; + + // Create a private serial dispatch queue in which the deleagte method will be called + // asynchronously. This is to ensure that if the client performs a long running operation in + // the delegate method, the queue on which the C++ callbacks is invoked is not blocked and is + // freed up to continue with its operations. + _callbackQueue = dispatch_queue_create( + [MPPVisionTaskRunner uniqueDispatchQueueNameWithSuffix:kTaskName], NULL); + + // Capturing `self` as weak in order to avoid `self` being kept in memory + // and cause a retain cycle, after self is set to `nil`. + MPPPoseLandmarker *__weak weakSelf = self; + packetsCallback = [=](absl::StatusOr liveStreamResult) { + [weakSelf processLiveStreamResult:liveStreamResult]; + }; + } + + _visionTaskRunner = [[MPPVisionTaskRunner alloc] initWithTaskInfo:taskInfo + runningMode:options.runningMode + roiAllowed:NO + packetsCallback:std::move(packetsCallback) + imageInputStreamName:kImageInStreamName + normRectInputStreamName:kNormRectInStreamName + error:error]; + + if (!_visionTaskRunner) { + return nil; + } + } + return self; +} + +- (instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error { + MPPPoseLandmarkerOptions *options = [[MPPPoseLandmarkerOptions alloc] init]; + + options.baseOptions.modelAssetPath = modelPath; + + return [self initWithOptions:options error:error]; +} + +- (nullable MPPPoseLandmarkerResult *)detectImage:(MPPImage *)image error:(NSError **)error { + std::optional outputPacketMap = [_visionTaskRunner processImage:image error:error]; + + return [MPPPoseLandmarker poseLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; +} + +- (nullable MPPPoseLandmarkerResult *)detectVideoFrame:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + std::optional outputPacketMap = + [_visionTaskRunner processVideoFrame:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; + + return [MPPPoseLandmarker poseLandmarkerResultWithOptionalOutputPacketMap:outputPacketMap]; +} + +- (BOOL)detectAsyncImage:(MPPImage *)image + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError **)error { + return [_visionTaskRunner processLiveStreamImage:image + timestampInMilliseconds:timestampInMilliseconds + error:error]; +} + ++ (NSArray *)poseLandmarks { + return MPPPoseLandmarksConnections; +} + +#pragma mark - Private + +- (void)processLiveStreamResult:(absl::StatusOr)liveStreamResult { + if (![self.poseLandmarkerLiveStreamDelegate + respondsToSelector:@selector(poseLandmarker: + didFinishDetectionWithResult:timestampInMilliseconds:error:)]) { + return; + } + + NSError *callbackError = nil; + if (![MPPCommonUtils checkCppError:liveStreamResult.status() toError:&callbackError]) { + dispatch_async(_callbackQueue, ^{ + [self.poseLandmarkerLiveStreamDelegate poseLandmarker:self + didFinishDetectionWithResult:nil + timestampInMilliseconds:Timestamp::Unset().Value() + error:callbackError]; + }); + return; + } + + PacketMap &outputPacketMap = liveStreamResult.value(); + if (outputPacketMap[kImageOutStreamName.cppString].IsEmpty()) { + return; + } + + MPPPoseLandmarkerResult *result = PoseLandmarkerResultWithOutputPacketMap(outputPacketMap); + + NSInteger timestampInMilliseconds = + outputPacketMap[kImageOutStreamName.cppString].Timestamp().Value() / + kMicroSecondsPerMillisecond; + dispatch_async(_callbackQueue, ^{ + [self.poseLandmarkerLiveStreamDelegate poseLandmarker:self + didFinishDetectionWithResult:result + timestampInMilliseconds:timestampInMilliseconds + error:callbackError]; + }); +} + ++ (nullable MPPPoseLandmarkerResult *)poseLandmarkerResultWithOptionalOutputPacketMap: + (std::optional &)outputPacketMap { + if (!outputPacketMap.has_value()) { + return nil; + } + + return PoseLandmarkerResultWithOutputPacketMap(outputPacketMap.value()); +} + +@end From 91095c2d6affa536b63fa0df88adb3862d4a54fb Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 7 Nov 2023 09:49:42 +0530 Subject: [PATCH 075/157] Added null check for segmentation masks in pose landmarker helper initializer --- .../sources/MPPPoseLandmarkerResult+Helpers.h | 4 ++- .../MPPPoseLandmarkerResult+Helpers.mm | 25 +++++++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h index 0b20dc32a..83379f676 100644 --- a/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h +++ b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h @@ -20,6 +20,8 @@ NS_ASSUME_NONNULL_BEGIN +static const int kMicroSecondsPerMillisecond = 1000; + @interface MPPPoseLandmarkerResult (Helpers) /** @@ -56,7 +58,7 @@ NS_ASSUME_NONNULL_BEGIN worldLandmarksProto: (const std::vector<::mediapipe::LandmarkList> &)worldLandmarksProto segmentationMasks:(const std::vector *)segmentationMasks - timestampInMilliSeconds:(NSInteger)timestampInMilliseconds; + timestampInMilliseconds:(NSInteger)timestampInMilliseconds; @end diff --git a/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.mm b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.mm index 6cd67ff9c..c83365c86 100644 --- a/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.mm +++ b/mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.mm @@ -21,8 +21,6 @@ using LandmarkListProto = ::mediapipe::LandmarkList; using NormalizedLandmarkListProto = ::mediapipe::NormalizedLandmarkList; using ::mediapipe::Image; using ::mediapipe::Packet; - -static const int kMicroSecondsPerMilliSecond = 1000; } // namespace @implementation MPPPoseLandmarkerResult (Helpers) @@ -41,7 +39,7 @@ static const int kMicroSecondsPerMilliSecond = 1000; worldLandmarksProto: (const std::vector &)worldLandmarksProto segmentationMasks:(const std::vector *)segmentationMasks - timestampInMilliSeconds:(NSInteger)timestampInMilliseconds { + timestampInMilliseconds:(NSInteger)timestampInMilliseconds { NSMutableArray *> *multiplePoseLandmarks = [NSMutableArray arrayWithCapacity:(NSUInteger)landmarksProto.size()]; @@ -69,6 +67,12 @@ static const int kMicroSecondsPerMilliSecond = 1000; [multiplePoseWorldLandmarks addObject:worldLandmarks]; } + if (!segmentationMasks) { + return [[MPPPoseLandmarkerResult alloc] initWithLandmarks:multiplePoseLandmarks + worldLandmarks:multiplePoseWorldLandmarks + segmentationMasks:nil + timestampInMilliseconds:timestampInMilliseconds]; + } NSMutableArray *confidenceMasks = [NSMutableArray arrayWithCapacity:(NSUInteger)segmentationMasks->size()]; @@ -83,12 +87,11 @@ static const int kMicroSecondsPerMilliSecond = 1000; shouldCopy:YES]]; } - MPPPoseLandmarkerResult *poseLandmarkerResult = - [[MPPPoseLandmarkerResult alloc] initWithLandmarks:multiplePoseLandmarks - worldLandmarks:multiplePoseWorldLandmarks - segmentationMasks:confidenceMasks - timestampInMilliseconds:timestampInMilliseconds]; - return poseLandmarkerResult; + return [[MPPPoseLandmarkerResult alloc] initWithLandmarks:multiplePoseLandmarks + worldLandmarks:multiplePoseWorldLandmarks + segmentationMasks:confidenceMasks + timestampInMilliseconds:timestampInMilliseconds]; + ; } + (MPPPoseLandmarkerResult *) @@ -96,7 +99,7 @@ static const int kMicroSecondsPerMilliSecond = 1000; worldLandmarksPacket:(const Packet &)worldLandmarksPacket segmentationMasksPacket:(const Packet *)segmentationMasksPacket { NSInteger timestampInMilliseconds = - (NSInteger)(landmarksPacket.Timestamp().Value() / kMicroSecondsPerMilliSecond); + (NSInteger)(landmarksPacket.Timestamp().Value() / kMicroSecondsPerMillisecond); if (landmarksPacket.IsEmpty()) { return [MPPPoseLandmarkerResult @@ -118,7 +121,7 @@ static const int kMicroSecondsPerMilliSecond = 1000; worldLandmarksProto:worldLandmarksPacket .Get>() segmentationMasks:segmentationMasks - timestampInMilliSeconds:timestampInMilliseconds]; + timestampInMilliseconds:timestampInMilliseconds]; } @end From 32571a37d21162353f6d2897484624015c6deeaa Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 7 Nov 2023 09:49:53 +0530 Subject: [PATCH 076/157] Added pose landmarker protobuf utils --- .../test/vision/pose_landmarker/utils/BUILD | 21 +++++++ .../MPPPoseLandmarkerResult+ProtobufHelpers.h | 26 ++++++++ ...MPPPoseLandmarkerResult+ProtobufHelpers.mm | 61 +++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/pose_landmarker/utils/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+ProtobufHelpers.h create mode 100644 mediapipe/tasks/ios/test/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+ProtobufHelpers.mm diff --git a/mediapipe/tasks/ios/test/vision/pose_landmarker/utils/BUILD b/mediapipe/tasks/ios/test/vision/pose_landmarker/utils/BUILD new file mode 100644 index 000000000..7c9314367 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/pose_landmarker/utils/BUILD @@ -0,0 +1,21 @@ +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPPoseLandmarkerResultProtobufHelpers", + srcs = ["sources/MPPPoseLandmarkerResult+ProtobufHelpers.mm"], + hdrs = ["sources/MPPPoseLandmarkerResult+ProtobufHelpers.h"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + deps = [ + "//mediapipe/tasks/cc/components/containers/proto:landmarks_detection_result_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/test/vision/utils:parse_proto_utils", + "//mediapipe/tasks/ios/vision/pose_landmarker:MPPPoseLandmarkerResult", + "//mediapipe/tasks/ios/vision/pose_landmarker/utils:MPPPoseLandmarkerResultHelpers", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+ProtobufHelpers.h b/mediapipe/tasks/ios/test/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+ProtobufHelpers.h new file mode 100644 index 000000000..3db43c41f --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+ProtobufHelpers.h @@ -0,0 +1,26 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarkerResult.h" + +NS_ASSUME_NONNULL_BEGIN +@interface MPPPoseLandmarkerResult (ProtobufHelpers) + ++ (MPPPoseLandmarkerResult *)poseLandmarkerResultFromProtobufFileWithName:(NSString *)fileName + shouldRemoveZPosition:(BOOL)removeZPosition; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/test/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+ProtobufHelpers.mm b/mediapipe/tasks/ios/test/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+ProtobufHelpers.mm new file mode 100644 index 000000000..38d855c36 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+ProtobufHelpers.mm @@ -0,0 +1,61 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/test/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+ProtobufHelpers.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+Helpers.h" + +#include "mediapipe/tasks/cc/components/containers/proto/landmarks_detection_result.pb.h" +#include "mediapipe/tasks/ios/test/vision/utils/sources/parse_proto_utils.h" + +namespace { +using LandmarksDetectionResultProto = + ::mediapipe::tasks::containers::proto::LandmarksDetectionResult; +using ::mediapipe::tasks::ios::test::vision::utils::get_proto_from_pbtxt; +} // anonymous namespace + +@implementation MPPPoseLandmarkerResult (ProtobufHelpers) + ++ (MPPPoseLandmarkerResult *)poseLandmarkerResultFromProtobufFileWithName:(NSString *)fileName + shouldRemoveZPosition:(BOOL)removeZPosition { + LandmarksDetectionResultProto landmarkDetectionResultProto; + + if (!get_proto_from_pbtxt(fileName.cppString, landmarkDetectionResultProto).ok()) { + return nil; + } + + if (removeZPosition) { + // Remove z position of landmarks, because they are not used in correctness testing. For video + // or live stream mode, the z positions varies a lot during tracking from frame to frame. + for (int i = 0; i < landmarkDetectionResultProto.landmarks().landmark().size(); i++) { + auto &landmark = *landmarkDetectionResultProto.mutable_landmarks()->mutable_landmark(i); + landmark.clear_z(); + } + } + + MPPPoseLandmarkerResult *result = [MPPPoseLandmarkerResult + poseLandmarkerResultWithLandmarksProto:{landmarkDetectionResultProto.landmarks()} + worldLandmarksProto:{landmarkDetectionResultProto.world_landmarks()} + segmentationMasks:nullptr + timestampInMilliseconds:0]; + + return [MPPPoseLandmarkerResult + poseLandmarkerResultWithLandmarksProto:{landmarkDetectionResultProto.landmarks()} + worldLandmarksProto:{landmarkDetectionResultProto.world_landmarks()} + segmentationMasks:nullptr + timestampInMilliseconds:0]; +} + +@end From b5b0d6eee74ab442762a160ef84cebd3be91f2ac Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 7 Nov 2023 11:13:25 +0530 Subject: [PATCH 077/157] Fixed graph name in iOS language detector --- .../ios/text/language_detector/sources/MPPLanguageDetector.mm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.mm b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.mm index 4c9628c82..02094e92e 100644 --- a/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.mm +++ b/mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.mm @@ -32,7 +32,7 @@ static NSString *const kClassificationsTag = @"CLASSIFICATIONS"; static NSString *const kTextInStreamName = @"text_in"; static NSString *const kTextTag = @"TEXT"; static NSString *const kTaskGraphName = - @"mediapipe.tasks.text.language_detector.LanguageDetectorGraph"; + @"mediapipe.tasks.text.text_classifier.TextClassifierGraph"; @interface MPPLanguageDetector () { /** iOS Text Task Runner */ From 9d9a5dc5e702e560cfc8a3f42786400d07e0b29b Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Tue, 7 Nov 2023 11:15:01 +0530 Subject: [PATCH 078/157] Added iOS language detector tests --- .../ios/test/text/language_detector/BUILD | 61 +++++++ .../MPPLanguageDetectorTests.mm | 162 ++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100644 mediapipe/tasks/ios/test/text/language_detector/BUILD create mode 100644 mediapipe/tasks/ios/test/text/language_detector/MPPLanguageDetectorTests.mm diff --git a/mediapipe/tasks/ios/test/text/language_detector/BUILD b/mediapipe/tasks/ios/test/text/language_detector/BUILD new file mode 100644 index 000000000..9f5c70bfd --- /dev/null +++ b/mediapipe/tasks/ios/test/text/language_detector/BUILD @@ -0,0 +1,61 @@ +load( + "@build_bazel_rules_apple//apple:ios.bzl", + "ios_unit_test", +) +load( + "@build_bazel_rules_swift//swift:swift.bzl", + "swift_library", +) +load( + "//mediapipe/framework/tool:ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPLanguageDetectorObjcTestLibrary", + testonly = 1, + srcs = ["MPPLanguageDetectorTests.mm"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + data = [ + "//mediapipe/tasks/testdata/text:language_detector", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/text/language_detector:MPPLanguageDetector", + "//mediapipe/tasks/ios/test/utils:MPPFileInfo", + ], +) + +ios_unit_test( + name = "MPPLanguageDetectorObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPLanguageDetectorObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/text/language_detector/MPPLanguageDetectorTests.mm b/mediapipe/tasks/ios/test/text/language_detector/MPPLanguageDetectorTests.mm new file mode 100644 index 000000000..51009fb75 --- /dev/null +++ b/mediapipe/tasks/ios/test/text/language_detector/MPPLanguageDetectorTests.mm @@ -0,0 +1,162 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/test/utils/sources/MPPFileInfo.h" +#import "mediapipe/tasks/ios/text/language_detector/sources/MPPLanguageDetector.h" + +static MPPFileInfo *const kLanguageDetectorModelFileInfo = + [[MPPFileInfo alloc] initWithName:@"language_detector" type:@"tflite"]; + +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertEqualObjects(error.localizedDescription, expectedError.localizedDescription) + +@interface MPPLanguageDetectorTests : XCTestCase +@end + +@implementation MPPLanguageDetectorTests + +- (void)testCreateLanguageDetectorFailsWithMissingModelPath { + MPPFileInfo *fileInfo = [[MPPFileInfo alloc] initWithName:@"" type:@""]; + + NSError *error = nil; + MPPLanguageDetector *languageDetector = + [[MPPLanguageDetector alloc] initWithModelPath:fileInfo.path error:&error]; + XCTAssertNil(languageDetector); + + NSError *expectedError = [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: ExternalFile must specify at least one of 'file_content', " + @"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'." + }]; + AssertEqualErrors(error, expectedError); +} + +- (void)testCreateLanguageDetectorFailsWithBothAllowlistAndDenylist { + MPPLanguageDetectorOptions *options = + [self languageDetectorOptionsWithModelFileInfo:kLanguageDetectorModelFileInfo]; + options.categoryAllowlist = @[ @"en" ]; + options.categoryDenylist = @[ @"en" ]; + + [self assertCreateLanguageDetectorWithOptions:options + failsWithExpectedError: + [NSError + errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: `category_allowlist` and " + @"`category_denylist` are mutually exclusive options." + }]]; +} + +- (void)testCreateLanguageDetectorFailsWithInvalidMaxResults { + MPPLanguageDetectorOptions *options = + [self languageDetectorOptionsWithModelFileInfo:kLanguageDetectorModelFileInfo]; + options.maxResults = 0; + + [self + assertCreateLanguageDetectorWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: Invalid `max_results` option: " + @"value must be != 0." + }]]; +} + +- (void)testClassifyWithL2CModelSucceeds { + MPPLanguageDetectorOptions *options = + [self languageDetectorOptionsWithModelFileInfo:kLanguageDetectorModelFileInfo]; + + MPPLanguageDetector *languageDetector = [self createLanguageDetectorWithOptionsSucceeds:options]; + NSString *enText = @"To be, or not to be, that is the question"; + NSArray *expectedEnLanguagePredictions = + @[ [[MPPLanguagePrediction alloc] initWithLanguageCode:@"en" probability:0.9998559f] ]; + + [self assertResultsOfDetectLanguageOfText:enText + usingLanguageDetector:languageDetector + approximatelyEqualsExpectedLanguagePredictions:expectedEnLanguagePredictions]; + + NSString *frText = @"Il y a beaucoup de bouches qui parlent et fort peu de têtes qui pensent."; + NSArray *expectedFrLanguagePredictions = + @[ [[MPPLanguagePrediction alloc] initWithLanguageCode:@"fr" probability:0.9997813f] ]; + + [self assertResultsOfDetectLanguageOfText:frText + usingLanguageDetector:languageDetector + approximatelyEqualsExpectedLanguagePredictions:expectedFrLanguagePredictions]; + + NSString *ruText = @"это какой-то английский язык"; + NSArray *expectedRuLanguagePredictions = + @[ [[MPPLanguagePrediction alloc] initWithLanguageCode:@"ru" probability:0.9933616f] ]; + + [self assertResultsOfDetectLanguageOfText:ruText + usingLanguageDetector:languageDetector + approximatelyEqualsExpectedLanguagePredictions:expectedRuLanguagePredictions]; +} + +#pragma mark Assert Segmenter Results +- (void)assertResultsOfDetectLanguageOfText:(NSString *)text + usingLanguageDetector:(MPPLanguageDetector *)languageDetector + approximatelyEqualsExpectedLanguagePredictions: + (NSArray *)expectedLanguagePredictions { + MPPLanguageDetectorResult *result = [languageDetector detectText:text error:nil]; + XCTAssertNotNil(result); + XCTAssertEqualWithAccuracy(result.languagePredictions[0].probability, + expectedLanguagePredictions[0].probability, 1e-3); + XCTAssertEqualObjects(result.languagePredictions[0].languageCode, + expectedLanguagePredictions[0].languageCode); +} + +#pragma mark Language Detector Initializers + +- (MPPLanguageDetectorOptions *)languageDetectorOptionsWithModelFileInfo:(MPPFileInfo *)fileInfo { + MPPLanguageDetectorOptions *options = [[MPPLanguageDetectorOptions alloc] init]; + options.baseOptions.modelAssetPath = fileInfo.path; + return options; +} + +- (MPPLanguageDetector *)createLanguageDetectorWithOptionsSucceeds: + (MPPLanguageDetectorOptions *)options { + NSError *error; + MPPLanguageDetector *languageDetector = [[MPPLanguageDetector alloc] initWithOptions:options + error:&error]; + XCTAssertNotNil(languageDetector); + XCTAssertNil(error); + + return languageDetector; +} + +- (void)assertCreateLanguageDetectorWithOptions:(MPPLanguageDetectorOptions *)options + failsWithExpectedError:(NSError *)expectedError { + NSError *error = nil; + MPPLanguageDetector *languageDetector = [[MPPLanguageDetector alloc] initWithOptions:options + error:&error]; + XCTAssertNil(languageDetector); + AssertEqualErrors(error, expectedError); +} + +@end \ No newline at end of file From d9080c0d3819deef26249650ac067bfa036e43da Mon Sep 17 00:00:00 2001 From: Kinar Date: Tue, 7 Nov 2023 07:02:08 -0800 Subject: [PATCH 079/157] Updated the Image Embedder C API and added tests for cosine similarity --- .../containers/embedding_result_converter.cc | 23 +++ .../containers/embedding_result_converter.h | 4 + .../c/text/text_embedder/text_embedder.cc | 27 +++ .../c/text/text_embedder/text_embedder.h | 9 + .../text/text_embedder/text_embedder_test.cc | 36 +++- .../c/vision/image_embedder/image_embedder.cc | 169 ++++++++++++++++- .../c/vision/image_embedder/image_embedder.h | 34 +++- .../image_embedder/image_embedder_test.cc | 176 +++++++++++++++++- 8 files changed, 459 insertions(+), 19 deletions(-) diff --git a/mediapipe/tasks/c/components/containers/embedding_result_converter.cc b/mediapipe/tasks/c/components/containers/embedding_result_converter.cc index ba72c0994..cd4850c18 100644 --- a/mediapipe/tasks/c/components/containers/embedding_result_converter.cc +++ b/mediapipe/tasks/c/components/containers/embedding_result_converter.cc @@ -66,6 +66,29 @@ void CppConvertToEmbeddingResult( } } +void ConvertToCppEmbedding( + const Embedding& in, // C struct as input + mediapipe::tasks::components::containers::Embedding* out) { + // Handle float embeddings + if (in.float_embedding != nullptr) { + out->float_embedding.assign(in.float_embedding, + in.float_embedding + in.values_count); + } + + // Handle quantized embeddings + if (in.quantized_embedding != nullptr) { + out->quantized_embedding.assign(in.quantized_embedding, + in.quantized_embedding + in.values_count); + } + + out->head_index = in.head_index; + + // Copy head_name if it is present. + if (in.head_name) { + out->head_name = std::make_optional(std::string(in.head_name)); + } +} + void CppCloseEmbeddingResult(EmbeddingResult* in) { for (uint32_t i = 0; i < in->embeddings_count; ++i) { auto embedding_in = in->embeddings[i]; diff --git a/mediapipe/tasks/c/components/containers/embedding_result_converter.h b/mediapipe/tasks/c/components/containers/embedding_result_converter.h index 15bcdbdd0..5ba4e4e2b 100644 --- a/mediapipe/tasks/c/components/containers/embedding_result_converter.h +++ b/mediapipe/tasks/c/components/containers/embedding_result_converter.h @@ -29,6 +29,10 @@ void CppConvertToEmbeddingResult( const mediapipe::tasks::components::containers::EmbeddingResult& in, EmbeddingResult* out); +void ConvertToCppEmbedding( + const Embedding& in, + mediapipe::tasks::components::containers::Embedding* out); + void CppCloseEmbedding(Embedding* in); void CppCloseEmbeddingResult(EmbeddingResult* in); diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder.cc b/mediapipe/tasks/c/text/text_embedder/text_embedder.cc index c98b958f5..85b15e923 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder.cc +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder.cc @@ -29,6 +29,8 @@ namespace mediapipe::tasks::c::text::text_embedder { namespace { + +using ::mediapipe::tasks::c::components::containers::ConvertToCppEmbedding; using ::mediapipe::tasks::c::components::containers::CppCloseEmbeddingResult; using ::mediapipe::tasks::c::components::containers:: CppConvertToEmbeddingResult; @@ -36,6 +38,7 @@ using ::mediapipe::tasks::c::components::processors:: CppConvertToEmbedderOptions; using ::mediapipe::tasks::c::core::CppConvertToBaseOptions; using ::mediapipe::tasks::text::text_embedder::TextEmbedder; +typedef ::mediapipe::tasks::components::containers::Embedding CppEmbedding; int CppProcessError(absl::Status status, char** error_msg) { if (error_msg) { @@ -91,6 +94,24 @@ int CppTextEmbedderClose(void* embedder, char** error_msg) { return 0; } +int CppTextEmbedderCosineSimilarity(const Embedding& u, const Embedding& v, + double* similarity, char** error_msg) { + CppEmbedding cpp_u; + ConvertToCppEmbedding(u, &cpp_u); + CppEmbedding cpp_v; + ConvertToCppEmbedding(v, &cpp_v); + auto status_or_similarity = + mediapipe::tasks::text::text_embedder::TextEmbedder::CosineSimilarity( + cpp_u, cpp_v); + if (status_or_similarity.ok()) { + *similarity = status_or_similarity.value(); + } else { + ABSL_LOG(ERROR) << "Cannot computer cosine similarity."; + return CppProcessError(status_or_similarity.status(), error_msg); + } + return 0; +} + } // namespace mediapipe::tasks::c::text::text_embedder extern "C" { @@ -116,4 +137,10 @@ int text_embedder_close(void* embedder, char** error_ms) { embedder, error_ms); } +int cosine_similarity(const Embedding& u, const Embedding& v, + double* similarity, char** error_msg) { + return mediapipe::tasks::c::text::text_embedder:: + CppTextEmbedderCosineSimilarity(u, v, similarity, error_msg); +} + } // extern "C" diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder.h b/mediapipe/tasks/c/text/text_embedder/text_embedder.h index c9ccf816b..7fee3c737 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder.h +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder.h @@ -67,6 +67,15 @@ MP_EXPORT void text_embedder_close_result(TextEmbedderResult* result); // allocated for the error message. MP_EXPORT int text_embedder_close(void* embedder, char** error_msg = nullptr); +// Utility function to compute cosine similarity [1] between two embeddings. +// May return an InvalidArgumentError if e.g. the embeddings are of different +// types (quantized vs. float), have different sizes, or have a an L2-norm of +// 0. +// +// [1]: https://en.wikipedia.org/wiki/Cosine_similarity +MP_EXPORT int cosine_similarity(const Embedding& u, const Embedding& v, + double* similarity, char** error_msg = nullptr); + #ifdef __cplusplus } // extern C #endif diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc index c823e01b4..c6b19d9f5 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc @@ -32,7 +32,12 @@ using testing::HasSubstr; constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/text/"; constexpr char kTestBertModelPath[] = "mobilebert_embedding_with_metadata.tflite"; -constexpr char kTestString[] = "It's beautiful outside."; +constexpr char kTestString0[] = + "When you go to this restaurant, they hold the pancake upside-down " + "before they hand it to you. It's a great gimmick."; +constexpr char kTestString1[] = + "Let's make a plan to steal the declaration of independence."; +constexpr float kPrecision = 1e-3; std::string GetFullPath(absl::string_view file_name) { return JoinPath("./", kTestDataDirectory, file_name); @@ -51,7 +56,7 @@ TEST(TextEmbedderTest, SmokeTest) { EXPECT_NE(embedder, nullptr); TextEmbedderResult result; - text_embedder_embed(embedder, kTestString, &result); + text_embedder_embed(embedder, kTestString0, &result); EXPECT_EQ(result.embeddings_count, 1); EXPECT_EQ(result.embeddings[0].values_count, 512); @@ -59,6 +64,33 @@ TEST(TextEmbedderTest, SmokeTest) { text_embedder_close(embedder); } +TEST(TextEmbedderTest, SucceedsWithCosineSimilarity) { + std::string model_path = GetFullPath(kTestBertModelPath); + TextEmbedderOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_path= */ model_path.c_str()}, + /* embedder_options= */ + {/* l2_normalize= */ false, + /* quantize= */ false}}; + + void* embedder = text_embedder_create(&options); + EXPECT_NE(embedder, nullptr); + + // Extract both embeddings. + TextEmbedderResult result0; + text_embedder_embed(embedder, kTestString0, &result0); + TextEmbedderResult result1; + text_embedder_embed(embedder, kTestString1, &result1); + + // Check cosine similarity. + double similarity; + cosine_similarity(result0.embeddings[0], result1.embeddings[0], + &similarity); + double expected_similarity = 0.98077; + EXPECT_LE(abs(similarity - expected_similarity), kPrecision); + text_embedder_close(embedder); +} + TEST(TextEmbedderTest, ErrorHandling) { // It is an error to set neither the asset buffer nor the path. TextEmbedderOptions options = { diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc b/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc index 4b42034cb..00a571e90 100644 --- a/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc @@ -15,6 +15,8 @@ limitations under the License. #include "mediapipe/tasks/c/vision/image_embedder/image_embedder.h" +#include +#include #include #include @@ -26,6 +28,7 @@ limitations under the License. #include "mediapipe/tasks/c/components/containers/embedding_result_converter.h" #include "mediapipe/tasks/c/components/processors/embedder_options_converter.h" #include "mediapipe/tasks/c/core/base_options_converter.h" +#include "mediapipe/tasks/cc/vision/core/running_mode.h" #include "mediapipe/tasks/cc/vision/image_embedder/image_embedder.h" #include "mediapipe/tasks/cc/vision/utils/image_utils.h" @@ -33,6 +36,7 @@ namespace mediapipe::tasks::c::vision::image_embedder { namespace { +using ::mediapipe::tasks::c::components::containers::ConvertToCppEmbedding; using ::mediapipe::tasks::c::components::containers::CppCloseEmbeddingResult; using ::mediapipe::tasks::c::components::containers:: CppConvertToEmbeddingResult; @@ -40,7 +44,11 @@ using ::mediapipe::tasks::c::components::processors:: CppConvertToEmbedderOptions; using ::mediapipe::tasks::c::core::CppConvertToBaseOptions; using ::mediapipe::tasks::vision::CreateImageFromBuffer; +using ::mediapipe::tasks::vision::core::RunningMode; using ::mediapipe::tasks::vision::image_embedder::ImageEmbedder; +typedef ::mediapipe::tasks::components::containers::Embedding CppEmbedding; +typedef ::mediapipe::tasks::vision::image_embedder::ImageEmbedderResult + CppImageEmbedderResult; int CppProcessError(absl::Status status, char** error_msg) { if (error_msg) { @@ -59,6 +67,54 @@ ImageEmbedder* CppImageEmbedderCreate(const ImageEmbedderOptions& options, CppConvertToBaseOptions(options.base_options, &cpp_options->base_options); CppConvertToEmbedderOptions(options.embedder_options, &cpp_options->embedder_options); + cpp_options->running_mode = static_cast(options.running_mode); + + // Enable callback for processing live stream data when the running mode is + // set to RunningMode::LIVE_STREAM. + if (cpp_options->running_mode == RunningMode::LIVE_STREAM) { + if (options.result_callback == nullptr) { + const absl::Status status = absl::InvalidArgumentError( + "Provided null pointer to callback function."); + ABSL_LOG(ERROR) << "Failed to create ImageEmbedder: " << status; + CppProcessError(status, error_msg); + return nullptr; + } + + ImageEmbedderOptions::result_callback_fn result_callback = + options.result_callback; + cpp_options->result_callback = + [result_callback](absl::StatusOr cpp_result, + const Image& image, int64_t timestamp) { + char* error_msg = nullptr; + + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) + << "Embedding extraction failed: " << cpp_result.status(); + CppProcessError(cpp_result.status(), &error_msg); + result_callback(nullptr, MpImage(), timestamp, error_msg); + free(error_msg); + return; + } + + // Result is valid for the lifetime of the callback function. + ImageEmbedderResult result; + CppConvertToEmbeddingResult(*cpp_result, &result); + + const auto& image_frame = image.GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = { + .format = static_cast<::ImageFormat>(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + result_callback(&result, mp_image, timestamp, + /* error_msg= */ nullptr); + + CppCloseEmbeddingResult(&result); + }; + } auto embedder = ImageEmbedder::Create(std::move(cpp_options)); if (!embedder.ok()) { @@ -72,10 +128,10 @@ ImageEmbedder* CppImageEmbedderCreate(const ImageEmbedderOptions& options, int CppImageEmbedderEmbed(void* embedder, const MpImage* image, ImageEmbedderResult* result, char** error_msg) { if (image->type == MpImage::GPU_BUFFER) { - absl::Status status = - absl::InvalidArgumentError("gpu buffer not supported yet"); + const absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet."); - ABSL_LOG(ERROR) << "Classification failed: " << status.message(); + ABSL_LOG(ERROR) << "Embedding extraction failed: " << status.message(); return CppProcessError(status, error_msg); } @@ -92,13 +148,75 @@ int CppImageEmbedderEmbed(void* embedder, const MpImage* image, auto cpp_embedder = static_cast(embedder); auto cpp_result = cpp_embedder->Embed(*img); if (!cpp_result.ok()) { - ABSL_LOG(ERROR) << "Classification failed: " << cpp_result.status(); + ABSL_LOG(ERROR) << "Embedding extraction failed: " << cpp_result.status(); return CppProcessError(cpp_result.status(), error_msg); } CppConvertToEmbeddingResult(*cpp_result, result); return 0; } +int CppImageEmbedderEmbedForVideo(void* embedder, const MpImage* image, + int64_t timestamp_ms, + ImageEmbedderResult* result, + char** error_msg) { + if (image->type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Embedding extraction failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image->image_frame.format), + image->image_frame.image_buffer, image->image_frame.width, + image->image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_embedder = static_cast(embedder); + auto cpp_result = cpp_embedder->EmbedForVideo(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Embedding extraction failed: " << cpp_result.status(); + return CppProcessError(cpp_result.status(), error_msg); + } + CppConvertToEmbeddingResult(*cpp_result, result); + return 0; +} + +int CppImageEmbedderEmbedAsync(void* embedder, const MpImage* image, + int64_t timestamp_ms, char** error_msg) { + if (image->type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Embedding extraction failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image->image_frame.format), + image->image_frame.image_buffer, image->image_frame.width, + image->image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_embedder = static_cast(embedder); + auto cpp_result = cpp_embedder->EmbedAsync(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Data preparation for the embedding extraction failed: " + << cpp_result; + return CppProcessError(cpp_result, error_msg); + } + return 0; +} + void CppImageEmbedderCloseResult(ImageEmbedderResult* result) { CppCloseEmbeddingResult(result); } @@ -114,6 +232,24 @@ int CppImageEmbedderClose(void* embedder, char** error_msg) { return 0; } +int CppImageEmbedderCosineSimilarity(const Embedding& u, const Embedding& v, + double* similarity, char** error_msg) { + CppEmbedding cpp_u; + ConvertToCppEmbedding(u, &cpp_u); + CppEmbedding cpp_v; + ConvertToCppEmbedding(v, &cpp_v); + auto status_or_similarity = + mediapipe::tasks::vision::image_embedder::ImageEmbedder::CosineSimilarity( + cpp_u, cpp_v); + if (status_or_similarity.ok()) { + *similarity = status_or_similarity.value(); + } else { + ABSL_LOG(ERROR) << "Cannot computer cosine similarity."; + return CppProcessError(status_or_similarity.status(), error_msg); + } + return 0; +} + } // namespace mediapipe::tasks::c::vision::image_embedder extern "C" { @@ -130,14 +266,35 @@ int image_embedder_embed_image(void* embedder, const MpImage* image, embedder, image, result, error_msg); } +int image_embedder_embed_for_video(void* embedder, const MpImage* image, + int64_t timestamp_ms, + ImageEmbedderResult* result, + char** error_msg) { + return mediapipe::tasks::c::vision::image_embedder:: + CppImageEmbedderEmbedForVideo(embedder, image, timestamp_ms, result, + error_msg); +} + +int image_embedder_embed_async(void* embedder, const MpImage* image, + int64_t timestamp_ms, char** error_msg) { + return mediapipe::tasks::c::vision::image_embedder:: + CppImageEmbedderEmbedAsync(embedder, image, timestamp_ms, error_msg); +} + void image_embedder_close_result(ImageEmbedderResult* result) { mediapipe::tasks::c::vision::image_embedder::CppImageEmbedderCloseResult( result); } -int image_embedder_close(void* embedder, char** error_ms) { +int image_embedder_close(void* embedder, char** error_msg) { return mediapipe::tasks::c::vision::image_embedder::CppImageEmbedderClose( - embedder, error_ms); + embedder, error_msg); +} + +int cosine_similarity(const Embedding& u, const Embedding& v, + double* similarity, char** error_msg) { + return mediapipe::tasks::c::vision::image_embedder:: + CppImageEmbedderCosineSimilarity(u, v, similarity, error_msg); } } // extern "C" diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder.h b/mediapipe/tasks/c/vision/image_embedder/image_embedder.h index cefe97b1b..550e30781 100644 --- a/mediapipe/tasks/c/vision/image_embedder/image_embedder.h +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder.h @@ -92,9 +92,16 @@ struct ImageEmbedderOptions { // The user-defined result callback for processing live stream data. // The result callback should only be specified when the running mode is set - // to RunningMode::LIVE_STREAM. - typedef void (*result_callback_fn)(ImageEmbedderResult*, const MpImage*, - int64_t); + // to RunningMode::LIVE_STREAM. Arguments of the callback function include: + // the pointer to embedding result, the image that result was obtained + // on, the timestamp relevant to embedding extraction results and pointer to + // error message in case of any failure. The validity of the passed arguments + // is true for the lifetime of the callback function. + // + // A caller is responsible for closing image embedder result. + typedef void (*result_callback_fn)(ImageEmbedderResult* result, + const MpImage image, int64_t timestamp_ms, + char* error_msg); result_callback_fn result_callback; }; @@ -110,12 +117,20 @@ MP_EXPORT void* image_embedder_create(struct ImageEmbedderOptions* options, // If an error occurs, returns an error code and sets the error parameter to an // an error message (if `error_msg` is not nullptr). You must free the memory // allocated for the error message. -// -// TODO: Add API for video and live stream processing. MP_EXPORT int image_embedder_embed_image(void* embedder, const MpImage* image, ImageEmbedderResult* result, char** error_msg = nullptr); +MP_EXPORT int image_embedder_embed_for_video(void* embedder, + const MpImage* image, + int64_t timestamp_ms, + ImageEmbedderResult* result, + char** error_msg = nullptr); + +MP_EXPORT int image_embedder_embed_async(void* embedder, const MpImage* image, + int64_t timestamp_ms, + char** error_msg = nullptr); + // Frees the memory allocated inside a ImageEmbedderResult result. // Does not free the result pointer itself. MP_EXPORT void image_embedder_close_result(ImageEmbedderResult* result); @@ -126,6 +141,15 @@ MP_EXPORT void image_embedder_close_result(ImageEmbedderResult* result); // allocated for the error message. MP_EXPORT int image_embedder_close(void* embedder, char** error_msg = nullptr); +// Utility function to compute cosine similarity [1] between two embeddings. +// May return an InvalidArgumentError if e.g. the embeddings are of different +// types (quantized vs. float), have different sizes, or have a an L2-norm of +// 0. +// +// [1]: https://en.wikipedia.org/wiki/Cosine_similarity +MP_EXPORT int cosine_similarity(const Embedding& u, const Embedding& v, + double* similarity, char** error_msg = nullptr); + #ifdef __cplusplus } // extern C #endif diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc b/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc index fb6b0b628..d31b8b736 100644 --- a/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc @@ -37,13 +37,27 @@ constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; constexpr char kModelName[] = "mobilenet_v3_small_100_224_embedder.tflite"; constexpr char kImageFile[] = "burger.jpg"; constexpr float kPrecision = 1e-6; +constexpr int kIterations = 100; std::string GetFullPath(absl::string_view file_name) { return JoinPath("./", kTestDataDirectory, file_name); } -TEST(ImageEmbedderTest, SmokeTest) { - const auto image = DecodeImageFromFile(GetFullPath("burger.jpg")); +// Utility function to check the sizes, head_index and head_names of a result +// produced by kMobileNetV3Embedder. +void CheckMobileNetV3Result(const ImageEmbedderResult& result, bool quantized) { + EXPECT_EQ(result.embeddings_count, 1); + EXPECT_EQ(result.embeddings[0].head_index, 0); + EXPECT_EQ(std::string{result.embeddings[0].head_name}, "feature"); + if (quantized) { + EXPECT_EQ(result.embeddings[0].values_count, 1024); + } else { + EXPECT_EQ(result.embeddings[0].values_count, 1024); + } +} + +TEST(ImageEmbedderTest, ImageModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); ASSERT_TRUE(image.ok()); const std::string model_path = GetFullPath(kModelName); @@ -53,8 +67,7 @@ TEST(ImageEmbedderTest, SmokeTest) { /* running_mode= */ RunningMode::IMAGE, /* embedder_options= */ {/* l2_normalize= */ true, - /* quantize= */ false}, - }; + /* quantize= */ false}}; void* embedder = image_embedder_create(&options); EXPECT_NE(embedder, nullptr); @@ -70,12 +83,163 @@ TEST(ImageEmbedderTest, SmokeTest) { ImageEmbedderResult result; image_embedder_embed_image(embedder, &mp_image, &result); - EXPECT_EQ(result.embeddings_count, 1); + CheckMobileNetV3Result(result, false); EXPECT_NEAR(result.embeddings[0].float_embedding[0], -0.0142344, kPrecision); image_embedder_close_result(&result); image_embedder_close(embedder); } +TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) { + const auto image = DecodeImageFromFile(GetFullPath("burger.jpg")); + ASSERT_TRUE(image.ok()); + const auto crop = DecodeImageFromFile(GetFullPath("burger_crop.jpg")); + ASSERT_TRUE(crop.ok()); + + const std::string model_path = GetFullPath(kModelName); + ImageEmbedderOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* embedder_options= */ + {/* l2_normalize= */ true, + /* quantize= */ false}}; + + void* embedder = image_embedder_create(&options); + EXPECT_NE(embedder, nullptr); + + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = { + .format = static_cast( + image->GetImageFrameSharedPtr()->Format()), + .image_buffer = image->GetImageFrameSharedPtr()->PixelData(), + .width = image->GetImageFrameSharedPtr()->Width(), + .height = image->GetImageFrameSharedPtr()->Height()}}; + + const MpImage mp_crop = { + .type = MpImage::IMAGE_FRAME, + .image_frame = { + .format = static_cast( + crop->GetImageFrameSharedPtr()->Format()), + .image_buffer = crop->GetImageFrameSharedPtr()->PixelData(), + .width = crop->GetImageFrameSharedPtr()->Width(), + .height = crop->GetImageFrameSharedPtr()->Height()}}; + + // Extract both embeddings. + ImageEmbedderResult image_result; + image_embedder_embed_image(embedder, &mp_image, &image_result); + ImageEmbedderResult crop_result; + image_embedder_embed_image(embedder, &mp_crop, &crop_result); + + // Check results. + CheckMobileNetV3Result(image_result, false); + CheckMobileNetV3Result(crop_result, false); + // Check cosine similarity. + double similarity; + cosine_similarity(image_result.embeddings[0], crop_result.embeddings[0], + &similarity); + double expected_similarity = 0.925519; + EXPECT_LE(abs(similarity - expected_similarity), kPrecision); + image_embedder_close_result(&image_result); + image_embedder_close_result(&crop_result); + image_embedder_close(embedder); +} + +TEST(ImageEmbedderTest, VideoModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + ImageEmbedderOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::VIDEO, + /* embedder_options= */ + {/* l2_normalize= */ true, + /* quantize= */ false}}; + + void* embedder = image_embedder_create(&options); + EXPECT_NE(embedder, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + ImageEmbedderResult result; + image_embedder_embed_for_video(embedder, &mp_image, i, &result); + CheckMobileNetV3Result(result, false); + EXPECT_NEAR(result.embeddings[0].float_embedding[0], -0.0142344, + kPrecision); + image_embedder_close_result(&result); + } + image_embedder_close(embedder); +} + +// A structure to support LiveStreamModeTest below. This structure holds a +// static method `Fn` for a callback function of C API. A `static` qualifier +// allows to take an address of the method to follow API style. Another static +// struct member is `last_timestamp` that is used to verify that current +// timestamp is greater than the previous one. +struct LiveStreamModeCallback { + static int64_t last_timestamp; + static void Fn(ImageEmbedderResult* embedder_result, const MpImage image, + int64_t timestamp, char* error_msg) { + ASSERT_NE(embedder_result, nullptr); + ASSERT_EQ(error_msg, nullptr); + CheckMobileNetV3Result(*embedder_result, false); + EXPECT_NEAR(embedder_result->embeddings[0].float_embedding[0], -0.0142344, + kPrecision); + EXPECT_GT(image.image_frame.width, 0); + EXPECT_GT(image.image_frame.height, 0); + EXPECT_GT(timestamp, last_timestamp); + last_timestamp++; + } +}; +int64_t LiveStreamModeCallback::last_timestamp = -1; + +TEST(ImageEmbedderTest, LiveStreamModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + + ImageEmbedderOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::LIVE_STREAM, + /* embedder_options= */ + {/* l2_normalize= */ true, + /* quantize= */ false}, + /* result_callback= */ LiveStreamModeCallback::Fn, + }; + + void* embedder = image_embedder_create(&options); + EXPECT_NE(embedder, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + EXPECT_GE(image_embedder_embed_async(embedder, &mp_image, i), 0); + } + image_embedder_close(embedder); + + // Due to the flow limiter, the total of outputs might be smaller than the + // number of iterations. + EXPECT_LE(LiveStreamModeCallback::last_timestamp, kIterations); + EXPECT_GT(LiveStreamModeCallback::last_timestamp, 0); +} + TEST(ImageEmbedderTest, InvalidArgumentHandling) { // It is an error to set neither the asset buffer nor the path. ImageEmbedderOptions options = { @@ -111,7 +275,7 @@ TEST(ImageEmbedderTest, FailedEmbeddingHandling) { ImageEmbedderResult result; char* error_msg; image_embedder_embed_image(embedder, &mp_image, &result, &error_msg); - EXPECT_THAT(error_msg, HasSubstr("gpu buffer not supported yet")); + EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet.")); free(error_msg); image_embedder_close(embedder); } From b0725b46fb81c389805375ae3a6e0490290e41d0 Mon Sep 17 00:00:00 2001 From: Kinar Date: Tue, 7 Nov 2023 07:12:58 -0800 Subject: [PATCH 080/157] Fixed merge conflicts --- mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc | 6 ++++-- .../tasks/c/vision/image_embedder/image_embedder_test.cc | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc index c4461df83..bce4ffe38 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc @@ -69,12 +69,14 @@ TEST(TextEmbedderTest, SucceedsWithCosineSimilarity) { std::string model_path = GetFullPath(kTestBertModelPath); TextEmbedderOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* embedder_options= */ {/* l2_normalize= */ false, /* quantize= */ false}}; - void* embedder = text_embedder_create(&options); + void* embedder = text_embedder_create(&options, + /* error_msg */ nullptr); EXPECT_NE(embedder, nullptr); // Extract both embeddings. @@ -89,7 +91,7 @@ TEST(TextEmbedderTest, SucceedsWithCosineSimilarity) { &similarity); double expected_similarity = 0.98077; EXPECT_LE(abs(similarity - expected_similarity), kPrecision); - text_embedder_close(embedder); + text_embedder_close(embedder, /* error_msg */ nullptr); } TEST(TextEmbedderTest, ErrorHandling) { diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc b/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc index d31b8b736..52a5a27a8 100644 --- a/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc @@ -63,6 +63,7 @@ TEST(ImageEmbedderTest, ImageModeTest) { const std::string model_path = GetFullPath(kModelName); ImageEmbedderOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* running_mode= */ RunningMode::IMAGE, /* embedder_options= */ @@ -98,6 +99,7 @@ TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) { const std::string model_path = GetFullPath(kModelName); ImageEmbedderOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* running_mode= */ RunningMode::IMAGE, /* embedder_options= */ @@ -152,6 +154,7 @@ TEST(ImageEmbedderTest, VideoModeTest) { const std::string model_path = GetFullPath(kModelName); ImageEmbedderOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* running_mode= */ RunningMode::VIDEO, /* embedder_options= */ @@ -210,6 +213,7 @@ TEST(ImageEmbedderTest, LiveStreamModeTest) { ImageEmbedderOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* running_mode= */ RunningMode::LIVE_STREAM, /* embedder_options= */ @@ -244,6 +248,7 @@ TEST(ImageEmbedderTest, InvalidArgumentHandling) { // It is an error to set neither the asset buffer nor the path. ImageEmbedderOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ nullptr}, /* embedder_options= */ {}, }; @@ -261,6 +266,7 @@ TEST(ImageEmbedderTest, FailedEmbeddingHandling) { const std::string model_path = GetFullPath(kModelName); ImageEmbedderOptions options = { /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, /* model_asset_path= */ model_path.c_str()}, /* running_mode= */ RunningMode::IMAGE, /* embedder_options= */ From 60fcfa74cc0f1de6650083fdaf68d69ab3abd9a5 Mon Sep 17 00:00:00 2001 From: Kinar Date: Tue, 7 Nov 2023 07:26:57 -0800 Subject: [PATCH 081/157] Fixed some typos in the error message --- mediapipe/tasks/c/text/text_embedder/text_embedder.cc | 2 +- mediapipe/tasks/c/vision/image_embedder/image_embedder.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder.cc b/mediapipe/tasks/c/text/text_embedder/text_embedder.cc index 85b15e923..b19998985 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder.cc +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder.cc @@ -106,7 +106,7 @@ int CppTextEmbedderCosineSimilarity(const Embedding& u, const Embedding& v, if (status_or_similarity.ok()) { *similarity = status_or_similarity.value(); } else { - ABSL_LOG(ERROR) << "Cannot computer cosine similarity."; + ABSL_LOG(ERROR) << "Cannot compute cosine similarity."; return CppProcessError(status_or_similarity.status(), error_msg); } return 0; diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc b/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc index 00a571e90..ee042ba00 100644 --- a/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc @@ -244,7 +244,7 @@ int CppImageEmbedderCosineSimilarity(const Embedding& u, const Embedding& v, if (status_or_similarity.ok()) { *similarity = status_or_similarity.value(); } else { - ABSL_LOG(ERROR) << "Cannot computer cosine similarity."; + ABSL_LOG(ERROR) << "Cannot compute cosine similarity."; return CppProcessError(status_or_similarity.status(), error_msg); } return 0; From 197358dfeea233adff986fc63b538a6f2bbe825b Mon Sep 17 00:00:00 2001 From: Kinar Date: Tue, 7 Nov 2023 07:34:08 -0800 Subject: [PATCH 082/157] Drop default arguments in Image Embedder C API --- .../c/vision/image_embedder/image_embedder.h | 18 ++++---- .../image_embedder/image_embedder_test.cc | 43 ++++++++++++------- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder.h b/mediapipe/tasks/c/vision/image_embedder/image_embedder.h index 550e30781..3fdb448f0 100644 --- a/mediapipe/tasks/c/vision/image_embedder/image_embedder.h +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder.h @@ -108,28 +108,28 @@ struct ImageEmbedderOptions { // Creates an ImageEmbedder from provided `options`. // Returns a pointer to the image embedder on success. // If an error occurs, returns `nullptr` and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT void* image_embedder_create(struct ImageEmbedderOptions* options, - char** error_msg = nullptr); + char** error_msg); // Performs embedding extraction on the input `image`. Returns `0` on success. // If an error occurs, returns an error code and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT int image_embedder_embed_image(void* embedder, const MpImage* image, ImageEmbedderResult* result, - char** error_msg = nullptr); + char** error_msg); MP_EXPORT int image_embedder_embed_for_video(void* embedder, const MpImage* image, int64_t timestamp_ms, ImageEmbedderResult* result, - char** error_msg = nullptr); + char** error_msg); MP_EXPORT int image_embedder_embed_async(void* embedder, const MpImage* image, int64_t timestamp_ms, - char** error_msg = nullptr); + char** error_msg); // Frees the memory allocated inside a ImageEmbedderResult result. // Does not free the result pointer itself. @@ -137,9 +137,9 @@ MP_EXPORT void image_embedder_close_result(ImageEmbedderResult* result); // Frees image embedder. // If an error occurs, returns an error code and sets the error parameter to an -// an error message (if `error_msg` is not nullptr). You must free the memory +// an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. -MP_EXPORT int image_embedder_close(void* embedder, char** error_msg = nullptr); +MP_EXPORT int image_embedder_close(void* embedder, char** error_msg); // Utility function to compute cosine similarity [1] between two embeddings. // May return an InvalidArgumentError if e.g. the embeddings are of different @@ -148,7 +148,7 @@ MP_EXPORT int image_embedder_close(void* embedder, char** error_msg = nullptr); // // [1]: https://en.wikipedia.org/wiki/Cosine_similarity MP_EXPORT int cosine_similarity(const Embedding& u, const Embedding& v, - double* similarity, char** error_msg = nullptr); + double* similarity, char** error_msg); #ifdef __cplusplus } // extern C diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc b/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc index 52a5a27a8..52a8b523a 100644 --- a/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc @@ -70,7 +70,8 @@ TEST(ImageEmbedderTest, ImageModeTest) { {/* l2_normalize= */ true, /* quantize= */ false}}; - void* embedder = image_embedder_create(&options); + void* embedder = image_embedder_create(&options, + /* error_msg */ nullptr); EXPECT_NE(embedder, nullptr); const MpImage mp_image = { @@ -83,11 +84,12 @@ TEST(ImageEmbedderTest, ImageModeTest) { .height = image->GetImageFrameSharedPtr()->Height()}}; ImageEmbedderResult result; - image_embedder_embed_image(embedder, &mp_image, &result); + image_embedder_embed_image(embedder, &mp_image, &result, + /* error_msg */ nullptr); CheckMobileNetV3Result(result, false); EXPECT_NEAR(result.embeddings[0].float_embedding[0], -0.0142344, kPrecision); image_embedder_close_result(&result); - image_embedder_close(embedder); + image_embedder_close(embedder, /* error_msg */ nullptr); } TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) { @@ -106,7 +108,8 @@ TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) { {/* l2_normalize= */ true, /* quantize= */ false}}; - void* embedder = image_embedder_create(&options); + void* embedder = image_embedder_create(&options, + /* error_msg */ nullptr); EXPECT_NE(embedder, nullptr); const MpImage mp_image = { @@ -129,9 +132,11 @@ TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) { // Extract both embeddings. ImageEmbedderResult image_result; - image_embedder_embed_image(embedder, &mp_image, &image_result); + image_embedder_embed_image(embedder, &mp_image, &image_result, + /* error_msg */ nullptr); ImageEmbedderResult crop_result; - image_embedder_embed_image(embedder, &mp_crop, &crop_result); + image_embedder_embed_image(embedder, &mp_crop, &crop_result, + /* error_msg */ nullptr); // Check results. CheckMobileNetV3Result(image_result, false); @@ -139,12 +144,12 @@ TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) { // Check cosine similarity. double similarity; cosine_similarity(image_result.embeddings[0], crop_result.embeddings[0], - &similarity); + &similarity, /* error_msg */ nullptr); double expected_similarity = 0.925519; EXPECT_LE(abs(similarity - expected_similarity), kPrecision); image_embedder_close_result(&image_result); image_embedder_close_result(&crop_result); - image_embedder_close(embedder); + image_embedder_close(embedder, /* error_msg */ nullptr); } TEST(ImageEmbedderTest, VideoModeTest) { @@ -161,7 +166,8 @@ TEST(ImageEmbedderTest, VideoModeTest) { {/* l2_normalize= */ true, /* quantize= */ false}}; - void* embedder = image_embedder_create(&options); + void* embedder = image_embedder_create(&options, + /* error_msg */ nullptr); EXPECT_NE(embedder, nullptr); const auto& image_frame = image->GetImageFrameSharedPtr(); @@ -174,13 +180,14 @@ TEST(ImageEmbedderTest, VideoModeTest) { for (int i = 0; i < kIterations; ++i) { ImageEmbedderResult result; - image_embedder_embed_for_video(embedder, &mp_image, i, &result); + image_embedder_embed_for_video(embedder, &mp_image, i, &result, + /* error_msg */ nullptr); CheckMobileNetV3Result(result, false); EXPECT_NEAR(result.embeddings[0].float_embedding[0], -0.0142344, kPrecision); image_embedder_close_result(&result); } - image_embedder_close(embedder); + image_embedder_close(embedder, /* error_msg */ nullptr); } // A structure to support LiveStreamModeTest below. This structure holds a @@ -222,7 +229,8 @@ TEST(ImageEmbedderTest, LiveStreamModeTest) { /* result_callback= */ LiveStreamModeCallback::Fn, }; - void* embedder = image_embedder_create(&options); + void* embedder = image_embedder_create(&options, + /* error_msg */ nullptr); EXPECT_NE(embedder, nullptr); const auto& image_frame = image->GetImageFrameSharedPtr(); @@ -234,9 +242,11 @@ TEST(ImageEmbedderTest, LiveStreamModeTest) { .height = image_frame->Height()}}; for (int i = 0; i < kIterations; ++i) { - EXPECT_GE(image_embedder_embed_async(embedder, &mp_image, i), 0); + EXPECT_GE(image_embedder_embed_async(embedder, &mp_image, i, + /* error_msg */ nullptr), + 0); } - image_embedder_close(embedder); + image_embedder_close(embedder, /* error_msg */ nullptr); // Due to the flow limiter, the total of outputs might be smaller than the // number of iterations. @@ -274,7 +284,8 @@ TEST(ImageEmbedderTest, FailedEmbeddingHandling) { /* quantize= */ false}, }; - void* embedder = image_embedder_create(&options); + void* embedder = image_embedder_create(&options, + /* error_msg */ nullptr); EXPECT_NE(embedder, nullptr); const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}}; @@ -283,7 +294,7 @@ TEST(ImageEmbedderTest, FailedEmbeddingHandling) { image_embedder_embed_image(embedder, &mp_image, &result, &error_msg); EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet.")); free(error_msg); - image_embedder_close(embedder); + image_embedder_close(embedder, /* error_msg */ nullptr); } } // namespace From 8d370f4f5bbb85e2c4dda43eef6546c8ecb2882b Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 7 Nov 2023 10:06:32 -0800 Subject: [PATCH 083/157] Remove const from input types of C API PiperOrigin-RevId: 580217902 --- mediapipe/tasks/c/core/base_options.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/c/core/base_options.h b/mediapipe/tasks/c/core/base_options.h index bbfd3ef7e..20c068a87 100644 --- a/mediapipe/tasks/c/core/base_options.h +++ b/mediapipe/tasks/c/core/base_options.h @@ -26,7 +26,7 @@ struct BaseOptions { const char* model_asset_buffer; // The size of the model assets buffer (or `0` if not set). - const unsigned int model_asset_buffer_count; + unsigned int model_asset_buffer_count; // The path to the model asset to open and mmap in memory. const char* model_asset_path; From c375761480d272505e97c4838c6dde01778981ee Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 7 Nov 2023 14:00:11 -0800 Subject: [PATCH 084/157] No public description PiperOrigin-RevId: 580292393 --- mediapipe/tasks/cc/core/model_resources.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mediapipe/tasks/cc/core/model_resources.h b/mediapipe/tasks/cc/core/model_resources.h index d8e8dada0..ab3897015 100644 --- a/mediapipe/tasks/cc/core/model_resources.h +++ b/mediapipe/tasks/cc/core/model_resources.h @@ -78,10 +78,10 @@ class ModelResources { ModelResources& operator=(const ModelResources&) = delete; // Returns the model resources tag. - std::string GetTag() const { return tag_; } + const std::string& GetTag() const { return tag_; } - // Returns a copy of the model file proto. - proto::ExternalFile GetModelFile() const { return *model_file_; } + // Returns the model file proto. + const proto::ExternalFile& GetModelFile() const { return *model_file_; } // Returns a pointer to tflite::model. const tflite::Model* GetTfLiteModel() const; From c442d6117e7b3743e3fc9fd680c77bdaf43a0c37 Mon Sep 17 00:00:00 2001 From: Kinar Date: Tue, 7 Nov 2023 14:23:15 -0800 Subject: [PATCH 085/157] Resolved issues and added a common header to hold all the necessary structures for the vision tasks --- .../containers/embedding_result_converter.cc | 4 +- .../containers/embedding_result_converter.h | 2 +- .../c/text/text_embedder/text_embedder.cc | 11 ++- .../c/text/text_embedder/text_embedder.h | 6 +- .../text/text_embedder/text_embedder_test.cc | 16 ++-- mediapipe/tasks/c/vision/core/BUILD | 22 ++++++ mediapipe/tasks/c/vision/core/common.h | 69 +++++++++++++++++ .../tasks/c/vision/image_classifier/BUILD | 1 + .../image_classifier/image_classifier.h | 71 ++++++++---------- mediapipe/tasks/c/vision/image_embedder/BUILD | 2 + .../c/vision/image_embedder/image_embedder.cc | 10 +-- .../c/vision/image_embedder/image_embedder.h | 74 +++++++++---------- .../image_embedder/image_embedder_test.cc | 5 +- 13 files changed, 185 insertions(+), 108 deletions(-) create mode 100644 mediapipe/tasks/c/vision/core/BUILD create mode 100644 mediapipe/tasks/c/vision/core/common.h diff --git a/mediapipe/tasks/c/components/containers/embedding_result_converter.cc b/mediapipe/tasks/c/components/containers/embedding_result_converter.cc index cd4850c18..2e552801d 100644 --- a/mediapipe/tasks/c/components/containers/embedding_result_converter.cc +++ b/mediapipe/tasks/c/components/containers/embedding_result_converter.cc @@ -66,7 +66,7 @@ void CppConvertToEmbeddingResult( } } -void ConvertToCppEmbedding( +void CppConvertToCppEmbedding( const Embedding& in, // C struct as input mediapipe::tasks::components::containers::Embedding* out) { // Handle float embeddings @@ -85,7 +85,7 @@ void ConvertToCppEmbedding( // Copy head_name if it is present. if (in.head_name) { - out->head_name = std::make_optional(std::string(in.head_name)); + out->head_name = std::string(in.head_name); } } diff --git a/mediapipe/tasks/c/components/containers/embedding_result_converter.h b/mediapipe/tasks/c/components/containers/embedding_result_converter.h index 5ba4e4e2b..0955a949d 100644 --- a/mediapipe/tasks/c/components/containers/embedding_result_converter.h +++ b/mediapipe/tasks/c/components/containers/embedding_result_converter.h @@ -29,7 +29,7 @@ void CppConvertToEmbeddingResult( const mediapipe::tasks::components::containers::EmbeddingResult& in, EmbeddingResult* out); -void ConvertToCppEmbedding( +void CppConvertToCppEmbedding( const Embedding& in, mediapipe::tasks::components::containers::Embedding* out); diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder.cc b/mediapipe/tasks/c/text/text_embedder/text_embedder.cc index b19998985..6a016dd37 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder.cc +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder.cc @@ -29,9 +29,8 @@ namespace mediapipe::tasks::c::text::text_embedder { namespace { - -using ::mediapipe::tasks::c::components::containers::ConvertToCppEmbedding; using ::mediapipe::tasks::c::components::containers::CppCloseEmbeddingResult; +using ::mediapipe::tasks::c::components::containers::CppConvertToCppEmbedding; using ::mediapipe::tasks::c::components::containers:: CppConvertToEmbeddingResult; using ::mediapipe::tasks::c::components::processors:: @@ -97,9 +96,9 @@ int CppTextEmbedderClose(void* embedder, char** error_msg) { int CppTextEmbedderCosineSimilarity(const Embedding& u, const Embedding& v, double* similarity, char** error_msg) { CppEmbedding cpp_u; - ConvertToCppEmbedding(u, &cpp_u); + CppConvertToCppEmbedding(u, &cpp_u); CppEmbedding cpp_v; - ConvertToCppEmbedding(v, &cpp_v); + CppConvertToCppEmbedding(v, &cpp_v); auto status_or_similarity = mediapipe::tasks::text::text_embedder::TextEmbedder::CosineSimilarity( cpp_u, cpp_v); @@ -137,8 +136,8 @@ int text_embedder_close(void* embedder, char** error_ms) { embedder, error_ms); } -int cosine_similarity(const Embedding& u, const Embedding& v, - double* similarity, char** error_msg) { +int text_embedder_cosine_similarity(const Embedding& u, const Embedding& v, + double* similarity, char** error_msg) { return mediapipe::tasks::c::text::text_embedder:: CppTextEmbedderCosineSimilarity(u, v, similarity, error_msg); } diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder.h b/mediapipe/tasks/c/text/text_embedder/text_embedder.h index 61a24044e..b737f47f1 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder.h +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder.h @@ -72,8 +72,10 @@ MP_EXPORT int text_embedder_close(void* embedder, char** error_msg); // 0. // // [1]: https://en.wikipedia.org/wiki/Cosine_similarity -MP_EXPORT int cosine_similarity(const Embedding& u, const Embedding& v, - double* similarity, char** error_msg = nullptr); +MP_EXPORT int text_embedder_cosine_similarity(const Embedding& u, + const Embedding& v, + double* similarity, + char** error_msg); #ifdef __cplusplus } // extern C diff --git a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc index bce4ffe38..57021773e 100644 --- a/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc +++ b/mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc @@ -33,10 +33,10 @@ constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/text/"; constexpr char kTestBertModelPath[] = "mobilebert_embedding_with_metadata.tflite"; constexpr char kTestString0[] = - "When you go to this restaurant, they hold the pancake upside-down " - "before they hand it to you. It's a great gimmick."; + "When you go to this restaurant, they hold the pancake upside-down " + "before they hand it to you. It's a great gimmick."; constexpr char kTestString1[] = - "Let's make a plan to steal the declaration of independence."; + "Let's make a plan to steal the declaration of independence."; constexpr float kPrecision = 1e-3; std::string GetFullPath(absl::string_view file_name) { @@ -81,14 +81,16 @@ TEST(TextEmbedderTest, SucceedsWithCosineSimilarity) { // Extract both embeddings. TextEmbedderResult result0; - text_embedder_embed(embedder, kTestString0, &result0, /* error_msg */ nullptr); + text_embedder_embed(embedder, kTestString0, &result0, + /* error_msg */ nullptr); TextEmbedderResult result1; - text_embedder_embed(embedder, kTestString1, &result1, /* error_msg */ nullptr); + text_embedder_embed(embedder, kTestString1, &result1, + /* error_msg */ nullptr); // Check cosine similarity. double similarity; - cosine_similarity(result0.embeddings[0], result1.embeddings[0], - &similarity); + text_embedder_cosine_similarity(result0.embeddings[0], result1.embeddings[0], + &similarity, nullptr); double expected_similarity = 0.98077; EXPECT_LE(abs(similarity - expected_similarity), kPrecision); text_embedder_close(embedder, /* error_msg */ nullptr); diff --git a/mediapipe/tasks/c/vision/core/BUILD b/mediapipe/tasks/c/vision/core/BUILD new file mode 100644 index 000000000..7d3b0f9a9 --- /dev/null +++ b/mediapipe/tasks/c/vision/core/BUILD @@ -0,0 +1,22 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +cc_library( + name = "common", + hdrs = ["common.h"], +) diff --git a/mediapipe/tasks/c/vision/core/common.h b/mediapipe/tasks/c/vision/core/common.h new file mode 100644 index 000000000..8e88e2244 --- /dev/null +++ b/mediapipe/tasks/c/vision/core/common.h @@ -0,0 +1,69 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + + +#ifndef MEDIAPIPE_TASKS_C_VISION_CORE_COMMON_H_ +#define MEDIAPIPE_TASKS_C_VISION_CORE_COMMON_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Supported image formats. +enum ImageFormat { + UNKNOWN = 0, + SRGB = 1, + SRGBA = 2, + GRAY8 = 3, + SBGRA = 11 // compatible with Flutter `bgra8888` format. +}; + +// Supported processing modes. +enum RunningMode { + IMAGE = 1, + VIDEO = 2, + LIVE_STREAM = 3, +}; + +// Structure to hold image frame. +struct ImageFrame { + enum ImageFormat format; + const uint8_t* image_buffer; + int width; + int height; +}; + +// TODO: Add GPU buffer declaration and processing logic for it. +struct GpuBuffer { + int width; + int height; +}; + +// The object to contain an image, realizes `OneOf` concept. +struct MpImage { + enum { IMAGE_FRAME, GPU_BUFFER } type; + union { + struct ImageFrame image_frame; + struct GpuBuffer gpu_buffer; + }; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_VISION_CORE_COMMON_H_ diff --git a/mediapipe/tasks/c/vision/image_classifier/BUILD b/mediapipe/tasks/c/vision/image_classifier/BUILD index a2c7ca290..08a0801d3 100644 --- a/mediapipe/tasks/c/vision/image_classifier/BUILD +++ b/mediapipe/tasks/c/vision/image_classifier/BUILD @@ -30,6 +30,7 @@ cc_library( "//mediapipe/tasks/c/components/processors:classifier_options_converter", "//mediapipe/tasks/c/core:base_options", "//mediapipe/tasks/c/core:base_options_converter", + "//mediapipe/tasks/c/vision/core:common", "//mediapipe/tasks/cc/vision/core:running_mode", "//mediapipe/tasks/cc/vision/image_classifier", "//mediapipe/tasks/cc/vision/utils:image_utils", diff --git a/mediapipe/tasks/c/vision/image_classifier/image_classifier.h b/mediapipe/tasks/c/vision/image_classifier/image_classifier.h index 51316bcbe..8d3231c2e 100644 --- a/mediapipe/tasks/c/vision/image_classifier/image_classifier.h +++ b/mediapipe/tasks/c/vision/image_classifier/image_classifier.h @@ -16,11 +16,10 @@ limitations under the License. #ifndef MEDIAPIPE_TASKS_C_VISION_IMAGE_CLASSIFIER_IMAGE_CLASSIFIER_H_ #define MEDIAPIPE_TASKS_C_VISION_IMAGE_CLASSIFIER_IMAGE_CLASSIFIER_H_ -#include - #include "mediapipe/tasks/c/components/containers/classification_result.h" #include "mediapipe/tasks/c/components/processors/classifier_options.h" #include "mediapipe/tasks/c/core/base_options.h" +#include "mediapipe/tasks/c/vision/core/common.h" #ifndef MP_EXPORT #define MP_EXPORT __attribute__((visibility("default"))) @@ -32,46 +31,7 @@ extern "C" { typedef ClassificationResult ImageClassifierResult; -// Supported image formats. -enum ImageFormat { - UNKNOWN = 0, - SRGB = 1, - SRGBA = 2, - GRAY8 = 3, - SBGRA = 11 // compatible with Flutter `bgra8888` format. -}; - -// Supported processing modes. -enum RunningMode { - IMAGE = 1, - VIDEO = 2, - LIVE_STREAM = 3, -}; - -// Structure to hold image frame. -struct ImageFrame { - enum ImageFormat format; - const uint8_t* image_buffer; - int width; - int height; -}; - -// TODO: Add GPU buffer declaration and proccessing logic for it. -struct GpuBuffer { - int width; - int height; -}; - -// The object to contain an image, realizes `OneOf` concept. -struct MpImage { - enum { IMAGE_FRAME, GPU_BUFFER } type; - union { - struct ImageFrame image_frame; - struct GpuBuffer gpu_buffer; - }; -}; - -// The options for configuring a Mediapipe image classifier task. +// The options for configuring a MediaPipe image classifier task. struct ImageClassifierOptions { // Base options for configuring MediaPipe Tasks, such as specifying the model // file with metadata, accelerator options, op resolver, etc. @@ -122,12 +82,39 @@ MP_EXPORT int image_classifier_classify_image(void* classifier, ImageClassifierResult* result, char** error_msg); +// Performs image classification on the provided video frame. +// Only use this method when the ImageClassifier is created with the video +// running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide the video frame's timestamp (in milliseconds). The input timestamps +// must be monotonically increasing. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. MP_EXPORT int image_classifier_classify_for_video(void* classifier, const MpImage* image, int64_t timestamp_ms, ImageClassifierResult* result, char** error_msg); +// Sends live image data to image classification, and the results will be +// available via the "result_callback" provided in the ImageClassifierOptions. +// Only use this method when the ImageClassifier is created with the live +// stream running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide a timestamp (in milliseconds) to indicate when the input image is +// sent to the object detector. The input timestamps must be monotonically +// increasing. +// The "result_callback" provides: +// - The classification results as an ImageClassifierResult object. +// - The const reference to the corresponding input image that the image +// classifier runs on. Note that the const reference to the image will no +// longer be valid when the callback returns. To access the image data +// outside of the callback, callers need to make a copy of the image. +// - The input timestamp in milliseconds. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. MP_EXPORT int image_classifier_classify_async(void* classifier, const MpImage* image, int64_t timestamp_ms, diff --git a/mediapipe/tasks/c/vision/image_embedder/BUILD b/mediapipe/tasks/c/vision/image_embedder/BUILD index 3300b4a0a..69f7cb08b 100644 --- a/mediapipe/tasks/c/vision/image_embedder/BUILD +++ b/mediapipe/tasks/c/vision/image_embedder/BUILD @@ -30,6 +30,8 @@ cc_library( "//mediapipe/tasks/c/components/processors:embedder_options_converter", "//mediapipe/tasks/c/core:base_options", "//mediapipe/tasks/c/core:base_options_converter", + "//mediapipe/tasks/c/vision/core:common", + "//mediapipe/tasks/cc/vision/core:running_mode", "//mediapipe/tasks/cc/vision/image_embedder", "//mediapipe/tasks/cc/vision/utils:image_utils", "@com_google_absl//absl/log:absl_log", diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc b/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc index ee042ba00..041962004 100644 --- a/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder.cc @@ -36,8 +36,8 @@ namespace mediapipe::tasks::c::vision::image_embedder { namespace { -using ::mediapipe::tasks::c::components::containers::ConvertToCppEmbedding; using ::mediapipe::tasks::c::components::containers::CppCloseEmbeddingResult; +using ::mediapipe::tasks::c::components::containers::CppConvertToCppEmbedding; using ::mediapipe::tasks::c::components::containers:: CppConvertToEmbeddingResult; using ::mediapipe::tasks::c::components::processors:: @@ -235,9 +235,9 @@ int CppImageEmbedderClose(void* embedder, char** error_msg) { int CppImageEmbedderCosineSimilarity(const Embedding& u, const Embedding& v, double* similarity, char** error_msg) { CppEmbedding cpp_u; - ConvertToCppEmbedding(u, &cpp_u); + CppConvertToCppEmbedding(u, &cpp_u); CppEmbedding cpp_v; - ConvertToCppEmbedding(v, &cpp_v); + CppConvertToCppEmbedding(v, &cpp_v); auto status_or_similarity = mediapipe::tasks::vision::image_embedder::ImageEmbedder::CosineSimilarity( cpp_u, cpp_v); @@ -291,8 +291,8 @@ int image_embedder_close(void* embedder, char** error_msg) { embedder, error_msg); } -int cosine_similarity(const Embedding& u, const Embedding& v, - double* similarity, char** error_msg) { +int image_embedder_cosine_similarity(const Embedding& u, const Embedding& v, + double* similarity, char** error_msg) { return mediapipe::tasks::c::vision::image_embedder:: CppImageEmbedderCosineSimilarity(u, v, similarity, error_msg); } diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder.h b/mediapipe/tasks/c/vision/image_embedder/image_embedder.h index 3fdb448f0..f198d9810 100644 --- a/mediapipe/tasks/c/vision/image_embedder/image_embedder.h +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder.h @@ -21,6 +21,7 @@ limitations under the License. #include "mediapipe/tasks/c/components/containers/embedding_result.h" #include "mediapipe/tasks/c/components/processors/embedder_options.h" #include "mediapipe/tasks/c/core/base_options.h" +#include "mediapipe/tasks/c/vision/core/common.h" #ifndef MP_EXPORT #define MP_EXPORT __attribute__((visibility("default"))) @@ -32,45 +33,6 @@ extern "C" { typedef EmbeddingResult ImageEmbedderResult; -// Supported image formats. -enum ImageFormat { - UNKNOWN = 0, - SRGB = 1, - SRGBA = 2, - GRAY8 = 3, - SBGRA = 11 // compatible with Flutter `bgra8888` format. -}; - -// Supported processing modes. -enum RunningMode { - IMAGE = 1, - VIDEO = 2, - LIVE_STREAM = 3, -}; - -// Structure to hold image frame. -struct ImageFrame { - enum ImageFormat format; - const uint8_t* image_buffer; - int width; - int height; -}; - -// TODO: Add GPU buffer declaration and proccessing logic for it. -struct GpuBuffer { - int width; - int height; -}; - -// The object to contain an image, realizes `OneOf` concept. -struct MpImage { - enum { IMAGE_FRAME, GPU_BUFFER } type; - union { - struct ImageFrame image_frame; - struct GpuBuffer gpu_buffer; - }; -}; - // The options for configuring a MediaPipe image embedder task. struct ImageEmbedderOptions { // Base options for configuring MediaPipe Tasks, such as specifying the model @@ -121,12 +83,40 @@ MP_EXPORT int image_embedder_embed_image(void* embedder, const MpImage* image, ImageEmbedderResult* result, char** error_msg); +// Performs embedding extraction on the provided video frame. +// Only use this method when the ImageEmbedder is created with the video +// running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide the video frame's timestamp (in milliseconds). The input timestamps +// must be monotonically increasing. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. MP_EXPORT int image_embedder_embed_for_video(void* embedder, const MpImage* image, int64_t timestamp_ms, ImageEmbedderResult* result, char** error_msg); +// Sends live image data to embedder, and the results will be available via +// the "result_callback" provided in the ImageEmbedderOptions. +// Only use this method when the ImageEmbedder is created with the live +// stream running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide a timestamp (in milliseconds) to indicate when the input image is +// sent to the object detector. The input timestamps must be monotonically +// increasing. +// The "result_callback" provides +// - The embedding results as a +// components::containers::proto::EmbeddingResult object. +// - The const reference to the corresponding input image that the image +// embedder runs on. Note that the const reference to the image will no +// longer be valid when the callback returns. To access the image data +// outside of the callback, callers need to make a copy of the image. +// - The input timestamp in milliseconds. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. MP_EXPORT int image_embedder_embed_async(void* embedder, const MpImage* image, int64_t timestamp_ms, char** error_msg); @@ -147,8 +137,10 @@ MP_EXPORT int image_embedder_close(void* embedder, char** error_msg); // 0. // // [1]: https://en.wikipedia.org/wiki/Cosine_similarity -MP_EXPORT int cosine_similarity(const Embedding& u, const Embedding& v, - double* similarity, char** error_msg); +MP_EXPORT int image_embedder_cosine_similarity(const Embedding& u, + const Embedding& v, + double* similarity, + char** error_msg); #ifdef __cplusplus } // extern C diff --git a/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc b/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc index 52a8b523a..c92a2427d 100644 --- a/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc +++ b/mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc @@ -143,8 +143,9 @@ TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) { CheckMobileNetV3Result(crop_result, false); // Check cosine similarity. double similarity; - cosine_similarity(image_result.embeddings[0], crop_result.embeddings[0], - &similarity, /* error_msg */ nullptr); + image_embedder_cosine_similarity(image_result.embeddings[0], + crop_result.embeddings[0], &similarity, + /* error_msg */ nullptr); double expected_similarity = 0.925519; EXPECT_LE(abs(similarity - expected_similarity), kPrecision); image_embedder_close_result(&image_result); From 6ea6f28250893f7d0e0550ecdf95845f72add9e9 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 7 Nov 2023 17:59:02 -0800 Subject: [PATCH 086/157] Creates GpuBuffers around pre-allocated AHardware_Buffer objects. PiperOrigin-RevId: 580358465 --- mediapipe/gpu/BUILD | 5 +++++ mediapipe/gpu/gpu_buffer_format.cc | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index b75c824b3..f39b8d3f7 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -516,6 +516,7 @@ cc_library( ":gpu_buffer_storage", ":image_frame_view", "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/port:ret_check", "@com_google_absl//absl/strings:str_format", ], ) @@ -1223,9 +1224,13 @@ mediapipe_cc_test( ], requires_full_emulation = True, deps = [ + ":gl_texture_buffer", + ":gl_texture_util", ":gpu_buffer_format", ":gpu_buffer_storage_ahwb", + ":gpu_test_base", "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/tool:test_util", ], ) diff --git a/mediapipe/gpu/gpu_buffer_format.cc b/mediapipe/gpu/gpu_buffer_format.cc index b099f4a25..510a9cd48 100644 --- a/mediapipe/gpu/gpu_buffer_format.cc +++ b/mediapipe/gpu/gpu_buffer_format.cc @@ -238,7 +238,7 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) { case GpuBufferFormat::kRGBAFloat128: return ImageFormat::VEC32F4; case GpuBufferFormat::kRGBA32: - // TODO: this likely maps to ImageFormat::SRGBA + return ImageFormat::SRGBA; case GpuBufferFormat::kGrayHalf16: case GpuBufferFormat::kOneComponent8Alpha: case GpuBufferFormat::kOneComponent8Red: From 81a07e2e32805a79c39d79a9682037908c980b5a Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 8 Nov 2023 05:51:19 -0800 Subject: [PATCH 087/157] No public description PiperOrigin-RevId: 580504831 --- mediapipe/gpu/gl_texture_buffer.h | 1 + 1 file changed, 1 insertion(+) diff --git a/mediapipe/gpu/gl_texture_buffer.h b/mediapipe/gpu/gl_texture_buffer.h index 7b9140646..4548fce5c 100644 --- a/mediapipe/gpu/gl_texture_buffer.h +++ b/mediapipe/gpu/gl_texture_buffer.h @@ -19,6 +19,7 @@ #define MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_ #include +#include #include "absl/memory/memory.h" #include "mediapipe/framework/formats/image_frame.h" From 000314a54597349b8b8b1c217659032bc09714bb Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 8 Nov 2023 12:15:02 -0800 Subject: [PATCH 088/157] No public description PiperOrigin-RevId: 580614241 --- mediapipe/framework/port.h | 2 +- mediapipe/gpu/gl_thread_collector.h | 2 +- .../calculators/tensors_to_segmentation_calculator.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mediapipe/framework/port.h b/mediapipe/framework/port.h index a18080637..e8b17e4d2 100644 --- a/mediapipe/framework/port.h +++ b/mediapipe/framework/port.h @@ -50,7 +50,7 @@ // but may or may not still be able to run other OpenGL code. #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) && \ (defined(__APPLE__) || defined(__EMSCRIPTEN__) || MEDIAPIPE_DISABLE_GPU || \ - MEDIAPIPE_USING_SWIFTSHADER) + MEDIAPIPE_USING_LEGACY_SWIFTSHADER) #define MEDIAPIPE_DISABLE_GL_COMPUTE #endif diff --git a/mediapipe/gpu/gl_thread_collector.h b/mediapipe/gpu/gl_thread_collector.h index 58a35503c..2cc7aa57a 100644 --- a/mediapipe/gpu/gl_thread_collector.h +++ b/mediapipe/gpu/gl_thread_collector.h @@ -17,7 +17,7 @@ #include -#if defined(MEDIAPIPE_USING_SWIFTSHADER) +#if defined(MEDIAPIPE_USING_LEGACY_SWIFTSHADER) #define MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR 1 #endif diff --git a/mediapipe/tasks/cc/vision/image_segmenter/calculators/tensors_to_segmentation_calculator.cc b/mediapipe/tasks/cc/vision/image_segmenter/calculators/tensors_to_segmentation_calculator.cc index d449bb123..69f74b469 100644 --- a/mediapipe/tasks/cc/vision/image_segmenter/calculators/tensors_to_segmentation_calculator.cc +++ b/mediapipe/tasks/cc/vision/image_segmenter/calculators/tensors_to_segmentation_calculator.cc @@ -45,7 +45,7 @@ limitations under the License. #ifdef __EMSCRIPTEN__ #define TASK_SEGMENTATION_USE_GL_POSTPROCESSING 1 #elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 && \ - !MEDIAPIPE_USING_SWIFTSHADER && defined(MEDIAPIPE_ANDROID) + !MEDIAPIPE_USING_LEGACY_SWIFTSHADER && defined(MEDIAPIPE_ANDROID) #define TASK_SEGMENTATION_USE_GL_POSTPROCESSING 1 #else #undef TASK_SEGMENTATION_USE_GL_POSTPROCESSING From ae606c155042dfa74f78a941e59811de23c85e07 Mon Sep 17 00:00:00 2001 From: Youchuan Hu Date: Wed, 8 Nov 2023 12:51:09 -0800 Subject: [PATCH 089/157] Refactor OpenCV path out of TensorsToSegmentationCalculator main file. ProcessCpu() is changed into an OpenCV converter that is owned by the calculator. The calculator should call converter.Convert() to get the conversion result. PiperOrigin-RevId: 580625461 --- mediapipe/calculators/tensor/BUILD | 82 ++++++- .../tensors_to_segmentation_calculator.cc | 223 +++++------------- .../tensors_to_segmentation_converter.h | 43 ++++ ...ensors_to_segmentation_converter_opencv.cc | 157 ++++++++++++ ...tensors_to_segmentation_converter_opencv.h | 31 +++ .../tensor/tensors_to_segmentation_utils.cc | 52 ++++ .../tensor/tensors_to_segmentation_utils.h | 34 +++ .../tensors_to_segmentation_utils_test.cc | 63 +++++ 8 files changed, 513 insertions(+), 172 deletions(-) create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter.h create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils.h create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index ac2ced837..76f5bdbf6 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -1414,6 +1414,8 @@ cc_library( }), deps = [ ":tensors_to_segmentation_calculator_cc_proto", + ":tensors_to_segmentation_converter", + ":tensors_to_segmentation_utils", "//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:port", @@ -1421,9 +1423,11 @@ cc_library( "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/util:resource_util", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", @@ -1434,6 +1438,7 @@ cc_library( "//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_simple_shaders", "//mediapipe/gpu:gpu_buffer", + "//mediapipe/gpu:gpu_buffer_format", "//mediapipe/gpu:shader_util", ], }) + selects.with_or({ @@ -1453,13 +1458,86 @@ cc_library( }) + select({ "//mediapipe/framework/port:disable_opencv": [], "//conditions:default": [ - "//mediapipe/framework/formats:image_opencv", - "//mediapipe/framework/port:opencv_imgproc", + ":tensors_to_segmentation_converter_opencv", ], }), alwayslink = 1, ) +cc_library( + name = "tensors_to_segmentation_utils", + srcs = ["tensors_to_segmentation_utils.cc"], + hdrs = ["tensors_to_segmentation_utils.h"], + copts = select({ + "//mediapipe:apple": [ + "-x objective-c++", + "-fobjc-arc", # enable reference-counting + ], + "//conditions:default": [], + }), + deps = [ + "//mediapipe/framework:port", + "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_test( + name = "tensors_to_segmentation_utils_test", + srcs = ["tensors_to_segmentation_utils_test.cc"], + deps = [ + ":tensors_to_segmentation_utils", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:status_matchers", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_library( + name = "tensors_to_segmentation_converter", + hdrs = ["tensors_to_segmentation_converter.h"], + copts = select({ + "//mediapipe:apple": [ + "-x objective-c++", + "-fobjc-arc", # enable reference-counting + ], + "//conditions:default": [], + }), + deps = [ + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:tensor", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_library( + name = "tensors_to_segmentation_converter_opencv", + srcs = ["tensors_to_segmentation_converter_opencv.cc"], + hdrs = ["tensors_to_segmentation_converter_opencv.h"], + copts = select({ + "//mediapipe:apple": [ + "-x objective-c++", + "-fobjc-arc", # enable reference-counting + ], + "//conditions:default": [], + }), + deps = [ + ":tensors_to_segmentation_calculator_cc_proto", + ":tensors_to_segmentation_converter", + ":tensors_to_segmentation_utils", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_opencv", + "//mediapipe/framework/formats:tensor", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], +) + cc_test( name = "tensors_to_segmentation_calculator_test", srcs = ["tensors_to_segmentation_calculator_test.cc"], diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc index 24fd1bd52..90d2e6246 100644 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc @@ -12,32 +12,35 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include +#include #include -#include "absl/strings/str_format.h" -#include "absl/types/span.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port.h" #include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/statusor.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/gpu/gpu_buffer_format.h" #include "mediapipe/gpu/gpu_origin.pb.h" -#include "mediapipe/util/resource_util.h" -#include "tensorflow/lite/interpreter.h" #if !MEDIAPIPE_DISABLE_GPU #include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/gpu_buffer.h" #include "mediapipe/gpu/shader_util.h" #endif // !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_OPENCV -#include "mediapipe/framework/formats/image_opencv.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h" #endif // !MEDIAPIPE_DISABLE_OPENCV #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 @@ -62,37 +65,9 @@ namespace { constexpr int kWorkgroupSize = 8; // Block size for GPU shader. enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; -// Commonly used to compute the number of blocks to launch in a kernel. -int NumGroups(const int size, const int group_size) { // NOLINT - return (size + group_size - 1) / group_size; -} - -bool CanUseGpu() { -#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED - // TODO: Configure GPU usage policy in individual calculators. - constexpr bool kAllowGpuProcessing = true; - return kAllowGpuProcessing; -#else - return false; -#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED -} - constexpr char kTensorsTag[] = "TENSORS"; constexpr char kOutputSizeTag[] = "OUTPUT_SIZE"; constexpr char kMaskTag[] = "MASK"; - -absl::StatusOr> GetHwcFromDims( - const std::vector& dims) { - if (dims.size() == 3) { - return std::make_tuple(dims[0], dims[1], dims[2]); - } else if (dims.size() == 4) { - // BHWC format check B == 1 - RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap"; - return std::make_tuple(dims[1], dims[2], dims[3]); - } else { - RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size(); - } -} } // namespace namespace mediapipe { @@ -156,19 +131,24 @@ class TensorsToSegmentationCalculator : public CalculatorBase { private: absl::Status LoadOptions(CalculatorContext* cc); absl::Status InitGpu(CalculatorContext* cc); - absl::Status ProcessGpu(CalculatorContext* cc); - absl::Status ProcessCpu(CalculatorContext* cc); + absl::Status ProcessGpu(CalculatorContext* cc, + const std::vector& input_tensors, + std::tuple hwc, int output_width, + int output_height); void GlRender(); bool DoesGpuTextureStartAtBottom() { return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT; } + absl::Status InitConverterIfNecessary() { + if (!cpu_converter_) { + MP_ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(options_)); + } + return absl::OkStatus(); + } -#if !MEDIAPIPE_DISABLE_OPENCV - template - absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); -#endif // !MEDIAPIPE_DISABLE_OPENCV - ::mediapipe::TensorsToSegmentationCalculatorOptions options_; + mediapipe::TensorsToSegmentationCalculatorOptions options_; + std::unique_ptr cpu_converter_; #if !MEDIAPIPE_DISABLE_GPU mediapipe::GlCalculatorHelper gpu_helper_; @@ -261,7 +241,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); int tensor_channels = std::get<2>(hwc); - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; + using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; switch (options_.activation()) { case Options::NONE: RET_CHECK_EQ(tensor_channels, 1); @@ -275,6 +255,17 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { } } + // Get dimensions. + MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); + auto [tensor_height, tensor_width, tensor_channels] = hwc; + int output_width = tensor_width, output_height = tensor_height; + if (cc->Inputs().HasTag(kOutputSizeTag)) { + const auto& size = + cc->Inputs().Tag(kOutputSizeTag).Get>(); + output_width = size.first; + output_height = size.second; + } + if (use_gpu) { #if !MEDIAPIPE_DISABLE_GPU if (!gpu_initialized_) { @@ -286,16 +277,25 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { #endif // !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { - MP_RETURN_IF_ERROR(ProcessGpu(cc)); - return absl::OkStatus(); - })); + MP_RETURN_IF_ERROR( + gpu_helper_.RunInGlContext([this, cc, &input_tensors, output_width, + output_height, hwc]() -> absl::Status { + MP_RETURN_IF_ERROR( + ProcessGpu(cc, input_tensors, hwc, output_width, output_height)); + return absl::OkStatus(); + })); #else RET_CHECK_FAIL() << "GPU processing disabled."; #endif // !MEDIAPIPE_DISABLE_GPU } else { #if !MEDIAPIPE_DISABLE_OPENCV - MP_RETURN_IF_ERROR(ProcessCpu(cc)); + // Lazily initialize converter. + MP_RETURN_IF_ERROR(InitConverterIfNecessary()); + MP_ASSIGN_OR_RETURN( + std::unique_ptr output_mask, + cpu_converter_->Convert(input_tensors, output_width, output_height)); + cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), + cc->InputTimestamp()); #else RET_CHECK_FAIL() << "OpenCV processing disabled."; #endif // !MEDIAPIPE_DISABLE_OPENCV @@ -329,132 +329,15 @@ absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) { return absl::OkStatus(); } -absl::Status TensorsToSegmentationCalculator::ProcessCpu( - CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_OPENCV - // Get input streams, and dimensions. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); - auto [tensor_height, tensor_width, tensor_channels] = hwc; - int output_width = tensor_width, output_height = tensor_height; - if (cc->Inputs().HasTag(kOutputSizeTag)) { - const auto& size = - cc->Inputs().Tag(kOutputSizeTag).Get>(); - output_width = size.first; - output_height = size.second; - } - - // Create initial working mask. - cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1); - - // Wrap input tensor. - auto raw_input_tensor = &input_tensors[0]; - auto raw_input_view = raw_input_tensor->GetCpuReadView(); - const float* raw_input_data = raw_input_view.buffer(); - cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height), - CV_MAKETYPE(CV_32F, tensor_channels), - const_cast(raw_input_data)); - - // Process mask tensor and apply activation function. - if (tensor_channels == 2) { - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else if (tensor_channels == 1) { - RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX != - options_.activation()); // Requires 2 channels. - if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE == - options_.activation()) // Pass-through optimization. - tensor_mat.copyTo(small_mask_mat); - else - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else { - RET_CHECK_FAIL() << "Unsupported number of tensor channels " - << tensor_channels; - } - - // Send out image as CPU packet. - std::shared_ptr mask_frame = std::make_shared( - ImageFormat::VEC32F1, output_width, output_height); - std::unique_ptr output_mask = absl::make_unique(mask_frame); - auto output_mat = formats::MatView(output_mask.get()); - // Upsample small mask into output. - cv::resize(small_mask_mat, *output_mat, - cv::Size(output_width, output_height)); - cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp()); -#endif // !MEDIAPIPE_DISABLE_OPENCV - - return absl::OkStatus(); -} - -#if !MEDIAPIPE_DISABLE_OPENCV -template -absl::Status TensorsToSegmentationCalculator::ApplyActivation( - cv::Mat& tensor_mat, cv::Mat* small_mask_mat) { - // Configure activation function. - const int output_layer_index = options_.output_layer_index(); - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; - const auto activation_fn = [&](const cv::Vec2f& mask_value) { - float new_mask_value = 0; - // TODO consider moving switch out of the loop, - // and also avoid float/Vec2f casting. - switch (options_.activation()) { - case Options::NONE: { - new_mask_value = mask_value[0]; - break; - } - case Options::SIGMOID: { - const float pixel0 = mask_value[0]; - new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0); - break; - } - case Options::SOFTMAX: { - const float pixel0 = mask_value[0]; - const float pixel1 = mask_value[1]; - const float max_pixel = std::max(pixel0, pixel1); - const float min_pixel = std::min(pixel0, pixel1); - const float softmax_denom = - /*exp(max_pixel - max_pixel)=*/1.0f + - std::exp(min_pixel - max_pixel); - new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) / - softmax_denom; - break; - } - } - return new_mask_value; - }; - - // Process mask tensor. - for (int i = 0; i < tensor_mat.rows; ++i) { - for (int j = 0; j < tensor_mat.cols; ++j) { - const T& input_pix = tensor_mat.at(i, j); - const float mask_value = activation_fn(input_pix); - small_mask_mat->at(i, j) = mask_value; - } - } - - return absl::OkStatus(); -} -#endif // !MEDIAPIPE_DISABLE_OPENCV - // Steps: // 1. receive tensor // 2. process segmentation tensor into small mask // 3. upsample small mask into output mask to be same size as input image absl::Status TensorsToSegmentationCalculator::ProcessGpu( - CalculatorContext* cc) { + CalculatorContext* cc, const std::vector& input_tensors, + std::tuple hwc, int output_width, int output_height) { #if !MEDIAPIPE_DISABLE_GPU - // Get input streams, and dimensions. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); auto [tensor_height, tensor_width, tensor_channels] = hwc; - int output_width = tensor_width, output_height = tensor_height; - if (cc->Inputs().HasTag(kOutputSizeTag)) { - const auto& size = - cc->Inputs().Tag(kOutputSizeTag).Get>(); - output_width = size.first; - output_height = size.second; - } // Create initial working mask texture. #if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31) @@ -632,7 +515,7 @@ void TensorsToSegmentationCalculator::GlRender() { absl::Status TensorsToSegmentationCalculator::LoadOptions( CalculatorContext* cc) { // Get calculator options specified in the graph. - options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>(); + options_ = cc->Options(); return absl::OkStatus(); } @@ -826,7 +709,7 @@ void main() { #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 // Shader defines. - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; + using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; const std::string output_layer_index = "\n#define OUTPUT_LAYER_INDEX int(" + std::to_string(options_.output_layer_index()) + ")"; diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h b/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h new file mode 100644 index 000000000..61d95dfe0 --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h @@ -0,0 +1,43 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ + +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/tensor.h" + +namespace mediapipe { + +class TensorsToSegmentationConverter { + public: + virtual ~TensorsToSegmentationConverter() = default; + + // Converts tensors to image mask. + // Returns a unique pointer containing the converted image. + // @input_tensors contains the tensors needed to be processed. + // @output_width/height describes output dimensions to reshape the output mask + // into. + virtual absl::StatusOr> Convert( + const std::vector& input_tensors, int output_width, + int output_height) = 0; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc new file mode 100644 index 000000000..1ee2e172b --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc @@ -0,0 +1,157 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_opencv.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_macros.h" + +namespace mediapipe { +namespace { + +class OpenCvProcessor : public TensorsToSegmentationConverter { + public: + absl::Status Init(const TensorsToSegmentationCalculatorOptions& options) { + options_ = options; + return absl::OkStatus(); + } + + absl::StatusOr> Convert( + const std::vector& input_tensors, int output_width, + int output_height) override; + + private: + template + absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); + + TensorsToSegmentationCalculatorOptions options_; +}; + +absl::StatusOr> OpenCvProcessor::Convert( + const std::vector& input_tensors, int output_width, + int output_height) { + MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); + auto [tensor_height, tensor_width, tensor_channels] = hwc; + // Create initial working mask. + cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1); + + // Wrap input tensor. + auto raw_input_tensor = &input_tensors[0]; + auto raw_input_view = raw_input_tensor->GetCpuReadView(); + const float* raw_input_data = raw_input_view.buffer(); + cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height), + CV_MAKETYPE(CV_32F, tensor_channels), + const_cast(raw_input_data)); + + // Process mask tensor and apply activation function. + if (tensor_channels == 2) { + MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); + } else if (tensor_channels == 1) { + RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX != + options_.activation()); // Requires 2 channels. + if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE == + options_.activation()) // Pass-through optimization. + tensor_mat.copyTo(small_mask_mat); + else + MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); + } else { + RET_CHECK_FAIL() << "Unsupported number of tensor channels " + << tensor_channels; + } + + // Send out image as CPU packet. + std::shared_ptr mask_frame = std::make_shared( + ImageFormat::VEC32F1, output_width, output_height); + auto output_mask = std::make_unique(mask_frame); + auto output_mat = formats::MatView(output_mask.get()); + // Upsample small mask into output. + cv::resize(small_mask_mat, *output_mat, + cv::Size(output_width, output_height)); + return output_mask; +} + +template +absl::Status OpenCvProcessor::ApplyActivation(cv::Mat& tensor_mat, + cv::Mat* small_mask_mat) { + // Configure activation function. + const int output_layer_index = options_.output_layer_index(); + using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; + const auto activation_fn = [&](const cv::Vec2f& mask_value) { + float new_mask_value = 0; + // TODO consider moving switch out of the loop, + // and also avoid float/Vec2f casting. + switch (options_.activation()) { + case Options::NONE: { + new_mask_value = mask_value[0]; + break; + } + case Options::SIGMOID: { + const float pixel0 = mask_value[0]; + new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0); + break; + } + case Options::SOFTMAX: { + const float pixel0 = mask_value[0]; + const float pixel1 = mask_value[1]; + const float max_pixel = std::max(pixel0, pixel1); + const float min_pixel = std::min(pixel0, pixel1); + const float softmax_denom = + /*exp(max_pixel - max_pixel)=*/1.0f + + std::exp(min_pixel - max_pixel); + new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) / + softmax_denom; + break; + } + } + return new_mask_value; + }; + + // Process mask tensor. + for (int i = 0; i < tensor_mat.rows; ++i) { + for (int j = 0; j < tensor_mat.cols; ++j) { + const T& input_pix = tensor_mat.at(i, j); + const float mask_value = activation_fn(input_pix); + small_mask_mat->at(i, j) = mask_value; + } + } + + return absl::OkStatus(); +} + +} // namespace + +absl::StatusOr> +CreateOpenCvConverter(const TensorsToSegmentationCalculatorOptions& options) { + auto converter = std::make_unique(); + MP_RETURN_IF_ERROR(converter->Init(options)); + return converter; +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h new file mode 100644 index 000000000..3ae41b5e0 --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h @@ -0,0 +1,31 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ + +#include + +#include "absl/status/statusor.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" + +namespace mediapipe { +// Creates OpenCV tensors-to-segmentation converter. +absl::StatusOr> +CreateOpenCvConverter( + const mediapipe::TensorsToSegmentationCalculatorOptions& options); +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc new file mode 100644 index 000000000..ab1e9c139 --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc @@ -0,0 +1,52 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" + +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/port.h" +#include "mediapipe/framework/port/ret_check.h" + +namespace mediapipe { + +int NumGroups(int size, int group_size) { + return (size + group_size - 1) / group_size; +} + +bool CanUseGpu() { +#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED + // TODO: Configure GPU usage policy in individual calculators. + constexpr bool kAllowGpuProcessing = true; + return kAllowGpuProcessing; +#else + return false; +#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED +} + +absl::StatusOr> GetHwcFromDims( + const std::vector& dims) { + if (dims.size() == 3) { + return std::make_tuple(dims[0], dims[1], dims[2]); + } else if (dims.size() == 4) { + // BHWC format check B == 1 + RET_CHECK_EQ(dims[0], 1) << "Expected batch to be 1 for BHWC heatmap"; + return std::make_tuple(dims[1], dims[2], dims[3]); + } else { + RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size(); + } +} +} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h new file mode 100644 index 000000000..44893073b --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h @@ -0,0 +1,34 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ + +#include +#include + +#include "absl/status/statusor.h" + +namespace mediapipe { + +// Commonly used to compute the number of blocks to launch in a kernel. +int NumGroups(const int size, const int group_size); // NOLINT + +bool CanUseGpu(); + +absl::StatusOr> GetHwcFromDims( + const std::vector& dims); +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc new file mode 100644 index 000000000..5535d159d --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc @@ -0,0 +1,63 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" + +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/status_matchers.h" + +namespace mediapipe { +namespace { + +using ::testing::HasSubstr; + +TEST(TensorsToSegmentationUtilsTest, NumGroupsWorksProperly) { + EXPECT_EQ(NumGroups(13, 4), 4); + EXPECT_EQ(NumGroups(4, 13), 1); +} + +TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsWorksProperly) { + std::vector dims_3 = {2, 3, 4}; + absl::StatusOr> result_1 = GetHwcFromDims(dims_3); + MP_ASSERT_OK(result_1); + EXPECT_EQ(result_1.value(), (std::make_tuple(2, 3, 4))); + std::vector dims_4 = {1, 3, 4, 5}; + absl::StatusOr> result_2 = GetHwcFromDims(dims_4); + MP_ASSERT_OK(result_2); + EXPECT_EQ(result_2.value(), (std::make_tuple(3, 4, 5))); +} + +TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsBatchCheckFail) { + std::vector dims_4 = {2, 3, 4, 5}; + absl::StatusOr> result = GetHwcFromDims(dims_4); + EXPECT_FALSE(result.ok()); + EXPECT_THAT(result.status().message(), + HasSubstr("Expected batch to be 1 for BHWC heatmap")); +} + +TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsInvalidShape) { + std::vector dims_5 = {1, 2, 3, 4, 5}; + absl::StatusOr> result = GetHwcFromDims(dims_5); + EXPECT_FALSE(result.ok()); + EXPECT_THAT(result.status().message(), + HasSubstr("Invalid shape for segmentation tensor")); +} + +} // namespace +} // namespace mediapipe From 252c7eef253dfe866e98b4f679fc4d6af616d0a0 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 8 Nov 2023 14:27:05 -0800 Subject: [PATCH 090/157] Add option to omit the checkpoint callback in text classifier. PiperOrigin-RevId: 580658724 --- .../python/core/utils/model_util.py | 19 ++++++++++++------- .../python/core/utils/model_util_test.py | 17 +++++++++++++++++ .../text/text_classifier/hyperparameters.py | 4 ++++ .../text/text_classifier/text_classifier.py | 4 +++- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/mediapipe/model_maker/python/core/utils/model_util.py b/mediapipe/model_maker/python/core/utils/model_util.py index 2b1eebf9f..32b509797 100644 --- a/mediapipe/model_maker/python/core/utils/model_util.py +++ b/mediapipe/model_maker/python/core/utils/model_util.py @@ -35,18 +35,23 @@ ESTIMITED_STEPS_PER_EPOCH = 1000 def get_default_callbacks( export_dir: str, + checkpoint_frequency: int = 5, ) -> Sequence[tf.keras.callbacks.Callback]: """Gets default callbacks.""" + callbacks = [] summary_dir = os.path.join(export_dir, 'summaries') summary_callback = tf.keras.callbacks.TensorBoard(summary_dir) + callbacks.append(summary_callback) - checkpoint_path = os.path.join(export_dir, 'checkpoint') - checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( - os.path.join(checkpoint_path, 'model-{epoch:04d}'), - save_weights_only=True, - period=5, - ) - return [summary_callback, checkpoint_callback] + if checkpoint_frequency > 0: + checkpoint_path = os.path.join(export_dir, 'checkpoint') + checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( + os.path.join(checkpoint_path, 'model-{epoch:04d}'), + save_weights_only=True, + period=checkpoint_frequency, + ) + callbacks.append(checkpoint_callback) + return callbacks def load_keras_model( diff --git a/mediapipe/model_maker/python/core/utils/model_util_test.py b/mediapipe/model_maker/python/core/utils/model_util_test.py index 57750624f..ed8ba85e5 100644 --- a/mediapipe/model_maker/python/core/utils/model_util_test.py +++ b/mediapipe/model_maker/python/core/utils/model_util_test.py @@ -25,6 +25,23 @@ from mediapipe.model_maker.python.core.utils import test_util class ModelUtilTest(tf.test.TestCase, parameterized.TestCase): + def test_get_default_callbacks(self): + callbacks = model_util.get_default_callbacks( + 'export_dir', checkpoint_frequency=5 + ) + self.assertLen(callbacks, 2) + self.assertIsInstance(callbacks[0], tf.keras.callbacks.TensorBoard) + self.assertEqual(callbacks[0].log_dir, 'export_dir/summaries') + self.assertIsInstance(callbacks[1], tf.keras.callbacks.ModelCheckpoint) + self.assertEqual(callbacks[1].period, 5) + + callbacks = model_util.get_default_callbacks( + 'export_dir_2', checkpoint_frequency=0 + ) + self.assertLen(callbacks, 1) + self.assertIsInstance(callbacks[0], tf.keras.callbacks.TensorBoard) + self.assertEqual(callbacks[0].log_dir, 'export_dir_2/summaries') + def test_load_keras_model(self): input_dim = 4 model = test_util.build_model(input_shape=[input_dim], num_classes=2) diff --git a/mediapipe/model_maker/python/text/text_classifier/hyperparameters.py b/mediapipe/model_maker/python/text/text_classifier/hyperparameters.py index 5d16564f5..a7dc05d5b 100644 --- a/mediapipe/model_maker/python/text/text_classifier/hyperparameters.py +++ b/mediapipe/model_maker/python/text/text_classifier/hyperparameters.py @@ -56,6 +56,8 @@ class BertHParams(hp.BaseHParams): value to 0. Defaults to 2.0. tokenizer: Tokenizer to use for preprocessing. Must be one of the enum options of SupportedBertTokenizers. Defaults to FULL_TOKENIZER. + checkpoint_frequency: Frequency(in epochs) of saving checkpoints during + training. Defaults to 0 which does not save training checkpoints. """ learning_rate: float = 3e-5 @@ -75,5 +77,7 @@ class BertHParams(hp.BaseHParams): bert_tokenizer.SupportedBertTokenizers.FULL_TOKENIZER ) + checkpoint_frequency: int = 0 + HParams = Union[BertHParams, AverageWordEmbeddingHParams] diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py index 348f4cfb6..386e9360e 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py @@ -372,7 +372,9 @@ class _BertClassifier(TextClassifier): ): super().__init__(model_spec, label_names, hparams.shuffle) self._hparams = hparams - self._callbacks = model_util.get_default_callbacks(self._hparams.export_dir) + self._callbacks = model_util.get_default_callbacks( + self._hparams.export_dir, self._hparams.checkpoint_frequency + ) self._model_options = model_options self._text_preprocessor: preprocessor.BertClassifierPreprocessor = None with self._hparams.get_strategy().scope(): From 7c5c21665242ee3a2f3ce2875d6379e74ec9a22c Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 8 Nov 2023 20:04:12 -0800 Subject: [PATCH 091/157] Exposes a handle to AHardwareBuffers through a new GpuBuffer view PiperOrigin-RevId: 580754933 --- mediapipe/framework/formats/BUILD | 9 +++++ mediapipe/framework/formats/ahwb_view.h | 54 +++++++++++++++++++++++++ mediapipe/framework/port.h | 5 +++ mediapipe/gpu/BUILD | 7 ++++ 4 files changed, 75 insertions(+) create mode 100644 mediapipe/framework/formats/ahwb_view.h diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index b36ea0211..047b95d32 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -124,6 +124,15 @@ cc_library( ], ) +cc_library( + name = "ahwb_view", + hdrs = ["ahwb_view.h"], + deps = [ + "//mediapipe/framework:port", + "//mediapipe/gpu:gpu_buffer_storage", + ], +) + cc_library( name = "affine_transform", srcs = ["affine_transform.cc"], diff --git a/mediapipe/framework/formats/ahwb_view.h b/mediapipe/framework/formats/ahwb_view.h new file mode 100644 index 000000000..0c8ad6323 --- /dev/null +++ b/mediapipe/framework/formats/ahwb_view.h @@ -0,0 +1,54 @@ +#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_ +#define MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_ + +#include "mediapipe/framework/port.h" +#ifdef MEDIAPIPE_GPU_BUFFER_USE_AHWB +#include + +#include "mediapipe/gpu/gpu_buffer_storage.h" + +namespace mediapipe { + +// Wrapper to facilitate short lived access to Android Hardware Buffer objects. +// Intended use cases: +// - Extracting an AHWB for processing in another library after it's produced by +// MediaPipe. +// - Sending AHWBs to compute devices that are able to map the memory for their +// own usage. +// The AHWB abstractions in GpuBuffer and Tensor are likely more suitable for +// other CPU/GPU uses of AHWBs. +class AhwbView { + public: + explicit AhwbView(AHardwareBuffer* handle) : handle_(handle) {} + // Non-copyable + AhwbView(const AhwbView&) = delete; + AhwbView& operator=(const AhwbView&) = delete; + // Non-movable + AhwbView(AhwbView&&) = delete; + + // Only supports synchronous usage. All users of GetHandle must finish + // accessing the buffer before this view object is destroyed to avoid race + // conditions. + // TODO: Support asynchronous usage. + const AHardwareBuffer* GetHandle() const { return handle_; } + + private: + const AHardwareBuffer* handle_; +}; + +namespace internal { +// Makes this class available as a GpuBuffer view. +template <> +class ViewProvider { + public: + virtual ~ViewProvider() = default; + virtual const AhwbView GetReadView(types) const = 0; + virtual AhwbView GetWriteView(types) = 0; +}; + +} // namespace internal + +} // namespace mediapipe + +#endif // MEDIAPIPE_GPU_BUFFER_USE_AHWB +#endif // MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_ diff --git a/mediapipe/framework/port.h b/mediapipe/framework/port.h index e8b17e4d2..1bb4d4cdf 100644 --- a/mediapipe/framework/port.h +++ b/mediapipe/framework/port.h @@ -104,4 +104,9 @@ #endif #endif // MEDIAPIPE_HAS_RTTI +// AHardware buffers are only available since Android API 26. +#if (__ANDROID_API__ >= 26) +#define MEDIAPIPE_GPU_BUFFER_USE_AHWB 1 +#endif + #endif // MEDIAPIPE_FRAMEWORK_PORT_H_ diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index f39b8d3f7..6ce4ec117 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -511,12 +511,19 @@ cc_library( ], }), deps = [ + ":gl_base_hdr", + ":gl_context", ":gl_texture_buffer", + ":gl_texture_view", ":gpu_buffer_format", ":gpu_buffer_storage", ":image_frame_view", + "//mediapipe/framework:port", + "//mediapipe/framework/formats:ahwb_view", "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/port:ret_check", + "//third_party/GL:EGL_headers", + "@com_google_absl//absl/log:absl_check", "@com_google_absl//absl/strings:str_format", ], ) From a9a169372ad191bedeea6c3ffbf9d1a4e97f615a Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 9 Nov 2023 07:59:46 -0800 Subject: [PATCH 092/157] Fixes multiple typos in the calculator's internal files. PiperOrigin-RevId: 580907788 --- mediapipe/calculators/audio/spectrogram_calculator.proto | 2 +- .../calculators/audio/time_series_framer_calculator.proto | 2 +- .../calculators/core/packet_sequencer_calculator_test.cc | 2 +- .../calculators/core/value_or_default_calculator_test.cc | 2 +- .../image/affine_transformation_runner_opencv.cc | 2 +- mediapipe/calculators/image/image_cropping_calculator.proto | 2 +- .../calculators/image/image_file_properties_calculator.cc | 6 +++--- mediapipe/calculators/image/warp_affine_calculator_test.cc | 2 +- mediapipe/calculators/image/yuv_to_image_calculator.cc | 2 +- mediapipe/calculators/tflite/testdata/README.md | 2 +- mediapipe/calculators/tflite/tflite_converter_calculator.cc | 2 +- .../calculators/tflite/tflite_converter_calculator.proto | 2 +- .../tflite_tensors_to_classification_calculator.proto | 2 +- .../tflite/tflite_tensors_to_detections_calculator.cc | 2 +- .../tflite/tflite_tensors_to_detections_calculator.proto | 2 +- .../tflite/tflite_tensors_to_landmarks_calculator.cc | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/mediapipe/calculators/audio/spectrogram_calculator.proto b/mediapipe/calculators/audio/spectrogram_calculator.proto index d8bca3f76..ac7181f4f 100644 --- a/mediapipe/calculators/audio/spectrogram_calculator.proto +++ b/mediapipe/calculators/audio/spectrogram_calculator.proto @@ -80,7 +80,7 @@ message SpectrogramCalculatorOptions { // If use_local_timestamp is true, the output packet's timestamp is based on // the last sample of the packet and it's inferred from the latest input // packet's timestamp. If false, the output packet's timestamp is based on - // the cumulative timestamping, which is inferred from the intial input + // the cumulative timestamping, which is inferred from the initial input // timestamp and the cumulative number of samples. optional bool use_local_timestamp = 8 [default = false]; } diff --git a/mediapipe/calculators/audio/time_series_framer_calculator.proto b/mediapipe/calculators/audio/time_series_framer_calculator.proto index 9e5b07462..16ecfc97c 100644 --- a/mediapipe/calculators/audio/time_series_framer_calculator.proto +++ b/mediapipe/calculators/audio/time_series_framer_calculator.proto @@ -66,7 +66,7 @@ message TimeSeriesFramerCalculatorOptions { // If use_local_timestamp is true, the output packet's timestamp is based on // the last sample of the packet and it's inferred from the latest input // packet's timestamp. If false, the output packet's timestamp is based on - // the cumulative timestamping, which is inferred from the intial input + // the cumulative timestamping, which is inferred from the initial input // timestamp and the cumulative number of samples. optional bool use_local_timestamp = 6 [default = false]; } diff --git a/mediapipe/calculators/core/packet_sequencer_calculator_test.cc b/mediapipe/calculators/core/packet_sequencer_calculator_test.cc index c08e6bb12..6502fa4e9 100644 --- a/mediapipe/calculators/core/packet_sequencer_calculator_test.cc +++ b/mediapipe/calculators/core/packet_sequencer_calculator_test.cc @@ -71,7 +71,7 @@ TEST_F(PacketSequencerCalculatorTest, IsRegistered) { CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator")); } -// Shows how control packets recieve timestamps before and after frame packets +// Shows how control packets receive timestamps before and after frame packets // have arrived. TEST_F(PacketSequencerCalculatorTest, ChannelEarly) { CalculatorGraphConfig::Node node_config = BuildNodeConfig(); diff --git a/mediapipe/calculators/core/value_or_default_calculator_test.cc b/mediapipe/calculators/core/value_or_default_calculator_test.cc index acd1415ad..12a043bc6 100644 --- a/mediapipe/calculators/core/value_or_default_calculator_test.cc +++ b/mediapipe/calculators/core/value_or_default_calculator_test.cc @@ -174,7 +174,7 @@ TEST(ValueOrDefaultCalculatorTest, DefaultAndValues) { ElementsAre(kDefaultValue, 1, 2, kDefaultValue, 3, kDefaultValue)); } -TEST(ValueOrDefaultCalculatorTest, TimestampsMissmatch) { +TEST(ValueOrDefaultCalculatorTest, TimestampsMismatch) { // Check that when we provide the inputs not on time - we don't get them. ValueOrDefaultRunner runner; const std::vector ticks = {1, 2, 5, 8, 12, 33, 231}; diff --git a/mediapipe/calculators/image/affine_transformation_runner_opencv.cc b/mediapipe/calculators/image/affine_transformation_runner_opencv.cc index c43d73ff7..b58e035ee 100644 --- a/mediapipe/calculators/image/affine_transformation_runner_opencv.cc +++ b/mediapipe/calculators/image/affine_transformation_runner_opencv.cc @@ -59,7 +59,7 @@ class OpenCvRunner const ImageFrame& input, const std::array& matrix, const AffineTransformation::Size& size, AffineTransformation::BorderMode border_mode) override { - // OpenCV warpAffine works in absolute coordinates, so the transfom (which + // OpenCV warpAffine works in absolute coordinates, so the transform (which // accepts and produces relative coordinates) should be adjusted to first // normalize coordinates and then scale them. // clang-format off diff --git a/mediapipe/calculators/image/image_cropping_calculator.proto b/mediapipe/calculators/image/image_cropping_calculator.proto index 55d3467d1..17e4cb3e8 100644 --- a/mediapipe/calculators/image/image_cropping_calculator.proto +++ b/mediapipe/calculators/image/image_cropping_calculator.proto @@ -24,7 +24,7 @@ message ImageCroppingCalculatorOptions { } // Output texture buffer dimensions. The values defined in the options will be - // overriden by the WIDTH and HEIGHT input streams if they exist. + // overridden by the WIDTH and HEIGHT input streams if they exist. optional int32 width = 1; optional int32 height = 2; diff --git a/mediapipe/calculators/image/image_file_properties_calculator.cc b/mediapipe/calculators/image/image_file_properties_calculator.cc index db01400cd..01a1bd2c1 100644 --- a/mediapipe/calculators/image/image_file_properties_calculator.cc +++ b/mediapipe/calculators/image/image_file_properties_calculator.cc @@ -77,7 +77,7 @@ absl::StatusOr ComputeFocalLengthInPixels(int image_width, return focal_length_pixels; } -absl::StatusOr GetImageFileProperites( +absl::StatusOr GetImageFileProperties( const std::string& image_bytes) { easyexif::EXIFInfo result; int code = result.parseFrom(image_bytes); @@ -151,7 +151,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase { if (cc->InputSidePackets().NumEntries() == 1) { const std::string& image_bytes = cc->InputSidePackets().Index(0).Get(); - MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes)); + MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes)); read_properties_ = true; } @@ -169,7 +169,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase { return absl::OkStatus(); } const std::string& image_bytes = cc->Inputs().Index(0).Get(); - MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes)); + MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes)); read_properties_ = true; } if (read_properties_) { diff --git a/mediapipe/calculators/image/warp_affine_calculator_test.cc b/mediapipe/calculators/image/warp_affine_calculator_test.cc index 8a4c2429e..90bf41233 100644 --- a/mediapipe/calculators/image/warp_affine_calculator_test.cc +++ b/mediapipe/calculators/image/warp_affine_calculator_test.cc @@ -284,7 +284,7 @@ std::array GetMatrix(cv::Mat input, mediapipe::NormalizedRect roi, .IgnoreError(); mediapipe::GetRotatedSubRectToRectTransformMatrix( roi_absolute, input.cols, input.rows, - /*flip_horizontaly=*/false, &transform_mat); + /*flip_horizontally=*/false, &transform_mat); return transform_mat; } diff --git a/mediapipe/calculators/image/yuv_to_image_calculator.cc b/mediapipe/calculators/image/yuv_to_image_calculator.cc index 6a82877c3..e177ba589 100644 --- a/mediapipe/calculators/image/yuv_to_image_calculator.cc +++ b/mediapipe/calculators/image/yuv_to_image_calculator.cc @@ -49,7 +49,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) { // The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka // YV21) format (as per the `fourcc()` property). This covers the most commonly // used YUV image formats used on mobile devices. Other formats are not -// supported and wil result in an `InvalidArgumentError`. +// supported and will result in an `InvalidArgumentError`. class YUVToImageCalculator : public Node { public: static constexpr Input kInput{"YUV_IMAGE"}; diff --git a/mediapipe/calculators/tflite/testdata/README.md b/mediapipe/calculators/tflite/testdata/README.md index c0efdcf07..ffafe0df9 100644 --- a/mediapipe/calculators/tflite/testdata/README.md +++ b/mediapipe/calculators/tflite/testdata/README.md @@ -1,2 +1,2 @@ The model files add.bin, add_quantized.bin -(and corresponding metatada json files) come from tensorflow/lite/testdata/ +(and corresponding metadata json files) come from tensorflow/lite/testdata/ diff --git a/mediapipe/calculators/tflite/tflite_converter_calculator.cc b/mediapipe/calculators/tflite/tflite_converter_calculator.cc index 7188cbc59..682dd3b7b 100644 --- a/mediapipe/calculators/tflite/tflite_converter_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_converter_calculator.cc @@ -95,7 +95,7 @@ struct GPUData { // into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer // or MTLBuffer. // -// This calculator is designed to be used with the TfLiteInferenceCalcualtor, +// This calculator is designed to be used with the TfLiteInferenceCalculator, // as a pre-processing step for calculator inputs. // // IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1], diff --git a/mediapipe/calculators/tflite/tflite_converter_calculator.proto b/mediapipe/calculators/tflite/tflite_converter_calculator.proto index 5ed70879d..930545831 100644 --- a/mediapipe/calculators/tflite/tflite_converter_calculator.proto +++ b/mediapipe/calculators/tflite/tflite_converter_calculator.proto @@ -31,7 +31,7 @@ message TfLiteConverterCalculatorOptions { // Custom settings to override the internal scaling factors `div` and `sub`. // Both values must be set to non-negative values. Will only take effect on // CPU AND when |use_custom_normalization| is set to true. When these custom - // values take effect, the |zero_center| setting above will be overriden, and + // values take effect, the |zero_center| setting above will be overridden, and // the normalized_value will be calculated as: // normalized_value = input / custom_div - custom_sub. optional bool use_custom_normalization = 6 [default = false]; diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto index c6c9d915d..aa141eee5 100644 --- a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto +++ b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto @@ -25,7 +25,7 @@ message TfLiteTensorsToClassificationCalculatorOptions { optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463; } - // Score threshold for perserving the class. + // Score threshold for preserving the class. optional float min_score_threshold = 1; // Number of highest scoring labels to output. If top_k is not positive then // all labels are used. diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc index 269661f73..0eaba9eb0 100644 --- a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc @@ -116,7 +116,7 @@ void ConvertAnchorsToRawValues(const std::vector& anchors, // tensors can have 2 or 3 tensors. First tensor is the predicted // raw boxes/keypoints. The size of the values must be (num_boxes // * num_predicted_values). Second tensor is the score tensor. The -// size of the valuse must be (num_boxes * num_classes). It's +// size of the values must be (num_boxes * num_classes). It's // optional to pass in a third tensor for anchors (e.g. for SSD // models) depend on the outputs of the detection model. The size // of anchor tensor must be (num_boxes * 4). diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto b/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto index 41ad903de..f054608a6 100644 --- a/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto +++ b/mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.proto @@ -69,6 +69,6 @@ message TfLiteTensorsToDetectionsCalculatorOptions { // representation has a bottom-left origin (e.g., in OpenGL). optional bool flip_vertically = 18 [default = false]; - // Score threshold for perserving decoded detections. + // Score threshold for preserving decoded detections. optional float min_score_thresh = 19; } diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc index 6740f0afa..c25776de3 100644 --- a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc @@ -158,7 +158,7 @@ absl::Status TfLiteTensorsToLandmarksCalculator::Open(CalculatorContext* cc) { RET_CHECK(options_.has_input_image_height() && options_.has_input_image_width()) << "Must provide input width/height for using flip_vertically option " - "when outputing landmarks in absolute coordinates."; + "when outputting landmarks in absolute coordinates."; } flip_horizontally_ = From 6532ce5c594cf8aea39ec7fe4252c7796740811a Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 9 Nov 2023 09:41:36 -0800 Subject: [PATCH 093/157] Refactor OpenCV path out of TensorsToSegmentationCalculator main file. ProcessCpu() is changed into an OpenCV converter that is owned by the calculator. The calculator should call converter.Convert() to get the conversion result. PiperOrigin-RevId: 580937591 --- mediapipe/calculators/tensor/BUILD | 82 +------ .../tensors_to_segmentation_calculator.cc | 223 +++++++++++++----- .../tensors_to_segmentation_converter.h | 43 ---- ...ensors_to_segmentation_converter_opencv.cc | 157 ------------ ...tensors_to_segmentation_converter_opencv.h | 31 --- .../tensor/tensors_to_segmentation_utils.cc | 52 ---- .../tensor/tensors_to_segmentation_utils.h | 34 --- .../tensors_to_segmentation_utils_test.cc | 63 ----- 8 files changed, 172 insertions(+), 513 deletions(-) delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter.h delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils.h delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index 76f5bdbf6..ac2ced837 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -1414,8 +1414,6 @@ cc_library( }), deps = [ ":tensors_to_segmentation_calculator_cc_proto", - ":tensors_to_segmentation_converter", - ":tensors_to_segmentation_utils", "//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:port", @@ -1423,11 +1421,9 @@ cc_library( "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/util:resource_util", - "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", @@ -1438,7 +1434,6 @@ cc_library( "//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_simple_shaders", "//mediapipe/gpu:gpu_buffer", - "//mediapipe/gpu:gpu_buffer_format", "//mediapipe/gpu:shader_util", ], }) + selects.with_or({ @@ -1458,86 +1453,13 @@ cc_library( }) + select({ "//mediapipe/framework/port:disable_opencv": [], "//conditions:default": [ - ":tensors_to_segmentation_converter_opencv", + "//mediapipe/framework/formats:image_opencv", + "//mediapipe/framework/port:opencv_imgproc", ], }), alwayslink = 1, ) -cc_library( - name = "tensors_to_segmentation_utils", - srcs = ["tensors_to_segmentation_utils.cc"], - hdrs = ["tensors_to_segmentation_utils.h"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - deps = [ - "//mediapipe/framework:port", - "//mediapipe/framework/port:ret_check", - "@com_google_absl//absl/status:statusor", - ], -) - -cc_test( - name = "tensors_to_segmentation_utils_test", - srcs = ["tensors_to_segmentation_utils_test.cc"], - deps = [ - ":tensors_to_segmentation_utils", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status_matchers", - "@com_google_absl//absl/status:statusor", - ], -) - -cc_library( - name = "tensors_to_segmentation_converter", - hdrs = ["tensors_to_segmentation_converter.h"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - deps = [ - "//mediapipe/framework/formats:image", - "//mediapipe/framework/formats:tensor", - "@com_google_absl//absl/status:statusor", - ], -) - -cc_library( - name = "tensors_to_segmentation_converter_opencv", - srcs = ["tensors_to_segmentation_converter_opencv.cc"], - hdrs = ["tensors_to_segmentation_converter_opencv.h"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), - deps = [ - ":tensors_to_segmentation_calculator_cc_proto", - ":tensors_to_segmentation_converter", - ":tensors_to_segmentation_utils", - "//mediapipe/framework/formats:image", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_opencv", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - ], -) - cc_test( name = "tensors_to_segmentation_calculator_test", srcs = ["tensors_to_segmentation_calculator_test.cc"], diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc index 90d2e6246..24fd1bd52 100644 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc @@ -12,35 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include -#include -#include #include -#include "absl/status/status.h" -#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/types/span.h" #include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port.h" #include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status_macros.h" -#include "mediapipe/gpu/gpu_buffer_format.h" +#include "mediapipe/framework/port/statusor.h" #include "mediapipe/gpu/gpu_origin.pb.h" +#include "mediapipe/util/resource_util.h" +#include "tensorflow/lite/interpreter.h" #if !MEDIAPIPE_DISABLE_GPU #include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_simple_shaders.h" +#include "mediapipe/gpu/gpu_buffer.h" #include "mediapipe/gpu/shader_util.h" #endif // !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_OPENCV -#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h" +#include "mediapipe/framework/formats/image_opencv.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" #endif // !MEDIAPIPE_DISABLE_OPENCV #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 @@ -65,9 +62,37 @@ namespace { constexpr int kWorkgroupSize = 8; // Block size for GPU shader. enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; +// Commonly used to compute the number of blocks to launch in a kernel. +int NumGroups(const int size, const int group_size) { // NOLINT + return (size + group_size - 1) / group_size; +} + +bool CanUseGpu() { +#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED + // TODO: Configure GPU usage policy in individual calculators. + constexpr bool kAllowGpuProcessing = true; + return kAllowGpuProcessing; +#else + return false; +#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED +} + constexpr char kTensorsTag[] = "TENSORS"; constexpr char kOutputSizeTag[] = "OUTPUT_SIZE"; constexpr char kMaskTag[] = "MASK"; + +absl::StatusOr> GetHwcFromDims( + const std::vector& dims) { + if (dims.size() == 3) { + return std::make_tuple(dims[0], dims[1], dims[2]); + } else if (dims.size() == 4) { + // BHWC format check B == 1 + RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap"; + return std::make_tuple(dims[1], dims[2], dims[3]); + } else { + RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size(); + } +} } // namespace namespace mediapipe { @@ -131,24 +156,19 @@ class TensorsToSegmentationCalculator : public CalculatorBase { private: absl::Status LoadOptions(CalculatorContext* cc); absl::Status InitGpu(CalculatorContext* cc); - absl::Status ProcessGpu(CalculatorContext* cc, - const std::vector& input_tensors, - std::tuple hwc, int output_width, - int output_height); + absl::Status ProcessGpu(CalculatorContext* cc); + absl::Status ProcessCpu(CalculatorContext* cc); void GlRender(); bool DoesGpuTextureStartAtBottom() { return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT; } - absl::Status InitConverterIfNecessary() { - if (!cpu_converter_) { - MP_ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(options_)); - } - return absl::OkStatus(); - } - mediapipe::TensorsToSegmentationCalculatorOptions options_; - std::unique_ptr cpu_converter_; +#if !MEDIAPIPE_DISABLE_OPENCV + template + absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); +#endif // !MEDIAPIPE_DISABLE_OPENCV + ::mediapipe::TensorsToSegmentationCalculatorOptions options_; #if !MEDIAPIPE_DISABLE_GPU mediapipe::GlCalculatorHelper gpu_helper_; @@ -241,7 +261,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); int tensor_channels = std::get<2>(hwc); - using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; + typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; switch (options_.activation()) { case Options::NONE: RET_CHECK_EQ(tensor_channels, 1); @@ -255,17 +275,6 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { } } - // Get dimensions. - MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); - auto [tensor_height, tensor_width, tensor_channels] = hwc; - int output_width = tensor_width, output_height = tensor_height; - if (cc->Inputs().HasTag(kOutputSizeTag)) { - const auto& size = - cc->Inputs().Tag(kOutputSizeTag).Get>(); - output_width = size.first; - output_height = size.second; - } - if (use_gpu) { #if !MEDIAPIPE_DISABLE_GPU if (!gpu_initialized_) { @@ -277,25 +286,16 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { #endif // !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR( - gpu_helper_.RunInGlContext([this, cc, &input_tensors, output_width, - output_height, hwc]() -> absl::Status { - MP_RETURN_IF_ERROR( - ProcessGpu(cc, input_tensors, hwc, output_width, output_height)); - return absl::OkStatus(); - })); + MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { + MP_RETURN_IF_ERROR(ProcessGpu(cc)); + return absl::OkStatus(); + })); #else RET_CHECK_FAIL() << "GPU processing disabled."; #endif // !MEDIAPIPE_DISABLE_GPU } else { #if !MEDIAPIPE_DISABLE_OPENCV - // Lazily initialize converter. - MP_RETURN_IF_ERROR(InitConverterIfNecessary()); - MP_ASSIGN_OR_RETURN( - std::unique_ptr output_mask, - cpu_converter_->Convert(input_tensors, output_width, output_height)); - cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), - cc->InputTimestamp()); + MP_RETURN_IF_ERROR(ProcessCpu(cc)); #else RET_CHECK_FAIL() << "OpenCV processing disabled."; #endif // !MEDIAPIPE_DISABLE_OPENCV @@ -329,15 +329,132 @@ absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) { return absl::OkStatus(); } +absl::Status TensorsToSegmentationCalculator::ProcessCpu( + CalculatorContext* cc) { +#if !MEDIAPIPE_DISABLE_OPENCV + // Get input streams, and dimensions. + const auto& input_tensors = + cc->Inputs().Tag(kTensorsTag).Get>(); + MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); + auto [tensor_height, tensor_width, tensor_channels] = hwc; + int output_width = tensor_width, output_height = tensor_height; + if (cc->Inputs().HasTag(kOutputSizeTag)) { + const auto& size = + cc->Inputs().Tag(kOutputSizeTag).Get>(); + output_width = size.first; + output_height = size.second; + } + + // Create initial working mask. + cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1); + + // Wrap input tensor. + auto raw_input_tensor = &input_tensors[0]; + auto raw_input_view = raw_input_tensor->GetCpuReadView(); + const float* raw_input_data = raw_input_view.buffer(); + cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height), + CV_MAKETYPE(CV_32F, tensor_channels), + const_cast(raw_input_data)); + + // Process mask tensor and apply activation function. + if (tensor_channels == 2) { + MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); + } else if (tensor_channels == 1) { + RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX != + options_.activation()); // Requires 2 channels. + if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE == + options_.activation()) // Pass-through optimization. + tensor_mat.copyTo(small_mask_mat); + else + MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); + } else { + RET_CHECK_FAIL() << "Unsupported number of tensor channels " + << tensor_channels; + } + + // Send out image as CPU packet. + std::shared_ptr mask_frame = std::make_shared( + ImageFormat::VEC32F1, output_width, output_height); + std::unique_ptr output_mask = absl::make_unique(mask_frame); + auto output_mat = formats::MatView(output_mask.get()); + // Upsample small mask into output. + cv::resize(small_mask_mat, *output_mat, + cv::Size(output_width, output_height)); + cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp()); +#endif // !MEDIAPIPE_DISABLE_OPENCV + + return absl::OkStatus(); +} + +#if !MEDIAPIPE_DISABLE_OPENCV +template +absl::Status TensorsToSegmentationCalculator::ApplyActivation( + cv::Mat& tensor_mat, cv::Mat* small_mask_mat) { + // Configure activation function. + const int output_layer_index = options_.output_layer_index(); + typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; + const auto activation_fn = [&](const cv::Vec2f& mask_value) { + float new_mask_value = 0; + // TODO consider moving switch out of the loop, + // and also avoid float/Vec2f casting. + switch (options_.activation()) { + case Options::NONE: { + new_mask_value = mask_value[0]; + break; + } + case Options::SIGMOID: { + const float pixel0 = mask_value[0]; + new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0); + break; + } + case Options::SOFTMAX: { + const float pixel0 = mask_value[0]; + const float pixel1 = mask_value[1]; + const float max_pixel = std::max(pixel0, pixel1); + const float min_pixel = std::min(pixel0, pixel1); + const float softmax_denom = + /*exp(max_pixel - max_pixel)=*/1.0f + + std::exp(min_pixel - max_pixel); + new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) / + softmax_denom; + break; + } + } + return new_mask_value; + }; + + // Process mask tensor. + for (int i = 0; i < tensor_mat.rows; ++i) { + for (int j = 0; j < tensor_mat.cols; ++j) { + const T& input_pix = tensor_mat.at(i, j); + const float mask_value = activation_fn(input_pix); + small_mask_mat->at(i, j) = mask_value; + } + } + + return absl::OkStatus(); +} +#endif // !MEDIAPIPE_DISABLE_OPENCV + // Steps: // 1. receive tensor // 2. process segmentation tensor into small mask // 3. upsample small mask into output mask to be same size as input image absl::Status TensorsToSegmentationCalculator::ProcessGpu( - CalculatorContext* cc, const std::vector& input_tensors, - std::tuple hwc, int output_width, int output_height) { + CalculatorContext* cc) { #if !MEDIAPIPE_DISABLE_GPU + // Get input streams, and dimensions. + const auto& input_tensors = + cc->Inputs().Tag(kTensorsTag).Get>(); + MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); auto [tensor_height, tensor_width, tensor_channels] = hwc; + int output_width = tensor_width, output_height = tensor_height; + if (cc->Inputs().HasTag(kOutputSizeTag)) { + const auto& size = + cc->Inputs().Tag(kOutputSizeTag).Get>(); + output_width = size.first; + output_height = size.second; + } // Create initial working mask texture. #if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31) @@ -515,7 +632,7 @@ void TensorsToSegmentationCalculator::GlRender() { absl::Status TensorsToSegmentationCalculator::LoadOptions( CalculatorContext* cc) { // Get calculator options specified in the graph. - options_ = cc->Options(); + options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>(); return absl::OkStatus(); } @@ -709,7 +826,7 @@ void main() { #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 // Shader defines. - using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; + typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; const std::string output_layer_index = "\n#define OUTPUT_LAYER_INDEX int(" + std::to_string(options_.output_layer_index()) + ")"; diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h b/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h deleted file mode 100644 index 61d95dfe0..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ - -#include -#include - -#include "absl/status/statusor.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/tensor.h" - -namespace mediapipe { - -class TensorsToSegmentationConverter { - public: - virtual ~TensorsToSegmentationConverter() = default; - - // Converts tensors to image mask. - // Returns a unique pointer containing the converted image. - // @input_tensors contains the tensors needed to be processed. - // @output_width/height describes output dimensions to reshape the output mask - // into. - virtual absl::StatusOr> Convert( - const std::vector& input_tensors, int output_width, - int output_height) = 0; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc deleted file mode 100644 index 1ee2e172b..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h" - -#include -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_opencv.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status_macros.h" - -namespace mediapipe { -namespace { - -class OpenCvProcessor : public TensorsToSegmentationConverter { - public: - absl::Status Init(const TensorsToSegmentationCalculatorOptions& options) { - options_ = options; - return absl::OkStatus(); - } - - absl::StatusOr> Convert( - const std::vector& input_tensors, int output_width, - int output_height) override; - - private: - template - absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); - - TensorsToSegmentationCalculatorOptions options_; -}; - -absl::StatusOr> OpenCvProcessor::Convert( - const std::vector& input_tensors, int output_width, - int output_height) { - MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); - auto [tensor_height, tensor_width, tensor_channels] = hwc; - // Create initial working mask. - cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1); - - // Wrap input tensor. - auto raw_input_tensor = &input_tensors[0]; - auto raw_input_view = raw_input_tensor->GetCpuReadView(); - const float* raw_input_data = raw_input_view.buffer(); - cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height), - CV_MAKETYPE(CV_32F, tensor_channels), - const_cast(raw_input_data)); - - // Process mask tensor and apply activation function. - if (tensor_channels == 2) { - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else if (tensor_channels == 1) { - RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX != - options_.activation()); // Requires 2 channels. - if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE == - options_.activation()) // Pass-through optimization. - tensor_mat.copyTo(small_mask_mat); - else - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else { - RET_CHECK_FAIL() << "Unsupported number of tensor channels " - << tensor_channels; - } - - // Send out image as CPU packet. - std::shared_ptr mask_frame = std::make_shared( - ImageFormat::VEC32F1, output_width, output_height); - auto output_mask = std::make_unique(mask_frame); - auto output_mat = formats::MatView(output_mask.get()); - // Upsample small mask into output. - cv::resize(small_mask_mat, *output_mat, - cv::Size(output_width, output_height)); - return output_mask; -} - -template -absl::Status OpenCvProcessor::ApplyActivation(cv::Mat& tensor_mat, - cv::Mat* small_mask_mat) { - // Configure activation function. - const int output_layer_index = options_.output_layer_index(); - using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; - const auto activation_fn = [&](const cv::Vec2f& mask_value) { - float new_mask_value = 0; - // TODO consider moving switch out of the loop, - // and also avoid float/Vec2f casting. - switch (options_.activation()) { - case Options::NONE: { - new_mask_value = mask_value[0]; - break; - } - case Options::SIGMOID: { - const float pixel0 = mask_value[0]; - new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0); - break; - } - case Options::SOFTMAX: { - const float pixel0 = mask_value[0]; - const float pixel1 = mask_value[1]; - const float max_pixel = std::max(pixel0, pixel1); - const float min_pixel = std::min(pixel0, pixel1); - const float softmax_denom = - /*exp(max_pixel - max_pixel)=*/1.0f + - std::exp(min_pixel - max_pixel); - new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) / - softmax_denom; - break; - } - } - return new_mask_value; - }; - - // Process mask tensor. - for (int i = 0; i < tensor_mat.rows; ++i) { - for (int j = 0; j < tensor_mat.cols; ++j) { - const T& input_pix = tensor_mat.at(i, j); - const float mask_value = activation_fn(input_pix); - small_mask_mat->at(i, j) = mask_value; - } - } - - return absl::OkStatus(); -} - -} // namespace - -absl::StatusOr> -CreateOpenCvConverter(const TensorsToSegmentationCalculatorOptions& options) { - auto converter = std::make_unique(); - MP_RETURN_IF_ERROR(converter->Init(options)); - return converter; -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h deleted file mode 100644 index 3ae41b5e0..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ - -#include - -#include "absl/status/statusor.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" - -namespace mediapipe { -// Creates OpenCV tensors-to-segmentation converter. -absl::StatusOr> -CreateOpenCvConverter( - const mediapipe::TensorsToSegmentationCalculatorOptions& options); -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc deleted file mode 100644 index ab1e9c139..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" - -#include -#include - -#include "absl/status/statusor.h" -#include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -int NumGroups(int size, int group_size) { - return (size + group_size - 1) / group_size; -} - -bool CanUseGpu() { -#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED - // TODO: Configure GPU usage policy in individual calculators. - constexpr bool kAllowGpuProcessing = true; - return kAllowGpuProcessing; -#else - return false; -#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED -} - -absl::StatusOr> GetHwcFromDims( - const std::vector& dims) { - if (dims.size() == 3) { - return std::make_tuple(dims[0], dims[1], dims[2]); - } else if (dims.size() == 4) { - // BHWC format check B == 1 - RET_CHECK_EQ(dims[0], 1) << "Expected batch to be 1 for BHWC heatmap"; - return std::make_tuple(dims[1], dims[2], dims[3]); - } else { - RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size(); - } -} -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h deleted file mode 100644 index 44893073b..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ - -#include -#include - -#include "absl/status/statusor.h" - -namespace mediapipe { - -// Commonly used to compute the number of blocks to launch in a kernel. -int NumGroups(const int size, const int group_size); // NOLINT - -bool CanUseGpu(); - -absl::StatusOr> GetHwcFromDims( - const std::vector& dims); -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc deleted file mode 100644 index 5535d159d..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" - -#include -#include - -#include "absl/status/statusor.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -using ::testing::HasSubstr; - -TEST(TensorsToSegmentationUtilsTest, NumGroupsWorksProperly) { - EXPECT_EQ(NumGroups(13, 4), 4); - EXPECT_EQ(NumGroups(4, 13), 1); -} - -TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsWorksProperly) { - std::vector dims_3 = {2, 3, 4}; - absl::StatusOr> result_1 = GetHwcFromDims(dims_3); - MP_ASSERT_OK(result_1); - EXPECT_EQ(result_1.value(), (std::make_tuple(2, 3, 4))); - std::vector dims_4 = {1, 3, 4, 5}; - absl::StatusOr> result_2 = GetHwcFromDims(dims_4); - MP_ASSERT_OK(result_2); - EXPECT_EQ(result_2.value(), (std::make_tuple(3, 4, 5))); -} - -TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsBatchCheckFail) { - std::vector dims_4 = {2, 3, 4, 5}; - absl::StatusOr> result = GetHwcFromDims(dims_4); - EXPECT_FALSE(result.ok()); - EXPECT_THAT(result.status().message(), - HasSubstr("Expected batch to be 1 for BHWC heatmap")); -} - -TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsInvalidShape) { - std::vector dims_5 = {1, 2, 3, 4, 5}; - absl::StatusOr> result = GetHwcFromDims(dims_5); - EXPECT_FALSE(result.ok()); - EXPECT_THAT(result.status().message(), - HasSubstr("Invalid shape for segmentation tensor")); -} - -} // namespace -} // namespace mediapipe From edca85c5d3fa4ef848340fcc9f88e9d95db05688 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 9 Nov 2023 13:34:48 -0800 Subject: [PATCH 094/157] Create shared utilities to construct category lists PiperOrigin-RevId: 581009898 --- .../tasks/components/containers/Category.java | 20 +++++++ .../containers/Classifications.java | 8 +-- .../com/google/mediapipe/tasks/vision/BUILD | 4 -- .../facelandmarker/FaceLandmarkerResult.java | 13 +---- .../GestureRecognizerResult.java | 12 +---- .../handlandmarker/HandLandmarkerResult.java | 13 +---- .../components/containers/AndroidManifest.xml | 24 +++++++++ .../tasks/components/containers/BUILD | 19 +++++++ .../components/containers/CategoryTest.java | 52 +++++++++++++++++++ 9 files changed, 122 insertions(+), 43 deletions(-) create mode 100644 mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/AndroidManifest.xml create mode 100644 mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/BUILD create mode 100644 mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/CategoryTest.java diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Category.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Category.java index 65996c2af..916ad1bed 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Category.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Category.java @@ -16,6 +16,10 @@ package com.google.mediapipe.tasks.components.containers; import com.google.auto.value.AutoValue; import com.google.mediapipe.formats.proto.ClassificationProto; +import com.google.mediapipe.formats.proto.ClassificationProto.Classification; +import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; /** @@ -49,6 +53,22 @@ public abstract class Category { return create(proto.getScore(), proto.getIndex(), proto.getLabel(), proto.getDisplayName()); } + /** + * Creates a list of {@link Category} objects from a {@link + * ClassificationProto.ClassificationList}. + * + * @param classificationListProto the {@link ClassificationProto.ClassificationList} protobuf + * message to convert. + * @return A list of {@link Category} objects. + */ + public static List createListFromProto(ClassificationList classificationListProto) { + List categoryList = new ArrayList<>(); + for (Classification classification : classificationListProto.getClassificationList()) { + categoryList.add(createFromProto(classification)); + } + return categoryList; + } + /** The probability score of this label category. */ public abstract float score(); diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Classifications.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Classifications.java index 9e53590d7..7c2a1fc21 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Classifications.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Classifications.java @@ -15,9 +15,7 @@ package com.google.mediapipe.tasks.components.containers; import com.google.auto.value.AutoValue; -import com.google.mediapipe.formats.proto.ClassificationProto; import com.google.mediapipe.tasks.components.containers.proto.ClassificationsProto; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -49,11 +47,7 @@ public abstract class Classifications { * @param proto the {@link ClassificationsProto.Classifications} protobuf message to convert. */ public static Classifications createFromProto(ClassificationsProto.Classifications proto) { - List categories = new ArrayList<>(); - for (ClassificationProto.Classification classificationProto : - proto.getClassificationList().getClassificationList()) { - categories.add(Category.createFromProto(classificationProto)); - } + List categories = Category.createListFromProto(proto.getClassificationList()); Optional headName = proto.hasHeadName() ? Optional.of(proto.getHeadName()) : Optional.empty(); return create(categories, proto.getHeadIndex(), headName); diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD index 60a9806e9..181b45dc8 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD @@ -208,11 +208,9 @@ android_library( deps = [ ":core", "//mediapipe/framework:calculator_options_java_proto_lite", - "//mediapipe/framework/formats:classification_java_proto_lite", "//mediapipe/framework/formats:landmark_java_proto_lite", "//mediapipe/java/com/google/mediapipe/framework:android_framework", "//mediapipe/java/com/google/mediapipe/framework/image", - "//mediapipe/tasks/cc/components/processors/proto:classifier_options_java_proto_lite", "//mediapipe/tasks/cc/core/proto:base_options_java_proto_lite", "//mediapipe/tasks/cc/vision/pose_detector/proto:pose_detector_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarker_graph_options_java_proto_lite", @@ -222,7 +220,6 @@ android_library( "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:normalized_landmark", "//mediapipe/tasks/java/com/google/mediapipe/tasks/core", "//third_party:autovalue", - "@maven//:androidx_annotation_annotation", "@maven//:com_google_guava_guava", ], ) @@ -246,7 +243,6 @@ android_library( "//mediapipe/framework/formats:landmark_java_proto_lite", "//mediapipe/java/com/google/mediapipe/framework:android_framework", "//mediapipe/java/com/google/mediapipe/framework/image", - "//mediapipe/tasks/cc/components/processors/proto:classifier_options_java_proto_lite", "//mediapipe/tasks/cc/core/proto:base_options_java_proto_lite", "//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_java_proto_lite", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java index 0429ecacb..98fb9376f 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java @@ -16,7 +16,6 @@ package com.google.mediapipe.tasks.vision.facelandmarker; import com.google.auto.value.AutoValue; import com.google.mediapipe.formats.proto.LandmarkProto; -import com.google.mediapipe.formats.proto.ClassificationProto.Classification; import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList; import com.google.mediapipe.tasks.components.containers.Category; import com.google.mediapipe.tasks.components.containers.NormalizedLandmark; @@ -68,16 +67,8 @@ public abstract class FaceLandmarkerResult implements TaskResult { if (multiFaceBendshapesProto.isPresent()) { List> blendshapes = new ArrayList<>(); for (ClassificationList faceBendshapeProto : multiFaceBendshapesProto.get()) { - List blendshape = new ArrayList<>(); - blendshapes.add(blendshape); - for (Classification classification : faceBendshapeProto.getClassificationList()) { - blendshape.add( - Category.create( - classification.getScore(), - classification.getIndex(), - classification.getLabel(), - classification.getDisplayName())); - } + List blendshape = Category.createListFromProto(faceBendshapeProto); + blendshapes.add(Collections.unmodifiableList(blendshape)); } multiFaceBlendshapes = Optional.of(Collections.unmodifiableList(blendshapes)); } diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerResult.java index c8d43e2ca..09ceac215 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerResult.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/gesturerecognizer/GestureRecognizerResult.java @@ -75,16 +75,8 @@ public abstract class GestureRecognizerResult implements TaskResult { } } for (ClassificationList handednessProto : handednessesProto) { - List handedness = new ArrayList<>(); - multiHandHandednesses.add(handedness); - for (Classification classification : handednessProto.getClassificationList()) { - handedness.add( - Category.create( - classification.getScore(), - classification.getIndex(), - classification.getLabel(), - classification.getDisplayName())); - } + List handedness = Category.createListFromProto(handednessProto); + multiHandHandednesses.add(Collections.unmodifiableList(handedness)); } for (ClassificationList gestureProto : gesturesProto) { List gestures = new ArrayList<>(); diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java index 14d2fa926..54ed04848 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java @@ -16,7 +16,6 @@ package com.google.mediapipe.tasks.vision.handlandmarker; import com.google.auto.value.AutoValue; import com.google.mediapipe.formats.proto.LandmarkProto; -import com.google.mediapipe.formats.proto.ClassificationProto.Classification; import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList; import com.google.mediapipe.tasks.components.containers.Category; import com.google.mediapipe.tasks.components.containers.Landmark; @@ -84,16 +83,8 @@ public abstract class HandLandmarkerResult implements TaskResult { } } for (ClassificationList handednessProto : handednessesProto) { - List handedness = new ArrayList<>(); - multiHandHandednesses.add(handedness); - for (Classification classification : handednessProto.getClassificationList()) { - handedness.add( - Category.create( - classification.getScore(), - classification.getIndex(), - classification.getLabel(), - classification.getDisplayName())); - } + List handedness = Category.createListFromProto(handednessProto); + multiHandHandednesses.add(Collections.unmodifiableList(handedness)); } return new AutoValue_HandLandmarkerResult( timestampMs, diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/AndroidManifest.xml b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/AndroidManifest.xml new file mode 100644 index 000000000..4a6416933 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/AndroidManifest.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/BUILD b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/BUILD new file mode 100644 index 000000000..7363a23e0 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/BUILD @@ -0,0 +1,19 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# TODO: Enable these tests in OSS diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/CategoryTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/CategoryTest.java new file mode 100644 index 000000000..ed501ac57 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/CategoryTest.java @@ -0,0 +1,52 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.components.containers; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.mediapipe.formats.proto.ClassificationProto.Classification; +import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(AndroidJUnit4.class) +public final class CategoryTest { + + @Test + public void create_succeedsWithClassificationProto() { + Classification input = + Classification.newBuilder() + .setScore(0.1f) + .setIndex(1) + .setLabel("label") + .setDisplayName("displayName") + .build(); + Category output = Category.createFromProto(input); + assertThat(output.score()).isEqualTo(0.1f); + assertThat(output.index()).isEqualTo(1); + assertThat(output.categoryName()).isEqualTo("label"); + assertThat(output.displayName()).isEqualTo("displayName"); + } + + @Test + public void create_succeedsWithClassificationListProto() { + Classification element = Classification.newBuilder().setScore(0.1f).build(); + ClassificationList input = ClassificationList.newBuilder().addClassification(element).build(); + List output = Category.createListFromProto(input); + assertThat(output).containsExactly(Category.create(0.1f, 0, "", "")); + } +} From 333125ac20cd0a1fe731d46ce585fc1dfb03dee3 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 9 Nov 2023 15:55:29 -0800 Subject: [PATCH 095/157] Add some convenience getters to EglManager. PiperOrigin-RevId: 581049412 --- .../java/com/google/mediapipe/glutil/EglManager.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mediapipe/java/com/google/mediapipe/glutil/EglManager.java b/mediapipe/java/com/google/mediapipe/glutil/EglManager.java index bad59ce3a..ae52dc895 100644 --- a/mediapipe/java/com/google/mediapipe/glutil/EglManager.java +++ b/mediapipe/java/com/google/mediapipe/glutil/EglManager.java @@ -114,6 +114,16 @@ public class EglManager { } } + /** Returns the managed {@link EGLDisplay}. */ + public EGLDisplay getEglDisplay() { + return eglDisplay; + } + + /** Returns the {@link EGL10}. */ + public EGL10 getEgl() { + return egl; + } + /** Returns the managed {@link EGLContext} */ public EGLContext getContext() { return eglContext; From 1038f8176d4edcd61d624d1da497a8ebb6d022bb Mon Sep 17 00:00:00 2001 From: Youchuan Hu Date: Thu, 9 Nov 2023 17:26:07 -0800 Subject: [PATCH 096/157] Refactor OpenCV path out of TensorsToSegmentationCalculator main file. ProcessCpu() is changed into an OpenCV converter that is owned by the calculator. The calculator should call converter.Convert() to get the conversion result. PiperOrigin-RevId: 581073731 --- mediapipe/calculators/tensor/BUILD | 68 +++++- .../tensors_to_segmentation_calculator.cc | 225 +++++------------- .../tensors_to_segmentation_converter.h | 43 ++++ ...ensors_to_segmentation_converter_opencv.cc | 157 ++++++++++++ ...tensors_to_segmentation_converter_opencv.h | 31 +++ .../tensor/tensors_to_segmentation_utils.cc | 52 ++++ .../tensor/tensors_to_segmentation_utils.h | 34 +++ .../tensors_to_segmentation_utils_test.cc | 63 +++++ 8 files changed, 495 insertions(+), 178 deletions(-) create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter.h create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils.h create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index ac2ced837..c0db4e35b 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -1405,15 +1405,10 @@ mediapipe_proto_library( cc_library( name = "tensors_to_segmentation_calculator", srcs = ["tensors_to_segmentation_calculator.cc"], - copts = select({ - "//mediapipe:apple": [ - "-x objective-c++", - "-fobjc-arc", # enable reference-counting - ], - "//conditions:default": [], - }), deps = [ ":tensors_to_segmentation_calculator_cc_proto", + ":tensors_to_segmentation_converter", + ":tensors_to_segmentation_utils", "//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:port", @@ -1421,9 +1416,11 @@ cc_library( "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/util:resource_util", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", @@ -1434,6 +1431,7 @@ cc_library( "//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_simple_shaders", "//mediapipe/gpu:gpu_buffer", + "//mediapipe/gpu:gpu_buffer_format", "//mediapipe/gpu:shader_util", ], }) + selects.with_or({ @@ -1453,13 +1451,65 @@ cc_library( }) + select({ "//mediapipe/framework/port:disable_opencv": [], "//conditions:default": [ - "//mediapipe/framework/formats:image_opencv", - "//mediapipe/framework/port:opencv_imgproc", + ":tensors_to_segmentation_converter_opencv", ], }), alwayslink = 1, ) +cc_library( + name = "tensors_to_segmentation_utils", + srcs = ["tensors_to_segmentation_utils.cc"], + hdrs = ["tensors_to_segmentation_utils.h"], + deps = [ + "//mediapipe/framework:port", + "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_test( + name = "tensors_to_segmentation_utils_test", + srcs = ["tensors_to_segmentation_utils_test.cc"], + deps = [ + ":tensors_to_segmentation_utils", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:status_matchers", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_library( + name = "tensors_to_segmentation_converter", + hdrs = ["tensors_to_segmentation_converter.h"], + deps = [ + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:tensor", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_library( + name = "tensors_to_segmentation_converter_opencv", + srcs = ["tensors_to_segmentation_converter_opencv.cc"], + hdrs = ["tensors_to_segmentation_converter_opencv.h"], + deps = [ + ":tensors_to_segmentation_calculator_cc_proto", + ":tensors_to_segmentation_converter", + ":tensors_to_segmentation_utils", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_opencv", + "//mediapipe/framework/formats:tensor", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], +) + cc_test( name = "tensors_to_segmentation_calculator_test", srcs = ["tensors_to_segmentation_calculator_test.cc"], diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc index 24fd1bd52..6164c7b0a 100644 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc @@ -12,32 +12,35 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include +#include #include -#include "absl/strings/str_format.h" -#include "absl/types/span.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port.h" #include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/statusor.h" +#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/gpu/gpu_origin.pb.h" -#include "mediapipe/util/resource_util.h" -#include "tensorflow/lite/interpreter.h" #if !MEDIAPIPE_DISABLE_GPU #include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/gpu_buffer.h" +#include "mediapipe/gpu/gpu_buffer_format.h" #include "mediapipe/gpu/shader_util.h" #endif // !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_OPENCV -#include "mediapipe/framework/formats/image_opencv.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h" #endif // !MEDIAPIPE_DISABLE_OPENCV #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 @@ -62,37 +65,9 @@ namespace { constexpr int kWorkgroupSize = 8; // Block size for GPU shader. enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; -// Commonly used to compute the number of blocks to launch in a kernel. -int NumGroups(const int size, const int group_size) { // NOLINT - return (size + group_size - 1) / group_size; -} - -bool CanUseGpu() { -#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED - // TODO: Configure GPU usage policy in individual calculators. - constexpr bool kAllowGpuProcessing = true; - return kAllowGpuProcessing; -#else - return false; -#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED -} - constexpr char kTensorsTag[] = "TENSORS"; constexpr char kOutputSizeTag[] = "OUTPUT_SIZE"; constexpr char kMaskTag[] = "MASK"; - -absl::StatusOr> GetHwcFromDims( - const std::vector& dims) { - if (dims.size() == 3) { - return std::make_tuple(dims[0], dims[1], dims[2]); - } else if (dims.size() == 4) { - // BHWC format check B == 1 - RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap"; - return std::make_tuple(dims[1], dims[2], dims[3]); - } else { - RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size(); - } -} } // namespace namespace mediapipe { @@ -156,19 +131,28 @@ class TensorsToSegmentationCalculator : public CalculatorBase { private: absl::Status LoadOptions(CalculatorContext* cc); absl::Status InitGpu(CalculatorContext* cc); - absl::Status ProcessGpu(CalculatorContext* cc); - absl::Status ProcessCpu(CalculatorContext* cc); + absl::Status ProcessGpu(CalculatorContext* cc, + const std::vector& input_tensors, + std::tuple hwc, int output_width, + int output_height); void GlRender(); bool DoesGpuTextureStartAtBottom() { return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT; } - + absl::Status InitConverterIfNecessary() { #if !MEDIAPIPE_DISABLE_OPENCV - template - absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); + if (!cpu_converter_) { + MP_ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(options_)); + } +#else + RET_CHECK_FAIL() << "OpenCV processing disabled."; #endif // !MEDIAPIPE_DISABLE_OPENCV - ::mediapipe::TensorsToSegmentationCalculatorOptions options_; + return absl::OkStatus(); + } + + mediapipe::TensorsToSegmentationCalculatorOptions options_; + std::unique_ptr cpu_converter_; #if !MEDIAPIPE_DISABLE_GPU mediapipe::GlCalculatorHelper gpu_helper_; @@ -261,7 +245,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); int tensor_channels = std::get<2>(hwc); - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; + using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; switch (options_.activation()) { case Options::NONE: RET_CHECK_EQ(tensor_channels, 1); @@ -275,6 +259,17 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { } } + // Get dimensions. + MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); + auto [tensor_height, tensor_width, tensor_channels] = hwc; + int output_width = tensor_width, output_height = tensor_height; + if (cc->Inputs().HasTag(kOutputSizeTag)) { + const auto& size = + cc->Inputs().Tag(kOutputSizeTag).Get>(); + output_width = size.first; + output_height = size.second; + } + if (use_gpu) { #if !MEDIAPIPE_DISABLE_GPU if (!gpu_initialized_) { @@ -286,16 +281,25 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { #endif // !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { - MP_RETURN_IF_ERROR(ProcessGpu(cc)); - return absl::OkStatus(); - })); + MP_RETURN_IF_ERROR( + gpu_helper_.RunInGlContext([this, cc, &input_tensors, output_width, + output_height, hwc]() -> absl::Status { + MP_RETURN_IF_ERROR( + ProcessGpu(cc, input_tensors, hwc, output_width, output_height)); + return absl::OkStatus(); + })); #else RET_CHECK_FAIL() << "GPU processing disabled."; #endif // !MEDIAPIPE_DISABLE_GPU } else { #if !MEDIAPIPE_DISABLE_OPENCV - MP_RETURN_IF_ERROR(ProcessCpu(cc)); + // Lazily initialize converter. + MP_RETURN_IF_ERROR(InitConverterIfNecessary()); + MP_ASSIGN_OR_RETURN( + std::unique_ptr output_mask, + cpu_converter_->Convert(input_tensors, output_width, output_height)); + cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), + cc->InputTimestamp()); #else RET_CHECK_FAIL() << "OpenCV processing disabled."; #endif // !MEDIAPIPE_DISABLE_OPENCV @@ -329,132 +333,15 @@ absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) { return absl::OkStatus(); } -absl::Status TensorsToSegmentationCalculator::ProcessCpu( - CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_OPENCV - // Get input streams, and dimensions. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); - auto [tensor_height, tensor_width, tensor_channels] = hwc; - int output_width = tensor_width, output_height = tensor_height; - if (cc->Inputs().HasTag(kOutputSizeTag)) { - const auto& size = - cc->Inputs().Tag(kOutputSizeTag).Get>(); - output_width = size.first; - output_height = size.second; - } - - // Create initial working mask. - cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1); - - // Wrap input tensor. - auto raw_input_tensor = &input_tensors[0]; - auto raw_input_view = raw_input_tensor->GetCpuReadView(); - const float* raw_input_data = raw_input_view.buffer(); - cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height), - CV_MAKETYPE(CV_32F, tensor_channels), - const_cast(raw_input_data)); - - // Process mask tensor and apply activation function. - if (tensor_channels == 2) { - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else if (tensor_channels == 1) { - RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX != - options_.activation()); // Requires 2 channels. - if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE == - options_.activation()) // Pass-through optimization. - tensor_mat.copyTo(small_mask_mat); - else - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else { - RET_CHECK_FAIL() << "Unsupported number of tensor channels " - << tensor_channels; - } - - // Send out image as CPU packet. - std::shared_ptr mask_frame = std::make_shared( - ImageFormat::VEC32F1, output_width, output_height); - std::unique_ptr output_mask = absl::make_unique(mask_frame); - auto output_mat = formats::MatView(output_mask.get()); - // Upsample small mask into output. - cv::resize(small_mask_mat, *output_mat, - cv::Size(output_width, output_height)); - cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp()); -#endif // !MEDIAPIPE_DISABLE_OPENCV - - return absl::OkStatus(); -} - -#if !MEDIAPIPE_DISABLE_OPENCV -template -absl::Status TensorsToSegmentationCalculator::ApplyActivation( - cv::Mat& tensor_mat, cv::Mat* small_mask_mat) { - // Configure activation function. - const int output_layer_index = options_.output_layer_index(); - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; - const auto activation_fn = [&](const cv::Vec2f& mask_value) { - float new_mask_value = 0; - // TODO consider moving switch out of the loop, - // and also avoid float/Vec2f casting. - switch (options_.activation()) { - case Options::NONE: { - new_mask_value = mask_value[0]; - break; - } - case Options::SIGMOID: { - const float pixel0 = mask_value[0]; - new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0); - break; - } - case Options::SOFTMAX: { - const float pixel0 = mask_value[0]; - const float pixel1 = mask_value[1]; - const float max_pixel = std::max(pixel0, pixel1); - const float min_pixel = std::min(pixel0, pixel1); - const float softmax_denom = - /*exp(max_pixel - max_pixel)=*/1.0f + - std::exp(min_pixel - max_pixel); - new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) / - softmax_denom; - break; - } - } - return new_mask_value; - }; - - // Process mask tensor. - for (int i = 0; i < tensor_mat.rows; ++i) { - for (int j = 0; j < tensor_mat.cols; ++j) { - const T& input_pix = tensor_mat.at(i, j); - const float mask_value = activation_fn(input_pix); - small_mask_mat->at(i, j) = mask_value; - } - } - - return absl::OkStatus(); -} -#endif // !MEDIAPIPE_DISABLE_OPENCV - // Steps: // 1. receive tensor // 2. process segmentation tensor into small mask // 3. upsample small mask into output mask to be same size as input image absl::Status TensorsToSegmentationCalculator::ProcessGpu( - CalculatorContext* cc) { + CalculatorContext* cc, const std::vector& input_tensors, + std::tuple hwc, int output_width, int output_height) { #if !MEDIAPIPE_DISABLE_GPU - // Get input streams, and dimensions. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); auto [tensor_height, tensor_width, tensor_channels] = hwc; - int output_width = tensor_width, output_height = tensor_height; - if (cc->Inputs().HasTag(kOutputSizeTag)) { - const auto& size = - cc->Inputs().Tag(kOutputSizeTag).Get>(); - output_width = size.first; - output_height = size.second; - } // Create initial working mask texture. #if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31) @@ -632,7 +519,7 @@ void TensorsToSegmentationCalculator::GlRender() { absl::Status TensorsToSegmentationCalculator::LoadOptions( CalculatorContext* cc) { // Get calculator options specified in the graph. - options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>(); + options_ = cc->Options(); return absl::OkStatus(); } @@ -826,7 +713,7 @@ void main() { #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 // Shader defines. - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; + using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; const std::string output_layer_index = "\n#define OUTPUT_LAYER_INDEX int(" + std::to_string(options_.output_layer_index()) + ")"; diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h b/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h new file mode 100644 index 000000000..61d95dfe0 --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h @@ -0,0 +1,43 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ + +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/tensor.h" + +namespace mediapipe { + +class TensorsToSegmentationConverter { + public: + virtual ~TensorsToSegmentationConverter() = default; + + // Converts tensors to image mask. + // Returns a unique pointer containing the converted image. + // @input_tensors contains the tensors needed to be processed. + // @output_width/height describes output dimensions to reshape the output mask + // into. + virtual absl::StatusOr> Convert( + const std::vector& input_tensors, int output_width, + int output_height) = 0; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc new file mode 100644 index 000000000..1ee2e172b --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc @@ -0,0 +1,157 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_opencv.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_macros.h" + +namespace mediapipe { +namespace { + +class OpenCvProcessor : public TensorsToSegmentationConverter { + public: + absl::Status Init(const TensorsToSegmentationCalculatorOptions& options) { + options_ = options; + return absl::OkStatus(); + } + + absl::StatusOr> Convert( + const std::vector& input_tensors, int output_width, + int output_height) override; + + private: + template + absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); + + TensorsToSegmentationCalculatorOptions options_; +}; + +absl::StatusOr> OpenCvProcessor::Convert( + const std::vector& input_tensors, int output_width, + int output_height) { + MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); + auto [tensor_height, tensor_width, tensor_channels] = hwc; + // Create initial working mask. + cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1); + + // Wrap input tensor. + auto raw_input_tensor = &input_tensors[0]; + auto raw_input_view = raw_input_tensor->GetCpuReadView(); + const float* raw_input_data = raw_input_view.buffer(); + cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height), + CV_MAKETYPE(CV_32F, tensor_channels), + const_cast(raw_input_data)); + + // Process mask tensor and apply activation function. + if (tensor_channels == 2) { + MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); + } else if (tensor_channels == 1) { + RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX != + options_.activation()); // Requires 2 channels. + if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE == + options_.activation()) // Pass-through optimization. + tensor_mat.copyTo(small_mask_mat); + else + MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); + } else { + RET_CHECK_FAIL() << "Unsupported number of tensor channels " + << tensor_channels; + } + + // Send out image as CPU packet. + std::shared_ptr mask_frame = std::make_shared( + ImageFormat::VEC32F1, output_width, output_height); + auto output_mask = std::make_unique(mask_frame); + auto output_mat = formats::MatView(output_mask.get()); + // Upsample small mask into output. + cv::resize(small_mask_mat, *output_mat, + cv::Size(output_width, output_height)); + return output_mask; +} + +template +absl::Status OpenCvProcessor::ApplyActivation(cv::Mat& tensor_mat, + cv::Mat* small_mask_mat) { + // Configure activation function. + const int output_layer_index = options_.output_layer_index(); + using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; + const auto activation_fn = [&](const cv::Vec2f& mask_value) { + float new_mask_value = 0; + // TODO consider moving switch out of the loop, + // and also avoid float/Vec2f casting. + switch (options_.activation()) { + case Options::NONE: { + new_mask_value = mask_value[0]; + break; + } + case Options::SIGMOID: { + const float pixel0 = mask_value[0]; + new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0); + break; + } + case Options::SOFTMAX: { + const float pixel0 = mask_value[0]; + const float pixel1 = mask_value[1]; + const float max_pixel = std::max(pixel0, pixel1); + const float min_pixel = std::min(pixel0, pixel1); + const float softmax_denom = + /*exp(max_pixel - max_pixel)=*/1.0f + + std::exp(min_pixel - max_pixel); + new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) / + softmax_denom; + break; + } + } + return new_mask_value; + }; + + // Process mask tensor. + for (int i = 0; i < tensor_mat.rows; ++i) { + for (int j = 0; j < tensor_mat.cols; ++j) { + const T& input_pix = tensor_mat.at(i, j); + const float mask_value = activation_fn(input_pix); + small_mask_mat->at(i, j) = mask_value; + } + } + + return absl::OkStatus(); +} + +} // namespace + +absl::StatusOr> +CreateOpenCvConverter(const TensorsToSegmentationCalculatorOptions& options) { + auto converter = std::make_unique(); + MP_RETURN_IF_ERROR(converter->Init(options)); + return converter; +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h new file mode 100644 index 000000000..3ae41b5e0 --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h @@ -0,0 +1,31 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ + +#include + +#include "absl/status/statusor.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" + +namespace mediapipe { +// Creates OpenCV tensors-to-segmentation converter. +absl::StatusOr> +CreateOpenCvConverter( + const mediapipe::TensorsToSegmentationCalculatorOptions& options); +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc new file mode 100644 index 000000000..ab1e9c139 --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc @@ -0,0 +1,52 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" + +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/port.h" +#include "mediapipe/framework/port/ret_check.h" + +namespace mediapipe { + +int NumGroups(int size, int group_size) { + return (size + group_size - 1) / group_size; +} + +bool CanUseGpu() { +#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED + // TODO: Configure GPU usage policy in individual calculators. + constexpr bool kAllowGpuProcessing = true; + return kAllowGpuProcessing; +#else + return false; +#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED +} + +absl::StatusOr> GetHwcFromDims( + const std::vector& dims) { + if (dims.size() == 3) { + return std::make_tuple(dims[0], dims[1], dims[2]); + } else if (dims.size() == 4) { + // BHWC format check B == 1 + RET_CHECK_EQ(dims[0], 1) << "Expected batch to be 1 for BHWC heatmap"; + return std::make_tuple(dims[1], dims[2], dims[3]); + } else { + RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size(); + } +} +} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h new file mode 100644 index 000000000..44893073b --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h @@ -0,0 +1,34 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ + +#include +#include + +#include "absl/status/statusor.h" + +namespace mediapipe { + +// Commonly used to compute the number of blocks to launch in a kernel. +int NumGroups(const int size, const int group_size); // NOLINT + +bool CanUseGpu(); + +absl::StatusOr> GetHwcFromDims( + const std::vector& dims); +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc new file mode 100644 index 000000000..5535d159d --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc @@ -0,0 +1,63 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" + +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/status_matchers.h" + +namespace mediapipe { +namespace { + +using ::testing::HasSubstr; + +TEST(TensorsToSegmentationUtilsTest, NumGroupsWorksProperly) { + EXPECT_EQ(NumGroups(13, 4), 4); + EXPECT_EQ(NumGroups(4, 13), 1); +} + +TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsWorksProperly) { + std::vector dims_3 = {2, 3, 4}; + absl::StatusOr> result_1 = GetHwcFromDims(dims_3); + MP_ASSERT_OK(result_1); + EXPECT_EQ(result_1.value(), (std::make_tuple(2, 3, 4))); + std::vector dims_4 = {1, 3, 4, 5}; + absl::StatusOr> result_2 = GetHwcFromDims(dims_4); + MP_ASSERT_OK(result_2); + EXPECT_EQ(result_2.value(), (std::make_tuple(3, 4, 5))); +} + +TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsBatchCheckFail) { + std::vector dims_4 = {2, 3, 4, 5}; + absl::StatusOr> result = GetHwcFromDims(dims_4); + EXPECT_FALSE(result.ok()); + EXPECT_THAT(result.status().message(), + HasSubstr("Expected batch to be 1 for BHWC heatmap")); +} + +TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsInvalidShape) { + std::vector dims_5 = {1, 2, 3, 4, 5}; + absl::StatusOr> result = GetHwcFromDims(dims_5); + EXPECT_FALSE(result.ok()); + EXPECT_THAT(result.status().message(), + HasSubstr("Invalid shape for segmentation tensor")); +} + +} // namespace +} // namespace mediapipe From fd4859c1787c5b502b65cfc294718aeea1493ddf Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 9 Nov 2023 20:04:36 -0800 Subject: [PATCH 097/157] Refactor OpenCV path out of TensorsToSegmentationCalculator main file. ProcessCpu() is changed into an OpenCV converter that is owned by the calculator. The calculator should call converter.Convert() to get the conversion result. PiperOrigin-RevId: 581103226 --- mediapipe/calculators/tensor/BUILD | 68 +----- .../tensors_to_segmentation_calculator.cc | 227 +++++++++++++----- .../tensors_to_segmentation_converter.h | 43 ---- ...ensors_to_segmentation_converter_opencv.cc | 157 ------------ ...tensors_to_segmentation_converter_opencv.h | 31 --- .../tensor/tensors_to_segmentation_utils.cc | 52 ---- .../tensor/tensors_to_segmentation_utils.h | 34 --- .../tensors_to_segmentation_utils_test.cc | 63 ----- 8 files changed, 179 insertions(+), 496 deletions(-) delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter.h delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils.h delete mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index c0db4e35b..ac2ced837 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -1405,10 +1405,15 @@ mediapipe_proto_library( cc_library( name = "tensors_to_segmentation_calculator", srcs = ["tensors_to_segmentation_calculator.cc"], + copts = select({ + "//mediapipe:apple": [ + "-x objective-c++", + "-fobjc-arc", # enable reference-counting + ], + "//conditions:default": [], + }), deps = [ ":tensors_to_segmentation_calculator_cc_proto", - ":tensors_to_segmentation_converter", - ":tensors_to_segmentation_utils", "//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:port", @@ -1416,11 +1421,9 @@ cc_library( "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/util:resource_util", - "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", @@ -1431,7 +1434,6 @@ cc_library( "//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_simple_shaders", "//mediapipe/gpu:gpu_buffer", - "//mediapipe/gpu:gpu_buffer_format", "//mediapipe/gpu:shader_util", ], }) + selects.with_or({ @@ -1451,65 +1453,13 @@ cc_library( }) + select({ "//mediapipe/framework/port:disable_opencv": [], "//conditions:default": [ - ":tensors_to_segmentation_converter_opencv", + "//mediapipe/framework/formats:image_opencv", + "//mediapipe/framework/port:opencv_imgproc", ], }), alwayslink = 1, ) -cc_library( - name = "tensors_to_segmentation_utils", - srcs = ["tensors_to_segmentation_utils.cc"], - hdrs = ["tensors_to_segmentation_utils.h"], - deps = [ - "//mediapipe/framework:port", - "//mediapipe/framework/port:ret_check", - "@com_google_absl//absl/status:statusor", - ], -) - -cc_test( - name = "tensors_to_segmentation_utils_test", - srcs = ["tensors_to_segmentation_utils_test.cc"], - deps = [ - ":tensors_to_segmentation_utils", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status_matchers", - "@com_google_absl//absl/status:statusor", - ], -) - -cc_library( - name = "tensors_to_segmentation_converter", - hdrs = ["tensors_to_segmentation_converter.h"], - deps = [ - "//mediapipe/framework/formats:image", - "//mediapipe/framework/formats:tensor", - "@com_google_absl//absl/status:statusor", - ], -) - -cc_library( - name = "tensors_to_segmentation_converter_opencv", - srcs = ["tensors_to_segmentation_converter_opencv.cc"], - hdrs = ["tensors_to_segmentation_converter_opencv.h"], - deps = [ - ":tensors_to_segmentation_calculator_cc_proto", - ":tensors_to_segmentation_converter", - ":tensors_to_segmentation_utils", - "//mediapipe/framework/formats:image", - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:image_opencv", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:opencv_core", - "//mediapipe/framework/port:opencv_imgproc", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - ], -) - cc_test( name = "tensors_to_segmentation_calculator_test", srcs = ["tensors_to_segmentation_calculator_test.cc"], diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc index 6164c7b0a..24fd1bd52 100644 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc @@ -12,35 +12,32 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include -#include -#include #include -#include "absl/status/status.h" -#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/types/span.h" #include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port.h" #include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/framework/port/statusor.h" #include "mediapipe/gpu/gpu_origin.pb.h" +#include "mediapipe/util/resource_util.h" +#include "tensorflow/lite/interpreter.h" #if !MEDIAPIPE_DISABLE_GPU #include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/gpu_buffer_format.h" +#include "mediapipe/gpu/gpu_buffer.h" #include "mediapipe/gpu/shader_util.h" #endif // !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_OPENCV -#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h" +#include "mediapipe/framework/formats/image_opencv.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" #endif // !MEDIAPIPE_DISABLE_OPENCV #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 @@ -65,9 +62,37 @@ namespace { constexpr int kWorkgroupSize = 8; // Block size for GPU shader. enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; +// Commonly used to compute the number of blocks to launch in a kernel. +int NumGroups(const int size, const int group_size) { // NOLINT + return (size + group_size - 1) / group_size; +} + +bool CanUseGpu() { +#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED + // TODO: Configure GPU usage policy in individual calculators. + constexpr bool kAllowGpuProcessing = true; + return kAllowGpuProcessing; +#else + return false; +#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED +} + constexpr char kTensorsTag[] = "TENSORS"; constexpr char kOutputSizeTag[] = "OUTPUT_SIZE"; constexpr char kMaskTag[] = "MASK"; + +absl::StatusOr> GetHwcFromDims( + const std::vector& dims) { + if (dims.size() == 3) { + return std::make_tuple(dims[0], dims[1], dims[2]); + } else if (dims.size() == 4) { + // BHWC format check B == 1 + RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap"; + return std::make_tuple(dims[1], dims[2], dims[3]); + } else { + RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size(); + } +} } // namespace namespace mediapipe { @@ -131,28 +156,19 @@ class TensorsToSegmentationCalculator : public CalculatorBase { private: absl::Status LoadOptions(CalculatorContext* cc); absl::Status InitGpu(CalculatorContext* cc); - absl::Status ProcessGpu(CalculatorContext* cc, - const std::vector& input_tensors, - std::tuple hwc, int output_width, - int output_height); + absl::Status ProcessGpu(CalculatorContext* cc); + absl::Status ProcessCpu(CalculatorContext* cc); void GlRender(); bool DoesGpuTextureStartAtBottom() { return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT; } - absl::Status InitConverterIfNecessary() { -#if !MEDIAPIPE_DISABLE_OPENCV - if (!cpu_converter_) { - MP_ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(options_)); - } -#else - RET_CHECK_FAIL() << "OpenCV processing disabled."; -#endif // !MEDIAPIPE_DISABLE_OPENCV - return absl::OkStatus(); - } - mediapipe::TensorsToSegmentationCalculatorOptions options_; - std::unique_ptr cpu_converter_; +#if !MEDIAPIPE_DISABLE_OPENCV + template + absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); +#endif // !MEDIAPIPE_DISABLE_OPENCV + ::mediapipe::TensorsToSegmentationCalculatorOptions options_; #if !MEDIAPIPE_DISABLE_GPU mediapipe::GlCalculatorHelper gpu_helper_; @@ -245,7 +261,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); int tensor_channels = std::get<2>(hwc); - using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; + typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; switch (options_.activation()) { case Options::NONE: RET_CHECK_EQ(tensor_channels, 1); @@ -259,17 +275,6 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { } } - // Get dimensions. - MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); - auto [tensor_height, tensor_width, tensor_channels] = hwc; - int output_width = tensor_width, output_height = tensor_height; - if (cc->Inputs().HasTag(kOutputSizeTag)) { - const auto& size = - cc->Inputs().Tag(kOutputSizeTag).Get>(); - output_width = size.first; - output_height = size.second; - } - if (use_gpu) { #if !MEDIAPIPE_DISABLE_GPU if (!gpu_initialized_) { @@ -281,25 +286,16 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { #endif // !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR( - gpu_helper_.RunInGlContext([this, cc, &input_tensors, output_width, - output_height, hwc]() -> absl::Status { - MP_RETURN_IF_ERROR( - ProcessGpu(cc, input_tensors, hwc, output_width, output_height)); - return absl::OkStatus(); - })); + MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { + MP_RETURN_IF_ERROR(ProcessGpu(cc)); + return absl::OkStatus(); + })); #else RET_CHECK_FAIL() << "GPU processing disabled."; #endif // !MEDIAPIPE_DISABLE_GPU } else { #if !MEDIAPIPE_DISABLE_OPENCV - // Lazily initialize converter. - MP_RETURN_IF_ERROR(InitConverterIfNecessary()); - MP_ASSIGN_OR_RETURN( - std::unique_ptr output_mask, - cpu_converter_->Convert(input_tensors, output_width, output_height)); - cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), - cc->InputTimestamp()); + MP_RETURN_IF_ERROR(ProcessCpu(cc)); #else RET_CHECK_FAIL() << "OpenCV processing disabled."; #endif // !MEDIAPIPE_DISABLE_OPENCV @@ -333,15 +329,132 @@ absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) { return absl::OkStatus(); } +absl::Status TensorsToSegmentationCalculator::ProcessCpu( + CalculatorContext* cc) { +#if !MEDIAPIPE_DISABLE_OPENCV + // Get input streams, and dimensions. + const auto& input_tensors = + cc->Inputs().Tag(kTensorsTag).Get>(); + MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); + auto [tensor_height, tensor_width, tensor_channels] = hwc; + int output_width = tensor_width, output_height = tensor_height; + if (cc->Inputs().HasTag(kOutputSizeTag)) { + const auto& size = + cc->Inputs().Tag(kOutputSizeTag).Get>(); + output_width = size.first; + output_height = size.second; + } + + // Create initial working mask. + cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1); + + // Wrap input tensor. + auto raw_input_tensor = &input_tensors[0]; + auto raw_input_view = raw_input_tensor->GetCpuReadView(); + const float* raw_input_data = raw_input_view.buffer(); + cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height), + CV_MAKETYPE(CV_32F, tensor_channels), + const_cast(raw_input_data)); + + // Process mask tensor and apply activation function. + if (tensor_channels == 2) { + MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); + } else if (tensor_channels == 1) { + RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX != + options_.activation()); // Requires 2 channels. + if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE == + options_.activation()) // Pass-through optimization. + tensor_mat.copyTo(small_mask_mat); + else + MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); + } else { + RET_CHECK_FAIL() << "Unsupported number of tensor channels " + << tensor_channels; + } + + // Send out image as CPU packet. + std::shared_ptr mask_frame = std::make_shared( + ImageFormat::VEC32F1, output_width, output_height); + std::unique_ptr output_mask = absl::make_unique(mask_frame); + auto output_mat = formats::MatView(output_mask.get()); + // Upsample small mask into output. + cv::resize(small_mask_mat, *output_mat, + cv::Size(output_width, output_height)); + cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp()); +#endif // !MEDIAPIPE_DISABLE_OPENCV + + return absl::OkStatus(); +} + +#if !MEDIAPIPE_DISABLE_OPENCV +template +absl::Status TensorsToSegmentationCalculator::ApplyActivation( + cv::Mat& tensor_mat, cv::Mat* small_mask_mat) { + // Configure activation function. + const int output_layer_index = options_.output_layer_index(); + typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; + const auto activation_fn = [&](const cv::Vec2f& mask_value) { + float new_mask_value = 0; + // TODO consider moving switch out of the loop, + // and also avoid float/Vec2f casting. + switch (options_.activation()) { + case Options::NONE: { + new_mask_value = mask_value[0]; + break; + } + case Options::SIGMOID: { + const float pixel0 = mask_value[0]; + new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0); + break; + } + case Options::SOFTMAX: { + const float pixel0 = mask_value[0]; + const float pixel1 = mask_value[1]; + const float max_pixel = std::max(pixel0, pixel1); + const float min_pixel = std::min(pixel0, pixel1); + const float softmax_denom = + /*exp(max_pixel - max_pixel)=*/1.0f + + std::exp(min_pixel - max_pixel); + new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) / + softmax_denom; + break; + } + } + return new_mask_value; + }; + + // Process mask tensor. + for (int i = 0; i < tensor_mat.rows; ++i) { + for (int j = 0; j < tensor_mat.cols; ++j) { + const T& input_pix = tensor_mat.at(i, j); + const float mask_value = activation_fn(input_pix); + small_mask_mat->at(i, j) = mask_value; + } + } + + return absl::OkStatus(); +} +#endif // !MEDIAPIPE_DISABLE_OPENCV + // Steps: // 1. receive tensor // 2. process segmentation tensor into small mask // 3. upsample small mask into output mask to be same size as input image absl::Status TensorsToSegmentationCalculator::ProcessGpu( - CalculatorContext* cc, const std::vector& input_tensors, - std::tuple hwc, int output_width, int output_height) { + CalculatorContext* cc) { #if !MEDIAPIPE_DISABLE_GPU + // Get input streams, and dimensions. + const auto& input_tensors = + cc->Inputs().Tag(kTensorsTag).Get>(); + MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); auto [tensor_height, tensor_width, tensor_channels] = hwc; + int output_width = tensor_width, output_height = tensor_height; + if (cc->Inputs().HasTag(kOutputSizeTag)) { + const auto& size = + cc->Inputs().Tag(kOutputSizeTag).Get>(); + output_width = size.first; + output_height = size.second; + } // Create initial working mask texture. #if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31) @@ -519,7 +632,7 @@ void TensorsToSegmentationCalculator::GlRender() { absl::Status TensorsToSegmentationCalculator::LoadOptions( CalculatorContext* cc) { // Get calculator options specified in the graph. - options_ = cc->Options(); + options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>(); return absl::OkStatus(); } @@ -713,7 +826,7 @@ void main() { #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 // Shader defines. - using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; + typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; const std::string output_layer_index = "\n#define OUTPUT_LAYER_INDEX int(" + std::to_string(options_.output_layer_index()) + ")"; diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h b/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h deleted file mode 100644 index 61d95dfe0..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ - -#include -#include - -#include "absl/status/statusor.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/tensor.h" - -namespace mediapipe { - -class TensorsToSegmentationConverter { - public: - virtual ~TensorsToSegmentationConverter() = default; - - // Converts tensors to image mask. - // Returns a unique pointer containing the converted image. - // @input_tensors contains the tensors needed to be processed. - // @output_width/height describes output dimensions to reshape the output mask - // into. - virtual absl::StatusOr> Convert( - const std::vector& input_tensors, int output_width, - int output_height) = 0; -}; - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc deleted file mode 100644 index 1ee2e172b..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h" - -#include -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" -#include "mediapipe/framework/formats/image.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/image_opencv.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/opencv_core_inc.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status_macros.h" - -namespace mediapipe { -namespace { - -class OpenCvProcessor : public TensorsToSegmentationConverter { - public: - absl::Status Init(const TensorsToSegmentationCalculatorOptions& options) { - options_ = options; - return absl::OkStatus(); - } - - absl::StatusOr> Convert( - const std::vector& input_tensors, int output_width, - int output_height) override; - - private: - template - absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); - - TensorsToSegmentationCalculatorOptions options_; -}; - -absl::StatusOr> OpenCvProcessor::Convert( - const std::vector& input_tensors, int output_width, - int output_height) { - MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); - auto [tensor_height, tensor_width, tensor_channels] = hwc; - // Create initial working mask. - cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1); - - // Wrap input tensor. - auto raw_input_tensor = &input_tensors[0]; - auto raw_input_view = raw_input_tensor->GetCpuReadView(); - const float* raw_input_data = raw_input_view.buffer(); - cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height), - CV_MAKETYPE(CV_32F, tensor_channels), - const_cast(raw_input_data)); - - // Process mask tensor and apply activation function. - if (tensor_channels == 2) { - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else if (tensor_channels == 1) { - RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX != - options_.activation()); // Requires 2 channels. - if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE == - options_.activation()) // Pass-through optimization. - tensor_mat.copyTo(small_mask_mat); - else - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else { - RET_CHECK_FAIL() << "Unsupported number of tensor channels " - << tensor_channels; - } - - // Send out image as CPU packet. - std::shared_ptr mask_frame = std::make_shared( - ImageFormat::VEC32F1, output_width, output_height); - auto output_mask = std::make_unique(mask_frame); - auto output_mat = formats::MatView(output_mask.get()); - // Upsample small mask into output. - cv::resize(small_mask_mat, *output_mat, - cv::Size(output_width, output_height)); - return output_mask; -} - -template -absl::Status OpenCvProcessor::ApplyActivation(cv::Mat& tensor_mat, - cv::Mat* small_mask_mat) { - // Configure activation function. - const int output_layer_index = options_.output_layer_index(); - using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; - const auto activation_fn = [&](const cv::Vec2f& mask_value) { - float new_mask_value = 0; - // TODO consider moving switch out of the loop, - // and also avoid float/Vec2f casting. - switch (options_.activation()) { - case Options::NONE: { - new_mask_value = mask_value[0]; - break; - } - case Options::SIGMOID: { - const float pixel0 = mask_value[0]; - new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0); - break; - } - case Options::SOFTMAX: { - const float pixel0 = mask_value[0]; - const float pixel1 = mask_value[1]; - const float max_pixel = std::max(pixel0, pixel1); - const float min_pixel = std::min(pixel0, pixel1); - const float softmax_denom = - /*exp(max_pixel - max_pixel)=*/1.0f + - std::exp(min_pixel - max_pixel); - new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) / - softmax_denom; - break; - } - } - return new_mask_value; - }; - - // Process mask tensor. - for (int i = 0; i < tensor_mat.rows; ++i) { - for (int j = 0; j < tensor_mat.cols; ++j) { - const T& input_pix = tensor_mat.at(i, j); - const float mask_value = activation_fn(input_pix); - small_mask_mat->at(i, j) = mask_value; - } - } - - return absl::OkStatus(); -} - -} // namespace - -absl::StatusOr> -CreateOpenCvConverter(const TensorsToSegmentationCalculatorOptions& options) { - auto converter = std::make_unique(); - MP_RETURN_IF_ERROR(converter->Init(options)); - return converter; -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h deleted file mode 100644 index 3ae41b5e0..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ - -#include - -#include "absl/status/statusor.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" -#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" - -namespace mediapipe { -// Creates OpenCV tensors-to-segmentation converter. -absl::StatusOr> -CreateOpenCvConverter( - const mediapipe::TensorsToSegmentationCalculatorOptions& options); -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc deleted file mode 100644 index ab1e9c139..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" - -#include -#include - -#include "absl/status/statusor.h" -#include "mediapipe/framework/port.h" -#include "mediapipe/framework/port/ret_check.h" - -namespace mediapipe { - -int NumGroups(int size, int group_size) { - return (size + group_size - 1) / group_size; -} - -bool CanUseGpu() { -#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED - // TODO: Configure GPU usage policy in individual calculators. - constexpr bool kAllowGpuProcessing = true; - return kAllowGpuProcessing; -#else - return false; -#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED -} - -absl::StatusOr> GetHwcFromDims( - const std::vector& dims) { - if (dims.size() == 3) { - return std::make_tuple(dims[0], dims[1], dims[2]); - } else if (dims.size() == 4) { - // BHWC format check B == 1 - RET_CHECK_EQ(dims[0], 1) << "Expected batch to be 1 for BHWC heatmap"; - return std::make_tuple(dims[1], dims[2], dims[3]); - } else { - RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size(); - } -} -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h deleted file mode 100644 index 44893073b..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ - -#include -#include - -#include "absl/status/statusor.h" - -namespace mediapipe { - -// Commonly used to compute the number of blocks to launch in a kernel. -int NumGroups(const int size, const int group_size); // NOLINT - -bool CanUseGpu(); - -absl::StatusOr> GetHwcFromDims( - const std::vector& dims); -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc deleted file mode 100644 index 5535d159d..000000000 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" - -#include -#include - -#include "absl/status/statusor.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" - -namespace mediapipe { -namespace { - -using ::testing::HasSubstr; - -TEST(TensorsToSegmentationUtilsTest, NumGroupsWorksProperly) { - EXPECT_EQ(NumGroups(13, 4), 4); - EXPECT_EQ(NumGroups(4, 13), 1); -} - -TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsWorksProperly) { - std::vector dims_3 = {2, 3, 4}; - absl::StatusOr> result_1 = GetHwcFromDims(dims_3); - MP_ASSERT_OK(result_1); - EXPECT_EQ(result_1.value(), (std::make_tuple(2, 3, 4))); - std::vector dims_4 = {1, 3, 4, 5}; - absl::StatusOr> result_2 = GetHwcFromDims(dims_4); - MP_ASSERT_OK(result_2); - EXPECT_EQ(result_2.value(), (std::make_tuple(3, 4, 5))); -} - -TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsBatchCheckFail) { - std::vector dims_4 = {2, 3, 4, 5}; - absl::StatusOr> result = GetHwcFromDims(dims_4); - EXPECT_FALSE(result.ok()); - EXPECT_THAT(result.status().message(), - HasSubstr("Expected batch to be 1 for BHWC heatmap")); -} - -TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsInvalidShape) { - std::vector dims_5 = {1, 2, 3, 4, 5}; - absl::StatusOr> result = GetHwcFromDims(dims_5); - EXPECT_FALSE(result.ok()); - EXPECT_THAT(result.status().message(), - HasSubstr("Invalid shape for segmentation tensor")); -} - -} // namespace -} // namespace mediapipe From d772bf8134d54d7106632205fd13b15ddd9532d2 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 10 Nov 2023 09:01:33 -0800 Subject: [PATCH 098/157] Add BinaryAUC metric and Best Checkpoint callback to Text Classifier PiperOrigin-RevId: 581276382 --- .../model_maker/python/core/utils/metrics.py | 17 +++++++++++++++++ .../python/core/utils/metrics_test.py | 12 ++++++++---- .../text/text_classifier/text_classifier.py | 17 ++++++++++++++--- .../text_classifier/text_classifier_test.py | 1 + 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/mediapipe/model_maker/python/core/utils/metrics.py b/mediapipe/model_maker/python/core/utils/metrics.py index 310146168..cf0be6d08 100644 --- a/mediapipe/model_maker/python/core/utils/metrics.py +++ b/mediapipe/model_maker/python/core/utils/metrics.py @@ -94,6 +94,23 @@ def _get_sparse_metric(metric: tf.metrics.Metric): return SparseMetric +class BinaryAUC(tf.keras.metrics.AUC): + """A Binary AUC metric for binary classification tasks. + + For update state, the shapes of y_true and y_pred are expected to be: + - y_true: [batch_size x 1] array of 0 for negatives and 1 for positives + - y_pred: [batch_size x 2] array of probabilities where y_pred[:,0] are the + probabilities of the 0th(negative) class and y_pred[:,1] are the + probabilities of the 1st(positive) class + + See https://www.tensorflow.org/api_docs/python/tf/keras/metrics/AUC for + details. + """ + + def update_state(self, y_true, y_pred, sample_weight=None): + super().update_state(y_true, y_pred[:, 1], sample_weight) + + SparseRecall = _get_sparse_metric(tf.metrics.Recall) SparsePrecision = _get_sparse_metric(tf.metrics.Precision) BinarySparseRecallAtPrecision = _get_binary_sparse_metric( diff --git a/mediapipe/model_maker/python/core/utils/metrics_test.py b/mediapipe/model_maker/python/core/utils/metrics_test.py index 842335273..2ea8769d2 100644 --- a/mediapipe/model_maker/python/core/utils/metrics_test.py +++ b/mediapipe/model_maker/python/core/utils/metrics_test.py @@ -14,6 +14,7 @@ from absl.testing import parameterized +import numpy as np import tensorflow as tf from mediapipe.model_maker.python.core.utils import metrics @@ -23,16 +24,15 @@ class SparseMetricTest(tf.test.TestCase, parameterized.TestCase): def setUp(self): super().setUp() - self.y_true = [0, 0, 1, 1, 0, 1] - self.y_pred = [ + self.y_true = np.array([0, 0, 1, 1, 0, 1]) + self.y_pred = np.array([ [0.9, 0.1], # 0, 0 y [0.8, 0.2], # 0, 0 y [0.7, 0.3], # 0, 1 n [0.6, 0.4], # 0, 1 n [0.3, 0.7], # 1, 0 y [0.3, 0.7], # 1, 1 y - ] - self.num_classes = 3 + ]) def _assert_metric_equals(self, metric, value): metric.update_state(self.y_true, self.y_pred) @@ -69,6 +69,10 @@ class SparseMetricTest(tf.test.TestCase, parameterized.TestCase): ): _ = metrics.BinarySparsePrecisionAtRecall(1.0, class_id=2) + def test_binary_auc(self): + metric = metrics.BinaryAUC(num_thresholds=1000) + self._assert_metric_equals(metric, 0.7222222) + if __name__ == '__main__': tf.test.main() diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py index 386e9360e..aea9224ff 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier.py @@ -372,9 +372,19 @@ class _BertClassifier(TextClassifier): ): super().__init__(model_spec, label_names, hparams.shuffle) self._hparams = hparams - self._callbacks = model_util.get_default_callbacks( - self._hparams.export_dir, self._hparams.checkpoint_frequency - ) + self._callbacks = list( + model_util.get_default_callbacks( + self._hparams.export_dir, self._hparams.checkpoint_frequency + ) + ) + [ + tf.keras.callbacks.ModelCheckpoint( + os.path.join(self._hparams.export_dir, "best_model"), + monitor="val_auc", + mode="max", + save_best_only=True, + save_weights_only=False, + ) + ] self._model_options = model_options self._text_preprocessor: preprocessor.BertClassifierPreprocessor = None with self._hparams.get_strategy().scope(): @@ -465,6 +475,7 @@ class _BertClassifier(TextClassifier): ), metrics.SparsePrecision(name="precision", dtype=tf.float32), metrics.SparseRecall(name="recall", dtype=tf.float32), + metrics.BinaryAUC(name="auc", num_thresholds=1000), ] if self._num_classes == 2: if self._hparams.desired_precisions: diff --git a/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py b/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py index fdc2613a9..02b3fe4d5 100644 --- a/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py +++ b/mediapipe/model_maker/python/text/text_classifier/text_classifier_test.py @@ -230,6 +230,7 @@ class TextClassifierTest(tf.test.TestCase, parameterized.TestCase): 'accuracy', 'recall', 'precision', + 'auc', 'precision_at_recall_0.2', 'recall_at_precision_0.9', ] From 64b21d758ed06872e51dfa39e1555258f94e0602 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 10 Nov 2023 10:00:56 -0800 Subject: [PATCH 099/157] Remove batch dimension from the output of tflite_with_tokenizer in text classifier. PiperOrigin-RevId: 581292824 --- .../python/text/text_classifier/model_with_tokenizer.py | 2 +- .../python/text/text_classifier/model_with_tokenizer_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer.py b/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer.py index 95328fb43..a96fe1b84 100644 --- a/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer.py +++ b/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer.py @@ -32,4 +32,4 @@ class ModelWithTokenizer(tf.keras.Model): x = self._tokenizer.process_fn(input_tensor) x = {k: tf.expand_dims(v, axis=0) for k, v in x.items()} x = self._model(x) - return x + return x[0] # TODO: Add back the batch dimension diff --git a/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer_test.py b/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer_test.py index f6c5d2477..1da09ab4e 100644 --- a/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer_test.py +++ b/mediapipe/model_maker/python/text/text_classifier/model_with_tokenizer_test.py @@ -97,7 +97,7 @@ class BertTokenizerTest(tf.test.TestCase): self._tokenizer, self._model ) output = model(tf.constant(["Example input".encode("utf-8")])) - self.assertAllEqual(output.shape, (1, 2)) + self.assertAllEqual(output.shape, (2,)) self.assertEqual(tf.reduce_sum(output), 1) From 5dec91226d42c6bd4f7b2c2050812b40f57c3b69 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 10 Nov 2023 11:36:44 -0800 Subject: [PATCH 100/157] No public description PiperOrigin-RevId: 581322099 --- mediapipe/framework/tool/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mediapipe/framework/tool/BUILD b/mediapipe/framework/tool/BUILD index 65c8a15c8..7a4b5a112 100644 --- a/mediapipe/framework/tool/BUILD +++ b/mediapipe/framework/tool/BUILD @@ -857,6 +857,7 @@ cc_library( mediapipe_cc_test( name = "switch_demux_calculator_test", srcs = ["switch_demux_calculator_test.cc"], + requires_full_emulation = False, deps = [ ":container_util", ":switch_demux_calculator", @@ -892,6 +893,7 @@ cc_library( mediapipe_cc_test( name = "switch_mux_calculator_test", srcs = ["switch_mux_calculator_test.cc"], + requires_full_emulation = False, deps = [ ":container_util", ":switch_mux_calculator", From 4ad67abd7093e064788f8b934a73ed3da67d57f8 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 10 Nov 2023 14:44:19 -0800 Subject: [PATCH 101/157] ...internal change... PiperOrigin-RevId: 581375224 --- mediapipe/modules/face_geometry/data/BUILD | 14 + .../data/face_model_with_iris.obj | 1862 ++++++ ...ne_metadata_including_iris_landmarks.pbtxt | 5160 +++++++++++++++++ 3 files changed, 7036 insertions(+) create mode 100644 mediapipe/modules/face_geometry/data/face_model_with_iris.obj create mode 100644 mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_including_iris_landmarks.pbtxt diff --git a/mediapipe/modules/face_geometry/data/BUILD b/mediapipe/modules/face_geometry/data/BUILD index 1661a2283..5a7d795a1 100644 --- a/mediapipe/modules/face_geometry/data/BUILD +++ b/mediapipe/modules/face_geometry/data/BUILD @@ -57,3 +57,17 @@ exports_files([ "canonical_face_model.obj", "canonical_face_model_uv_visualization.png", ]) + +# This metadata contains the 468 landmarks copied from geometry_pipeline_metadata_landmarks.pbtxt +# plus 10 extra landmarks representing the two irises. The position of each iris landmark was +# calculated using neighbor vertices through linear interpolation. Visual inspection suggests +# these positions are good. The UV coordinates are also estimated using a similar approach. +encode_binary_proto( + name = "geometry_pipeline_metadata_including_iris_landmarks", + input = "geometry_pipeline_metadata_including_iris_landmarks.pbtxt", + message_type = "mediapipe.face_geometry.GeometryPipelineMetadata", + output = "geometry_pipeline_metadata_including_iris_landmarks.binarypb", + deps = [ + "//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_proto", + ], +) diff --git a/mediapipe/modules/face_geometry/data/face_model_with_iris.obj b/mediapipe/modules/face_geometry/data/face_model_with_iris.obj new file mode 100644 index 000000000..29174b749 --- /dev/null +++ b/mediapipe/modules/face_geometry/data/face_model_with_iris.obj @@ -0,0 +1,1862 @@ +v 0.000000 -3.406404 5.979507 +v 0.000000 -1.126865 7.475604 +v 0.000000 -2.089024 6.058267 +v -0.463928 0.955357 6.633583 +v 0.000000 -0.463170 7.586580 +v 0.000000 0.365669 7.242870 +v 0.000000 2.473255 5.788627 +v -4.253081 2.577646 3.279702 +v 0.000000 4.019042 5.284764 +v 0.000000 4.885979 5.385258 +v 0.000000 8.261778 4.481535 +v 0.000000 -3.706811 5.864924 +v 0.000000 -3.918301 5.569430 +v 0.000000 -3.994436 5.219482 +v 0.000000 -4.542400 5.404754 +v 0.000000 -4.745577 5.529457 +v 0.000000 -5.019567 5.601448 +v 0.000000 -5.365123 5.535441 +v 0.000000 -6.149624 5.071372 +v 0.000000 -1.501095 7.112196 +v -0.416106 -1.466449 6.447657 +v -7.087960 5.434801 0.099620 +v -2.628639 2.035898 3.848121 +v -3.198363 1.985815 3.796952 +v -3.775151 2.039402 3.646194 +v -4.465819 2.422950 3.155168 +v -2.164289 2.189867 3.851822 +v -3.208229 3.223926 4.115822 +v -2.673803 3.205337 4.092203 +v -3.745193 3.165286 3.972409 +v -4.161018 3.059069 3.719554 +v -5.062006 1.934418 2.776093 +v -2.266659 -7.425768 4.389812 +v -4.445859 2.663991 3.173422 +v -7.214530 2.263009 0.073150 +v -5.799793 2.349546 2.204059 +v -2.844939 -0.720868 4.433130 +v -0.711452 -3.329355 5.877044 +v -0.606033 -3.924562 5.444923 +v -1.431615 -3.500953 5.496189 +v -1.914910 -3.803146 5.028930 +v -1.131043 -3.973937 5.189648 +v -1.563548 -4.082763 4.842263 +v -2.650112 -5.003649 4.188483 +v -0.427049 -1.094134 7.360529 +v -0.496396 -0.475659 7.440358 +v -5.253307 3.881582 3.363159 +v -1.718698 0.974609 4.558359 +v -1.608635 -0.942516 5.814193 +v -1.651267 -0.610868 5.581319 +v -4.765501 -0.701554 3.534632 +v -0.478306 0.295766 7.101013 +v -3.734964 4.508230 4.550454 +v -4.588603 4.302037 4.048484 +v -6.279331 6.615427 1.425850 +v -1.220941 4.142165 5.106035 +v -2.193489 3.100317 4.000575 +v -3.102642 -4.352984 4.095905 +v -6.719682 -4.788645 -1.745401 +v -1.193824 -1.306795 5.737747 +v -0.729766 -1.593712 5.833208 +v -2.456206 -4.342621 4.283884 +v -2.204823 -4.304508 4.162499 +v -4.985894 4.802461 3.751977 +v -1.592294 -1.257709 5.456949 +v -2.644548 4.524654 4.921559 +v -2.760292 5.100971 5.015990 +v -3.523964 8.005976 3.729163 +v -5.599763 5.715470 2.724259 +v -3.063932 6.566144 4.529981 +v -5.720968 4.254584 2.830852 +v -6.374393 4.785590 1.591691 +v -0.672728 -3.688016 5.737804 +v -1.262560 -3.787691 5.417779 +v -1.732553 -3.952767 5.000579 +v -1.043625 -1.464973 5.662455 +v -2.321234 -4.329069 4.258156 +v -2.056846 -4.477671 4.520883 +v -2.153084 -4.276322 4.038093 +v -0.946874 -1.035249 6.512274 +v -1.469132 -4.036351 4.604908 +v -1.024340 -3.989851 4.926693 +v -0.533422 -3.993222 5.138202 +v -0.769720 -6.095394 4.985883 +v -0.699606 -5.291850 5.448304 +v -0.669687 -4.949770 5.509612 +v -0.630947 -4.695101 5.449371 +v -0.583218 -4.517982 5.339869 +v -1.537170 -4.423206 4.745470 +v -1.615600 -4.475942 4.813632 +v -1.729053 -4.618680 4.854463 +v -1.838624 -4.828746 4.823737 +v -2.368250 -3.106237 4.868096 +v -7.542244 -1.049282 -2.431321 +v 0.000000 -1.724003 6.601390 +v -1.826614 -4.399531 4.399021 +v -1.929558 -4.411831 4.497052 +v -0.597442 -2.013686 5.866456 +v -1.405627 -1.714196 5.241087 +v -0.662449 -1.819321 5.863759 +v -2.342340 0.572222 4.294303 +v -3.327324 0.104863 4.113860 +v -1.726175 -0.919165 5.273355 +v -5.133204 7.485602 2.660442 +v -4.538641 6.319907 3.683424 +v -3.986562 5.109487 4.466315 +v -2.169681 -5.440433 4.455874 +v -1.395634 5.011963 5.316032 +v -1.619500 6.599217 4.921106 +v -1.891399 8.236377 4.274997 +v -4.195832 2.235205 3.375099 +v -5.733342 1.411738 2.431726 +v -1.859887 2.355757 3.843181 +v -4.988612 3.074654 3.083858 +v -1.303263 1.416453 4.831091 +v -1.305757 -0.672779 6.415959 +v -6.465170 0.937119 1.689873 +v -5.258659 0.945811 2.974312 +v -4.432338 0.722096 3.522615 +v -3.300681 0.861641 3.872784 +v -2.430178 1.131492 4.039035 +v -1.820731 1.467954 4.224124 +v -0.563221 2.307693 5.566789 +v -6.338145 -0.529279 1.881175 +v -5.587698 3.208071 2.687839 +v -0.242624 -1.462857 7.071491 +v -1.611251 0.339326 4.895421 +v -7.743095 2.364999 -2.005167 +v -1.391142 1.851048 4.448999 +v -1.785794 -0.978284 4.850470 +v -4.670959 2.664461 3.084075 +v -1.333970 -0.283761 6.097047 +v -7.270895 -2.890917 -2.252455 +v -1.856432 2.585245 3.757904 +v -0.923388 0.073076 6.671944 +v -5.000589 -6.135128 1.892523 +v -5.085276 -7.178590 0.714711 +v -7.159291 -0.811820 -0.072044 +v -5.843051 -5.248023 0.924091 +v -6.847258 3.662916 0.724695 +v -2.412942 -8.258853 4.119213 +v -0.179909 -1.689864 6.573301 +v -2.103655 -0.163946 4.566119 +v -6.407571 2.236021 1.560843 +v -3.670075 2.360153 3.635230 +v -3.177186 2.294265 3.775704 +v -2.196121 -4.598322 4.479786 +v -6.234883 -1.944430 1.663542 +v -1.292924 -9.295920 4.094063 +v -3.210651 -8.533278 2.802001 +v -4.068926 -7.993109 1.925119 +v 0.000000 6.545390 5.027311 +v 0.000000 -9.403378 4.264492 +v -2.724032 2.315802 3.777151 +v -2.288460 2.398891 3.697603 +v -1.998311 2.496547 3.689148 +v -6.130040 3.399261 2.038516 +v -2.288460 2.886504 3.775031 +v -2.724032 2.961810 3.871767 +v -3.177186 2.964136 3.876973 +v -3.670075 2.927714 3.724325 +v -4.018389 2.857357 3.482983 +v -7.555811 4.106811 -0.991917 +v -4.018389 2.483695 3.440898 +v 0.000000 -2.521945 5.932265 +v -1.776217 -2.683946 5.213116 +v -1.222237 -1.182444 5.952465 +v -0.731493 -2.536683 5.815343 +v 0.000000 3.271027 5.236015 +v -4.135272 -6.996638 2.671970 +v -3.311811 -7.660815 3.382963 +v -1.313701 -8.639995 4.702456 +v -5.940524 -6.223629 -0.631468 +v -1.998311 2.743838 3.744030 +v -0.901447 1.236992 5.754256 +v 0.000000 -8.765243 4.891441 +v -2.308977 -8.974196 3.609070 +v -6.954154 -2.439843 -0.131163 +v -1.098819 -4.458788 5.120727 +v -1.181124 -4.579996 5.189564 +v -1.255818 -4.787901 5.237051 +v -1.325085 -5.106507 5.205010 +v -1.546388 -5.819392 4.757893 +v -1.953754 -4.183892 4.431713 +v -2.117802 -4.137093 4.555096 +v -2.285339 -4.051196 4.582438 +v -2.850160 -3.665720 4.484994 +v -5.278538 -2.238942 2.861224 +v -0.946709 1.907628 5.196779 +v -1.314173 3.104912 4.231404 +v -1.780000 2.860000 3.881555 +v -1.845110 -4.098880 4.247264 +v -5.436187 -4.030482 2.109852 +v -0.766444 3.182131 4.861453 +v -1.938616 -6.614410 4.521085 +v 0.000000 1.059413 6.774605 +v -0.516573 1.583572 6.148363 +v 0.000000 1.728369 6.316750 +v -1.246815 0.230297 5.681036 +v 0.000000 -7.942194 5.181173 +v 0.000000 -6.991499 5.153478 +v -0.997827 -6.930921 4.979576 +v -3.288807 -5.382514 3.795752 +v -2.311631 -1.566237 4.590085 +v -2.680250 -6.111567 4.096152 +v -3.832928 -1.537326 4.137731 +v -2.961860 -2.274215 4.440943 +v -4.386901 -2.683286 3.643886 +v -1.217295 -7.834465 4.969286 +v -1.542374 -0.136843 5.201008 +v -3.878377 -6.041764 3.311079 +v -3.084037 -6.809842 3.814195 +v -3.747321 -4.503545 3.726453 +v -6.094129 -3.205991 1.473482 +v -4.588995 -4.728726 2.983221 +v -6.583231 -3.941269 0.070268 +v -3.492580 -3.195820 4.130198 +v -1.255543 0.802341 5.307551 +v -1.126122 -0.933602 6.538785 +v -1.443109 -1.142774 5.905127 +v -0.923043 -0.529042 7.003423 +v -1.755386 3.529117 4.327696 +v -2.632589 3.713828 4.364629 +v -3.388062 3.721976 4.309028 +v -4.075766 3.675413 4.076063 +v -4.622910 3.474691 3.646321 +v -5.171755 2.535753 2.670867 +v -7.297331 0.763172 -0.048769 +v -4.706828 1.651000 3.109532 +v -4.071712 1.476821 3.476944 +v -3.269817 1.470659 3.731945 +v -2.527572 1.617311 3.865444 +v -1.970894 1.858505 3.961782 +v -1.579543 2.097941 4.084996 +v -7.664182 0.673132 -2.435867 +v -1.397041 -1.340139 5.630378 +v -0.884838 0.658740 6.233232 +v -0.767097 -0.968035 7.077932 +v -0.460213 -1.334106 6.787447 +v -0.748618 -1.067994 6.798303 +v -1.236408 -1.585568 5.480490 +v -0.387306 -1.409990 6.957705 +v -0.319925 -1.607931 6.508676 +v -1.639633 2.556298 3.863736 +v -1.255645 2.467144 4.203800 +v -1.031362 2.382663 4.615849 +v -4.253081 2.772296 3.315305 +v -4.530000 2.910000 3.339685 +v 0.463928 0.955357 6.633583 +v 4.253081 2.577646 3.279702 +v 0.416106 -1.466449 6.447657 +v 7.087960 5.434801 0.099620 +v 2.628639 2.035898 3.848121 +v 3.198363 1.985815 3.796952 +v 3.775151 2.039402 3.646194 +v 4.465819 2.422950 3.155168 +v 2.164289 2.189867 3.851822 +v 3.208229 3.223926 4.115822 +v 2.673803 3.205337 4.092203 +v 3.745193 3.165286 3.972409 +v 4.161018 3.059069 3.719554 +v 5.062006 1.934418 2.776093 +v 2.266659 -7.425768 4.389812 +v 4.445859 2.663991 3.173422 +v 7.214530 2.263009 0.073150 +v 5.799793 2.349546 2.204059 +v 2.844939 -0.720868 4.433130 +v 0.711452 -3.329355 5.877044 +v 0.606033 -3.924562 5.444923 +v 1.431615 -3.500953 5.496189 +v 1.914910 -3.803146 5.028930 +v 1.131043 -3.973937 5.189648 +v 1.563548 -4.082763 4.842263 +v 2.650112 -5.003649 4.188483 +v 0.427049 -1.094134 7.360529 +v 0.496396 -0.475659 7.440358 +v 5.253307 3.881582 3.363159 +v 1.718698 0.974609 4.558359 +v 1.608635 -0.942516 5.814193 +v 1.651267 -0.610868 5.581319 +v 4.765501 -0.701554 3.534632 +v 0.478306 0.295766 7.101013 +v 3.734964 4.508230 4.550454 +v 4.588603 4.302037 4.048484 +v 6.279331 6.615427 1.425850 +v 1.220941 4.142165 5.106035 +v 2.193489 3.100317 4.000575 +v 3.102642 -4.352984 4.095905 +v 6.719682 -4.788645 -1.745401 +v 1.193824 -1.306795 5.737747 +v 0.729766 -1.593712 5.833208 +v 2.456206 -4.342621 4.283884 +v 2.204823 -4.304508 4.162499 +v 4.985894 4.802461 3.751977 +v 1.592294 -1.257709 5.456949 +v 2.644548 4.524654 4.921559 +v 2.760292 5.100971 5.015990 +v 3.523964 8.005976 3.729163 +v 5.599763 5.715470 2.724259 +v 3.063932 6.566144 4.529981 +v 5.720968 4.254584 2.830852 +v 6.374393 4.785590 1.591691 +v 0.672728 -3.688016 5.737804 +v 1.262560 -3.787691 5.417779 +v 1.732553 -3.952767 5.000579 +v 1.043625 -1.464973 5.662455 +v 2.321234 -4.329069 4.258156 +v 2.056846 -4.477671 4.520883 +v 2.153084 -4.276322 4.038093 +v 0.946874 -1.035249 6.512274 +v 1.469132 -4.036351 4.604908 +v 1.024340 -3.989851 4.926693 +v 0.533422 -3.993222 5.138202 +v 0.769720 -6.095394 4.985883 +v 0.699606 -5.291850 5.448304 +v 0.669687 -4.949770 5.509612 +v 0.630947 -4.695101 5.449371 +v 0.583218 -4.517982 5.339869 +v 1.537170 -4.423206 4.745470 +v 1.615600 -4.475942 4.813632 +v 1.729053 -4.618680 4.854463 +v 1.838624 -4.828746 4.823737 +v 2.368250 -3.106237 4.868096 +v 7.542244 -1.049282 -2.431321 +v 1.826614 -4.399531 4.399021 +v 1.929558 -4.411831 4.497052 +v 0.597442 -2.013686 5.866456 +v 1.405627 -1.714196 5.241087 +v 0.662449 -1.819321 5.863759 +v 2.342340 0.572222 4.294303 +v 3.327324 0.104863 4.113860 +v 1.726175 -0.919165 5.273355 +v 5.133204 7.485602 2.660442 +v 4.538641 6.319907 3.683424 +v 3.986562 5.109487 4.466315 +v 2.169681 -5.440433 4.455874 +v 1.395634 5.011963 5.316032 +v 1.619500 6.599217 4.921106 +v 1.891399 8.236377 4.274997 +v 4.195832 2.235205 3.375099 +v 5.733342 1.411738 2.431726 +v 1.859887 2.355757 3.843181 +v 4.988612 3.074654 3.083858 +v 1.303263 1.416453 4.831091 +v 1.305757 -0.672779 6.415959 +v 6.465170 0.937119 1.689873 +v 5.258659 0.945811 2.974312 +v 4.432338 0.722096 3.522615 +v 3.300681 0.861641 3.872784 +v 2.430178 1.131492 4.039035 +v 1.820731 1.467954 4.224124 +v 0.563221 2.307693 5.566789 +v 6.338145 -0.529279 1.881175 +v 5.587698 3.208071 2.687839 +v 0.242624 -1.462857 7.071491 +v 1.611251 0.339326 4.895421 +v 7.743095 2.364999 -2.005167 +v 1.391142 1.851048 4.448999 +v 1.785794 -0.978284 4.850470 +v 4.670959 2.664461 3.084075 +v 1.333970 -0.283761 6.097047 +v 7.270895 -2.890917 -2.252455 +v 1.856432 2.585245 3.757904 +v 0.923388 0.073076 6.671944 +v 5.000589 -6.135128 1.892523 +v 5.085276 -7.178590 0.714711 +v 7.159291 -0.811820 -0.072044 +v 5.843051 -5.248023 0.924091 +v 6.847258 3.662916 0.724695 +v 2.412942 -8.258853 4.119213 +v 0.179909 -1.689864 6.573301 +v 2.103655 -0.163946 4.566119 +v 6.407571 2.236021 1.560843 +v 3.670075 2.360153 3.635230 +v 3.177186 2.294265 3.775704 +v 2.196121 -4.598322 4.479786 +v 6.234883 -1.944430 1.663542 +v 1.292924 -9.295920 4.094063 +v 3.210651 -8.533278 2.802001 +v 4.068926 -7.993109 1.925119 +v 2.724032 2.315802 3.777151 +v 2.288460 2.398891 3.697603 +v 1.998311 2.496547 3.689148 +v 6.130040 3.399261 2.038516 +v 2.288460 2.886504 3.775031 +v 2.724032 2.961810 3.871767 +v 3.177186 2.964136 3.876973 +v 3.670075 2.927714 3.724325 +v 4.018389 2.857357 3.482983 +v 7.555811 4.106811 -0.991917 +v 4.018389 2.483695 3.440898 +v 1.776217 -2.683946 5.213116 +v 1.222237 -1.182444 5.952465 +v 0.731493 -2.536683 5.815343 +v 4.135272 -6.996638 2.671970 +v 3.311811 -7.660815 3.382963 +v 1.313701 -8.639995 4.702456 +v 5.940524 -6.223629 -0.631468 +v 1.998311 2.743838 3.744030 +v 0.901447 1.236992 5.754256 +v 2.308977 -8.974196 3.609070 +v 6.954154 -2.439843 -0.131163 +v 1.098819 -4.458788 5.120727 +v 1.181124 -4.579996 5.189564 +v 1.255818 -4.787901 5.237051 +v 1.325085 -5.106507 5.205010 +v 1.546388 -5.819392 4.757893 +v 1.953754 -4.183892 4.431713 +v 2.117802 -4.137093 4.555096 +v 2.285339 -4.051196 4.582438 +v 2.850160 -3.665720 4.484994 +v 5.278538 -2.238942 2.861224 +v 0.946709 1.907628 5.196779 +v 1.314173 3.104912 4.231404 +v 1.780000 2.860000 3.881555 +v 1.845110 -4.098880 4.247264 +v 5.436187 -4.030482 2.109852 +v 0.766444 3.182131 4.861453 +v 1.938616 -6.614410 4.521085 +v 0.516573 1.583572 6.148363 +v 1.246815 0.230297 5.681036 +v 0.997827 -6.930921 4.979576 +v 3.288807 -5.382514 3.795752 +v 2.311631 -1.566237 4.590085 +v 2.680250 -6.111567 4.096152 +v 3.832928 -1.537326 4.137731 +v 2.961860 -2.274215 4.440943 +v 4.386901 -2.683286 3.643886 +v 1.217295 -7.834465 4.969286 +v 1.542374 -0.136843 5.201008 +v 3.878377 -6.041764 3.311079 +v 3.084037 -6.809842 3.814195 +v 3.747321 -4.503545 3.726453 +v 6.094129 -3.205991 1.473482 +v 4.588995 -4.728726 2.983221 +v 6.583231 -3.941269 0.070268 +v 3.492580 -3.195820 4.130198 +v 1.255543 0.802341 5.307551 +v 1.126122 -0.933602 6.538785 +v 1.443109 -1.142774 5.905127 +v 0.923043 -0.529042 7.003423 +v 1.755386 3.529117 4.327696 +v 2.632589 3.713828 4.364629 +v 3.388062 3.721976 4.309028 +v 4.075766 3.675413 4.076063 +v 4.622910 3.474691 3.646321 +v 5.171755 2.535753 2.670867 +v 7.297331 0.763172 -0.048769 +v 4.706828 1.651000 3.109532 +v 4.071712 1.476821 3.476944 +v 3.269817 1.470659 3.731945 +v 2.527572 1.617311 3.865444 +v 1.970894 1.858505 3.961782 +v 1.579543 2.097941 4.084996 +v 7.664182 0.673132 -2.435867 +v 1.397041 -1.340139 5.630378 +v 0.884838 0.658740 6.233232 +v 0.767097 -0.968035 7.077932 +v 0.460213 -1.334106 6.787447 +v 0.748618 -1.067994 6.798303 +v 1.236408 -1.585568 5.480490 +v 0.387306 -1.409990 6.957705 +v 0.319925 -1.607931 6.508676 +v 1.639633 2.556298 3.863736 +v 1.255645 2.467144 4.203800 +v 1.031362 2.382663 4.615849 +v 4.253081 2.772296 3.315305 +v 4.530000 2.910000 3.339685 +v -3.18175 2.635786 3.826339 +v -2.58175 2.635786 3.824459 +v -3.18175 3.235786 3.876973 +v -3.78175 2.635786 3.679778 +v -3.18175 2.035786 3.775704 +v 3.181751 2.635786 3.826339 +v 3.781751 2.635786 3.679777 +v 3.181751 3.235786 3.876973 +v 2.581751 2.635786 3.824459 +v 3.181751 2.035786 3.775704 +vt 0.427942 0.304722 +vt 0.526878 0.295374 +vt 0.444832 0.269206 +vt 0.607600 0.322297 +vt 0.377046 0.677222 +vt 0.473033 0.304722 +vt 0.526913 0.282143 +vt 0.447112 0.284192 +vt 0.599262 0.318931 +vt 0.414712 0.664780 +vt 0.473122 0.295374 +vt 0.527671 0.263774 +vt 0.448020 0.295368 +vt 0.593203 0.314324 +vt 0.467288 0.470075 +vt 0.473087 0.282143 +vt 0.534090 0.220859 +vt 0.448662 0.304722 +vt 0.569944 0.232965 +vt 0.437114 0.441104 +vt 0.472329 0.263774 +vt 0.524613 0.307634 +vt 0.114210 0.384978 +vt 0.555168 0.269206 +vt 0.455528 0.451377 +vt 0.465828 0.220810 +vt 0.547818 0.307634 +vt 0.375437 0.075808 +vt 0.552888 0.284192 +vt 0.429884 0.533478 +vt 0.475387 0.307634 +vt 0.568842 0.307634 +vt 0.499877 0.091010 +vt 0.551980 0.295368 +vt 0.336768 0.355267 +vt 0.452182 0.307634 +vt 0.539958 0.442861 +vt 0.455607 0.548199 +vt 0.551338 0.304722 +vt 0.133823 0.317299 +vt 0.431158 0.307634 +vt 0.596371 0.306047 +vt 0.408772 0.626106 +vt 0.885770 0.384971 +vt 0.279777 0.285342 +vt 0.460042 0.442861 +vt 0.596961 0.293460 +vt 0.128294 0.208059 +vt 0.624563 0.075808 +vt 0.189096 0.353700 +vt 0.403629 0.306047 +vt 0.611897 0.306039 +vt 0.440512 0.097581 +vt 0.544341 0.548416 +vt 0.324548 0.296007 +vt 0.403039 0.293460 +vt 0.554692 0.419934 +vt 0.335279 0.147180 +vt 0.591234 0.626106 +vt 0.354128 0.187447 +vt 0.388103 0.306039 +vt 0.577238 0.326110 +vt 0.288719 0.180054 +vt 0.871706 0.208059 +vt 0.445308 0.419934 +vt 0.553172 0.331473 +vt 0.499923 0.648476 +vt 0.559100 0.097368 +vt 0.422762 0.326110 +vt 0.527121 0.333802 +vt 0.465844 0.379359 +vt 0.664630 0.147129 +vt 0.446828 0.331473 +vt 0.826722 0.721245 +vt 0.445682 0.433923 +vt 0.711218 0.180025 +vt 0.472879 0.333802 +vt 0.770391 0.700444 +vt 0.415838 0.375804 +vt 0.534154 0.379360 +vt 0.173287 0.721252 +vt 0.635536 0.810751 +vt 0.499988 0.381566 +vt 0.554318 0.433923 +vt 0.229622 0.700459 +vt 0.770092 0.767979 +vt 0.301415 0.612551 +vt 0.584177 0.375893 +vt 0.364501 0.810886 +vt 0.668509 0.880086 +vt 0.058133 0.680924 +vt 0.698585 0.612551 +vt 0.229924 0.767997 +vt 0.616907 0.744114 +vt 0.301415 0.636844 +vt 0.941867 0.680924 +vt 0.331431 0.880286 +vt 0.614083 0.718613 +vt 0.318785 0.641660 +vt 0.698585 0.636844 +vt 0.383103 0.744160 +vt 0.577414 0.436833 +vt 0.343364 0.644643 +vt 0.681215 0.641660 +vt 0.385919 0.718636 +vt 0.722943 0.728037 +vt 0.365962 0.644029 +vt 0.656636 0.644643 +vt 0.422552 0.436767 +vt 0.607591 0.305797 +vt 0.388665 0.637716 +vt 0.634038 0.644029 +vt 0.277076 0.728068 +vt 0.618026 0.305289 +vt 0.194993 0.657898 +vt 0.611335 0.637716 +vt 0.392389 0.305797 +vt 0.542902 0.415208 +vt 0.410373 0.608920 +vt 0.805016 0.657892 +vt 0.381974 0.305289 +vt 0.557261 0.427174 +vt 0.393207 0.604463 +vt 0.589660 0.608938 +vt 0.457098 0.415208 +vt 0.932695 0.269895 +vt 0.366170 0.601178 +vt 0.606793 0.604463 +vt 0.442739 0.427174 +vt 0.645429 0.303293 +vt 0.499977 0.045547 +vt 0.633830 0.601178 +vt 0.067305 0.269895 +vt 0.607610 0.646112 +vt 0.500023 0.809424 +vt 0.733752 0.130299 +vt 0.354490 0.303216 +vt 0.552386 0.697432 +vt 0.266248 0.130299 +vt 0.681008 0.101715 +vt 0.392390 0.646112 +vt 0.830705 0.806186 +vt 0.318993 0.101715 +vt 0.568013 0.055435 +vt 0.447580 0.697390 +vt 0.703624 0.706729 +vt 0.430987 0.055935 +vt 0.812086 0.411461 +vt 0.169295 0.806186 +vt 0.662801 0.717082 +vt 0.187885 0.411462 +vt 0.603900 0.289783 +vt 0.296392 0.706757 +vt 0.516446 0.500361 +vt 0.396100 0.289783 +vt 0.656636 0.599403 +vt 0.337212 0.717117 +vt 0.723330 0.636627 +vt 0.723087 0.467946 +vt 0.343364 0.599403 +vt 0.681215 0.603765 +vt 0.483370 0.500413 +vt 0.710288 0.631747 +vt 0.578632 0.466377 +vt 0.318785 0.603765 +vt 0.825608 0.602325 +vt 0.276896 0.467943 +vt 0.549756 0.600249 +vt 0.570338 0.451425 +vt 0.174399 0.602329 +vt 0.617942 0.491684 +vt 0.421352 0.466259 +vt 0.560698 0.604668 +vt 0.598631 0.545021 +vt 0.382385 0.491427 +vt 0.508953 0.420562 +vt 0.429819 0.451385 +vt 0.573595 0.610193 +vt 0.742247 0.685493 +vt 0.490967 0.420622 +vt 0.614074 0.116754 +vt 0.401223 0.544828 +vt 0.517472 0.422123 +vt 0.515097 0.472748 +vt 0.385764 0.116846 +vt 0.865595 0.666313 +vt 0.257765 0.685510 +vt 0.516311 0.436946 +vt 0.513050 0.452718 +vt 0.134410 0.666317 +vt 0.816351 0.259740 +vt 0.485301 0.472605 +vt 0.566036 0.417671 +vt 0.624852 0.271901 +vt 0.183610 0.259743 +vt 0.892441 0.459239 +vt 0.486717 0.452371 +vt 0.531529 0.444943 +vt 0.571228 0.317308 +vt 0.107550 0.459245 +vt 0.801779 0.168062 +vt 0.374971 0.272195 +vt 0.523913 0.436170 +vt 0.549626 0.319139 +vt 0.198221 0.168062 +vt 0.760966 0.220247 +vt 0.428771 0.317309 +vt 0.526564 0.453882 +vt 0.585384 0.333459 +vt 0.238979 0.220255 +vt 0.537728 0.494615 +vt 0.450374 0.319139 +vt 0.541366 0.521101 +vt 0.560215 0.342771 +vt 0.462783 0.494253 +vt 0.580985 0.612840 +vt 0.414617 0.333459 +vt 0.567192 0.430580 +vt 0.525850 0.319809 +vt 0.419054 0.612845 +vt 0.967686 0.355643 +vt 0.439785 0.342771 +vt 0.992440 0.519223 +vt 0.528249 0.349596 +vt 0.032314 0.355643 +vt 0.560611 0.480983 +vt 0.474155 0.319808 +vt 0.579658 0.590055 +vt 0.643998 0.465512 +vt 0.439121 0.481042 +vt 0.733530 0.623023 +vt 0.471751 0.349596 +vt 0.603876 0.583413 +vt 0.790082 0.608646 +vt 0.266470 0.623023 +vt 0.602995 0.451312 +vt 0.355808 0.465594 +vt 0.633505 0.573912 +vt 0.893693 0.600040 +vt 0.396993 0.451203 +vt 0.573500 0.580000 +vt 0.209925 0.608647 +vt 0.666525 0.566134 +vt 0.719902 0.624400 +vt 0.426243 0.579569 +vt 0.980531 0.598436 +vt 0.106310 0.600044 +vt 0.702114 0.566837 +vt 0.602918 0.157137 +vt 0.019469 0.598436 +vt 0.595293 0.514976 +vt 0.280098 0.624400 +vt 0.732392 0.575453 +vt 0.752212 0.589195 +vt 0.404670 0.514867 +vt 0.509127 0.437282 +vt 0.396889 0.157245 +vt 0.897013 0.531231 +vt 0.702097 0.646409 +vt 0.490726 0.437599 +vt 0.771046 0.651041 +vt 0.247792 0.589190 +vt 0.758757 0.617213 +vt 0.680678 0.652735 +vt 0.228962 0.651049 +vt 0.810748 0.476074 +vt 0.297903 0.646409 +vt 0.716482 0.666799 +vt 0.629906 0.653924 +vt 0.189241 0.476076 +vt 0.523481 0.594373 +vt 0.319322 0.652735 +vt 0.687132 0.677654 +vt 0.654766 0.655989 +vt 0.476410 0.594194 +vt 0.600862 0.567527 +vt 0.370094 0.653924 +vt 0.655896 0.679837 +vt 0.606630 0.596295 +vt 0.398964 0.567345 +vt 0.631101 0.552846 +vt 0.345234 0.655989 +vt 0.622953 0.677221 +vt 0.725342 0.610869 +vt 0.368756 0.552793 +vt 0.667113 0.539327 +vt 0.393362 0.596294 +vt 0.585271 0.664823 +vt 0.688880 0.590540 +vt 0.332828 0.539288 +vt 0.713757 0.532373 +vt 0.274658 0.610869 +vt 0.531987 0.469860 +vt 0.661242 0.586975 +vt 0.286267 0.532325 +vt 0.752702 0.542818 +vt 0.311120 0.590540 +vt 0.562759 0.441215 +vt 0.634070 0.590424 +vt 0.247308 0.542806 +vt 0.821442 0.542444 +vt 0.313951 0.224692 +vt 0.338758 0.586975 +vt 0.544562 0.451624 +vt 0.895093 0.745859 +vt 0.178560 0.542446 +vt 0.551868 0.463430 +vt 0.410986 0.491277 +vt 0.365930 0.590424 +vt 0.570082 0.533674 +vt 0.526227 0.426090 +vt 0.448340 0.463064 +vt 0.572156 0.562348 +vt 0.447750 0.137523 +vt 0.104907 0.745859 +vt 0.663187 0.355403 +vt 0.710288 0.619236 +vt 0.427685 0.562039 +vt 0.742870 0.644554 +vt 0.295284 0.378419 +vt 0.473773 0.426090 +vt 0.866152 0.317295 +vt 0.517862 0.528052 +vt 0.257135 0.644560 +vt 0.587247 0.601068 +vt 0.357155 0.395730 +vt 0.499816 0.437019 +vt 0.720122 0.285333 +vt 0.276670 0.636627 +vt 0.412782 0.601030 +vt 0.781070 0.564595 +vt 0.319688 0.429262 +vt 0.499968 0.218629 +vt 0.810858 0.353695 +vt 0.289712 0.631747 +vt 0.218937 0.564589 +vt 0.711045 0.601048 +vt 0.374293 0.219815 +vt 0.499977 0.262981 +vt 0.675343 0.296022 +vt 0.450067 0.599566 +vt 0.288955 0.601048 +vt 0.588166 0.890956 +vt 0.378909 0.425990 +vt 0.499977 0.280615 +vt 0.645735 0.187360 +vt 0.438999 0.603505 +vt 0.412198 0.891099 +vt 0.570304 0.812129 +vt 0.344549 0.254561 +vt 0.499977 0.294066 +vt 0.685945 0.224643 +vt 0.426450 0.610201 +vt 0.429765 0.812166 +vt 0.558266 0.738328 +vt 0.456549 0.180799 +vt 0.499977 0.304722 +vt 0.589072 0.491363 +vt 0.482483 0.422151 +vt 0.441728 0.738324 +vt 0.600409 0.250995 +vt 0.499913 0.178271 +vt 0.500023 0.307652 +vt 0.552012 0.137408 +vt 0.483518 0.437016 +vt 0.399510 0.251079 +vt 0.672684 0.743419 +vt 0.499886 0.133083 +vt 0.500016 0.320776 +vt 0.704663 0.378470 +vt 0.433991 0.417638 +vt 0.327338 0.743473 +vt 0.709250 0.798492 +vt 0.432112 0.506411 +vt 0.500023 0.333766 +vt 0.642764 0.395662 +vt 0.468472 0.444943 +vt 0.290777 0.798554 +vt 0.757824 0.852324 +vt 0.499974 0.560363 +vt 0.500023 0.892950 +vt 0.680198 0.429281 +vt 0.476088 0.436170 +vt 0.242176 0.852324 +vt 0.588354 0.453138 +vt 0.479154 0.557346 +vt 0.499987 0.730081 +vt 0.625560 0.219688 +vt 0.473466 0.454256 +vt 0.411671 0.453035 +vt 0.665586 0.504049 +vt 0.499989 0.530175 +vt 0.499955 0.687602 +vt 0.621009 0.425982 +vt 0.458639 0.520911 +vt 0.334562 0.503927 +vt 0.627543 0.526648 +vt 0.411362 0.195673 +vt 0.289712 0.619236 +vt 0.655317 0.254485 +vt 0.432949 0.430482 +vt 0.372120 0.526586 +vt 0.536915 0.406214 +vt 0.468268 0.647329 +vt 0.499523 0.598938 +vt 0.543283 0.180745 +vt 0.007561 0.519223 +vt 0.463080 0.406216 +vt 0.577268 0.414065 +vt 0.228018 0.316428 +vt 0.499910 0.501747 +vt 0.567985 0.506521 +vt 0.420121 0.589772 +vt 0.422729 0.414015 +vt 0.531915 0.398463 +vt 0.413386 0.307634 +vt 0.500151 0.472844 +vt 0.520797 0.557435 +vt 0.396012 0.583304 +vt 0.468080 0.398465 +vt 0.590372 0.298177 +vt 0.416164 0.631286 +vt 0.482113 0.528021 +vt 0.588371 0.195559 +vt 0.366427 0.573884 +vt 0.409626 0.298177 +vt 0.586800 0.304600 +vt 0.436392 0.640113 +vt 0.499974 0.397628 +vt 0.531597 0.647517 +vt 0.333434 0.566122 +vt 0.413200 0.304600 +vt 0.986046 0.439966 +vt 0.452770 0.579150 +vt 0.500026 0.452513 +vt 0.771915 0.316422 +vt 0.297879 0.566824 +vt 0.499914 0.419853 +vt 0.609945 0.360090 +vt 0.247923 0.398667 +vt 0.499977 0.347466 +vt 0.586614 0.307634 +vt 0.267612 0.575440 +vt 0.013954 0.439966 +vt 0.581691 0.279937 +vt 0.367856 0.336081 +vt 0.583841 0.631286 +vt 0.102986 0.531237 +vt 0.390095 0.360427 +vt 0.576838 0.288154 +vt 0.392400 0.322297 +vt 0.563544 0.640172 +vt 0.241246 0.617214 +vt 0.418309 0.279937 +vt 0.573521 0.296460 +vt 0.400738 0.318931 +vt 0.547226 0.579605 +vt 0.283526 0.666810 +vt 0.423162 0.288154 +vt 0.572058 0.304722 +vt 0.406787 0.314327 +vt 0.752033 0.398685 +vt 0.312876 0.677668 +vt 0.426479 0.296460 +vt 0.526967 0.304722 +vt 0.430012 0.233191 +vt 0.631938 0.336500 +vt 0.344108 0.679849 +vt 0.523494 0.653066 +vt 0.619766 0.484153 +vt 0.448126 0.441797 +vt 0.564397 0.650577 +vt 0.629396 0.4879675 +vt 0.500005 0.5319235 +vt 0.528836 0.3630495 +vt 0.601042 0.688245 +vt 0.489588 0.725148 +vt 0.626117 0.4614805 +f 174/43 156/119 134/220 +f 247/335 34/252 8/399 +f 383/124 399/59 363/216 +f 264/244 467/163 250/317 +f 309/42 416/442 325/427 +f 79/51 96/432 192/416 +f 357/246 390/96 265/239 +f 128/250 35/247 163/91 +f 369/186 265/239 390/96 +f 140/190 163/91 35/247 +f 268/224 1/441 303/70 +f 38/232 73/77 1/441 +f 12/375 303/70 1/441 +f 12/375 1/441 73/77 +f 350/281 452/238 351/276 +f 121/285 122/280 232/425 +f 453/233 351/276 452/238 +f 233/419 232/425 122/280 +f 268/224 303/70 270/214 +f 38/232 40/222 73/77 +f 304/66 270/214 303/70 +f 74/73 73/77 40/222 +f 358/241 344/313 351/276 +f 129/245 122/280 115/318 +f 278/174 351/276 344/313 +f 48/182 115/318 122/280 +f 351/276 453/233 358/241 +f 122/280 129/245 233/419 +f 454/228 358/241 453/233 +f 234/413 233/419 129/245 +f 300/82 334/373 298/90 +f 70/89 68/97 105/378 +f 333/379 298/90 334/373 +f 104/384 105/378 68/97 +f 176/33 153/131 397/68 +f 176/33 172/53 153/131 +f 378/144 397/68 153/131 +f 149/147 153/131 172/53 +f 382/128 385/116 383/124 +f 155/123 156/119 158/111 +f 399/59 383/124 385/116 +f 174/43 158/111 156/119 +f 281/159 348/291 331/391 +f 51/167 102/396 119/295 +f 349/286 331/391 348/291 +f 120/290 119/295 102/396 +f 270/214 304/66 271/209 +f 40/222 41/217 74/73 +f 305/62 271/209 304/66 +f 75/69 74/73 41/217 +f 10/387 337/355 152/135 +f 10/387 152/135 108/360 +f 338/349 152/135 337/355 +f 109/354 108/360 152/135 +f 345/307 279/169 361/226 +f 116/312 132/230 49/177 +f 280/164 361/226 279/169 +f 50/172 49/177 132/230 +f 263/249 432/346 419/424 +f 33/257 195/398 212/60 +f 425/388 419/424 432/346 +f 205/338 212/60 195/398 +f 305/62 409/9 271/209 +f 75/69 41/217 185/456 +f 410/4 271/209 409/9 +f 186/451 185/456 41/217 +f 273/199 311/32 408/14 +f 43/207 184/461 81/41 +f 416/442 408/14 311/32 +f 192/416 81/41 184/461 +f 323/439 271/209 411/467 +f 93/449 187/446 41/217 +f 410/4 411/467 271/209 +f 186/451 41/217 187/446 +f 348/291 450/248 349/286 +f 119/295 120/290 230/437 +f 451/243 349/286 450/248 +f 231/431 230/437 120/290 +f 435/328 433/340 431/352 +f 215/45 211/302 213/55 +f 423/400 431/352 433/340 +f 203/350 213/55 211/302 +f 314/17 315/12 19/333 +f 84/26 19/333 85/21 +f 18/339 19/333 315/12 +f 18/339 85/21 19/333 +f 308/47 376/152 307/52 +f 78/56 77/61 147/155 +f 292/114 307/52 376/152 +f 62/121 147/155 77/61 +f 260/264 388/104 261/259 +f 30/272 31/267 161/99 +f 389/100 261/259 388/104 +f 162/95 161/99 31/267 +f 287/134 415/447 385/116 +f 57/141 158/111 191/422 +f 399/59 385/116 415/447 +f 174/43 191/422 158/111 +f 419/424 425/388 407/19 +f 195/398 183/466 205/338 +f 336/361 407/19 425/388 +f 107/366 205/338 183/466 +f 368/191 417/436 365/206 +f 139/195 136/210 193/410 +f 435/328 365/206 417/436 +f 215/45 193/410 136/210 +f 392/88 424/394 328/409 +f 166/79 99/414 204/344 +f 359/236 328/409 424/394 +f 130/240 204/344 99/414 +f 299/86 302/74 285/142 +f 69/93 55/149 72/81 +f 252/305 285/142 302/74 +f 22/315 72/81 55/149 +f 5/417 276/184 6/411 +f 5/417 6/411 46/192 +f 282/154 6/411 276/184 +f 52/162 46/192 6/411 +f 255/289 374/161 254/294 +f 25/297 24/303 145/165 +f 375/156 254/294 374/161 +f 146/160 145/165 24/303 +f 321/450 322/445 308/47 +f 91/459 78/56 92/454 +f 376/152 308/47 322/445 +f 147/155 92/454 78/56 +f 281/159 426/382 412/462 +f 51/167 188/440 206/332 +f 428/370 412/462 426/382 +f 208/320 206/332 188/440 +f 422/406 314/17 201/362 +f 202/356 201/362 84/26 +f 19/333 201/362 314/17 +f 19/333 84/26 201/362 +f 336/361 322/445 407/19 +f 107/366 183/466 92/454 +f 406/24 407/19 322/445 +f 182/3 92/454 183/466 +f 406/24 322/445 405/29 +f 182/3 181/8 92/454 +f 321/450 405/29 322/445 +f 91/459 92/454 181/8 +f 18/339 315/12 17/345 +f 18/339 17/345 85/21 +f 316/7 17/345 315/12 +f 86/16 85/21 17/345 +f 426/382 267/229 427/376 +f 206/332 207/326 37/237 +f 424/394 427/376 267/229 +f 204/344 37/237 207/326 +f 370/181 397/68 401/49 +f 141/185 177/28 172/53 +f 378/144 401/49 397/68 +f 149/147 172/53 177/28 +f 392/88 270/214 323/439 +f 166/79 93/449 40/222 +f 271/209 323/439 270/214 +f 41/217 40/222 93/449 +f 418/430 466/168 414/452 +f 194/404 190/428 246/341 +f 465/173 414/452 466/168 +f 245/347 246/341 190/428 +f 258/274 259/269 387/108 +f 28/282 160/103 29/277 +f 386/112 387/108 259/269 +f 159/107 29/277 160/103 +f 261/259 389/100 468/158 +f 31/267 248/329 162/95 +f 467/163 468/158 389/100 +f 247/335 162/95 248/329 +f 249/323 457/213 420/418 +f 4/423 197/386 237/395 +f 400/54 420/418 457/213 +f 175/38 237/395 197/386 +f 334/373 299/86 333/379 +f 105/378 104/384 69/93 +f 285/142 333/379 299/86 +f 55/149 69/93 104/384 +f 286/138 9/393 418/430 +f 56/145 194/404 9/393 +f 169/67 418/430 9/393 +f 169/67 9/393 194/404 +f 341/331 262/254 347/296 +f 112/336 118/300 32/262 +f 449/253 347/296 262/254 +f 229/443 32/262 118/300 +f 286/138 418/430 442/288 +f 56/145 222/10 194/404 +f 414/452 442/288 418/430 +f 190/428 194/404 222/10 +f 328/409 461/193 327/415 +f 99/414 98/420 241/371 +f 329/403 327/415 461/193 +f 100/408 241/371 98/420 +f 278/174 356/251 330/397 +f 48/182 101/402 127/255 +f 372/171 330/397 356/251 +f 143/175 127/255 101/402 +f 310/37 393/84 439/304 +f 80/46 219/25 167/75 +f 440/298 439/304 393/84 +f 220/20 167/75 219/25 +f 382/128 383/124 257/279 +f 155/123 27/287 156/119 +f 342/325 257/279 383/124 +f 113/330 156/119 27/287 +f 361/226 280/164 421/412 +f 132/230 199/374 50/172 +f 430/358 421/412 280/164 +f 210/308 50/172 199/374 +f 366/201 365/206 380/136 +f 137/205 151/139 136/210 +f 395/76 380/136 365/206 +f 170/63 136/210 151/139 +f 356/251 278/174 438/310 +f 127/255 218/30 48/182 +f 344/313 438/310 278/174 +f 115/318 48/182 218/30 +f 444/278 445/273 283/150 +f 224/468 53/157 225/463 +f 284/146 283/150 445/273 +f 54/153 225/463 53/157 +f 282/154 276/184 364/211 +f 52/162 135/215 46/192 +f 441/293 364/211 276/184 +f 221/15 46/192 135/215 +f 432/346 263/249 396/72 +f 212/60 171/58 33/257 +f 370/181 396/72 263/249 +f 141/185 33/257 171/58 +f 338/349 300/82 339/343 +f 109/354 110/348 70/89 +f 298/90 339/343 300/82 +f 68/97 70/89 110/348 +f 336/361 274/194 322/445 +f 107/366 92/454 44/202 +f 376/152 322/445 274/194 +f 147/155 44/202 92/454 +f 349/286 451/243 350/281 +f 120/290 121/285 231/431 +f 452/238 350/281 451/243 +f 232/425 231/431 121/285 +f 468/158 360/231 343/319 +f 248/329 114/324 131/235 +f 447/263 343/319 360/231 +f 227/453 131/235 114/324 +f 283/150 284/146 335/367 +f 53/157 106/372 54/153 +f 294/106 335/367 284/146 +f 64/113 54/153 106/372 +f 251/311 459/203 463/183 +f 21/321 243/359 239/383 +f 462/188 463/183 459/203 +f 242/365 239/383 243/359 +f 277/179 354/261 301/78 +f 47/187 71/85 125/265 +f 384/120 301/78 354/261 +f 157/115 125/265 71/85 +f 326/421 293/110 325/427 +f 97/426 96/432 63/117 +f 309/42 325/427 293/110 +f 79/51 63/117 96/432 +f 284/146 277/179 294/106 +f 54/153 64/113 47/187 +f 301/78 294/106 277/179 +f 71/85 47/187 64/113 +f 448/258 265/239 346/301 +f 228/448 117/306 35/247 +f 373/166 346/301 265/239 +f 144/170 35/247 117/306 +f 353/266 346/301 347/296 +f 124/270 118/300 117/306 +f 341/331 347/296 346/301 +f 112/336 117/306 118/300 +f 2/435 20/327 275/189 +f 2/435 45/197 20/327 +f 355/256 275/189 20/327 +f 126/260 20/327 45/197 +f 249/323 282/154 457/213 +f 4/423 237/395 52/162 +f 364/211 457/213 282/154 +f 135/215 52/162 237/395 +f 426/382 427/376 428/370 +f 206/332 208/320 207/326 +f 437/316 428/370 427/376 +f 217/35 207/326 208/320 +f 381/132 382/128 253/299 +f 154/127 23/309 155/123 +f 257/279 253/299 382/128 +f 27/287 155/123 23/309 +f 392/88 394/80 270/214 +f 166/79 40/222 168/71 +f 268/224 270/214 394/80 +f 38/232 168/71 40/222 +f 200/368 429/364 201/362 +f 200/368 201/362 209/314 +f 422/406 201/362 429/364 +f 202/356 209/314 201/362 +f 331/391 330/397 267/229 +f 102/396 37/237 101/402 +f 372/171 267/229 330/397 +f 143/175 101/402 37/237 +f 423/400 433/340 274/194 +f 203/350 44/202 213/55 +f 288/130 274/194 433/340 +f 58/137 213/55 44/202 +f 291/118 251/311 329/403 +f 61/125 100/408 21/321 +f 463/183 329/403 251/311 +f 243/359 21/321 100/408 +f 259/269 287/134 386/112 +f 29/277 159/107 57/141 +f 385/116 386/112 287/134 +f 158/111 57/141 159/107 +f 343/319 447/263 354/261 +f 114/324 125/265 227/453 +f 266/234 354/261 447/263 +f 36/242 227/453 125/265 +f 258/274 387/108 260/264 +f 28/282 30/272 160/103 +f 388/104 260/264 387/108 +f 161/99 160/103 30/272 +f 431/352 423/400 432/346 +f 211/302 212/60 203/350 +f 425/388 432/346 423/400 +f 205/338 203/350 212/60 +f 446/268 343/319 277/179 +f 226/458 47/187 114/324 +f 354/261 277/179 343/319 +f 125/265 114/324 47/187 +f 425/388 423/400 336/361 +f 205/338 107/366 203/350 +f 274/194 336/361 423/400 +f 44/202 203/350 107/366 +f 307/52 293/110 308/47 +f 77/61 78/56 63/117 +f 326/421 308/47 293/110 +f 97/426 63/117 78/56 +f 367/196 448/258 353/266 +f 138/200 124/270 228/448 +f 346/301 353/266 448/258 +f 117/306 228/448 124/270 +f 303/70 269/219 304/66 +f 73/77 74/73 39/227 +f 272/204 304/66 269/219 +f 42/212 39/227 74/73 +f 372/171 359/236 267/229 +f 143/175 37/237 130/240 +f 424/394 267/229 359/236 +f 204/344 130/240 37/237 +f 328/409 295/102 461/193 +f 99/414 241/371 65/109 +f 456/218 461/193 295/102 +f 236/401 65/109 241/371 +f 295/102 332/385 279/169 +f 65/109 49/177 103/390 +f 280/164 279/169 332/385 +f 50/172 103/390 49/177 +f 304/66 272/204 305/62 +f 74/73 75/69 42/212 +f 273/199 305/62 272/204 +f 43/207 42/212 75/69 +f 428/370 437/316 435/328 +f 208/320 215/45 217/35 +f 433/340 435/328 437/316 +f 213/55 217/35 215/45 +f 305/62 273/199 409/9 +f 75/69 185/456 43/207 +f 408/14 409/9 273/199 +f 184/461 43/207 185/456 +f 395/76 431/352 396/72 +f 170/63 171/58 211/302 +f 432/346 396/72 431/352 +f 212/60 211/302 171/58 +f 396/72 370/181 379/140 +f 171/58 150/143 141/185 +f 401/49 379/140 370/181 +f 177/28 141/185 150/143 +f 297/94 335/367 300/82 +f 67/101 70/89 106/372 +f 334/373 300/82 335/367 +f 105/378 106/372 70/89 +f 418/430 169/67 352/271 +f 194/404 123/275 169/67 +f 7/405 352/271 169/67 +f 7/405 169/67 123/275 +f 281/159 412/462 353/266 +f 51/167 124/270 188/440 +f 377/148 353/266 412/462 +f 148/151 188/440 124/270 +f 320/455 321/450 326/421 +f 90/464 97/426 91/459 +f 308/47 326/421 321/450 +f 78/56 91/459 97/426 +f 286/138 296/98 337/355 +f 56/145 108/360 66/105 +f 297/94 337/355 296/98 +f 67/101 66/105 108/360 +f 405/29 321/450 404/34 +f 181/8 180/13 91/459 +f 320/455 404/34 321/450 +f 90/464 91/459 180/13 +f 331/391 349/286 330/397 +f 102/396 101/402 120/290 +f 350/281 330/397 349/286 +f 121/285 120/290 101/402 +f 335/367 294/106 334/373 +f 106/372 105/378 64/113 +f 299/86 334/373 294/106 +f 69/93 64/113 105/378 +f 324/433 455/223 367/196 +f 94/444 138/200 235/407 +f 448/258 367/196 455/223 +f 228/448 235/407 138/200 +f 17/345 316/7 16/351 +f 17/345 16/351 86/16 +f 317/2 16/351 316/7 +f 87/11 86/16 16/351 +f 430/358 280/164 359/236 +f 210/308 130/240 50/172 +f 332/385 359/236 280/164 +f 103/390 50/172 130/240 +f 16/351 317/2 15/357 +f 16/351 15/357 87/11 +f 318/465 15/357 317/2 +f 88/6 87/11 15/357 +f 9/393 286/138 10/387 +f 9/393 10/387 56/145 +f 337/355 10/387 286/138 +f 108/360 56/145 10/387 +f 330/397 350/281 278/174 +f 101/402 48/182 121/285 +f 351/276 278/174 350/281 +f 122/280 121/285 48/182 +f 253/299 254/294 381/132 +f 23/309 154/127 24/303 +f 375/156 381/132 254/294 +f 146/160 24/303 154/127 +f 403/39 404/34 319/460 +f 179/18 89/1 180/13 +f 320/455 319/460 404/34 +f 90/464 180/13 89/1 +f 352/271 7/405 420/418 +f 123/275 197/386 7/405 +f 198/380 420/418 7/405 +f 198/380 7/405 197/386 +f 325/427 319/460 326/421 +f 96/432 97/426 89/1 +f 320/455 326/421 319/460 +f 90/464 89/1 97/426 +f 398/64 368/191 366/201 +f 173/48 137/205 139/195 +f 365/206 366/201 368/191 +f 136/210 139/195 137/205 +f 289/126 436/322 398/64 +f 59/133 173/48 216/40 +f 368/191 398/64 436/322 +f 139/195 216/40 173/48 +f 439/304 440/298 345/307 +f 219/25 116/312 220/20 +f 279/169 345/307 440/298 +f 49/177 220/20 116/312 +f 272/204 312/27 273/199 +f 42/212 43/207 82/36 +f 311/32 273/199 312/27 +f 81/41 82/36 43/207 +f 6/411 282/154 196/392 +f 6/411 196/392 52/162 +f 249/323 196/392 282/154 +f 4/423 52/162 196/392 +f 274/194 288/130 376/152 +f 44/202 147/155 58/137 +f 292/114 376/152 288/130 +f 62/121 58/137 147/155 +f 397/68 429/364 176/33 +f 172/53 176/33 209/314 +f 200/368 176/33 429/364 +f 200/368 209/314 176/33 +f 269/219 313/22 272/204 +f 39/227 42/212 83/31 +f 312/27 272/204 313/22 +f 82/36 83/31 42/212 +f 445/273 446/268 284/146 +f 225/463 54/153 226/458 +f 277/179 284/146 446/268 +f 47/187 226/458 54/153 +f 255/289 340/337 374/161 +f 25/297 145/165 111/342 +f 391/92 374/161 340/337 +f 164/87 111/342 145/165 +f 296/98 283/150 297/94 +f 66/105 67/101 53/157 +f 335/367 297/94 283/150 +f 106/372 53/157 67/101 +f 347/296 449/253 348/291 +f 118/300 119/295 229/443 +f 450/248 348/291 449/253 +f 230/437 229/443 119/295 +f 455/223 357/246 448/258 +f 235/407 228/448 128/250 +f 265/239 448/258 357/246 +f 35/247 128/250 228/448 +f 337/355 297/94 338/349 +f 108/360 109/354 67/101 +f 300/82 338/349 297/94 +f 70/89 67/101 109/354 +f 152/135 338/349 11/381 +f 152/135 11/381 109/354 +f 339/343 11/381 338/349 +f 110/348 109/354 11/381 +f 279/169 440/298 295/102 +f 49/177 65/109 220/20 +f 456/218 295/102 440/298 +f 236/401 220/20 65/109 +f 408/14 416/442 293/110 +f 184/461 63/117 192/416 +f 309/42 293/110 416/442 +f 79/51 192/416 63/117 +f 359/236 372/171 430/358 +f 130/240 210/308 143/175 +f 356/251 430/358 372/171 +f 127/255 143/175 210/308 +f 346/301 373/166 341/331 +f 117/306 112/336 144/170 +f 266/234 341/331 373/166 +f 36/242 144/170 112/336 +f 389/100 391/92 467/163 +f 162/95 247/335 164/87 +f 250/317 467/163 391/92 +f 8/399 164/87 247/335 +f 353/266 347/296 281/159 +f 124/270 51/167 118/300 +f 348/291 281/159 347/296 +f 119/295 118/300 51/167 +f 296/98 443/283 283/150 +f 66/105 53/157 223/5 +f 444/278 283/150 443/283 +f 224/468 223/5 53/157 +f 20/327 95/438 355/256 +f 20/327 126/260 95/438 +f 371/176 355/256 95/438 +f 142/180 95/438 126/260 +f 296/98 286/138 443/283 +f 66/105 223/5 56/145 +f 442/288 443/283 286/138 +f 222/10 56/145 223/5 +f 420/418 198/380 249/323 +f 197/386 4/423 198/380 +f 196/392 249/323 198/380 +f 196/392 198/380 4/423 +f 360/231 264/244 256/284 +f 131/235 26/292 34/252 +f 250/317 256/284 264/244 +f 8/399 34/252 26/292 +f 276/184 275/189 441/293 +f 46/192 221/15 45/197 +f 458/208 441/293 275/189 +f 238/389 45/197 221/15 +f 301/78 384/120 302/74 +f 71/85 72/81 157/115 +f 369/186 302/74 384/120 +f 140/190 157/115 72/81 +f 418/430 352/271 466/168 +f 194/404 246/341 123/275 +f 413/457 466/168 352/271 +f 189/434 123/275 246/341 +f 467/163 264/244 468/158 +f 247/335 248/329 34/252 +f 360/231 468/158 264/244 +f 131/235 34/252 248/329 +f 390/96 252/305 369/186 +f 163/91 140/190 22/315 +f 302/74 369/186 252/305 +f 72/81 22/315 140/190 +f 375/156 387/108 381/132 +f 146/160 154/127 160/103 +f 386/112 381/132 387/108 +f 159/107 160/103 154/127 +f 380/136 395/76 379/140 +f 151/139 150/143 170/63 +f 396/72 379/140 395/76 +f 171/58 170/63 150/143 +f 352/271 420/418 413/457 +f 123/275 189/434 197/386 +f 400/54 413/457 420/418 +f 175/38 197/386 189/434 +f 427/376 323/439 437/316 +f 207/326 217/35 93/449 +f 411/467 437/316 323/439 +f 187/446 93/449 217/35 +f 388/104 374/161 389/100 +f 161/99 162/95 145/165 +f 391/92 389/100 374/161 +f 164/87 145/165 162/95 +f 394/80 327/415 165/83 +f 168/71 165/83 98/420 +f 3/429 165/83 327/415 +f 3/429 98/420 165/83 +f 355/256 371/176 462/188 +f 126/260 242/365 142/180 +f 463/183 462/188 371/176 +f 243/359 142/180 242/365 +f 1/441 268/224 165/83 +f 1/441 165/83 38/232 +f 394/80 165/83 268/224 +f 168/71 38/232 165/83 +f 12/375 13/369 303/70 +f 12/375 73/77 13/369 +f 269/219 303/70 13/369 +f 39/227 13/369 73/77 +f 387/108 375/156 388/104 +f 160/103 161/99 146/160 +f 374/161 388/104 375/156 +f 145/165 146/160 161/99 +f 13/369 14/363 269/219 +f 13/369 39/227 14/363 +f 313/22 269/219 14/363 +f 83/31 14/363 39/227 +f 294/106 301/78 299/86 +f 64/113 69/93 71/85 +f 302/74 299/86 301/78 +f 72/81 71/85 69/93 +f 341/331 266/234 262/254 +f 112/336 32/262 36/242 +f 447/263 262/254 266/234 +f 227/453 36/242 32/262 +f 381/132 386/112 382/128 +f 154/127 155/123 159/107 +f 385/116 382/128 386/112 +f 158/111 159/107 155/123 +f 281/159 331/391 426/382 +f 51/167 206/332 102/396 +f 267/229 426/382 331/391 +f 37/237 102/396 206/332 +f 424/394 392/88 427/376 +f 204/344 207/326 166/79 +f 323/439 427/376 392/88 +f 93/449 166/79 207/326 +f 430/358 356/251 421/412 +f 210/308 199/374 127/255 +f 438/310 421/412 356/251 +f 218/30 127/255 199/374 +f 392/88 328/409 394/80 +f 166/79 168/71 99/414 +f 327/415 394/80 328/409 +f 98/420 99/414 168/71 +f 458/208 439/304 441/293 +f 238/389 221/15 219/25 +f 345/307 441/293 439/304 +f 116/312 219/25 221/15 +f 383/124 363/216 342/325 +f 156/119 113/330 134/220 +f 464/178 342/325 363/216 +f 244/353 134/220 113/330 +f 458/208 462/188 460/198 +f 238/389 240/377 242/365 +f 459/203 460/198 462/188 +f 239/383 242/365 240/377 +f 435/328 431/352 365/206 +f 215/45 136/210 211/302 +f 395/76 365/206 431/352 +f 170/63 211/302 136/210 +f 415/447 464/178 399/59 +f 191/422 174/43 244/353 +f 363/216 399/59 464/178 +f 134/220 244/353 174/43 +f 263/249 429/364 370/181 +f 33/257 141/185 209/314 +f 397/68 370/181 429/364 +f 172/53 209/314 141/185 +f 458/208 275/189 462/188 +f 238/389 242/365 45/197 +f 355/256 462/188 275/189 +f 126/260 45/197 242/365 +f 317/2 404/34 318/465 +f 87/11 88/6 180/13 +f 403/39 318/465 404/34 +f 179/18 180/13 88/6 +f 316/7 405/29 317/2 +f 86/16 87/11 181/8 +f 404/34 317/2 405/29 +f 180/13 181/8 87/11 +f 315/12 406/24 316/7 +f 85/21 86/16 182/3 +f 405/29 316/7 406/24 +f 181/8 182/3 86/16 +f 314/17 407/19 315/12 +f 84/26 85/21 183/466 +f 406/24 315/12 407/19 +f 182/3 183/466 85/21 +f 419/424 407/19 422/406 +f 195/398 202/356 183/466 +f 314/17 422/406 407/19 +f 84/26 183/466 202/356 +f 367/196 402/44 324/433 +f 138/200 94/444 178/23 +f 362/221 324/433 402/44 +f 133/225 178/23 94/444 +f 409/9 408/14 307/52 +f 185/456 77/61 184/461 +f 293/110 307/52 408/14 +f 63/117 184/461 77/61 +f 409/9 307/52 410/4 +f 185/456 186/451 77/61 +f 292/114 410/4 307/52 +f 62/121 77/61 186/451 +f 411/467 410/4 288/130 +f 187/446 58/137 186/451 +f 292/114 288/130 410/4 +f 62/121 186/451 58/137 +f 437/316 411/467 433/340 +f 217/35 213/55 187/446 +f 288/130 433/340 411/467 +f 58/137 187/446 213/55 +f 435/328 417/436 428/370 +f 215/45 208/320 193/410 +f 412/462 428/370 417/436 +f 188/440 193/410 208/320 +f 265/239 369/186 373/166 +f 35/247 144/170 140/190 +f 384/120 373/166 369/186 +f 157/115 140/190 144/170 +f 458/208 460/198 439/304 +f 238/389 219/25 240/377 +f 310/37 439/304 460/198 +f 80/46 240/377 219/25 +f 353/266 377/148 367/196 +f 124/270 138/200 148/151 +f 402/44 367/196 377/148 +f 178/23 148/151 138/200 +f 5/417 2/435 276/184 +f 5/417 46/192 2/435 +f 275/189 276/184 2/435 +f 45/197 2/435 46/192 +f 429/364 263/249 422/406 +f 209/314 202/356 33/257 +f 419/424 422/406 263/249 +f 195/398 33/257 202/356 +f 328/409 359/236 295/102 +f 99/414 65/109 130/240 +f 332/385 295/102 359/236 +f 103/390 130/240 65/109 +f 368/191 436/322 417/436 +f 139/195 193/410 216/40 +f 434/334 417/436 436/322 +f 214/50 216/40 193/410 +f 456/218 440/298 290/122 +f 236/401 60/129 220/20 +f 393/84 290/122 440/298 +f 167/75 220/20 60/129 +f 329/403 463/183 327/415 +f 100/408 98/420 243/359 +f 371/176 327/415 463/183 +f 142/180 243/359 98/420 +f 327/415 371/176 3/429 +f 98/420 3/429 142/180 +f 95/438 3/429 371/176 +f 95/438 142/180 3/429 +f 461/193 456/218 306/57 +f 241/371 76/65 236/401 +f 290/122 306/57 456/218 +f 60/129 236/401 76/65 +f 449/253 340/337 450/248 +f 229/443 230/437 111/342 +f 255/289 450/248 340/337 +f 25/297 111/342 230/437 +f 262/254 447/263 256/284 +f 32/262 26/292 227/453 +f 360/231 256/284 447/263 +f 131/235 227/453 26/292 +f 450/248 255/289 451/243 +f 230/437 231/431 25/297 +f 254/294 451/243 255/289 +f 24/303 25/297 231/431 +f 451/243 254/294 452/238 +f 231/431 232/425 24/303 +f 253/299 452/238 254/294 +f 23/309 24/303 232/425 +f 452/238 253/299 453/233 +f 232/425 233/419 23/309 +f 257/279 453/233 253/299 +f 27/287 23/309 233/419 +f 257/279 342/325 453/233 +f 27/287 233/419 113/330 +f 454/228 453/233 342/325 +f 234/413 113/330 233/419 +f 414/452 465/173 415/447 +f 190/428 191/422 245/347 +f 464/178 415/447 465/173 +f 244/353 245/347 191/422 +f 442/288 414/452 287/134 +f 222/10 57/141 190/428 +f 415/447 287/134 414/452 +f 191/422 190/428 57/141 +f 442/288 287/134 443/283 +f 222/10 223/5 57/141 +f 259/269 443/283 287/134 +f 29/277 57/141 223/5 +f 443/283 259/269 444/278 +f 223/5 224/468 29/277 +f 258/274 444/278 259/269 +f 28/282 29/277 224/468 +f 445/273 444/278 260/264 +f 225/463 30/272 224/468 +f 258/274 260/264 444/278 +f 28/282 224/468 30/272 +f 260/264 261/259 445/273 +f 30/272 225/463 31/267 +f 446/268 445/273 261/259 +f 226/458 31/267 225/463 +f 261/259 468/158 446/268 +f 31/267 226/458 248/329 +f 343/319 446/268 468/158 +f 114/324 248/329 226/458 +f 251/311 310/37 459/203 +f 21/321 239/383 80/46 +f 460/198 459/203 310/37 +f 240/377 80/46 239/383 +f 291/118 306/57 393/84 +f 61/125 167/75 76/65 +f 290/122 393/84 306/57 +f 60/129 76/65 167/75 +f 461/193 306/57 329/403 +f 241/371 100/408 76/65 +f 291/118 329/403 306/57 +f 61/125 76/65 100/408 +f 377/148 434/334 402/44 +f 148/151 178/23 214/50 +f 436/322 402/44 434/334 +f 216/40 214/50 178/23 +f 251/311 291/118 310/37 +f 21/321 80/46 61/125 +f 393/84 310/37 291/118 +f 167/75 61/125 80/46 +f 412/462 417/436 377/148 +f 188/440 148/151 193/410 +f 434/334 377/148 417/436 +f 214/50 193/410 148/151 +f 342/325 464/178 454/228 +f 113/330 234/413 244/353 +f 465/173 454/228 464/178 +f 245/347 244/353 234/413 +f 454/228 465/173 358/241 +f 234/413 129/245 245/347 +f 466/168 358/241 465/173 +f 246/341 245/347 129/245 +f 413/457 344/313 466/168 +f 189/434 246/341 115/318 +f 358/241 466/168 344/313 +f 129/245 115/318 246/341 +f 438/310 344/313 400/54 +f 218/30 175/38 115/318 +f 413/457 400/54 344/313 +f 189/434 115/318 175/38 +f 364/211 441/293 361/226 +f 135/215 132/230 221/15 +f 345/307 361/226 441/293 +f 116/312 221/15 132/230 +f 457/213 421/412 400/54 +f 237/395 175/38 199/374 +f 438/310 400/54 421/412 +f 218/30 199/374 175/38 +f 457/213 364/211 421/412 +f 237/395 199/374 135/215 +f 361/226 421/412 364/211 +f 132/230 135/215 199/374 +f 362/221 402/44 289/126 +f 133/225 59/133 178/23 +f 436/322 289/126 402/44 +f 216/40 178/23 59/133 +f 354/261 266/234 384/120 +f 125/265 157/115 36/242 +f 373/166 384/120 266/234 +f 144/170 36/242 157/115 +f 256/284 250/317 340/337 +f 26/292 111/342 8/399 +f 391/92 340/337 250/317 +f 164/87 8/399 111/342 +f 262/254 256/284 449/253 +f 32/262 229/443 26/292 +f 340/337 449/253 256/284 +f 111/342 26/292 229/443 +f 15/357 318/465 14/363 +f 15/357 14/363 88/6 +f 313/22 14/363 318/465 +f 83/31 88/6 14/363 +f 318/465 403/39 313/22 +f 88/6 83/31 179/18 +f 312/27 313/22 403/39 +f 82/36 179/18 83/31 +f 403/39 319/460 312/27 +f 179/18 82/36 89/1 +f 311/32 312/27 319/460 +f 81/41 89/1 82/36 +f 319/460 325/427 311/32 +f 89/1 81/41 96/432 +f 416/442 311/32 325/427 +f 192/416 96/432 81/41 +f 469/100 472/100 473/100 +f 470/100 469/100 473/100 +f 471/100 469/100 470/100 +f 471/100 472/100 469/100 +f 474/100 477/100 478/100 +f 475/100 474/100 478/100 +f 476/100 474/100 475/100 +f 476/100 477/100 474/100 diff --git a/mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_including_iris_landmarks.pbtxt b/mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_including_iris_landmarks.pbtxt new file mode 100644 index 000000000..23b9a91e5 --- /dev/null +++ b/mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_including_iris_landmarks.pbtxt @@ -0,0 +1,5160 @@ +# Copyright 2020 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +input_source: FACE_LANDMARK_PIPELINE +procrustes_landmark_basis { landmark_id: 4 weight: 0.070909939706326 } +procrustes_landmark_basis { landmark_id: 6 weight: 0.032100144773722 } +procrustes_landmark_basis { landmark_id: 10 weight: 0.008446550928056 } +procrustes_landmark_basis { landmark_id: 33 weight: 0.058724168688059 } +procrustes_landmark_basis { landmark_id: 54 weight: 0.007667080033571 } +procrustes_landmark_basis { landmark_id: 67 weight: 0.009078059345484 } +procrustes_landmark_basis { landmark_id: 117 weight: 0.009791937656701 } +procrustes_landmark_basis { landmark_id: 119 weight: 0.014565368182957 } +procrustes_landmark_basis { landmark_id: 121 weight: 0.018591361120343 } +procrustes_landmark_basis { landmark_id: 127 weight: 0.005197994410992 } +procrustes_landmark_basis { landmark_id: 129 weight: 0.120625205338001 } +procrustes_landmark_basis { landmark_id: 132 weight: 0.005560018587857 } +procrustes_landmark_basis { landmark_id: 133 weight: 0.05328618362546 } +procrustes_landmark_basis { landmark_id: 136 weight: 0.066890455782413 } +procrustes_landmark_basis { landmark_id: 143 weight: 0.014816547743976 } +procrustes_landmark_basis { landmark_id: 147 weight: 0.014262833632529 } +procrustes_landmark_basis { landmark_id: 198 weight: 0.025462191551924 } +procrustes_landmark_basis { landmark_id: 205 weight: 0.047252278774977 } +procrustes_landmark_basis { landmark_id: 263 weight: 0.058724168688059 } +procrustes_landmark_basis { landmark_id: 284 weight: 0.007667080033571 } +procrustes_landmark_basis { landmark_id: 297 weight: 0.009078059345484 } +procrustes_landmark_basis { landmark_id: 346 weight: 0.009791937656701 } +procrustes_landmark_basis { landmark_id: 348 weight: 0.014565368182957 } +procrustes_landmark_basis { landmark_id: 350 weight: 0.018591361120343 } +procrustes_landmark_basis { landmark_id: 356 weight: 0.005197994410992 } +procrustes_landmark_basis { landmark_id: 358 weight: 0.120625205338001 } +procrustes_landmark_basis { landmark_id: 361 weight: 0.005560018587857 } +procrustes_landmark_basis { landmark_id: 362 weight: 0.05328618362546 } +procrustes_landmark_basis { landmark_id: 365 weight: 0.066890455782413 } +procrustes_landmark_basis { landmark_id: 372 weight: 0.014816547743976 } +procrustes_landmark_basis { landmark_id: 376 weight: 0.014262833632529 } +procrustes_landmark_basis { landmark_id: 420 weight: 0.025462191551924 } +procrustes_landmark_basis { landmark_id: 425 weight: 0.047252278774977 } +canonical_mesh: { + vertex_type: VERTEX_PT + primitive_type: TRIANGLE + vertex_buffer: 0.000000 + vertex_buffer: -3.406404 + vertex_buffer: 5.979507 + vertex_buffer: 0.499977 + vertex_buffer: 0.652534 + vertex_buffer: 0.000000 + vertex_buffer: -1.126865 + vertex_buffer: 7.475604 + vertex_buffer: 0.500026 + vertex_buffer: 0.547487 + vertex_buffer: 0.000000 + vertex_buffer: -2.089024 + vertex_buffer: 6.058267 + vertex_buffer: 0.499974 + vertex_buffer: 0.602372 + vertex_buffer: -0.463928 + vertex_buffer: 0.955357 + vertex_buffer: 6.633583 + vertex_buffer: 0.482113 + vertex_buffer: 0.471979 + vertex_buffer: 0.000000 + vertex_buffer: -0.463170 + vertex_buffer: 7.586580 + vertex_buffer: 0.500151 + vertex_buffer: 0.527156 + vertex_buffer: 0.000000 + vertex_buffer: 0.365669 + vertex_buffer: 7.242870 + vertex_buffer: 0.499910 + vertex_buffer: 0.498253 + vertex_buffer: 0.000000 + vertex_buffer: 2.473255 + vertex_buffer: 5.788627 + vertex_buffer: 0.499523 + vertex_buffer: 0.401062 + vertex_buffer: -4.253081 + vertex_buffer: 2.577646 + vertex_buffer: 3.279702 + vertex_buffer: 0.289712 + vertex_buffer: 0.380764 + vertex_buffer: 0.000000 + vertex_buffer: 4.019042 + vertex_buffer: 5.284764 + vertex_buffer: 0.499955 + vertex_buffer: 0.312398 + vertex_buffer: 0.000000 + vertex_buffer: 4.885979 + vertex_buffer: 5.385258 + vertex_buffer: 0.499987 + vertex_buffer: 0.269919 + vertex_buffer: 0.000000 + vertex_buffer: 8.261778 + vertex_buffer: 4.481535 + vertex_buffer: 0.500023 + vertex_buffer: 0.107050 + vertex_buffer: 0.000000 + vertex_buffer: -3.706811 + vertex_buffer: 5.864924 + vertex_buffer: 0.500023 + vertex_buffer: 0.666234 + vertex_buffer: 0.000000 + vertex_buffer: -3.918301 + vertex_buffer: 5.569430 + vertex_buffer: 0.500016 + vertex_buffer: 0.679224 + vertex_buffer: 0.000000 + vertex_buffer: -3.994436 + vertex_buffer: 5.219482 + vertex_buffer: 0.500023 + vertex_buffer: 0.692348 + vertex_buffer: 0.000000 + vertex_buffer: -4.542400 + vertex_buffer: 5.404754 + vertex_buffer: 0.499977 + vertex_buffer: 0.695278 + vertex_buffer: 0.000000 + vertex_buffer: -4.745577 + vertex_buffer: 5.529457 + vertex_buffer: 0.499977 + vertex_buffer: 0.705934 + vertex_buffer: 0.000000 + vertex_buffer: -5.019567 + vertex_buffer: 5.601448 + vertex_buffer: 0.499977 + vertex_buffer: 0.719385 + vertex_buffer: 0.000000 + vertex_buffer: -5.365123 + vertex_buffer: 5.535441 + vertex_buffer: 0.499977 + vertex_buffer: 0.737019 + vertex_buffer: 0.000000 + vertex_buffer: -6.149624 + vertex_buffer: 5.071372 + vertex_buffer: 0.499968 + vertex_buffer: 0.781371 + vertex_buffer: 0.000000 + vertex_buffer: -1.501095 + vertex_buffer: 7.112196 + vertex_buffer: 0.499816 + vertex_buffer: 0.562981 + vertex_buffer: -0.416106 + vertex_buffer: -1.466449 + vertex_buffer: 6.447657 + vertex_buffer: 0.473773 + vertex_buffer: 0.573910 + vertex_buffer: -7.087960 + vertex_buffer: 5.434801 + vertex_buffer: 0.099620 + vertex_buffer: 0.104907 + vertex_buffer: 0.254141 + vertex_buffer: -2.628639 + vertex_buffer: 2.035898 + vertex_buffer: 3.848121 + vertex_buffer: 0.365930 + vertex_buffer: 0.409576 + vertex_buffer: -3.198363 + vertex_buffer: 1.985815 + vertex_buffer: 3.796952 + vertex_buffer: 0.338758 + vertex_buffer: 0.413025 + vertex_buffer: -3.775151 + vertex_buffer: 2.039402 + vertex_buffer: 3.646194 + vertex_buffer: 0.311120 + vertex_buffer: 0.409460 + vertex_buffer: -4.465819 + vertex_buffer: 2.422950 + vertex_buffer: 3.155168 + vertex_buffer: 0.274658 + vertex_buffer: 0.389131 + vertex_buffer: -2.164289 + vertex_buffer: 2.189867 + vertex_buffer: 3.851822 + vertex_buffer: 0.393362 + vertex_buffer: 0.403706 + vertex_buffer: -3.208229 + vertex_buffer: 3.223926 + vertex_buffer: 4.115822 + vertex_buffer: 0.345234 + vertex_buffer: 0.344011 + vertex_buffer: -2.673803 + vertex_buffer: 3.205337 + vertex_buffer: 4.092203 + vertex_buffer: 0.370094 + vertex_buffer: 0.346076 + vertex_buffer: -3.745193 + vertex_buffer: 3.165286 + vertex_buffer: 3.972409 + vertex_buffer: 0.319322 + vertex_buffer: 0.347265 + vertex_buffer: -4.161018 + vertex_buffer: 3.059069 + vertex_buffer: 3.719554 + vertex_buffer: 0.297903 + vertex_buffer: 0.353591 + vertex_buffer: -5.062006 + vertex_buffer: 1.934418 + vertex_buffer: 2.776093 + vertex_buffer: 0.247792 + vertex_buffer: 0.410810 + vertex_buffer: -2.266659 + vertex_buffer: -7.425768 + vertex_buffer: 4.389812 + vertex_buffer: 0.396889 + vertex_buffer: 0.842755 + vertex_buffer: -4.445859 + vertex_buffer: 2.663991 + vertex_buffer: 3.173422 + vertex_buffer: 0.280098 + vertex_buffer: 0.375600 + vertex_buffer: -7.214530 + vertex_buffer: 2.263009 + vertex_buffer: 0.073150 + vertex_buffer: 0.106310 + vertex_buffer: 0.399956 + vertex_buffer: -5.799793 + vertex_buffer: 2.349546 + vertex_buffer: 2.204059 + vertex_buffer: 0.209925 + vertex_buffer: 0.391353 + vertex_buffer: -2.844939 + vertex_buffer: -0.720868 + vertex_buffer: 4.433130 + vertex_buffer: 0.355808 + vertex_buffer: 0.534406 + vertex_buffer: -0.711452 + vertex_buffer: -3.329355 + vertex_buffer: 5.877044 + vertex_buffer: 0.471751 + vertex_buffer: 0.650404 + vertex_buffer: -0.606033 + vertex_buffer: -3.924562 + vertex_buffer: 5.444923 + vertex_buffer: 0.474155 + vertex_buffer: 0.680192 + vertex_buffer: -1.431615 + vertex_buffer: -3.500953 + vertex_buffer: 5.496189 + vertex_buffer: 0.439785 + vertex_buffer: 0.657229 + vertex_buffer: -1.914910 + vertex_buffer: -3.803146 + vertex_buffer: 5.028930 + vertex_buffer: 0.414617 + vertex_buffer: 0.666541 + vertex_buffer: -1.131043 + vertex_buffer: -3.973937 + vertex_buffer: 5.189648 + vertex_buffer: 0.450374 + vertex_buffer: 0.680861 + vertex_buffer: -1.563548 + vertex_buffer: -4.082763 + vertex_buffer: 4.842263 + vertex_buffer: 0.428771 + vertex_buffer: 0.682691 + vertex_buffer: -2.650112 + vertex_buffer: -5.003649 + vertex_buffer: 4.188483 + vertex_buffer: 0.374971 + vertex_buffer: 0.727805 + vertex_buffer: -0.427049 + vertex_buffer: -1.094134 + vertex_buffer: 7.360529 + vertex_buffer: 0.486717 + vertex_buffer: 0.547629 + vertex_buffer: -0.496396 + vertex_buffer: -0.475659 + vertex_buffer: 7.440358 + vertex_buffer: 0.485301 + vertex_buffer: 0.527395 + vertex_buffer: -5.253307 + vertex_buffer: 3.881582 + vertex_buffer: 3.363159 + vertex_buffer: 0.257765 + vertex_buffer: 0.314490 + vertex_buffer: -1.718698 + vertex_buffer: 0.974609 + vertex_buffer: 4.558359 + vertex_buffer: 0.401223 + vertex_buffer: 0.455172 + vertex_buffer: -1.608635 + vertex_buffer: -0.942516 + vertex_buffer: 5.814193 + vertex_buffer: 0.429819 + vertex_buffer: 0.548615 + vertex_buffer: -1.651267 + vertex_buffer: -0.610868 + vertex_buffer: 5.581319 + vertex_buffer: 0.421352 + vertex_buffer: 0.533741 + vertex_buffer: -4.765501 + vertex_buffer: -0.701554 + vertex_buffer: 3.534632 + vertex_buffer: 0.276896 + vertex_buffer: 0.532057 + vertex_buffer: -0.478306 + vertex_buffer: 0.295766 + vertex_buffer: 7.101013 + vertex_buffer: 0.483370 + vertex_buffer: 0.499587 + vertex_buffer: -3.734964 + vertex_buffer: 4.508230 + vertex_buffer: 4.550454 + vertex_buffer: 0.337212 + vertex_buffer: 0.282883 + vertex_buffer: -4.588603 + vertex_buffer: 4.302037 + vertex_buffer: 4.048484 + vertex_buffer: 0.296392 + vertex_buffer: 0.293243 + vertex_buffer: -6.279331 + vertex_buffer: 6.615427 + vertex_buffer: 1.425850 + vertex_buffer: 0.169295 + vertex_buffer: 0.193814 + vertex_buffer: -1.220941 + vertex_buffer: 4.142165 + vertex_buffer: 5.106035 + vertex_buffer: 0.447580 + vertex_buffer: 0.302610 + vertex_buffer: -2.193489 + vertex_buffer: 3.100317 + vertex_buffer: 4.000575 + vertex_buffer: 0.392390 + vertex_buffer: 0.353888 + vertex_buffer: -3.102642 + vertex_buffer: -4.352984 + vertex_buffer: 4.095905 + vertex_buffer: 0.354490 + vertex_buffer: 0.696784 + vertex_buffer: -6.719682 + vertex_buffer: -4.788645 + vertex_buffer: -1.745401 + vertex_buffer: 0.067305 + vertex_buffer: 0.730105 + vertex_buffer: -1.193824 + vertex_buffer: -1.306795 + vertex_buffer: 5.737747 + vertex_buffer: 0.442739 + vertex_buffer: 0.572826 + vertex_buffer: -0.729766 + vertex_buffer: -1.593712 + vertex_buffer: 5.833208 + vertex_buffer: 0.457098 + vertex_buffer: 0.584792 + vertex_buffer: -2.456206 + vertex_buffer: -4.342621 + vertex_buffer: 4.283884 + vertex_buffer: 0.381974 + vertex_buffer: 0.694711 + vertex_buffer: -2.204823 + vertex_buffer: -4.304508 + vertex_buffer: 4.162499 + vertex_buffer: 0.392389 + vertex_buffer: 0.694203 + vertex_buffer: -4.985894 + vertex_buffer: 4.802461 + vertex_buffer: 3.751977 + vertex_buffer: 0.277076 + vertex_buffer: 0.271932 + vertex_buffer: -1.592294 + vertex_buffer: -1.257709 + vertex_buffer: 5.456949 + vertex_buffer: 0.422552 + vertex_buffer: 0.563233 + vertex_buffer: -2.644548 + vertex_buffer: 4.524654 + vertex_buffer: 4.921559 + vertex_buffer: 0.385919 + vertex_buffer: 0.281364 + vertex_buffer: -2.760292 + vertex_buffer: 5.100971 + vertex_buffer: 5.015990 + vertex_buffer: 0.383103 + vertex_buffer: 0.255840 + vertex_buffer: -3.523964 + vertex_buffer: 8.005976 + vertex_buffer: 3.729163 + vertex_buffer: 0.331431 + vertex_buffer: 0.119714 + vertex_buffer: -5.599763 + vertex_buffer: 5.715470 + vertex_buffer: 2.724259 + vertex_buffer: 0.229924 + vertex_buffer: 0.232003 + vertex_buffer: -3.063932 + vertex_buffer: 6.566144 + vertex_buffer: 4.529981 + vertex_buffer: 0.364501 + vertex_buffer: 0.189114 + vertex_buffer: -5.720968 + vertex_buffer: 4.254584 + vertex_buffer: 2.830852 + vertex_buffer: 0.229622 + vertex_buffer: 0.299541 + vertex_buffer: -6.374393 + vertex_buffer: 4.785590 + vertex_buffer: 1.591691 + vertex_buffer: 0.173287 + vertex_buffer: 0.278748 + vertex_buffer: -0.672728 + vertex_buffer: -3.688016 + vertex_buffer: 5.737804 + vertex_buffer: 0.472879 + vertex_buffer: 0.666198 + vertex_buffer: -1.262560 + vertex_buffer: -3.787691 + vertex_buffer: 5.417779 + vertex_buffer: 0.446828 + vertex_buffer: 0.668527 + vertex_buffer: -1.732553 + vertex_buffer: -3.952767 + vertex_buffer: 5.000579 + vertex_buffer: 0.422762 + vertex_buffer: 0.673890 + vertex_buffer: -1.043625 + vertex_buffer: -1.464973 + vertex_buffer: 5.662455 + vertex_buffer: 0.445308 + vertex_buffer: 0.580066 + vertex_buffer: -2.321234 + vertex_buffer: -4.329069 + vertex_buffer: 4.258156 + vertex_buffer: 0.388103 + vertex_buffer: 0.693961 + vertex_buffer: -2.056846 + vertex_buffer: -4.477671 + vertex_buffer: 4.520883 + vertex_buffer: 0.403039 + vertex_buffer: 0.706540 + vertex_buffer: -2.153084 + vertex_buffer: -4.276322 + vertex_buffer: 4.038093 + vertex_buffer: 0.403629 + vertex_buffer: 0.693953 + vertex_buffer: -0.946874 + vertex_buffer: -1.035249 + vertex_buffer: 6.512274 + vertex_buffer: 0.460042 + vertex_buffer: 0.557139 + vertex_buffer: -1.469132 + vertex_buffer: -4.036351 + vertex_buffer: 4.604908 + vertex_buffer: 0.431158 + vertex_buffer: 0.692366 + vertex_buffer: -1.024340 + vertex_buffer: -3.989851 + vertex_buffer: 4.926693 + vertex_buffer: 0.452182 + vertex_buffer: 0.692366 + vertex_buffer: -0.533422 + vertex_buffer: -3.993222 + vertex_buffer: 5.138202 + vertex_buffer: 0.475387 + vertex_buffer: 0.692366 + vertex_buffer: -0.769720 + vertex_buffer: -6.095394 + vertex_buffer: 4.985883 + vertex_buffer: 0.465828 + vertex_buffer: 0.779190 + vertex_buffer: -0.699606 + vertex_buffer: -5.291850 + vertex_buffer: 5.448304 + vertex_buffer: 0.472329 + vertex_buffer: 0.736226 + vertex_buffer: -0.669687 + vertex_buffer: -4.949770 + vertex_buffer: 5.509612 + vertex_buffer: 0.473087 + vertex_buffer: 0.717857 + vertex_buffer: -0.630947 + vertex_buffer: -4.695101 + vertex_buffer: 5.449371 + vertex_buffer: 0.473122 + vertex_buffer: 0.704626 + vertex_buffer: -0.583218 + vertex_buffer: -4.517982 + vertex_buffer: 5.339869 + vertex_buffer: 0.473033 + vertex_buffer: 0.695278 + vertex_buffer: -1.537170 + vertex_buffer: -4.423206 + vertex_buffer: 4.745470 + vertex_buffer: 0.427942 + vertex_buffer: 0.695278 + vertex_buffer: -1.615600 + vertex_buffer: -4.475942 + vertex_buffer: 4.813632 + vertex_buffer: 0.426479 + vertex_buffer: 0.703540 + vertex_buffer: -1.729053 + vertex_buffer: -4.618680 + vertex_buffer: 4.854463 + vertex_buffer: 0.423162 + vertex_buffer: 0.711846 + vertex_buffer: -1.838624 + vertex_buffer: -4.828746 + vertex_buffer: 4.823737 + vertex_buffer: 0.418309 + vertex_buffer: 0.720063 + vertex_buffer: -2.368250 + vertex_buffer: -3.106237 + vertex_buffer: 4.868096 + vertex_buffer: 0.390095 + vertex_buffer: 0.639573 + vertex_buffer: -7.542244 + vertex_buffer: -1.049282 + vertex_buffer: -2.431321 + vertex_buffer: 0.013954 + vertex_buffer: 0.560034 + vertex_buffer: 0.000000 + vertex_buffer: -1.724003 + vertex_buffer: 6.601390 + vertex_buffer: 0.499914 + vertex_buffer: 0.580147 + vertex_buffer: -1.826614 + vertex_buffer: -4.399531 + vertex_buffer: 4.399021 + vertex_buffer: 0.413200 + vertex_buffer: 0.695400 + vertex_buffer: -1.929558 + vertex_buffer: -4.411831 + vertex_buffer: 4.497052 + vertex_buffer: 0.409626 + vertex_buffer: 0.701823 + vertex_buffer: -0.597442 + vertex_buffer: -2.013686 + vertex_buffer: 5.866456 + vertex_buffer: 0.468080 + vertex_buffer: 0.601535 + vertex_buffer: -1.405627 + vertex_buffer: -1.714196 + vertex_buffer: 5.241087 + vertex_buffer: 0.422729 + vertex_buffer: 0.585985 + vertex_buffer: -0.662449 + vertex_buffer: -1.819321 + vertex_buffer: 5.863759 + vertex_buffer: 0.463080 + vertex_buffer: 0.593784 + vertex_buffer: -2.342340 + vertex_buffer: 0.572222 + vertex_buffer: 4.294303 + vertex_buffer: 0.372120 + vertex_buffer: 0.473414 + vertex_buffer: -3.327324 + vertex_buffer: 0.104863 + vertex_buffer: 4.113860 + vertex_buffer: 0.334562 + vertex_buffer: 0.496073 + vertex_buffer: -1.726175 + vertex_buffer: -0.919165 + vertex_buffer: 5.273355 + vertex_buffer: 0.411671 + vertex_buffer: 0.546965 + vertex_buffer: -5.133204 + vertex_buffer: 7.485602 + vertex_buffer: 2.660442 + vertex_buffer: 0.242176 + vertex_buffer: 0.147676 + vertex_buffer: -4.538641 + vertex_buffer: 6.319907 + vertex_buffer: 3.683424 + vertex_buffer: 0.290777 + vertex_buffer: 0.201446 + vertex_buffer: -3.986562 + vertex_buffer: 5.109487 + vertex_buffer: 4.466315 + vertex_buffer: 0.327338 + vertex_buffer: 0.256527 + vertex_buffer: -2.169681 + vertex_buffer: -5.440433 + vertex_buffer: 4.455874 + vertex_buffer: 0.399510 + vertex_buffer: 0.748921 + vertex_buffer: -1.395634 + vertex_buffer: 5.011963 + vertex_buffer: 5.316032 + vertex_buffer: 0.441728 + vertex_buffer: 0.261676 + vertex_buffer: -1.619500 + vertex_buffer: 6.599217 + vertex_buffer: 4.921106 + vertex_buffer: 0.429765 + vertex_buffer: 0.187834 + vertex_buffer: -1.891399 + vertex_buffer: 8.236377 + vertex_buffer: 4.274997 + vertex_buffer: 0.412198 + vertex_buffer: 0.108901 + vertex_buffer: -4.195832 + vertex_buffer: 2.235205 + vertex_buffer: 3.375099 + vertex_buffer: 0.288955 + vertex_buffer: 0.398952 + vertex_buffer: -5.733342 + vertex_buffer: 1.411738 + vertex_buffer: 2.431726 + vertex_buffer: 0.218937 + vertex_buffer: 0.435411 + vertex_buffer: -1.859887 + vertex_buffer: 2.355757 + vertex_buffer: 3.843181 + vertex_buffer: 0.412782 + vertex_buffer: 0.398970 + vertex_buffer: -4.988612 + vertex_buffer: 3.074654 + vertex_buffer: 3.083858 + vertex_buffer: 0.257135 + vertex_buffer: 0.355440 + vertex_buffer: -1.303263 + vertex_buffer: 1.416453 + vertex_buffer: 4.831091 + vertex_buffer: 0.427685 + vertex_buffer: 0.437961 + vertex_buffer: -1.305757 + vertex_buffer: -0.672779 + vertex_buffer: 6.415959 + vertex_buffer: 0.448340 + vertex_buffer: 0.536936 + vertex_buffer: -6.465170 + vertex_buffer: 0.937119 + vertex_buffer: 1.689873 + vertex_buffer: 0.178560 + vertex_buffer: 0.457554 + vertex_buffer: -5.258659 + vertex_buffer: 0.945811 + vertex_buffer: 2.974312 + vertex_buffer: 0.247308 + vertex_buffer: 0.457194 + vertex_buffer: -4.432338 + vertex_buffer: 0.722096 + vertex_buffer: 3.522615 + vertex_buffer: 0.286267 + vertex_buffer: 0.467675 + vertex_buffer: -3.300681 + vertex_buffer: 0.861641 + vertex_buffer: 3.872784 + vertex_buffer: 0.332828 + vertex_buffer: 0.460712 + vertex_buffer: -2.430178 + vertex_buffer: 1.131492 + vertex_buffer: 4.039035 + vertex_buffer: 0.368756 + vertex_buffer: 0.447207 + vertex_buffer: -1.820731 + vertex_buffer: 1.467954 + vertex_buffer: 4.224124 + vertex_buffer: 0.398964 + vertex_buffer: 0.432655 + vertex_buffer: -0.563221 + vertex_buffer: 2.307693 + vertex_buffer: 5.566789 + vertex_buffer: 0.476410 + vertex_buffer: 0.405806 + vertex_buffer: -6.338145 + vertex_buffer: -0.529279 + vertex_buffer: 1.881175 + vertex_buffer: 0.189241 + vertex_buffer: 0.523924 + vertex_buffer: -5.587698 + vertex_buffer: 3.208071 + vertex_buffer: 2.687839 + vertex_buffer: 0.228962 + vertex_buffer: 0.348951 + vertex_buffer: -0.242624 + vertex_buffer: -1.462857 + vertex_buffer: 7.071491 + vertex_buffer: 0.490726 + vertex_buffer: 0.562401 + vertex_buffer: -1.611251 + vertex_buffer: 0.339326 + vertex_buffer: 4.895421 + vertex_buffer: 0.404670 + vertex_buffer: 0.485133 + vertex_buffer: -7.743095 + vertex_buffer: 2.364999 + vertex_buffer: -2.005167 + vertex_buffer: 0.019469 + vertex_buffer: 0.401564 + vertex_buffer: -1.391142 + vertex_buffer: 1.851048 + vertex_buffer: 4.448999 + vertex_buffer: 0.426243 + vertex_buffer: 0.420431 + vertex_buffer: -1.785794 + vertex_buffer: -0.978284 + vertex_buffer: 4.850470 + vertex_buffer: 0.396993 + vertex_buffer: 0.548797 + vertex_buffer: -4.670959 + vertex_buffer: 2.664461 + vertex_buffer: 3.084075 + vertex_buffer: 0.266470 + vertex_buffer: 0.376977 + vertex_buffer: -1.333970 + vertex_buffer: -0.283761 + vertex_buffer: 6.097047 + vertex_buffer: 0.439121 + vertex_buffer: 0.518958 + vertex_buffer: -7.270895 + vertex_buffer: -2.890917 + vertex_buffer: -2.252455 + vertex_buffer: 0.032314 + vertex_buffer: 0.644357 + vertex_buffer: -1.856432 + vertex_buffer: 2.585245 + vertex_buffer: 3.757904 + vertex_buffer: 0.419054 + vertex_buffer: 0.387155 + vertex_buffer: -0.923388 + vertex_buffer: 0.073076 + vertex_buffer: 6.671944 + vertex_buffer: 0.462783 + vertex_buffer: 0.505747 + vertex_buffer: -5.000589 + vertex_buffer: -6.135128 + vertex_buffer: 1.892523 + vertex_buffer: 0.238979 + vertex_buffer: 0.779745 + vertex_buffer: -5.085276 + vertex_buffer: -7.178590 + vertex_buffer: 0.714711 + vertex_buffer: 0.198221 + vertex_buffer: 0.831938 + vertex_buffer: -7.159291 + vertex_buffer: -0.811820 + vertex_buffer: -0.072044 + vertex_buffer: 0.107550 + vertex_buffer: 0.540755 + vertex_buffer: -5.843051 + vertex_buffer: -5.248023 + vertex_buffer: 0.924091 + vertex_buffer: 0.183610 + vertex_buffer: 0.740257 + vertex_buffer: -6.847258 + vertex_buffer: 3.662916 + vertex_buffer: 0.724695 + vertex_buffer: 0.134410 + vertex_buffer: 0.333683 + vertex_buffer: -2.412942 + vertex_buffer: -8.258853 + vertex_buffer: 4.119213 + vertex_buffer: 0.385764 + vertex_buffer: 0.883154 + vertex_buffer: -0.179909 + vertex_buffer: -1.689864 + vertex_buffer: 6.573301 + vertex_buffer: 0.490967 + vertex_buffer: 0.579378 + vertex_buffer: -2.103655 + vertex_buffer: -0.163946 + vertex_buffer: 4.566119 + vertex_buffer: 0.382385 + vertex_buffer: 0.508573 + vertex_buffer: -6.407571 + vertex_buffer: 2.236021 + vertex_buffer: 1.560843 + vertex_buffer: 0.174399 + vertex_buffer: 0.397671 + vertex_buffer: -3.670075 + vertex_buffer: 2.360153 + vertex_buffer: 3.635230 + vertex_buffer: 0.318785 + vertex_buffer: 0.396235 + vertex_buffer: -3.177186 + vertex_buffer: 2.294265 + vertex_buffer: 3.775704 + vertex_buffer: 0.343364 + vertex_buffer: 0.400597 + vertex_buffer: -2.196121 + vertex_buffer: -4.598322 + vertex_buffer: 4.479786 + vertex_buffer: 0.396100 + vertex_buffer: 0.710217 + vertex_buffer: -6.234883 + vertex_buffer: -1.944430 + vertex_buffer: 1.663542 + vertex_buffer: 0.187885 + vertex_buffer: 0.588538 + vertex_buffer: -1.292924 + vertex_buffer: -9.295920 + vertex_buffer: 4.094063 + vertex_buffer: 0.430987 + vertex_buffer: 0.944065 + vertex_buffer: -3.210651 + vertex_buffer: -8.533278 + vertex_buffer: 2.802001 + vertex_buffer: 0.318993 + vertex_buffer: 0.898285 + vertex_buffer: -4.068926 + vertex_buffer: -7.993109 + vertex_buffer: 1.925119 + vertex_buffer: 0.266248 + vertex_buffer: 0.869701 + vertex_buffer: 0.000000 + vertex_buffer: 6.545390 + vertex_buffer: 5.027311 + vertex_buffer: 0.500023 + vertex_buffer: 0.190576 + vertex_buffer: 0.000000 + vertex_buffer: -9.403378 + vertex_buffer: 4.264492 + vertex_buffer: 0.499977 + vertex_buffer: 0.954453 + vertex_buffer: -2.724032 + vertex_buffer: 2.315802 + vertex_buffer: 3.777151 + vertex_buffer: 0.366170 + vertex_buffer: 0.398822 + vertex_buffer: -2.288460 + vertex_buffer: 2.398891 + vertex_buffer: 3.697603 + vertex_buffer: 0.393207 + vertex_buffer: 0.395537 + vertex_buffer: -1.998311 + vertex_buffer: 2.496547 + vertex_buffer: 3.689148 + vertex_buffer: 0.410373 + vertex_buffer: 0.391080 + vertex_buffer: -6.130040 + vertex_buffer: 3.399261 + vertex_buffer: 2.038516 + vertex_buffer: 0.194993 + vertex_buffer: 0.342102 + vertex_buffer: -2.288460 + vertex_buffer: 2.886504 + vertex_buffer: 3.775031 + vertex_buffer: 0.388665 + vertex_buffer: 0.362284 + vertex_buffer: -2.724032 + vertex_buffer: 2.961810 + vertex_buffer: 3.871767 + vertex_buffer: 0.365962 + vertex_buffer: 0.355971 + vertex_buffer: -3.177186 + vertex_buffer: 2.964136 + vertex_buffer: 3.876973 + vertex_buffer: 0.343364 + vertex_buffer: 0.355357 + vertex_buffer: -3.670075 + vertex_buffer: 2.927714 + vertex_buffer: 3.724325 + vertex_buffer: 0.318785 + vertex_buffer: 0.358340 + vertex_buffer: -4.018389 + vertex_buffer: 2.857357 + vertex_buffer: 3.482983 + vertex_buffer: 0.301415 + vertex_buffer: 0.363156 + vertex_buffer: -7.555811 + vertex_buffer: 4.106811 + vertex_buffer: -0.991917 + vertex_buffer: 0.058133 + vertex_buffer: 0.319076 + vertex_buffer: -4.018389 + vertex_buffer: 2.483695 + vertex_buffer: 3.440898 + vertex_buffer: 0.301415 + vertex_buffer: 0.387449 + vertex_buffer: 0.000000 + vertex_buffer: -2.521945 + vertex_buffer: 5.932265 + vertex_buffer: 0.499988 + vertex_buffer: 0.618434 + vertex_buffer: -1.776217 + vertex_buffer: -2.683946 + vertex_buffer: 5.213116 + vertex_buffer: 0.415838 + vertex_buffer: 0.624196 + vertex_buffer: -1.222237 + vertex_buffer: -1.182444 + vertex_buffer: 5.952465 + vertex_buffer: 0.445682 + vertex_buffer: 0.566077 + vertex_buffer: -0.731493 + vertex_buffer: -2.536683 + vertex_buffer: 5.815343 + vertex_buffer: 0.465844 + vertex_buffer: 0.620641 + vertex_buffer: 0.000000 + vertex_buffer: 3.271027 + vertex_buffer: 5.236015 + vertex_buffer: 0.499923 + vertex_buffer: 0.351524 + vertex_buffer: -4.135272 + vertex_buffer: -6.996638 + vertex_buffer: 2.671970 + vertex_buffer: 0.288719 + vertex_buffer: 0.819946 + vertex_buffer: -3.311811 + vertex_buffer: -7.660815 + vertex_buffer: 3.382963 + vertex_buffer: 0.335279 + vertex_buffer: 0.852820 + vertex_buffer: -1.313701 + vertex_buffer: -8.639995 + vertex_buffer: 4.702456 + vertex_buffer: 0.440512 + vertex_buffer: 0.902419 + vertex_buffer: -5.940524 + vertex_buffer: -6.223629 + vertex_buffer: -0.631468 + vertex_buffer: 0.128294 + vertex_buffer: 0.791941 + vertex_buffer: -1.998311 + vertex_buffer: 2.743838 + vertex_buffer: 3.744030 + vertex_buffer: 0.408772 + vertex_buffer: 0.373894 + vertex_buffer: -0.901447 + vertex_buffer: 1.236992 + vertex_buffer: 5.754256 + vertex_buffer: 0.455607 + vertex_buffer: 0.451801 + vertex_buffer: 0.000000 + vertex_buffer: -8.765243 + vertex_buffer: 4.891441 + vertex_buffer: 0.499877 + vertex_buffer: 0.908990 + vertex_buffer: -2.308977 + vertex_buffer: -8.974196 + vertex_buffer: 3.609070 + vertex_buffer: 0.375437 + vertex_buffer: 0.924192 + vertex_buffer: -6.954154 + vertex_buffer: -2.439843 + vertex_buffer: -0.131163 + vertex_buffer: 0.114210 + vertex_buffer: 0.615022 + vertex_buffer: -1.098819 + vertex_buffer: -4.458788 + vertex_buffer: 5.120727 + vertex_buffer: 0.448662 + vertex_buffer: 0.695278 + vertex_buffer: -1.181124 + vertex_buffer: -4.579996 + vertex_buffer: 5.189564 + vertex_buffer: 0.448020 + vertex_buffer: 0.704632 + vertex_buffer: -1.255818 + vertex_buffer: -4.787901 + vertex_buffer: 5.237051 + vertex_buffer: 0.447112 + vertex_buffer: 0.715808 + vertex_buffer: -1.325085 + vertex_buffer: -5.106507 + vertex_buffer: 5.205010 + vertex_buffer: 0.444832 + vertex_buffer: 0.730794 + vertex_buffer: -1.546388 + vertex_buffer: -5.819392 + vertex_buffer: 4.757893 + vertex_buffer: 0.430012 + vertex_buffer: 0.766809 + vertex_buffer: -1.953754 + vertex_buffer: -4.183892 + vertex_buffer: 4.431713 + vertex_buffer: 0.406787 + vertex_buffer: 0.685673 + vertex_buffer: -2.117802 + vertex_buffer: -4.137093 + vertex_buffer: 4.555096 + vertex_buffer: 0.400738 + vertex_buffer: 0.681069 + vertex_buffer: -2.285339 + vertex_buffer: -4.051196 + vertex_buffer: 4.582438 + vertex_buffer: 0.392400 + vertex_buffer: 0.677703 + vertex_buffer: -2.850160 + vertex_buffer: -3.665720 + vertex_buffer: 4.484994 + vertex_buffer: 0.367856 + vertex_buffer: 0.663919 + vertex_buffer: -5.278538 + vertex_buffer: -2.238942 + vertex_buffer: 2.861224 + vertex_buffer: 0.247923 + vertex_buffer: 0.601333 + vertex_buffer: -0.946709 + vertex_buffer: 1.907628 + vertex_buffer: 5.196779 + vertex_buffer: 0.452770 + vertex_buffer: 0.420850 + vertex_buffer: -1.314173 + vertex_buffer: 3.104912 + vertex_buffer: 4.231404 + vertex_buffer: 0.436392 + vertex_buffer: 0.359887 + vertex_buffer: -1.780000 + vertex_buffer: 2.860000 + vertex_buffer: 3.881555 + vertex_buffer: 0.416164 + vertex_buffer: 0.368714 + vertex_buffer: -1.845110 + vertex_buffer: -4.098880 + vertex_buffer: 4.247264 + vertex_buffer: 0.413386 + vertex_buffer: 0.692366 + vertex_buffer: -5.436187 + vertex_buffer: -4.030482 + vertex_buffer: 2.109852 + vertex_buffer: 0.228018 + vertex_buffer: 0.683572 + vertex_buffer: -0.766444 + vertex_buffer: 3.182131 + vertex_buffer: 4.861453 + vertex_buffer: 0.468268 + vertex_buffer: 0.352671 + vertex_buffer: -1.938616 + vertex_buffer: -6.614410 + vertex_buffer: 4.521085 + vertex_buffer: 0.411362 + vertex_buffer: 0.804327 + vertex_buffer: 0.000000 + vertex_buffer: 1.059413 + vertex_buffer: 6.774605 + vertex_buffer: 0.499989 + vertex_buffer: 0.469825 + vertex_buffer: -0.516573 + vertex_buffer: 1.583572 + vertex_buffer: 6.148363 + vertex_buffer: 0.479154 + vertex_buffer: 0.442654 + vertex_buffer: 0.000000 + vertex_buffer: 1.728369 + vertex_buffer: 6.316750 + vertex_buffer: 0.499974 + vertex_buffer: 0.439637 + vertex_buffer: -1.246815 + vertex_buffer: 0.230297 + vertex_buffer: 5.681036 + vertex_buffer: 0.432112 + vertex_buffer: 0.493589 + vertex_buffer: 0.000000 + vertex_buffer: -7.942194 + vertex_buffer: 5.181173 + vertex_buffer: 0.499886 + vertex_buffer: 0.866917 + vertex_buffer: 0.000000 + vertex_buffer: -6.991499 + vertex_buffer: 5.153478 + vertex_buffer: 0.499913 + vertex_buffer: 0.821729 + vertex_buffer: -0.997827 + vertex_buffer: -6.930921 + vertex_buffer: 4.979576 + vertex_buffer: 0.456549 + vertex_buffer: 0.819201 + vertex_buffer: -3.288807 + vertex_buffer: -5.382514 + vertex_buffer: 3.795752 + vertex_buffer: 0.344549 + vertex_buffer: 0.745439 + vertex_buffer: -2.311631 + vertex_buffer: -1.566237 + vertex_buffer: 4.590085 + vertex_buffer: 0.378909 + vertex_buffer: 0.574010 + vertex_buffer: -2.680250 + vertex_buffer: -6.111567 + vertex_buffer: 4.096152 + vertex_buffer: 0.374293 + vertex_buffer: 0.780185 + vertex_buffer: -3.832928 + vertex_buffer: -1.537326 + vertex_buffer: 4.137731 + vertex_buffer: 0.319688 + vertex_buffer: 0.570738 + vertex_buffer: -2.961860 + vertex_buffer: -2.274215 + vertex_buffer: 4.440943 + vertex_buffer: 0.357155 + vertex_buffer: 0.604270 + vertex_buffer: -4.386901 + vertex_buffer: -2.683286 + vertex_buffer: 3.643886 + vertex_buffer: 0.295284 + vertex_buffer: 0.621581 + vertex_buffer: -1.217295 + vertex_buffer: -7.834465 + vertex_buffer: 4.969286 + vertex_buffer: 0.447750 + vertex_buffer: 0.862477 + vertex_buffer: -1.542374 + vertex_buffer: -0.136843 + vertex_buffer: 5.201008 + vertex_buffer: 0.410986 + vertex_buffer: 0.508723 + vertex_buffer: -3.878377 + vertex_buffer: -6.041764 + vertex_buffer: 3.311079 + vertex_buffer: 0.313951 + vertex_buffer: 0.775308 + vertex_buffer: -3.084037 + vertex_buffer: -6.809842 + vertex_buffer: 3.814195 + vertex_buffer: 0.354128 + vertex_buffer: 0.812553 + vertex_buffer: -3.747321 + vertex_buffer: -4.503545 + vertex_buffer: 3.726453 + vertex_buffer: 0.324548 + vertex_buffer: 0.703993 + vertex_buffer: -6.094129 + vertex_buffer: -3.205991 + vertex_buffer: 1.473482 + vertex_buffer: 0.189096 + vertex_buffer: 0.646300 + vertex_buffer: -4.588995 + vertex_buffer: -4.728726 + vertex_buffer: 2.983221 + vertex_buffer: 0.279777 + vertex_buffer: 0.714658 + vertex_buffer: -6.583231 + vertex_buffer: -3.941269 + vertex_buffer: 0.070268 + vertex_buffer: 0.133823 + vertex_buffer: 0.682701 + vertex_buffer: -3.492580 + vertex_buffer: -3.195820 + vertex_buffer: 4.130198 + vertex_buffer: 0.336768 + vertex_buffer: 0.644733 + vertex_buffer: -1.255543 + vertex_buffer: 0.802341 + vertex_buffer: 5.307551 + vertex_buffer: 0.429884 + vertex_buffer: 0.466522 + vertex_buffer: -1.126122 + vertex_buffer: -0.933602 + vertex_buffer: 6.538785 + vertex_buffer: 0.455528 + vertex_buffer: 0.548623 + vertex_buffer: -1.443109 + vertex_buffer: -1.142774 + vertex_buffer: 5.905127 + vertex_buffer: 0.437114 + vertex_buffer: 0.558896 + vertex_buffer: -0.923043 + vertex_buffer: -0.529042 + vertex_buffer: 7.003423 + vertex_buffer: 0.467288 + vertex_buffer: 0.529925 + vertex_buffer: -1.755386 + vertex_buffer: 3.529117 + vertex_buffer: 4.327696 + vertex_buffer: 0.414712 + vertex_buffer: 0.335220 + vertex_buffer: -2.632589 + vertex_buffer: 3.713828 + vertex_buffer: 4.364629 + vertex_buffer: 0.377046 + vertex_buffer: 0.322778 + vertex_buffer: -3.388062 + vertex_buffer: 3.721976 + vertex_buffer: 4.309028 + vertex_buffer: 0.344108 + vertex_buffer: 0.320151 + vertex_buffer: -4.075766 + vertex_buffer: 3.675413 + vertex_buffer: 4.076063 + vertex_buffer: 0.312876 + vertex_buffer: 0.322332 + vertex_buffer: -4.622910 + vertex_buffer: 3.474691 + vertex_buffer: 3.646321 + vertex_buffer: 0.283526 + vertex_buffer: 0.333190 + vertex_buffer: -5.171755 + vertex_buffer: 2.535753 + vertex_buffer: 2.670867 + vertex_buffer: 0.241246 + vertex_buffer: 0.382786 + vertex_buffer: -7.297331 + vertex_buffer: 0.763172 + vertex_buffer: -0.048769 + vertex_buffer: 0.102986 + vertex_buffer: 0.468763 + vertex_buffer: -4.706828 + vertex_buffer: 1.651000 + vertex_buffer: 3.109532 + vertex_buffer: 0.267612 + vertex_buffer: 0.424560 + vertex_buffer: -4.071712 + vertex_buffer: 1.476821 + vertex_buffer: 3.476944 + vertex_buffer: 0.297879 + vertex_buffer: 0.433176 + vertex_buffer: -3.269817 + vertex_buffer: 1.470659 + vertex_buffer: 3.731945 + vertex_buffer: 0.333434 + vertex_buffer: 0.433878 + vertex_buffer: -2.527572 + vertex_buffer: 1.617311 + vertex_buffer: 3.865444 + vertex_buffer: 0.366427 + vertex_buffer: 0.426116 + vertex_buffer: -1.970894 + vertex_buffer: 1.858505 + vertex_buffer: 3.961782 + vertex_buffer: 0.396012 + vertex_buffer: 0.416696 + vertex_buffer: -1.579543 + vertex_buffer: 2.097941 + vertex_buffer: 4.084996 + vertex_buffer: 0.420121 + vertex_buffer: 0.410228 + vertex_buffer: -7.664182 + vertex_buffer: 0.673132 + vertex_buffer: -2.435867 + vertex_buffer: 0.007561 + vertex_buffer: 0.480777 + vertex_buffer: -1.397041 + vertex_buffer: -1.340139 + vertex_buffer: 5.630378 + vertex_buffer: 0.432949 + vertex_buffer: 0.569518 + vertex_buffer: -0.884838 + vertex_buffer: 0.658740 + vertex_buffer: 6.233232 + vertex_buffer: 0.458639 + vertex_buffer: 0.479089 + vertex_buffer: -0.767097 + vertex_buffer: -0.968035 + vertex_buffer: 7.077932 + vertex_buffer: 0.473466 + vertex_buffer: 0.545744 + vertex_buffer: -0.460213 + vertex_buffer: -1.334106 + vertex_buffer: 6.787447 + vertex_buffer: 0.476088 + vertex_buffer: 0.563830 + vertex_buffer: -0.748618 + vertex_buffer: -1.067994 + vertex_buffer: 6.798303 + vertex_buffer: 0.468472 + vertex_buffer: 0.555057 + vertex_buffer: -1.236408 + vertex_buffer: -1.585568 + vertex_buffer: 5.480490 + vertex_buffer: 0.433991 + vertex_buffer: 0.582362 + vertex_buffer: -0.387306 + vertex_buffer: -1.409990 + vertex_buffer: 6.957705 + vertex_buffer: 0.483518 + vertex_buffer: 0.562984 + vertex_buffer: -0.319925 + vertex_buffer: -1.607931 + vertex_buffer: 6.508676 + vertex_buffer: 0.482483 + vertex_buffer: 0.577849 + vertex_buffer: -1.639633 + vertex_buffer: 2.556298 + vertex_buffer: 3.863736 + vertex_buffer: 0.426450 + vertex_buffer: 0.389799 + vertex_buffer: -1.255645 + vertex_buffer: 2.467144 + vertex_buffer: 4.203800 + vertex_buffer: 0.438999 + vertex_buffer: 0.396495 + vertex_buffer: -1.031362 + vertex_buffer: 2.382663 + vertex_buffer: 4.615849 + vertex_buffer: 0.450067 + vertex_buffer: 0.400434 + vertex_buffer: -4.253081 + vertex_buffer: 2.772296 + vertex_buffer: 3.315305 + vertex_buffer: 0.289712 + vertex_buffer: 0.368253 + vertex_buffer: -4.530000 + vertex_buffer: 2.910000 + vertex_buffer: 3.339685 + vertex_buffer: 0.276670 + vertex_buffer: 0.363373 + vertex_buffer: 0.463928 + vertex_buffer: 0.955357 + vertex_buffer: 6.633583 + vertex_buffer: 0.517862 + vertex_buffer: 0.471948 + vertex_buffer: 4.253081 + vertex_buffer: 2.577646 + vertex_buffer: 3.279702 + vertex_buffer: 0.710288 + vertex_buffer: 0.380764 + vertex_buffer: 0.416106 + vertex_buffer: -1.466449 + vertex_buffer: 6.447657 + vertex_buffer: 0.526227 + vertex_buffer: 0.573910 + vertex_buffer: 7.087960 + vertex_buffer: 5.434801 + vertex_buffer: 0.099620 + vertex_buffer: 0.895093 + vertex_buffer: 0.254141 + vertex_buffer: 2.628639 + vertex_buffer: 2.035898 + vertex_buffer: 3.848121 + vertex_buffer: 0.634070 + vertex_buffer: 0.409576 + vertex_buffer: 3.198363 + vertex_buffer: 1.985815 + vertex_buffer: 3.796952 + vertex_buffer: 0.661242 + vertex_buffer: 0.413025 + vertex_buffer: 3.775151 + vertex_buffer: 2.039402 + vertex_buffer: 3.646194 + vertex_buffer: 0.688880 + vertex_buffer: 0.409460 + vertex_buffer: 4.465819 + vertex_buffer: 2.422950 + vertex_buffer: 3.155168 + vertex_buffer: 0.725342 + vertex_buffer: 0.389131 + vertex_buffer: 2.164289 + vertex_buffer: 2.189867 + vertex_buffer: 3.851822 + vertex_buffer: 0.606630 + vertex_buffer: 0.403705 + vertex_buffer: 3.208229 + vertex_buffer: 3.223926 + vertex_buffer: 4.115822 + vertex_buffer: 0.654766 + vertex_buffer: 0.344011 + vertex_buffer: 2.673803 + vertex_buffer: 3.205337 + vertex_buffer: 4.092203 + vertex_buffer: 0.629906 + vertex_buffer: 0.346076 + vertex_buffer: 3.745193 + vertex_buffer: 3.165286 + vertex_buffer: 3.972409 + vertex_buffer: 0.680678 + vertex_buffer: 0.347265 + vertex_buffer: 4.161018 + vertex_buffer: 3.059069 + vertex_buffer: 3.719554 + vertex_buffer: 0.702097 + vertex_buffer: 0.353591 + vertex_buffer: 5.062006 + vertex_buffer: 1.934418 + vertex_buffer: 2.776093 + vertex_buffer: 0.752212 + vertex_buffer: 0.410805 + vertex_buffer: 2.266659 + vertex_buffer: -7.425768 + vertex_buffer: 4.389812 + vertex_buffer: 0.602918 + vertex_buffer: 0.842863 + vertex_buffer: 4.445859 + vertex_buffer: 2.663991 + vertex_buffer: 3.173422 + vertex_buffer: 0.719902 + vertex_buffer: 0.375600 + vertex_buffer: 7.214530 + vertex_buffer: 2.263009 + vertex_buffer: 0.073150 + vertex_buffer: 0.893693 + vertex_buffer: 0.399960 + vertex_buffer: 5.799793 + vertex_buffer: 2.349546 + vertex_buffer: 2.204059 + vertex_buffer: 0.790082 + vertex_buffer: 0.391354 + vertex_buffer: 2.844939 + vertex_buffer: -0.720868 + vertex_buffer: 4.433130 + vertex_buffer: 0.643998 + vertex_buffer: 0.534488 + vertex_buffer: 0.711452 + vertex_buffer: -3.329355 + vertex_buffer: 5.877044 + vertex_buffer: 0.528249 + vertex_buffer: 0.650404 + vertex_buffer: 0.606033 + vertex_buffer: -3.924562 + vertex_buffer: 5.444923 + vertex_buffer: 0.525850 + vertex_buffer: 0.680191 + vertex_buffer: 1.431615 + vertex_buffer: -3.500953 + vertex_buffer: 5.496189 + vertex_buffer: 0.560215 + vertex_buffer: 0.657229 + vertex_buffer: 1.914910 + vertex_buffer: -3.803146 + vertex_buffer: 5.028930 + vertex_buffer: 0.585384 + vertex_buffer: 0.666541 + vertex_buffer: 1.131043 + vertex_buffer: -3.973937 + vertex_buffer: 5.189648 + vertex_buffer: 0.549626 + vertex_buffer: 0.680861 + vertex_buffer: 1.563548 + vertex_buffer: -4.082763 + vertex_buffer: 4.842263 + vertex_buffer: 0.571228 + vertex_buffer: 0.682692 + vertex_buffer: 2.650112 + vertex_buffer: -5.003649 + vertex_buffer: 4.188483 + vertex_buffer: 0.624852 + vertex_buffer: 0.728099 + vertex_buffer: 0.427049 + vertex_buffer: -1.094134 + vertex_buffer: 7.360529 + vertex_buffer: 0.513050 + vertex_buffer: 0.547282 + vertex_buffer: 0.496396 + vertex_buffer: -0.475659 + vertex_buffer: 7.440358 + vertex_buffer: 0.515097 + vertex_buffer: 0.527252 + vertex_buffer: 5.253307 + vertex_buffer: 3.881582 + vertex_buffer: 3.363159 + vertex_buffer: 0.742247 + vertex_buffer: 0.314507 + vertex_buffer: 1.718698 + vertex_buffer: 0.974609 + vertex_buffer: 4.558359 + vertex_buffer: 0.598631 + vertex_buffer: 0.454979 + vertex_buffer: 1.608635 + vertex_buffer: -0.942516 + vertex_buffer: 5.814193 + vertex_buffer: 0.570338 + vertex_buffer: 0.548575 + vertex_buffer: 1.651267 + vertex_buffer: -0.610868 + vertex_buffer: 5.581319 + vertex_buffer: 0.578632 + vertex_buffer: 0.533623 + vertex_buffer: 4.765501 + vertex_buffer: -0.701554 + vertex_buffer: 3.534632 + vertex_buffer: 0.723087 + vertex_buffer: 0.532054 + vertex_buffer: 0.478306 + vertex_buffer: 0.295766 + vertex_buffer: 7.101013 + vertex_buffer: 0.516446 + vertex_buffer: 0.499639 + vertex_buffer: 3.734964 + vertex_buffer: 4.508230 + vertex_buffer: 4.550454 + vertex_buffer: 0.662801 + vertex_buffer: 0.282918 + vertex_buffer: 4.588603 + vertex_buffer: 4.302037 + vertex_buffer: 4.048484 + vertex_buffer: 0.703624 + vertex_buffer: 0.293271 + vertex_buffer: 6.279331 + vertex_buffer: 6.615427 + vertex_buffer: 1.425850 + vertex_buffer: 0.830705 + vertex_buffer: 0.193814 + vertex_buffer: 1.220941 + vertex_buffer: 4.142165 + vertex_buffer: 5.106035 + vertex_buffer: 0.552386 + vertex_buffer: 0.302568 + vertex_buffer: 2.193489 + vertex_buffer: 3.100317 + vertex_buffer: 4.000575 + vertex_buffer: 0.607610 + vertex_buffer: 0.353888 + vertex_buffer: 3.102642 + vertex_buffer: -4.352984 + vertex_buffer: 4.095905 + vertex_buffer: 0.645429 + vertex_buffer: 0.696707 + vertex_buffer: 6.719682 + vertex_buffer: -4.788645 + vertex_buffer: -1.745401 + vertex_buffer: 0.932695 + vertex_buffer: 0.730105 + vertex_buffer: 1.193824 + vertex_buffer: -1.306795 + vertex_buffer: 5.737747 + vertex_buffer: 0.557261 + vertex_buffer: 0.572826 + vertex_buffer: 0.729766 + vertex_buffer: -1.593712 + vertex_buffer: 5.833208 + vertex_buffer: 0.542902 + vertex_buffer: 0.584792 + vertex_buffer: 2.456206 + vertex_buffer: -4.342621 + vertex_buffer: 4.283884 + vertex_buffer: 0.618026 + vertex_buffer: 0.694711 + vertex_buffer: 2.204823 + vertex_buffer: -4.304508 + vertex_buffer: 4.162499 + vertex_buffer: 0.607591 + vertex_buffer: 0.694203 + vertex_buffer: 4.985894 + vertex_buffer: 4.802461 + vertex_buffer: 3.751977 + vertex_buffer: 0.722943 + vertex_buffer: 0.271963 + vertex_buffer: 1.592294 + vertex_buffer: -1.257709 + vertex_buffer: 5.456949 + vertex_buffer: 0.577414 + vertex_buffer: 0.563167 + vertex_buffer: 2.644548 + vertex_buffer: 4.524654 + vertex_buffer: 4.921559 + vertex_buffer: 0.614083 + vertex_buffer: 0.281387 + vertex_buffer: 2.760292 + vertex_buffer: 5.100971 + vertex_buffer: 5.015990 + vertex_buffer: 0.616907 + vertex_buffer: 0.255886 + vertex_buffer: 3.523964 + vertex_buffer: 8.005976 + vertex_buffer: 3.729163 + vertex_buffer: 0.668509 + vertex_buffer: 0.119914 + vertex_buffer: 5.599763 + vertex_buffer: 5.715470 + vertex_buffer: 2.724259 + vertex_buffer: 0.770092 + vertex_buffer: 0.232021 + vertex_buffer: 3.063932 + vertex_buffer: 6.566144 + vertex_buffer: 4.529981 + vertex_buffer: 0.635536 + vertex_buffer: 0.189249 + vertex_buffer: 5.720968 + vertex_buffer: 4.254584 + vertex_buffer: 2.830852 + vertex_buffer: 0.770391 + vertex_buffer: 0.299556 + vertex_buffer: 6.374393 + vertex_buffer: 4.785590 + vertex_buffer: 1.591691 + vertex_buffer: 0.826722 + vertex_buffer: 0.278755 + vertex_buffer: 0.672728 + vertex_buffer: -3.688016 + vertex_buffer: 5.737804 + vertex_buffer: 0.527121 + vertex_buffer: 0.666198 + vertex_buffer: 1.262560 + vertex_buffer: -3.787691 + vertex_buffer: 5.417779 + vertex_buffer: 0.553172 + vertex_buffer: 0.668527 + vertex_buffer: 1.732553 + vertex_buffer: -3.952767 + vertex_buffer: 5.000579 + vertex_buffer: 0.577238 + vertex_buffer: 0.673890 + vertex_buffer: 1.043625 + vertex_buffer: -1.464973 + vertex_buffer: 5.662455 + vertex_buffer: 0.554692 + vertex_buffer: 0.580066 + vertex_buffer: 2.321234 + vertex_buffer: -4.329069 + vertex_buffer: 4.258156 + vertex_buffer: 0.611897 + vertex_buffer: 0.693961 + vertex_buffer: 2.056846 + vertex_buffer: -4.477671 + vertex_buffer: 4.520883 + vertex_buffer: 0.596961 + vertex_buffer: 0.706540 + vertex_buffer: 2.153084 + vertex_buffer: -4.276322 + vertex_buffer: 4.038093 + vertex_buffer: 0.596371 + vertex_buffer: 0.693953 + vertex_buffer: 0.946874 + vertex_buffer: -1.035249 + vertex_buffer: 6.512274 + vertex_buffer: 0.539958 + vertex_buffer: 0.557139 + vertex_buffer: 1.469132 + vertex_buffer: -4.036351 + vertex_buffer: 4.604908 + vertex_buffer: 0.568842 + vertex_buffer: 0.692366 + vertex_buffer: 1.024340 + vertex_buffer: -3.989851 + vertex_buffer: 4.926693 + vertex_buffer: 0.547818 + vertex_buffer: 0.692366 + vertex_buffer: 0.533422 + vertex_buffer: -3.993222 + vertex_buffer: 5.138202 + vertex_buffer: 0.524613 + vertex_buffer: 0.692366 + vertex_buffer: 0.769720 + vertex_buffer: -6.095394 + vertex_buffer: 4.985883 + vertex_buffer: 0.534090 + vertex_buffer: 0.779141 + vertex_buffer: 0.699606 + vertex_buffer: -5.291850 + vertex_buffer: 5.448304 + vertex_buffer: 0.527671 + vertex_buffer: 0.736226 + vertex_buffer: 0.669687 + vertex_buffer: -4.949770 + vertex_buffer: 5.509612 + vertex_buffer: 0.526913 + vertex_buffer: 0.717857 + vertex_buffer: 0.630947 + vertex_buffer: -4.695101 + vertex_buffer: 5.449371 + vertex_buffer: 0.526878 + vertex_buffer: 0.704626 + vertex_buffer: 0.583218 + vertex_buffer: -4.517982 + vertex_buffer: 5.339869 + vertex_buffer: 0.526967 + vertex_buffer: 0.695278 + vertex_buffer: 1.537170 + vertex_buffer: -4.423206 + vertex_buffer: 4.745470 + vertex_buffer: 0.572058 + vertex_buffer: 0.695278 + vertex_buffer: 1.615600 + vertex_buffer: -4.475942 + vertex_buffer: 4.813632 + vertex_buffer: 0.573521 + vertex_buffer: 0.703540 + vertex_buffer: 1.729053 + vertex_buffer: -4.618680 + vertex_buffer: 4.854463 + vertex_buffer: 0.576838 + vertex_buffer: 0.711846 + vertex_buffer: 1.838624 + vertex_buffer: -4.828746 + vertex_buffer: 4.823737 + vertex_buffer: 0.581691 + vertex_buffer: 0.720063 + vertex_buffer: 2.368250 + vertex_buffer: -3.106237 + vertex_buffer: 4.868096 + vertex_buffer: 0.609945 + vertex_buffer: 0.639910 + vertex_buffer: 7.542244 + vertex_buffer: -1.049282 + vertex_buffer: -2.431321 + vertex_buffer: 0.986046 + vertex_buffer: 0.560034 + vertex_buffer: 1.826614 + vertex_buffer: -4.399531 + vertex_buffer: 4.399021 + vertex_buffer: 0.586800 + vertex_buffer: 0.695400 + vertex_buffer: 1.929558 + vertex_buffer: -4.411831 + vertex_buffer: 4.497052 + vertex_buffer: 0.590372 + vertex_buffer: 0.701823 + vertex_buffer: 0.597442 + vertex_buffer: -2.013686 + vertex_buffer: 5.866456 + vertex_buffer: 0.531915 + vertex_buffer: 0.601537 + vertex_buffer: 1.405627 + vertex_buffer: -1.714196 + vertex_buffer: 5.241087 + vertex_buffer: 0.577268 + vertex_buffer: 0.585935 + vertex_buffer: 0.662449 + vertex_buffer: -1.819321 + vertex_buffer: 5.863759 + vertex_buffer: 0.536915 + vertex_buffer: 0.593786 + vertex_buffer: 2.342340 + vertex_buffer: 0.572222 + vertex_buffer: 4.294303 + vertex_buffer: 0.627543 + vertex_buffer: 0.473352 + vertex_buffer: 3.327324 + vertex_buffer: 0.104863 + vertex_buffer: 4.113860 + vertex_buffer: 0.665586 + vertex_buffer: 0.495951 + vertex_buffer: 1.726175 + vertex_buffer: -0.919165 + vertex_buffer: 5.273355 + vertex_buffer: 0.588354 + vertex_buffer: 0.546862 + vertex_buffer: 5.133204 + vertex_buffer: 7.485602 + vertex_buffer: 2.660442 + vertex_buffer: 0.757824 + vertex_buffer: 0.147676 + vertex_buffer: 4.538641 + vertex_buffer: 6.319907 + vertex_buffer: 3.683424 + vertex_buffer: 0.709250 + vertex_buffer: 0.201508 + vertex_buffer: 3.986562 + vertex_buffer: 5.109487 + vertex_buffer: 4.466315 + vertex_buffer: 0.672684 + vertex_buffer: 0.256581 + vertex_buffer: 2.169681 + vertex_buffer: -5.440433 + vertex_buffer: 4.455874 + vertex_buffer: 0.600409 + vertex_buffer: 0.749005 + vertex_buffer: 1.395634 + vertex_buffer: 5.011963 + vertex_buffer: 5.316032 + vertex_buffer: 0.558266 + vertex_buffer: 0.261672 + vertex_buffer: 1.619500 + vertex_buffer: 6.599217 + vertex_buffer: 4.921106 + vertex_buffer: 0.570304 + vertex_buffer: 0.187871 + vertex_buffer: 1.891399 + vertex_buffer: 8.236377 + vertex_buffer: 4.274997 + vertex_buffer: 0.588166 + vertex_buffer: 0.109044 + vertex_buffer: 4.195832 + vertex_buffer: 2.235205 + vertex_buffer: 3.375099 + vertex_buffer: 0.711045 + vertex_buffer: 0.398952 + vertex_buffer: 5.733342 + vertex_buffer: 1.411738 + vertex_buffer: 2.431726 + vertex_buffer: 0.781070 + vertex_buffer: 0.435405 + vertex_buffer: 1.859887 + vertex_buffer: 2.355757 + vertex_buffer: 3.843181 + vertex_buffer: 0.587247 + vertex_buffer: 0.398932 + vertex_buffer: 4.988612 + vertex_buffer: 3.074654 + vertex_buffer: 3.083858 + vertex_buffer: 0.742870 + vertex_buffer: 0.355446 + vertex_buffer: 1.303263 + vertex_buffer: 1.416453 + vertex_buffer: 4.831091 + vertex_buffer: 0.572156 + vertex_buffer: 0.437652 + vertex_buffer: 1.305757 + vertex_buffer: -0.672779 + vertex_buffer: 6.415959 + vertex_buffer: 0.551868 + vertex_buffer: 0.536570 + vertex_buffer: 6.465170 + vertex_buffer: 0.937119 + vertex_buffer: 1.689873 + vertex_buffer: 0.821442 + vertex_buffer: 0.457556 + vertex_buffer: 5.258659 + vertex_buffer: 0.945811 + vertex_buffer: 2.974312 + vertex_buffer: 0.752702 + vertex_buffer: 0.457182 + vertex_buffer: 4.432338 + vertex_buffer: 0.722096 + vertex_buffer: 3.522615 + vertex_buffer: 0.713757 + vertex_buffer: 0.467627 + vertex_buffer: 3.300681 + vertex_buffer: 0.861641 + vertex_buffer: 3.872784 + vertex_buffer: 0.667113 + vertex_buffer: 0.460673 + vertex_buffer: 2.430178 + vertex_buffer: 1.131492 + vertex_buffer: 4.039035 + vertex_buffer: 0.631101 + vertex_buffer: 0.447154 + vertex_buffer: 1.820731 + vertex_buffer: 1.467954 + vertex_buffer: 4.224124 + vertex_buffer: 0.600862 + vertex_buffer: 0.432473 + vertex_buffer: 0.563221 + vertex_buffer: 2.307693 + vertex_buffer: 5.566789 + vertex_buffer: 0.523481 + vertex_buffer: 0.405627 + vertex_buffer: 6.338145 + vertex_buffer: -0.529279 + vertex_buffer: 1.881175 + vertex_buffer: 0.810748 + vertex_buffer: 0.523926 + vertex_buffer: 5.587698 + vertex_buffer: 3.208071 + vertex_buffer: 2.687839 + vertex_buffer: 0.771046 + vertex_buffer: 0.348959 + vertex_buffer: 0.242624 + vertex_buffer: -1.462857 + vertex_buffer: 7.071491 + vertex_buffer: 0.509127 + vertex_buffer: 0.562718 + vertex_buffer: 1.611251 + vertex_buffer: 0.339326 + vertex_buffer: 4.895421 + vertex_buffer: 0.595293 + vertex_buffer: 0.485024 + vertex_buffer: 7.743095 + vertex_buffer: 2.364999 + vertex_buffer: -2.005167 + vertex_buffer: 0.980531 + vertex_buffer: 0.401564 + vertex_buffer: 1.391142 + vertex_buffer: 1.851048 + vertex_buffer: 4.448999 + vertex_buffer: 0.573500 + vertex_buffer: 0.420000 + vertex_buffer: 1.785794 + vertex_buffer: -0.978284 + vertex_buffer: 4.850470 + vertex_buffer: 0.602995 + vertex_buffer: 0.548688 + vertex_buffer: 4.670959 + vertex_buffer: 2.664461 + vertex_buffer: 3.084075 + vertex_buffer: 0.733530 + vertex_buffer: 0.376977 + vertex_buffer: 1.333970 + vertex_buffer: -0.283761 + vertex_buffer: 6.097047 + vertex_buffer: 0.560611 + vertex_buffer: 0.519017 + vertex_buffer: 7.270895 + vertex_buffer: -2.890917 + vertex_buffer: -2.252455 + vertex_buffer: 0.967686 + vertex_buffer: 0.644357 + vertex_buffer: 1.856432 + vertex_buffer: 2.585245 + vertex_buffer: 3.757904 + vertex_buffer: 0.580985 + vertex_buffer: 0.387160 + vertex_buffer: 0.923388 + vertex_buffer: 0.073076 + vertex_buffer: 6.671944 + vertex_buffer: 0.537728 + vertex_buffer: 0.505385 + vertex_buffer: 5.000589 + vertex_buffer: -6.135128 + vertex_buffer: 1.892523 + vertex_buffer: 0.760966 + vertex_buffer: 0.779753 + vertex_buffer: 5.085276 + vertex_buffer: -7.178590 + vertex_buffer: 0.714711 + vertex_buffer: 0.801779 + vertex_buffer: 0.831938 + vertex_buffer: 7.159291 + vertex_buffer: -0.811820 + vertex_buffer: -0.072044 + vertex_buffer: 0.892441 + vertex_buffer: 0.540761 + vertex_buffer: 5.843051 + vertex_buffer: -5.248023 + vertex_buffer: 0.924091 + vertex_buffer: 0.816351 + vertex_buffer: 0.740260 + vertex_buffer: 6.847258 + vertex_buffer: 3.662916 + vertex_buffer: 0.724695 + vertex_buffer: 0.865595 + vertex_buffer: 0.333687 + vertex_buffer: 2.412942 + vertex_buffer: -8.258853 + vertex_buffer: 4.119213 + vertex_buffer: 0.614074 + vertex_buffer: 0.883246 + vertex_buffer: 0.179909 + vertex_buffer: -1.689864 + vertex_buffer: 6.573301 + vertex_buffer: 0.508953 + vertex_buffer: 0.579438 + vertex_buffer: 2.103655 + vertex_buffer: -0.163946 + vertex_buffer: 4.566119 + vertex_buffer: 0.617942 + vertex_buffer: 0.508316 + vertex_buffer: 6.407571 + vertex_buffer: 2.236021 + vertex_buffer: 1.560843 + vertex_buffer: 0.825608 + vertex_buffer: 0.397675 + vertex_buffer: 3.670075 + vertex_buffer: 2.360153 + vertex_buffer: 3.635230 + vertex_buffer: 0.681215 + vertex_buffer: 0.396235 + vertex_buffer: 3.177186 + vertex_buffer: 2.294265 + vertex_buffer: 3.775704 + vertex_buffer: 0.656636 + vertex_buffer: 0.400597 + vertex_buffer: 2.196121 + vertex_buffer: -4.598322 + vertex_buffer: 4.479786 + vertex_buffer: 0.603900 + vertex_buffer: 0.710217 + vertex_buffer: 6.234883 + vertex_buffer: -1.944430 + vertex_buffer: 1.663542 + vertex_buffer: 0.812086 + vertex_buffer: 0.588539 + vertex_buffer: 1.292924 + vertex_buffer: -9.295920 + vertex_buffer: 4.094063 + vertex_buffer: 0.568013 + vertex_buffer: 0.944565 + vertex_buffer: 3.210651 + vertex_buffer: -8.533278 + vertex_buffer: 2.802001 + vertex_buffer: 0.681008 + vertex_buffer: 0.898285 + vertex_buffer: 4.068926 + vertex_buffer: -7.993109 + vertex_buffer: 1.925119 + vertex_buffer: 0.733752 + vertex_buffer: 0.869701 + vertex_buffer: 2.724032 + vertex_buffer: 2.315802 + vertex_buffer: 3.777151 + vertex_buffer: 0.633830 + vertex_buffer: 0.398822 + vertex_buffer: 2.288460 + vertex_buffer: 2.398891 + vertex_buffer: 3.697603 + vertex_buffer: 0.606793 + vertex_buffer: 0.395537 + vertex_buffer: 1.998311 + vertex_buffer: 2.496547 + vertex_buffer: 3.689148 + vertex_buffer: 0.589660 + vertex_buffer: 0.391062 + vertex_buffer: 6.130040 + vertex_buffer: 3.399261 + vertex_buffer: 2.038516 + vertex_buffer: 0.805016 + vertex_buffer: 0.342108 + vertex_buffer: 2.288460 + vertex_buffer: 2.886504 + vertex_buffer: 3.775031 + vertex_buffer: 0.611335 + vertex_buffer: 0.362284 + vertex_buffer: 2.724032 + vertex_buffer: 2.961810 + vertex_buffer: 3.871767 + vertex_buffer: 0.634038 + vertex_buffer: 0.355971 + vertex_buffer: 3.177186 + vertex_buffer: 2.964136 + vertex_buffer: 3.876973 + vertex_buffer: 0.656636 + vertex_buffer: 0.355357 + vertex_buffer: 3.670075 + vertex_buffer: 2.927714 + vertex_buffer: 3.724325 + vertex_buffer: 0.681215 + vertex_buffer: 0.358340 + vertex_buffer: 4.018389 + vertex_buffer: 2.857357 + vertex_buffer: 3.482983 + vertex_buffer: 0.698585 + vertex_buffer: 0.363156 + vertex_buffer: 7.555811 + vertex_buffer: 4.106811 + vertex_buffer: -0.991917 + vertex_buffer: 0.941867 + vertex_buffer: 0.319076 + vertex_buffer: 4.018389 + vertex_buffer: 2.483695 + vertex_buffer: 3.440898 + vertex_buffer: 0.698585 + vertex_buffer: 0.387449 + vertex_buffer: 1.776217 + vertex_buffer: -2.683946 + vertex_buffer: 5.213116 + vertex_buffer: 0.584177 + vertex_buffer: 0.624107 + vertex_buffer: 1.222237 + vertex_buffer: -1.182444 + vertex_buffer: 5.952465 + vertex_buffer: 0.554318 + vertex_buffer: 0.566077 + vertex_buffer: 0.731493 + vertex_buffer: -2.536683 + vertex_buffer: 5.815343 + vertex_buffer: 0.534154 + vertex_buffer: 0.620640 + vertex_buffer: 4.135272 + vertex_buffer: -6.996638 + vertex_buffer: 2.671970 + vertex_buffer: 0.711218 + vertex_buffer: 0.819975 + vertex_buffer: 3.311811 + vertex_buffer: -7.660815 + vertex_buffer: 3.382963 + vertex_buffer: 0.664630 + vertex_buffer: 0.852871 + vertex_buffer: 1.313701 + vertex_buffer: -8.639995 + vertex_buffer: 4.702456 + vertex_buffer: 0.559100 + vertex_buffer: 0.902632 + vertex_buffer: 5.940524 + vertex_buffer: -6.223629 + vertex_buffer: -0.631468 + vertex_buffer: 0.871706 + vertex_buffer: 0.791941 + vertex_buffer: 1.998311 + vertex_buffer: 2.743838 + vertex_buffer: 3.744030 + vertex_buffer: 0.591234 + vertex_buffer: 0.373894 + vertex_buffer: 0.901447 + vertex_buffer: 1.236992 + vertex_buffer: 5.754256 + vertex_buffer: 0.544341 + vertex_buffer: 0.451584 + vertex_buffer: 2.308977 + vertex_buffer: -8.974196 + vertex_buffer: 3.609070 + vertex_buffer: 0.624563 + vertex_buffer: 0.924192 + vertex_buffer: 6.954154 + vertex_buffer: -2.439843 + vertex_buffer: -0.131163 + vertex_buffer: 0.885770 + vertex_buffer: 0.615029 + vertex_buffer: 1.098819 + vertex_buffer: -4.458788 + vertex_buffer: 5.120727 + vertex_buffer: 0.551338 + vertex_buffer: 0.695278 + vertex_buffer: 1.181124 + vertex_buffer: -4.579996 + vertex_buffer: 5.189564 + vertex_buffer: 0.551980 + vertex_buffer: 0.704632 + vertex_buffer: 1.255818 + vertex_buffer: -4.787901 + vertex_buffer: 5.237051 + vertex_buffer: 0.552888 + vertex_buffer: 0.715808 + vertex_buffer: 1.325085 + vertex_buffer: -5.106507 + vertex_buffer: 5.205010 + vertex_buffer: 0.555168 + vertex_buffer: 0.730794 + vertex_buffer: 1.546388 + vertex_buffer: -5.819392 + vertex_buffer: 4.757893 + vertex_buffer: 0.569944 + vertex_buffer: 0.767035 + vertex_buffer: 1.953754 + vertex_buffer: -4.183892 + vertex_buffer: 4.431713 + vertex_buffer: 0.593203 + vertex_buffer: 0.685676 + vertex_buffer: 2.117802 + vertex_buffer: -4.137093 + vertex_buffer: 4.555096 + vertex_buffer: 0.599262 + vertex_buffer: 0.681069 + vertex_buffer: 2.285339 + vertex_buffer: -4.051196 + vertex_buffer: 4.582438 + vertex_buffer: 0.607600 + vertex_buffer: 0.677703 + vertex_buffer: 2.850160 + vertex_buffer: -3.665720 + vertex_buffer: 4.484994 + vertex_buffer: 0.631938 + vertex_buffer: 0.663500 + vertex_buffer: 5.278538 + vertex_buffer: -2.238942 + vertex_buffer: 2.861224 + vertex_buffer: 0.752033 + vertex_buffer: 0.601315 + vertex_buffer: 0.946709 + vertex_buffer: 1.907628 + vertex_buffer: 5.196779 + vertex_buffer: 0.547226 + vertex_buffer: 0.420395 + vertex_buffer: 1.314173 + vertex_buffer: 3.104912 + vertex_buffer: 4.231404 + vertex_buffer: 0.563544 + vertex_buffer: 0.359828 + vertex_buffer: 1.780000 + vertex_buffer: 2.860000 + vertex_buffer: 3.881555 + vertex_buffer: 0.583841 + vertex_buffer: 0.368714 + vertex_buffer: 1.845110 + vertex_buffer: -4.098880 + vertex_buffer: 4.247264 + vertex_buffer: 0.586614 + vertex_buffer: 0.692366 + vertex_buffer: 5.436187 + vertex_buffer: -4.030482 + vertex_buffer: 2.109852 + vertex_buffer: 0.771915 + vertex_buffer: 0.683578 + vertex_buffer: 0.766444 + vertex_buffer: 3.182131 + vertex_buffer: 4.861453 + vertex_buffer: 0.531597 + vertex_buffer: 0.352483 + vertex_buffer: 1.938616 + vertex_buffer: -6.614410 + vertex_buffer: 4.521085 + vertex_buffer: 0.588371 + vertex_buffer: 0.804441 + vertex_buffer: 0.516573 + vertex_buffer: 1.583572 + vertex_buffer: 6.148363 + vertex_buffer: 0.520797 + vertex_buffer: 0.442565 + vertex_buffer: 1.246815 + vertex_buffer: 0.230297 + vertex_buffer: 5.681036 + vertex_buffer: 0.567985 + vertex_buffer: 0.493479 + vertex_buffer: 0.997827 + vertex_buffer: -6.930921 + vertex_buffer: 4.979576 + vertex_buffer: 0.543283 + vertex_buffer: 0.819255 + vertex_buffer: 3.288807 + vertex_buffer: -5.382514 + vertex_buffer: 3.795752 + vertex_buffer: 0.655317 + vertex_buffer: 0.745515 + vertex_buffer: 2.311631 + vertex_buffer: -1.566237 + vertex_buffer: 4.590085 + vertex_buffer: 0.621009 + vertex_buffer: 0.574018 + vertex_buffer: 2.680250 + vertex_buffer: -6.111567 + vertex_buffer: 4.096152 + vertex_buffer: 0.625560 + vertex_buffer: 0.780312 + vertex_buffer: 3.832928 + vertex_buffer: -1.537326 + vertex_buffer: 4.137731 + vertex_buffer: 0.680198 + vertex_buffer: 0.570719 + vertex_buffer: 2.961860 + vertex_buffer: -2.274215 + vertex_buffer: 4.440943 + vertex_buffer: 0.642764 + vertex_buffer: 0.604338 + vertex_buffer: 4.386901 + vertex_buffer: -2.683286 + vertex_buffer: 3.643886 + vertex_buffer: 0.704663 + vertex_buffer: 0.621530 + vertex_buffer: 1.217295 + vertex_buffer: -7.834465 + vertex_buffer: 4.969286 + vertex_buffer: 0.552012 + vertex_buffer: 0.862592 + vertex_buffer: 1.542374 + vertex_buffer: -0.136843 + vertex_buffer: 5.201008 + vertex_buffer: 0.589072 + vertex_buffer: 0.508637 + vertex_buffer: 3.878377 + vertex_buffer: -6.041764 + vertex_buffer: 3.311079 + vertex_buffer: 0.685945 + vertex_buffer: 0.775357 + vertex_buffer: 3.084037 + vertex_buffer: -6.809842 + vertex_buffer: 3.814195 + vertex_buffer: 0.645735 + vertex_buffer: 0.812640 + vertex_buffer: 3.747321 + vertex_buffer: -4.503545 + vertex_buffer: 3.726453 + vertex_buffer: 0.675343 + vertex_buffer: 0.703978 + vertex_buffer: 6.094129 + vertex_buffer: -3.205991 + vertex_buffer: 1.473482 + vertex_buffer: 0.810858 + vertex_buffer: 0.646305 + vertex_buffer: 4.588995 + vertex_buffer: -4.728726 + vertex_buffer: 2.983221 + vertex_buffer: 0.720122 + vertex_buffer: 0.714667 + vertex_buffer: 6.583231 + vertex_buffer: -3.941269 + vertex_buffer: 0.070268 + vertex_buffer: 0.866152 + vertex_buffer: 0.682705 + vertex_buffer: 3.492580 + vertex_buffer: -3.195820 + vertex_buffer: 4.130198 + vertex_buffer: 0.663187 + vertex_buffer: 0.644597 + vertex_buffer: 1.255543 + vertex_buffer: 0.802341 + vertex_buffer: 5.307551 + vertex_buffer: 0.570082 + vertex_buffer: 0.466326 + vertex_buffer: 1.126122 + vertex_buffer: -0.933602 + vertex_buffer: 6.538785 + vertex_buffer: 0.544562 + vertex_buffer: 0.548376 + vertex_buffer: 1.443109 + vertex_buffer: -1.142774 + vertex_buffer: 5.905127 + vertex_buffer: 0.562759 + vertex_buffer: 0.558785 + vertex_buffer: 0.923043 + vertex_buffer: -0.529042 + vertex_buffer: 7.003423 + vertex_buffer: 0.531987 + vertex_buffer: 0.530140 + vertex_buffer: 1.755386 + vertex_buffer: 3.529117 + vertex_buffer: 4.327696 + vertex_buffer: 0.585271 + vertex_buffer: 0.335177 + vertex_buffer: 2.632589 + vertex_buffer: 3.713828 + vertex_buffer: 4.364629 + vertex_buffer: 0.622953 + vertex_buffer: 0.322779 + vertex_buffer: 3.388062 + vertex_buffer: 3.721976 + vertex_buffer: 4.309028 + vertex_buffer: 0.655896 + vertex_buffer: 0.320163 + vertex_buffer: 4.075766 + vertex_buffer: 3.675413 + vertex_buffer: 4.076063 + vertex_buffer: 0.687132 + vertex_buffer: 0.322346 + vertex_buffer: 4.622910 + vertex_buffer: 3.474691 + vertex_buffer: 3.646321 + vertex_buffer: 0.716482 + vertex_buffer: 0.333201 + vertex_buffer: 5.171755 + vertex_buffer: 2.535753 + vertex_buffer: 2.670867 + vertex_buffer: 0.758757 + vertex_buffer: 0.382787 + vertex_buffer: 7.297331 + vertex_buffer: 0.763172 + vertex_buffer: -0.048769 + vertex_buffer: 0.897013 + vertex_buffer: 0.468769 + vertex_buffer: 4.706828 + vertex_buffer: 1.651000 + vertex_buffer: 3.109532 + vertex_buffer: 0.732392 + vertex_buffer: 0.424547 + vertex_buffer: 4.071712 + vertex_buffer: 1.476821 + vertex_buffer: 3.476944 + vertex_buffer: 0.702114 + vertex_buffer: 0.433163 + vertex_buffer: 3.269817 + vertex_buffer: 1.470659 + vertex_buffer: 3.731945 + vertex_buffer: 0.666525 + vertex_buffer: 0.433866 + vertex_buffer: 2.527572 + vertex_buffer: 1.617311 + vertex_buffer: 3.865444 + vertex_buffer: 0.633505 + vertex_buffer: 0.426088 + vertex_buffer: 1.970894 + vertex_buffer: 1.858505 + vertex_buffer: 3.961782 + vertex_buffer: 0.603876 + vertex_buffer: 0.416587 + vertex_buffer: 1.579543 + vertex_buffer: 2.097941 + vertex_buffer: 4.084996 + vertex_buffer: 0.579658 + vertex_buffer: 0.409945 + vertex_buffer: 7.664182 + vertex_buffer: 0.673132 + vertex_buffer: -2.435867 + vertex_buffer: 0.992440 + vertex_buffer: 0.480777 + vertex_buffer: 1.397041 + vertex_buffer: -1.340139 + vertex_buffer: 5.630378 + vertex_buffer: 0.567192 + vertex_buffer: 0.569420 + vertex_buffer: 0.884838 + vertex_buffer: 0.658740 + vertex_buffer: 6.233232 + vertex_buffer: 0.541366 + vertex_buffer: 0.478899 + vertex_buffer: 0.767097 + vertex_buffer: -0.968035 + vertex_buffer: 7.077932 + vertex_buffer: 0.526564 + vertex_buffer: 0.546118 + vertex_buffer: 0.460213 + vertex_buffer: -1.334106 + vertex_buffer: 6.787447 + vertex_buffer: 0.523913 + vertex_buffer: 0.563830 + vertex_buffer: 0.748618 + vertex_buffer: -1.067994 + vertex_buffer: 6.798303 + vertex_buffer: 0.531529 + vertex_buffer: 0.555057 + vertex_buffer: 1.236408 + vertex_buffer: -1.585568 + vertex_buffer: 5.480490 + vertex_buffer: 0.566036 + vertex_buffer: 0.582329 + vertex_buffer: 0.387306 + vertex_buffer: -1.409990 + vertex_buffer: 6.957705 + vertex_buffer: 0.516311 + vertex_buffer: 0.563054 + vertex_buffer: 0.319925 + vertex_buffer: -1.607931 + vertex_buffer: 6.508676 + vertex_buffer: 0.517472 + vertex_buffer: 0.577877 + vertex_buffer: 1.639633 + vertex_buffer: 2.556298 + vertex_buffer: 3.863736 + vertex_buffer: 0.573595 + vertex_buffer: 0.389807 + vertex_buffer: 1.255645 + vertex_buffer: 2.467144 + vertex_buffer: 4.203800 + vertex_buffer: 0.560698 + vertex_buffer: 0.395332 + vertex_buffer: 1.031362 + vertex_buffer: 2.382663 + vertex_buffer: 4.615849 + vertex_buffer: 0.549756 + vertex_buffer: 0.399751 + vertex_buffer: 4.253081 + vertex_buffer: 2.772296 + vertex_buffer: 3.315305 + vertex_buffer: 0.710288 + vertex_buffer: 0.368253 + vertex_buffer: 4.530000 + vertex_buffer: 2.910000 + vertex_buffer: 3.339685 + vertex_buffer: 0.723330 + vertex_buffer: 0.363373 + vertex_buffer: -3.18175 + vertex_buffer: 2.635786 + vertex_buffer: 3.826339 + vertex_buffer: 0.523494 + vertex_buffer: 0.653066 + vertex_buffer: -2.58175 + vertex_buffer: 2.635786 + vertex_buffer: 3.824459 + vertex_buffer: 0.619766 + vertex_buffer: 0.484153 + vertex_buffer: -3.18175 + vertex_buffer: 3.235786 + vertex_buffer: 3.876973 + vertex_buffer: 0.448126 + vertex_buffer: 0.441797 + vertex_buffer: -3.78175 + vertex_buffer: 2.635786 + vertex_buffer: 3.679778 + vertex_buffer: 0.564397 + vertex_buffer: 0.650577 + vertex_buffer: -3.18175 + vertex_buffer: 2.035786 + vertex_buffer: 3.775704 + vertex_buffer: 0.629396 + vertex_buffer: 0.487967 + vertex_buffer: 3.181751 + vertex_buffer: 2.635786 + vertex_buffer: 3.826339 + vertex_buffer: 0.500005 + vertex_buffer: 0.531923 + vertex_buffer: 3.781751 + vertex_buffer: 2.635786 + vertex_buffer: 3.679777 + vertex_buffer: 0.528836 + vertex_buffer: 0.363049 + vertex_buffer: 3.181751 + vertex_buffer: 3.235786 + vertex_buffer: 3.876973 + vertex_buffer: 0.601042 + vertex_buffer: 0.688245 + vertex_buffer: 2.581751 + vertex_buffer: 2.635786 + vertex_buffer: 3.824459 + vertex_buffer: 0.489588 + vertex_buffer: 0.725148 + vertex_buffer: 3.181751 + vertex_buffer: 2.035786 + vertex_buffer: 3.775704 + vertex_buffer: 0.626117 + vertex_buffer: 0.461480 + index_buffer: 173 + index_buffer: 155 + index_buffer: 133 + index_buffer: 246 + index_buffer: 33 + index_buffer: 7 + index_buffer: 382 + index_buffer: 398 + index_buffer: 362 + index_buffer: 263 + index_buffer: 466 + index_buffer: 249 + index_buffer: 308 + index_buffer: 415 + index_buffer: 324 + index_buffer: 78 + index_buffer: 95 + index_buffer: 191 + index_buffer: 356 + index_buffer: 389 + index_buffer: 264 + index_buffer: 127 + index_buffer: 34 + index_buffer: 162 + index_buffer: 368 + index_buffer: 264 + index_buffer: 389 + index_buffer: 139 + index_buffer: 162 + index_buffer: 34 + index_buffer: 267 + index_buffer: 0 + index_buffer: 302 + index_buffer: 37 + index_buffer: 72 + index_buffer: 0 + index_buffer: 11 + index_buffer: 302 + index_buffer: 0 + index_buffer: 11 + index_buffer: 0 + index_buffer: 72 + index_buffer: 349 + index_buffer: 451 + index_buffer: 350 + index_buffer: 120 + index_buffer: 121 + index_buffer: 231 + index_buffer: 452 + index_buffer: 350 + index_buffer: 451 + index_buffer: 232 + index_buffer: 231 + index_buffer: 121 + index_buffer: 267 + index_buffer: 302 + index_buffer: 269 + index_buffer: 37 + index_buffer: 39 + index_buffer: 72 + index_buffer: 303 + index_buffer: 269 + index_buffer: 302 + index_buffer: 73 + index_buffer: 72 + index_buffer: 39 + index_buffer: 357 + index_buffer: 343 + index_buffer: 350 + index_buffer: 128 + index_buffer: 121 + index_buffer: 114 + index_buffer: 277 + index_buffer: 350 + index_buffer: 343 + index_buffer: 47 + index_buffer: 114 + index_buffer: 121 + index_buffer: 350 + index_buffer: 452 + index_buffer: 357 + index_buffer: 121 + index_buffer: 128 + index_buffer: 232 + index_buffer: 453 + index_buffer: 357 + index_buffer: 452 + index_buffer: 233 + index_buffer: 232 + index_buffer: 128 + index_buffer: 299 + index_buffer: 333 + index_buffer: 297 + index_buffer: 69 + index_buffer: 67 + index_buffer: 104 + index_buffer: 332 + index_buffer: 297 + index_buffer: 333 + index_buffer: 103 + index_buffer: 104 + index_buffer: 67 + index_buffer: 175 + index_buffer: 152 + index_buffer: 396 + index_buffer: 175 + index_buffer: 171 + index_buffer: 152 + index_buffer: 377 + index_buffer: 396 + index_buffer: 152 + index_buffer: 148 + index_buffer: 152 + index_buffer: 171 + index_buffer: 381 + index_buffer: 384 + index_buffer: 382 + index_buffer: 154 + index_buffer: 155 + index_buffer: 157 + index_buffer: 398 + index_buffer: 382 + index_buffer: 384 + index_buffer: 173 + index_buffer: 157 + index_buffer: 155 + index_buffer: 280 + index_buffer: 347 + index_buffer: 330 + index_buffer: 50 + index_buffer: 101 + index_buffer: 118 + index_buffer: 348 + index_buffer: 330 + index_buffer: 347 + index_buffer: 119 + index_buffer: 118 + index_buffer: 101 + index_buffer: 269 + index_buffer: 303 + index_buffer: 270 + index_buffer: 39 + index_buffer: 40 + index_buffer: 73 + index_buffer: 304 + index_buffer: 270 + index_buffer: 303 + index_buffer: 74 + index_buffer: 73 + index_buffer: 40 + index_buffer: 9 + index_buffer: 336 + index_buffer: 151 + index_buffer: 9 + index_buffer: 151 + index_buffer: 107 + index_buffer: 337 + index_buffer: 151 + index_buffer: 336 + index_buffer: 108 + index_buffer: 107 + index_buffer: 151 + index_buffer: 344 + index_buffer: 278 + index_buffer: 360 + index_buffer: 115 + index_buffer: 131 + index_buffer: 48 + index_buffer: 279 + index_buffer: 360 + index_buffer: 278 + index_buffer: 49 + index_buffer: 48 + index_buffer: 131 + index_buffer: 262 + index_buffer: 431 + index_buffer: 418 + index_buffer: 32 + index_buffer: 194 + index_buffer: 211 + index_buffer: 424 + index_buffer: 418 + index_buffer: 431 + index_buffer: 204 + index_buffer: 211 + index_buffer: 194 + index_buffer: 304 + index_buffer: 408 + index_buffer: 270 + index_buffer: 74 + index_buffer: 40 + index_buffer: 184 + index_buffer: 409 + index_buffer: 270 + index_buffer: 408 + index_buffer: 185 + index_buffer: 184 + index_buffer: 40 + index_buffer: 272 + index_buffer: 310 + index_buffer: 407 + index_buffer: 42 + index_buffer: 183 + index_buffer: 80 + index_buffer: 415 + index_buffer: 407 + index_buffer: 310 + index_buffer: 191 + index_buffer: 80 + index_buffer: 183 + index_buffer: 322 + index_buffer: 270 + index_buffer: 410 + index_buffer: 92 + index_buffer: 186 + index_buffer: 40 + index_buffer: 409 + index_buffer: 410 + index_buffer: 270 + index_buffer: 185 + index_buffer: 40 + index_buffer: 186 + index_buffer: 347 + index_buffer: 449 + index_buffer: 348 + index_buffer: 118 + index_buffer: 119 + index_buffer: 229 + index_buffer: 450 + index_buffer: 348 + index_buffer: 449 + index_buffer: 230 + index_buffer: 229 + index_buffer: 119 + index_buffer: 434 + index_buffer: 432 + index_buffer: 430 + index_buffer: 214 + index_buffer: 210 + index_buffer: 212 + index_buffer: 422 + index_buffer: 430 + index_buffer: 432 + index_buffer: 202 + index_buffer: 212 + index_buffer: 210 + index_buffer: 313 + index_buffer: 314 + index_buffer: 18 + index_buffer: 83 + index_buffer: 18 + index_buffer: 84 + index_buffer: 17 + index_buffer: 18 + index_buffer: 314 + index_buffer: 17 + index_buffer: 84 + index_buffer: 18 + index_buffer: 307 + index_buffer: 375 + index_buffer: 306 + index_buffer: 77 + index_buffer: 76 + index_buffer: 146 + index_buffer: 291 + index_buffer: 306 + index_buffer: 375 + index_buffer: 61 + index_buffer: 146 + index_buffer: 76 + index_buffer: 259 + index_buffer: 387 + index_buffer: 260 + index_buffer: 29 + index_buffer: 30 + index_buffer: 160 + index_buffer: 388 + index_buffer: 260 + index_buffer: 387 + index_buffer: 161 + index_buffer: 160 + index_buffer: 30 + index_buffer: 286 + index_buffer: 414 + index_buffer: 384 + index_buffer: 56 + index_buffer: 157 + index_buffer: 190 + index_buffer: 398 + index_buffer: 384 + index_buffer: 414 + index_buffer: 173 + index_buffer: 190 + index_buffer: 157 + index_buffer: 418 + index_buffer: 424 + index_buffer: 406 + index_buffer: 194 + index_buffer: 182 + index_buffer: 204 + index_buffer: 335 + index_buffer: 406 + index_buffer: 424 + index_buffer: 106 + index_buffer: 204 + index_buffer: 182 + index_buffer: 367 + index_buffer: 416 + index_buffer: 364 + index_buffer: 138 + index_buffer: 135 + index_buffer: 192 + index_buffer: 434 + index_buffer: 364 + index_buffer: 416 + index_buffer: 214 + index_buffer: 192 + index_buffer: 135 + index_buffer: 391 + index_buffer: 423 + index_buffer: 327 + index_buffer: 165 + index_buffer: 98 + index_buffer: 203 + index_buffer: 358 + index_buffer: 327 + index_buffer: 423 + index_buffer: 129 + index_buffer: 203 + index_buffer: 98 + index_buffer: 298 + index_buffer: 301 + index_buffer: 284 + index_buffer: 68 + index_buffer: 54 + index_buffer: 71 + index_buffer: 251 + index_buffer: 284 + index_buffer: 301 + index_buffer: 21 + index_buffer: 71 + index_buffer: 54 + index_buffer: 4 + index_buffer: 275 + index_buffer: 5 + index_buffer: 4 + index_buffer: 5 + index_buffer: 45 + index_buffer: 281 + index_buffer: 5 + index_buffer: 275 + index_buffer: 51 + index_buffer: 45 + index_buffer: 5 + index_buffer: 254 + index_buffer: 373 + index_buffer: 253 + index_buffer: 24 + index_buffer: 23 + index_buffer: 144 + index_buffer: 374 + index_buffer: 253 + index_buffer: 373 + index_buffer: 145 + index_buffer: 144 + index_buffer: 23 + index_buffer: 320 + index_buffer: 321 + index_buffer: 307 + index_buffer: 90 + index_buffer: 77 + index_buffer: 91 + index_buffer: 375 + index_buffer: 307 + index_buffer: 321 + index_buffer: 146 + index_buffer: 91 + index_buffer: 77 + index_buffer: 280 + index_buffer: 425 + index_buffer: 411 + index_buffer: 50 + index_buffer: 187 + index_buffer: 205 + index_buffer: 427 + index_buffer: 411 + index_buffer: 425 + index_buffer: 207 + index_buffer: 205 + index_buffer: 187 + index_buffer: 421 + index_buffer: 313 + index_buffer: 200 + index_buffer: 201 + index_buffer: 200 + index_buffer: 83 + index_buffer: 18 + index_buffer: 200 + index_buffer: 313 + index_buffer: 18 + index_buffer: 83 + index_buffer: 200 + index_buffer: 335 + index_buffer: 321 + index_buffer: 406 + index_buffer: 106 + index_buffer: 182 + index_buffer: 91 + index_buffer: 405 + index_buffer: 406 + index_buffer: 321 + index_buffer: 181 + index_buffer: 91 + index_buffer: 182 + index_buffer: 405 + index_buffer: 321 + index_buffer: 404 + index_buffer: 181 + index_buffer: 180 + index_buffer: 91 + index_buffer: 320 + index_buffer: 404 + index_buffer: 321 + index_buffer: 90 + index_buffer: 91 + index_buffer: 180 + index_buffer: 17 + index_buffer: 314 + index_buffer: 16 + index_buffer: 17 + index_buffer: 16 + index_buffer: 84 + index_buffer: 315 + index_buffer: 16 + index_buffer: 314 + index_buffer: 85 + index_buffer: 84 + index_buffer: 16 + index_buffer: 425 + index_buffer: 266 + index_buffer: 426 + index_buffer: 205 + index_buffer: 206 + index_buffer: 36 + index_buffer: 423 + index_buffer: 426 + index_buffer: 266 + index_buffer: 203 + index_buffer: 36 + index_buffer: 206 + index_buffer: 369 + index_buffer: 396 + index_buffer: 400 + index_buffer: 140 + index_buffer: 176 + index_buffer: 171 + index_buffer: 377 + index_buffer: 400 + index_buffer: 396 + index_buffer: 148 + index_buffer: 171 + index_buffer: 176 + index_buffer: 391 + index_buffer: 269 + index_buffer: 322 + index_buffer: 165 + index_buffer: 92 + index_buffer: 39 + index_buffer: 270 + index_buffer: 322 + index_buffer: 269 + index_buffer: 40 + index_buffer: 39 + index_buffer: 92 + index_buffer: 417 + index_buffer: 465 + index_buffer: 413 + index_buffer: 193 + index_buffer: 189 + index_buffer: 245 + index_buffer: 464 + index_buffer: 413 + index_buffer: 465 + index_buffer: 244 + index_buffer: 245 + index_buffer: 189 + index_buffer: 257 + index_buffer: 258 + index_buffer: 386 + index_buffer: 27 + index_buffer: 159 + index_buffer: 28 + index_buffer: 385 + index_buffer: 386 + index_buffer: 258 + index_buffer: 158 + index_buffer: 28 + index_buffer: 159 + index_buffer: 260 + index_buffer: 388 + index_buffer: 467 + index_buffer: 30 + index_buffer: 247 + index_buffer: 161 + index_buffer: 466 + index_buffer: 467 + index_buffer: 388 + index_buffer: 246 + index_buffer: 161 + index_buffer: 247 + index_buffer: 248 + index_buffer: 456 + index_buffer: 419 + index_buffer: 3 + index_buffer: 196 + index_buffer: 236 + index_buffer: 399 + index_buffer: 419 + index_buffer: 456 + index_buffer: 174 + index_buffer: 236 + index_buffer: 196 + index_buffer: 333 + index_buffer: 298 + index_buffer: 332 + index_buffer: 104 + index_buffer: 103 + index_buffer: 68 + index_buffer: 284 + index_buffer: 332 + index_buffer: 298 + index_buffer: 54 + index_buffer: 68 + index_buffer: 103 + index_buffer: 285 + index_buffer: 8 + index_buffer: 417 + index_buffer: 55 + index_buffer: 193 + index_buffer: 8 + index_buffer: 168 + index_buffer: 417 + index_buffer: 8 + index_buffer: 168 + index_buffer: 8 + index_buffer: 193 + index_buffer: 340 + index_buffer: 261 + index_buffer: 346 + index_buffer: 111 + index_buffer: 117 + index_buffer: 31 + index_buffer: 448 + index_buffer: 346 + index_buffer: 261 + index_buffer: 228 + index_buffer: 31 + index_buffer: 117 + index_buffer: 285 + index_buffer: 417 + index_buffer: 441 + index_buffer: 55 + index_buffer: 221 + index_buffer: 193 + index_buffer: 413 + index_buffer: 441 + index_buffer: 417 + index_buffer: 189 + index_buffer: 193 + index_buffer: 221 + index_buffer: 327 + index_buffer: 460 + index_buffer: 326 + index_buffer: 98 + index_buffer: 97 + index_buffer: 240 + index_buffer: 328 + index_buffer: 326 + index_buffer: 460 + index_buffer: 99 + index_buffer: 240 + index_buffer: 97 + index_buffer: 277 + index_buffer: 355 + index_buffer: 329 + index_buffer: 47 + index_buffer: 100 + index_buffer: 126 + index_buffer: 371 + index_buffer: 329 + index_buffer: 355 + index_buffer: 142 + index_buffer: 126 + index_buffer: 100 + index_buffer: 309 + index_buffer: 392 + index_buffer: 438 + index_buffer: 79 + index_buffer: 218 + index_buffer: 166 + index_buffer: 439 + index_buffer: 438 + index_buffer: 392 + index_buffer: 219 + index_buffer: 166 + index_buffer: 218 + index_buffer: 381 + index_buffer: 382 + index_buffer: 256 + index_buffer: 154 + index_buffer: 26 + index_buffer: 155 + index_buffer: 341 + index_buffer: 256 + index_buffer: 382 + index_buffer: 112 + index_buffer: 155 + index_buffer: 26 + index_buffer: 360 + index_buffer: 279 + index_buffer: 420 + index_buffer: 131 + index_buffer: 198 + index_buffer: 49 + index_buffer: 429 + index_buffer: 420 + index_buffer: 279 + index_buffer: 209 + index_buffer: 49 + index_buffer: 198 + index_buffer: 365 + index_buffer: 364 + index_buffer: 379 + index_buffer: 136 + index_buffer: 150 + index_buffer: 135 + index_buffer: 394 + index_buffer: 379 + index_buffer: 364 + index_buffer: 169 + index_buffer: 135 + index_buffer: 150 + index_buffer: 355 + index_buffer: 277 + index_buffer: 437 + index_buffer: 126 + index_buffer: 217 + index_buffer: 47 + index_buffer: 343 + index_buffer: 437 + index_buffer: 277 + index_buffer: 114 + index_buffer: 47 + index_buffer: 217 + index_buffer: 443 + index_buffer: 444 + index_buffer: 282 + index_buffer: 223 + index_buffer: 52 + index_buffer: 224 + index_buffer: 283 + index_buffer: 282 + index_buffer: 444 + index_buffer: 53 + index_buffer: 224 + index_buffer: 52 + index_buffer: 281 + index_buffer: 275 + index_buffer: 363 + index_buffer: 51 + index_buffer: 134 + index_buffer: 45 + index_buffer: 440 + index_buffer: 363 + index_buffer: 275 + index_buffer: 220 + index_buffer: 45 + index_buffer: 134 + index_buffer: 431 + index_buffer: 262 + index_buffer: 395 + index_buffer: 211 + index_buffer: 170 + index_buffer: 32 + index_buffer: 369 + index_buffer: 395 + index_buffer: 262 + index_buffer: 140 + index_buffer: 32 + index_buffer: 170 + index_buffer: 337 + index_buffer: 299 + index_buffer: 338 + index_buffer: 108 + index_buffer: 109 + index_buffer: 69 + index_buffer: 297 + index_buffer: 338 + index_buffer: 299 + index_buffer: 67 + index_buffer: 69 + index_buffer: 109 + index_buffer: 335 + index_buffer: 273 + index_buffer: 321 + index_buffer: 106 + index_buffer: 91 + index_buffer: 43 + index_buffer: 375 + index_buffer: 321 + index_buffer: 273 + index_buffer: 146 + index_buffer: 43 + index_buffer: 91 + index_buffer: 348 + index_buffer: 450 + index_buffer: 349 + index_buffer: 119 + index_buffer: 120 + index_buffer: 230 + index_buffer: 451 + index_buffer: 349 + index_buffer: 450 + index_buffer: 231 + index_buffer: 230 + index_buffer: 120 + index_buffer: 467 + index_buffer: 359 + index_buffer: 342 + index_buffer: 247 + index_buffer: 113 + index_buffer: 130 + index_buffer: 446 + index_buffer: 342 + index_buffer: 359 + index_buffer: 226 + index_buffer: 130 + index_buffer: 113 + index_buffer: 282 + index_buffer: 283 + index_buffer: 334 + index_buffer: 52 + index_buffer: 105 + index_buffer: 53 + index_buffer: 293 + index_buffer: 334 + index_buffer: 283 + index_buffer: 63 + index_buffer: 53 + index_buffer: 105 + index_buffer: 250 + index_buffer: 458 + index_buffer: 462 + index_buffer: 20 + index_buffer: 242 + index_buffer: 238 + index_buffer: 461 + index_buffer: 462 + index_buffer: 458 + index_buffer: 241 + index_buffer: 238 + index_buffer: 242 + index_buffer: 276 + index_buffer: 353 + index_buffer: 300 + index_buffer: 46 + index_buffer: 70 + index_buffer: 124 + index_buffer: 383 + index_buffer: 300 + index_buffer: 353 + index_buffer: 156 + index_buffer: 124 + index_buffer: 70 + index_buffer: 325 + index_buffer: 292 + index_buffer: 324 + index_buffer: 96 + index_buffer: 95 + index_buffer: 62 + index_buffer: 308 + index_buffer: 324 + index_buffer: 292 + index_buffer: 78 + index_buffer: 62 + index_buffer: 95 + index_buffer: 283 + index_buffer: 276 + index_buffer: 293 + index_buffer: 53 + index_buffer: 63 + index_buffer: 46 + index_buffer: 300 + index_buffer: 293 + index_buffer: 276 + index_buffer: 70 + index_buffer: 46 + index_buffer: 63 + index_buffer: 447 + index_buffer: 264 + index_buffer: 345 + index_buffer: 227 + index_buffer: 116 + index_buffer: 34 + index_buffer: 372 + index_buffer: 345 + index_buffer: 264 + index_buffer: 143 + index_buffer: 34 + index_buffer: 116 + index_buffer: 352 + index_buffer: 345 + index_buffer: 346 + index_buffer: 123 + index_buffer: 117 + index_buffer: 116 + index_buffer: 340 + index_buffer: 346 + index_buffer: 345 + index_buffer: 111 + index_buffer: 116 + index_buffer: 117 + index_buffer: 1 + index_buffer: 19 + index_buffer: 274 + index_buffer: 1 + index_buffer: 44 + index_buffer: 19 + index_buffer: 354 + index_buffer: 274 + index_buffer: 19 + index_buffer: 125 + index_buffer: 19 + index_buffer: 44 + index_buffer: 248 + index_buffer: 281 + index_buffer: 456 + index_buffer: 3 + index_buffer: 236 + index_buffer: 51 + index_buffer: 363 + index_buffer: 456 + index_buffer: 281 + index_buffer: 134 + index_buffer: 51 + index_buffer: 236 + index_buffer: 425 + index_buffer: 426 + index_buffer: 427 + index_buffer: 205 + index_buffer: 207 + index_buffer: 206 + index_buffer: 436 + index_buffer: 427 + index_buffer: 426 + index_buffer: 216 + index_buffer: 206 + index_buffer: 207 + index_buffer: 380 + index_buffer: 381 + index_buffer: 252 + index_buffer: 153 + index_buffer: 22 + index_buffer: 154 + index_buffer: 256 + index_buffer: 252 + index_buffer: 381 + index_buffer: 26 + index_buffer: 154 + index_buffer: 22 + index_buffer: 391 + index_buffer: 393 + index_buffer: 269 + index_buffer: 165 + index_buffer: 39 + index_buffer: 167 + index_buffer: 267 + index_buffer: 269 + index_buffer: 393 + index_buffer: 37 + index_buffer: 167 + index_buffer: 39 + index_buffer: 199 + index_buffer: 428 + index_buffer: 200 + index_buffer: 199 + index_buffer: 200 + index_buffer: 208 + index_buffer: 421 + index_buffer: 200 + index_buffer: 428 + index_buffer: 201 + index_buffer: 208 + index_buffer: 200 + index_buffer: 330 + index_buffer: 329 + index_buffer: 266 + index_buffer: 101 + index_buffer: 36 + index_buffer: 100 + index_buffer: 371 + index_buffer: 266 + index_buffer: 329 + index_buffer: 142 + index_buffer: 100 + index_buffer: 36 + index_buffer: 422 + index_buffer: 432 + index_buffer: 273 + index_buffer: 202 + index_buffer: 43 + index_buffer: 212 + index_buffer: 287 + index_buffer: 273 + index_buffer: 432 + index_buffer: 57 + index_buffer: 212 + index_buffer: 43 + index_buffer: 290 + index_buffer: 250 + index_buffer: 328 + index_buffer: 60 + index_buffer: 99 + index_buffer: 20 + index_buffer: 462 + index_buffer: 328 + index_buffer: 250 + index_buffer: 242 + index_buffer: 20 + index_buffer: 99 + index_buffer: 258 + index_buffer: 286 + index_buffer: 385 + index_buffer: 28 + index_buffer: 158 + index_buffer: 56 + index_buffer: 384 + index_buffer: 385 + index_buffer: 286 + index_buffer: 157 + index_buffer: 56 + index_buffer: 158 + index_buffer: 342 + index_buffer: 446 + index_buffer: 353 + index_buffer: 113 + index_buffer: 124 + index_buffer: 226 + index_buffer: 265 + index_buffer: 353 + index_buffer: 446 + index_buffer: 35 + index_buffer: 226 + index_buffer: 124 + index_buffer: 257 + index_buffer: 386 + index_buffer: 259 + index_buffer: 27 + index_buffer: 29 + index_buffer: 159 + index_buffer: 387 + index_buffer: 259 + index_buffer: 386 + index_buffer: 160 + index_buffer: 159 + index_buffer: 29 + index_buffer: 430 + index_buffer: 422 + index_buffer: 431 + index_buffer: 210 + index_buffer: 211 + index_buffer: 202 + index_buffer: 424 + index_buffer: 431 + index_buffer: 422 + index_buffer: 204 + index_buffer: 202 + index_buffer: 211 + index_buffer: 445 + index_buffer: 342 + index_buffer: 276 + index_buffer: 225 + index_buffer: 46 + index_buffer: 113 + index_buffer: 353 + index_buffer: 276 + index_buffer: 342 + index_buffer: 124 + index_buffer: 113 + index_buffer: 46 + index_buffer: 424 + index_buffer: 422 + index_buffer: 335 + index_buffer: 204 + index_buffer: 106 + index_buffer: 202 + index_buffer: 273 + index_buffer: 335 + index_buffer: 422 + index_buffer: 43 + index_buffer: 202 + index_buffer: 106 + index_buffer: 306 + index_buffer: 292 + index_buffer: 307 + index_buffer: 76 + index_buffer: 77 + index_buffer: 62 + index_buffer: 325 + index_buffer: 307 + index_buffer: 292 + index_buffer: 96 + index_buffer: 62 + index_buffer: 77 + index_buffer: 366 + index_buffer: 447 + index_buffer: 352 + index_buffer: 137 + index_buffer: 123 + index_buffer: 227 + index_buffer: 345 + index_buffer: 352 + index_buffer: 447 + index_buffer: 116 + index_buffer: 227 + index_buffer: 123 + index_buffer: 302 + index_buffer: 268 + index_buffer: 303 + index_buffer: 72 + index_buffer: 73 + index_buffer: 38 + index_buffer: 271 + index_buffer: 303 + index_buffer: 268 + index_buffer: 41 + index_buffer: 38 + index_buffer: 73 + index_buffer: 371 + index_buffer: 358 + index_buffer: 266 + index_buffer: 142 + index_buffer: 36 + index_buffer: 129 + index_buffer: 423 + index_buffer: 266 + index_buffer: 358 + index_buffer: 203 + index_buffer: 129 + index_buffer: 36 + index_buffer: 327 + index_buffer: 294 + index_buffer: 460 + index_buffer: 98 + index_buffer: 240 + index_buffer: 64 + index_buffer: 455 + index_buffer: 460 + index_buffer: 294 + index_buffer: 235 + index_buffer: 64 + index_buffer: 240 + index_buffer: 294 + index_buffer: 331 + index_buffer: 278 + index_buffer: 64 + index_buffer: 48 + index_buffer: 102 + index_buffer: 279 + index_buffer: 278 + index_buffer: 331 + index_buffer: 49 + index_buffer: 102 + index_buffer: 48 + index_buffer: 303 + index_buffer: 271 + index_buffer: 304 + index_buffer: 73 + index_buffer: 74 + index_buffer: 41 + index_buffer: 272 + index_buffer: 304 + index_buffer: 271 + index_buffer: 42 + index_buffer: 41 + index_buffer: 74 + index_buffer: 427 + index_buffer: 436 + index_buffer: 434 + index_buffer: 207 + index_buffer: 214 + index_buffer: 216 + index_buffer: 432 + index_buffer: 434 + index_buffer: 436 + index_buffer: 212 + index_buffer: 216 + index_buffer: 214 + index_buffer: 304 + index_buffer: 272 + index_buffer: 408 + index_buffer: 74 + index_buffer: 184 + index_buffer: 42 + index_buffer: 407 + index_buffer: 408 + index_buffer: 272 + index_buffer: 183 + index_buffer: 42 + index_buffer: 184 + index_buffer: 394 + index_buffer: 430 + index_buffer: 395 + index_buffer: 169 + index_buffer: 170 + index_buffer: 210 + index_buffer: 431 + index_buffer: 395 + index_buffer: 430 + index_buffer: 211 + index_buffer: 210 + index_buffer: 170 + index_buffer: 395 + index_buffer: 369 + index_buffer: 378 + index_buffer: 170 + index_buffer: 149 + index_buffer: 140 + index_buffer: 400 + index_buffer: 378 + index_buffer: 369 + index_buffer: 176 + index_buffer: 140 + index_buffer: 149 + index_buffer: 296 + index_buffer: 334 + index_buffer: 299 + index_buffer: 66 + index_buffer: 69 + index_buffer: 105 + index_buffer: 333 + index_buffer: 299 + index_buffer: 334 + index_buffer: 104 + index_buffer: 105 + index_buffer: 69 + index_buffer: 417 + index_buffer: 168 + index_buffer: 351 + index_buffer: 193 + index_buffer: 122 + index_buffer: 168 + index_buffer: 6 + index_buffer: 351 + index_buffer: 168 + index_buffer: 6 + index_buffer: 168 + index_buffer: 122 + index_buffer: 280 + index_buffer: 411 + index_buffer: 352 + index_buffer: 50 + index_buffer: 123 + index_buffer: 187 + index_buffer: 376 + index_buffer: 352 + index_buffer: 411 + index_buffer: 147 + index_buffer: 187 + index_buffer: 123 + index_buffer: 319 + index_buffer: 320 + index_buffer: 325 + index_buffer: 89 + index_buffer: 96 + index_buffer: 90 + index_buffer: 307 + index_buffer: 325 + index_buffer: 320 + index_buffer: 77 + index_buffer: 90 + index_buffer: 96 + index_buffer: 285 + index_buffer: 295 + index_buffer: 336 + index_buffer: 55 + index_buffer: 107 + index_buffer: 65 + index_buffer: 296 + index_buffer: 336 + index_buffer: 295 + index_buffer: 66 + index_buffer: 65 + index_buffer: 107 + index_buffer: 404 + index_buffer: 320 + index_buffer: 403 + index_buffer: 180 + index_buffer: 179 + index_buffer: 90 + index_buffer: 319 + index_buffer: 403 + index_buffer: 320 + index_buffer: 89 + index_buffer: 90 + index_buffer: 179 + index_buffer: 330 + index_buffer: 348 + index_buffer: 329 + index_buffer: 101 + index_buffer: 100 + index_buffer: 119 + index_buffer: 349 + index_buffer: 329 + index_buffer: 348 + index_buffer: 120 + index_buffer: 119 + index_buffer: 100 + index_buffer: 334 + index_buffer: 293 + index_buffer: 333 + index_buffer: 105 + index_buffer: 104 + index_buffer: 63 + index_buffer: 298 + index_buffer: 333 + index_buffer: 293 + index_buffer: 68 + index_buffer: 63 + index_buffer: 104 + index_buffer: 323 + index_buffer: 454 + index_buffer: 366 + index_buffer: 93 + index_buffer: 137 + index_buffer: 234 + index_buffer: 447 + index_buffer: 366 + index_buffer: 454 + index_buffer: 227 + index_buffer: 234 + index_buffer: 137 + index_buffer: 16 + index_buffer: 315 + index_buffer: 15 + index_buffer: 16 + index_buffer: 15 + index_buffer: 85 + index_buffer: 316 + index_buffer: 15 + index_buffer: 315 + index_buffer: 86 + index_buffer: 85 + index_buffer: 15 + index_buffer: 429 + index_buffer: 279 + index_buffer: 358 + index_buffer: 209 + index_buffer: 129 + index_buffer: 49 + index_buffer: 331 + index_buffer: 358 + index_buffer: 279 + index_buffer: 102 + index_buffer: 49 + index_buffer: 129 + index_buffer: 15 + index_buffer: 316 + index_buffer: 14 + index_buffer: 15 + index_buffer: 14 + index_buffer: 86 + index_buffer: 317 + index_buffer: 14 + index_buffer: 316 + index_buffer: 87 + index_buffer: 86 + index_buffer: 14 + index_buffer: 8 + index_buffer: 285 + index_buffer: 9 + index_buffer: 8 + index_buffer: 9 + index_buffer: 55 + index_buffer: 336 + index_buffer: 9 + index_buffer: 285 + index_buffer: 107 + index_buffer: 55 + index_buffer: 9 + index_buffer: 329 + index_buffer: 349 + index_buffer: 277 + index_buffer: 100 + index_buffer: 47 + index_buffer: 120 + index_buffer: 350 + index_buffer: 277 + index_buffer: 349 + index_buffer: 121 + index_buffer: 120 + index_buffer: 47 + index_buffer: 252 + index_buffer: 253 + index_buffer: 380 + index_buffer: 22 + index_buffer: 153 + index_buffer: 23 + index_buffer: 374 + index_buffer: 380 + index_buffer: 253 + index_buffer: 145 + index_buffer: 23 + index_buffer: 153 + index_buffer: 402 + index_buffer: 403 + index_buffer: 318 + index_buffer: 178 + index_buffer: 88 + index_buffer: 179 + index_buffer: 319 + index_buffer: 318 + index_buffer: 403 + index_buffer: 89 + index_buffer: 179 + index_buffer: 88 + index_buffer: 351 + index_buffer: 6 + index_buffer: 419 + index_buffer: 122 + index_buffer: 196 + index_buffer: 6 + index_buffer: 197 + index_buffer: 419 + index_buffer: 6 + index_buffer: 197 + index_buffer: 6 + index_buffer: 196 + index_buffer: 324 + index_buffer: 318 + index_buffer: 325 + index_buffer: 95 + index_buffer: 96 + index_buffer: 88 + index_buffer: 319 + index_buffer: 325 + index_buffer: 318 + index_buffer: 89 + index_buffer: 88 + index_buffer: 96 + index_buffer: 397 + index_buffer: 367 + index_buffer: 365 + index_buffer: 172 + index_buffer: 136 + index_buffer: 138 + index_buffer: 364 + index_buffer: 365 + index_buffer: 367 + index_buffer: 135 + index_buffer: 138 + index_buffer: 136 + index_buffer: 288 + index_buffer: 435 + index_buffer: 397 + index_buffer: 58 + index_buffer: 172 + index_buffer: 215 + index_buffer: 367 + index_buffer: 397 + index_buffer: 435 + index_buffer: 138 + index_buffer: 215 + index_buffer: 172 + index_buffer: 438 + index_buffer: 439 + index_buffer: 344 + index_buffer: 218 + index_buffer: 115 + index_buffer: 219 + index_buffer: 278 + index_buffer: 344 + index_buffer: 439 + index_buffer: 48 + index_buffer: 219 + index_buffer: 115 + index_buffer: 271 + index_buffer: 311 + index_buffer: 272 + index_buffer: 41 + index_buffer: 42 + index_buffer: 81 + index_buffer: 310 + index_buffer: 272 + index_buffer: 311 + index_buffer: 80 + index_buffer: 81 + index_buffer: 42 + index_buffer: 5 + index_buffer: 281 + index_buffer: 195 + index_buffer: 5 + index_buffer: 195 + index_buffer: 51 + index_buffer: 248 + index_buffer: 195 + index_buffer: 281 + index_buffer: 3 + index_buffer: 51 + index_buffer: 195 + index_buffer: 273 + index_buffer: 287 + index_buffer: 375 + index_buffer: 43 + index_buffer: 146 + index_buffer: 57 + index_buffer: 291 + index_buffer: 375 + index_buffer: 287 + index_buffer: 61 + index_buffer: 57 + index_buffer: 146 + index_buffer: 396 + index_buffer: 428 + index_buffer: 175 + index_buffer: 171 + index_buffer: 175 + index_buffer: 208 + index_buffer: 199 + index_buffer: 175 + index_buffer: 428 + index_buffer: 199 + index_buffer: 208 + index_buffer: 175 + index_buffer: 268 + index_buffer: 312 + index_buffer: 271 + index_buffer: 38 + index_buffer: 41 + index_buffer: 82 + index_buffer: 311 + index_buffer: 271 + index_buffer: 312 + index_buffer: 81 + index_buffer: 82 + index_buffer: 41 + index_buffer: 444 + index_buffer: 445 + index_buffer: 283 + index_buffer: 224 + index_buffer: 53 + index_buffer: 225 + index_buffer: 276 + index_buffer: 283 + index_buffer: 445 + index_buffer: 46 + index_buffer: 225 + index_buffer: 53 + index_buffer: 254 + index_buffer: 339 + index_buffer: 373 + index_buffer: 24 + index_buffer: 144 + index_buffer: 110 + index_buffer: 390 + index_buffer: 373 + index_buffer: 339 + index_buffer: 163 + index_buffer: 110 + index_buffer: 144 + index_buffer: 295 + index_buffer: 282 + index_buffer: 296 + index_buffer: 65 + index_buffer: 66 + index_buffer: 52 + index_buffer: 334 + index_buffer: 296 + index_buffer: 282 + index_buffer: 105 + index_buffer: 52 + index_buffer: 66 + index_buffer: 346 + index_buffer: 448 + index_buffer: 347 + index_buffer: 117 + index_buffer: 118 + index_buffer: 228 + index_buffer: 449 + index_buffer: 347 + index_buffer: 448 + index_buffer: 229 + index_buffer: 228 + index_buffer: 118 + index_buffer: 454 + index_buffer: 356 + index_buffer: 447 + index_buffer: 234 + index_buffer: 227 + index_buffer: 127 + index_buffer: 264 + index_buffer: 447 + index_buffer: 356 + index_buffer: 34 + index_buffer: 127 + index_buffer: 227 + index_buffer: 336 + index_buffer: 296 + index_buffer: 337 + index_buffer: 107 + index_buffer: 108 + index_buffer: 66 + index_buffer: 299 + index_buffer: 337 + index_buffer: 296 + index_buffer: 69 + index_buffer: 66 + index_buffer: 108 + index_buffer: 151 + index_buffer: 337 + index_buffer: 10 + index_buffer: 151 + index_buffer: 10 + index_buffer: 108 + index_buffer: 338 + index_buffer: 10 + index_buffer: 337 + index_buffer: 109 + index_buffer: 108 + index_buffer: 10 + index_buffer: 278 + index_buffer: 439 + index_buffer: 294 + index_buffer: 48 + index_buffer: 64 + index_buffer: 219 + index_buffer: 455 + index_buffer: 294 + index_buffer: 439 + index_buffer: 235 + index_buffer: 219 + index_buffer: 64 + index_buffer: 407 + index_buffer: 415 + index_buffer: 292 + index_buffer: 183 + index_buffer: 62 + index_buffer: 191 + index_buffer: 308 + index_buffer: 292 + index_buffer: 415 + index_buffer: 78 + index_buffer: 191 + index_buffer: 62 + index_buffer: 358 + index_buffer: 371 + index_buffer: 429 + index_buffer: 129 + index_buffer: 209 + index_buffer: 142 + index_buffer: 355 + index_buffer: 429 + index_buffer: 371 + index_buffer: 126 + index_buffer: 142 + index_buffer: 209 + index_buffer: 345 + index_buffer: 372 + index_buffer: 340 + index_buffer: 116 + index_buffer: 111 + index_buffer: 143 + index_buffer: 265 + index_buffer: 340 + index_buffer: 372 + index_buffer: 35 + index_buffer: 143 + index_buffer: 111 + index_buffer: 388 + index_buffer: 390 + index_buffer: 466 + index_buffer: 161 + index_buffer: 246 + index_buffer: 163 + index_buffer: 249 + index_buffer: 466 + index_buffer: 390 + index_buffer: 7 + index_buffer: 163 + index_buffer: 246 + index_buffer: 352 + index_buffer: 346 + index_buffer: 280 + index_buffer: 123 + index_buffer: 50 + index_buffer: 117 + index_buffer: 347 + index_buffer: 280 + index_buffer: 346 + index_buffer: 118 + index_buffer: 117 + index_buffer: 50 + index_buffer: 295 + index_buffer: 442 + index_buffer: 282 + index_buffer: 65 + index_buffer: 52 + index_buffer: 222 + index_buffer: 443 + index_buffer: 282 + index_buffer: 442 + index_buffer: 223 + index_buffer: 222 + index_buffer: 52 + index_buffer: 19 + index_buffer: 94 + index_buffer: 354 + index_buffer: 19 + index_buffer: 125 + index_buffer: 94 + index_buffer: 370 + index_buffer: 354 + index_buffer: 94 + index_buffer: 141 + index_buffer: 94 + index_buffer: 125 + index_buffer: 295 + index_buffer: 285 + index_buffer: 442 + index_buffer: 65 + index_buffer: 222 + index_buffer: 55 + index_buffer: 441 + index_buffer: 442 + index_buffer: 285 + index_buffer: 221 + index_buffer: 55 + index_buffer: 222 + index_buffer: 419 + index_buffer: 197 + index_buffer: 248 + index_buffer: 196 + index_buffer: 3 + index_buffer: 197 + index_buffer: 195 + index_buffer: 248 + index_buffer: 197 + index_buffer: 195 + index_buffer: 197 + index_buffer: 3 + index_buffer: 359 + index_buffer: 263 + index_buffer: 255 + index_buffer: 130 + index_buffer: 25 + index_buffer: 33 + index_buffer: 249 + index_buffer: 255 + index_buffer: 263 + index_buffer: 7 + index_buffer: 33 + index_buffer: 25 + index_buffer: 275 + index_buffer: 274 + index_buffer: 440 + index_buffer: 45 + index_buffer: 220 + index_buffer: 44 + index_buffer: 457 + index_buffer: 440 + index_buffer: 274 + index_buffer: 237 + index_buffer: 44 + index_buffer: 220 + index_buffer: 300 + index_buffer: 383 + index_buffer: 301 + index_buffer: 70 + index_buffer: 71 + index_buffer: 156 + index_buffer: 368 + index_buffer: 301 + index_buffer: 383 + index_buffer: 139 + index_buffer: 156 + index_buffer: 71 + index_buffer: 417 + index_buffer: 351 + index_buffer: 465 + index_buffer: 193 + index_buffer: 245 + index_buffer: 122 + index_buffer: 412 + index_buffer: 465 + index_buffer: 351 + index_buffer: 188 + index_buffer: 122 + index_buffer: 245 + index_buffer: 466 + index_buffer: 263 + index_buffer: 467 + index_buffer: 246 + index_buffer: 247 + index_buffer: 33 + index_buffer: 359 + index_buffer: 467 + index_buffer: 263 + index_buffer: 130 + index_buffer: 33 + index_buffer: 247 + index_buffer: 389 + index_buffer: 251 + index_buffer: 368 + index_buffer: 162 + index_buffer: 139 + index_buffer: 21 + index_buffer: 301 + index_buffer: 368 + index_buffer: 251 + index_buffer: 71 + index_buffer: 21 + index_buffer: 139 + index_buffer: 374 + index_buffer: 386 + index_buffer: 380 + index_buffer: 145 + index_buffer: 153 + index_buffer: 159 + index_buffer: 385 + index_buffer: 380 + index_buffer: 386 + index_buffer: 158 + index_buffer: 159 + index_buffer: 153 + index_buffer: 379 + index_buffer: 394 + index_buffer: 378 + index_buffer: 150 + index_buffer: 149 + index_buffer: 169 + index_buffer: 395 + index_buffer: 378 + index_buffer: 394 + index_buffer: 170 + index_buffer: 169 + index_buffer: 149 + index_buffer: 351 + index_buffer: 419 + index_buffer: 412 + index_buffer: 122 + index_buffer: 188 + index_buffer: 196 + index_buffer: 399 + index_buffer: 412 + index_buffer: 419 + index_buffer: 174 + index_buffer: 196 + index_buffer: 188 + index_buffer: 426 + index_buffer: 322 + index_buffer: 436 + index_buffer: 206 + index_buffer: 216 + index_buffer: 92 + index_buffer: 410 + index_buffer: 436 + index_buffer: 322 + index_buffer: 186 + index_buffer: 92 + index_buffer: 216 + index_buffer: 387 + index_buffer: 373 + index_buffer: 388 + index_buffer: 160 + index_buffer: 161 + index_buffer: 144 + index_buffer: 390 + index_buffer: 388 + index_buffer: 373 + index_buffer: 163 + index_buffer: 144 + index_buffer: 161 + index_buffer: 393 + index_buffer: 326 + index_buffer: 164 + index_buffer: 167 + index_buffer: 164 + index_buffer: 97 + index_buffer: 2 + index_buffer: 164 + index_buffer: 326 + index_buffer: 2 + index_buffer: 97 + index_buffer: 164 + index_buffer: 354 + index_buffer: 370 + index_buffer: 461 + index_buffer: 125 + index_buffer: 241 + index_buffer: 141 + index_buffer: 462 + index_buffer: 461 + index_buffer: 370 + index_buffer: 242 + index_buffer: 141 + index_buffer: 241 + index_buffer: 0 + index_buffer: 267 + index_buffer: 164 + index_buffer: 0 + index_buffer: 164 + index_buffer: 37 + index_buffer: 393 + index_buffer: 164 + index_buffer: 267 + index_buffer: 167 + index_buffer: 37 + index_buffer: 164 + index_buffer: 11 + index_buffer: 12 + index_buffer: 302 + index_buffer: 11 + index_buffer: 72 + index_buffer: 12 + index_buffer: 268 + index_buffer: 302 + index_buffer: 12 + index_buffer: 38 + index_buffer: 12 + index_buffer: 72 + index_buffer: 386 + index_buffer: 374 + index_buffer: 387 + index_buffer: 159 + index_buffer: 160 + index_buffer: 145 + index_buffer: 373 + index_buffer: 387 + index_buffer: 374 + index_buffer: 144 + index_buffer: 145 + index_buffer: 160 + index_buffer: 12 + index_buffer: 13 + index_buffer: 268 + index_buffer: 12 + index_buffer: 38 + index_buffer: 13 + index_buffer: 312 + index_buffer: 268 + index_buffer: 13 + index_buffer: 82 + index_buffer: 13 + index_buffer: 38 + index_buffer: 293 + index_buffer: 300 + index_buffer: 298 + index_buffer: 63 + index_buffer: 68 + index_buffer: 70 + index_buffer: 301 + index_buffer: 298 + index_buffer: 300 + index_buffer: 71 + index_buffer: 70 + index_buffer: 68 + index_buffer: 340 + index_buffer: 265 + index_buffer: 261 + index_buffer: 111 + index_buffer: 31 + index_buffer: 35 + index_buffer: 446 + index_buffer: 261 + index_buffer: 265 + index_buffer: 226 + index_buffer: 35 + index_buffer: 31 + index_buffer: 380 + index_buffer: 385 + index_buffer: 381 + index_buffer: 153 + index_buffer: 154 + index_buffer: 158 + index_buffer: 384 + index_buffer: 381 + index_buffer: 385 + index_buffer: 157 + index_buffer: 158 + index_buffer: 154 + index_buffer: 280 + index_buffer: 330 + index_buffer: 425 + index_buffer: 50 + index_buffer: 205 + index_buffer: 101 + index_buffer: 266 + index_buffer: 425 + index_buffer: 330 + index_buffer: 36 + index_buffer: 101 + index_buffer: 205 + index_buffer: 423 + index_buffer: 391 + index_buffer: 426 + index_buffer: 203 + index_buffer: 206 + index_buffer: 165 + index_buffer: 322 + index_buffer: 426 + index_buffer: 391 + index_buffer: 92 + index_buffer: 165 + index_buffer: 206 + index_buffer: 429 + index_buffer: 355 + index_buffer: 420 + index_buffer: 209 + index_buffer: 198 + index_buffer: 126 + index_buffer: 437 + index_buffer: 420 + index_buffer: 355 + index_buffer: 217 + index_buffer: 126 + index_buffer: 198 + index_buffer: 391 + index_buffer: 327 + index_buffer: 393 + index_buffer: 165 + index_buffer: 167 + index_buffer: 98 + index_buffer: 326 + index_buffer: 393 + index_buffer: 327 + index_buffer: 97 + index_buffer: 98 + index_buffer: 167 + index_buffer: 457 + index_buffer: 438 + index_buffer: 440 + index_buffer: 237 + index_buffer: 220 + index_buffer: 218 + index_buffer: 344 + index_buffer: 440 + index_buffer: 438 + index_buffer: 115 + index_buffer: 218 + index_buffer: 220 + index_buffer: 382 + index_buffer: 362 + index_buffer: 341 + index_buffer: 155 + index_buffer: 112 + index_buffer: 133 + index_buffer: 463 + index_buffer: 341 + index_buffer: 362 + index_buffer: 243 + index_buffer: 133 + index_buffer: 112 + index_buffer: 457 + index_buffer: 461 + index_buffer: 459 + index_buffer: 237 + index_buffer: 239 + index_buffer: 241 + index_buffer: 458 + index_buffer: 459 + index_buffer: 461 + index_buffer: 238 + index_buffer: 241 + index_buffer: 239 + index_buffer: 434 + index_buffer: 430 + index_buffer: 364 + index_buffer: 214 + index_buffer: 135 + index_buffer: 210 + index_buffer: 394 + index_buffer: 364 + index_buffer: 430 + index_buffer: 169 + index_buffer: 210 + index_buffer: 135 + index_buffer: 414 + index_buffer: 463 + index_buffer: 398 + index_buffer: 190 + index_buffer: 173 + index_buffer: 243 + index_buffer: 362 + index_buffer: 398 + index_buffer: 463 + index_buffer: 133 + index_buffer: 243 + index_buffer: 173 + index_buffer: 262 + index_buffer: 428 + index_buffer: 369 + index_buffer: 32 + index_buffer: 140 + index_buffer: 208 + index_buffer: 396 + index_buffer: 369 + index_buffer: 428 + index_buffer: 171 + index_buffer: 208 + index_buffer: 140 + index_buffer: 457 + index_buffer: 274 + index_buffer: 461 + index_buffer: 237 + index_buffer: 241 + index_buffer: 44 + index_buffer: 354 + index_buffer: 461 + index_buffer: 274 + index_buffer: 125 + index_buffer: 44 + index_buffer: 241 + index_buffer: 316 + index_buffer: 403 + index_buffer: 317 + index_buffer: 86 + index_buffer: 87 + index_buffer: 179 + index_buffer: 402 + index_buffer: 317 + index_buffer: 403 + index_buffer: 178 + index_buffer: 179 + index_buffer: 87 + index_buffer: 315 + index_buffer: 404 + index_buffer: 316 + index_buffer: 85 + index_buffer: 86 + index_buffer: 180 + index_buffer: 403 + index_buffer: 316 + index_buffer: 404 + index_buffer: 179 + index_buffer: 180 + index_buffer: 86 + index_buffer: 314 + index_buffer: 405 + index_buffer: 315 + index_buffer: 84 + index_buffer: 85 + index_buffer: 181 + index_buffer: 404 + index_buffer: 315 + index_buffer: 405 + index_buffer: 180 + index_buffer: 181 + index_buffer: 85 + index_buffer: 313 + index_buffer: 406 + index_buffer: 314 + index_buffer: 83 + index_buffer: 84 + index_buffer: 182 + index_buffer: 405 + index_buffer: 314 + index_buffer: 406 + index_buffer: 181 + index_buffer: 182 + index_buffer: 84 + index_buffer: 418 + index_buffer: 406 + index_buffer: 421 + index_buffer: 194 + index_buffer: 201 + index_buffer: 182 + index_buffer: 313 + index_buffer: 421 + index_buffer: 406 + index_buffer: 83 + index_buffer: 182 + index_buffer: 201 + index_buffer: 366 + index_buffer: 401 + index_buffer: 323 + index_buffer: 137 + index_buffer: 93 + index_buffer: 177 + index_buffer: 361 + index_buffer: 323 + index_buffer: 401 + index_buffer: 132 + index_buffer: 177 + index_buffer: 93 + index_buffer: 408 + index_buffer: 407 + index_buffer: 306 + index_buffer: 184 + index_buffer: 76 + index_buffer: 183 + index_buffer: 292 + index_buffer: 306 + index_buffer: 407 + index_buffer: 62 + index_buffer: 183 + index_buffer: 76 + index_buffer: 408 + index_buffer: 306 + index_buffer: 409 + index_buffer: 184 + index_buffer: 185 + index_buffer: 76 + index_buffer: 291 + index_buffer: 409 + index_buffer: 306 + index_buffer: 61 + index_buffer: 76 + index_buffer: 185 + index_buffer: 410 + index_buffer: 409 + index_buffer: 287 + index_buffer: 186 + index_buffer: 57 + index_buffer: 185 + index_buffer: 291 + index_buffer: 287 + index_buffer: 409 + index_buffer: 61 + index_buffer: 185 + index_buffer: 57 + index_buffer: 436 + index_buffer: 410 + index_buffer: 432 + index_buffer: 216 + index_buffer: 212 + index_buffer: 186 + index_buffer: 287 + index_buffer: 432 + index_buffer: 410 + index_buffer: 57 + index_buffer: 186 + index_buffer: 212 + index_buffer: 434 + index_buffer: 416 + index_buffer: 427 + index_buffer: 214 + index_buffer: 207 + index_buffer: 192 + index_buffer: 411 + index_buffer: 427 + index_buffer: 416 + index_buffer: 187 + index_buffer: 192 + index_buffer: 207 + index_buffer: 264 + index_buffer: 368 + index_buffer: 372 + index_buffer: 34 + index_buffer: 143 + index_buffer: 139 + index_buffer: 383 + index_buffer: 372 + index_buffer: 368 + index_buffer: 156 + index_buffer: 139 + index_buffer: 143 + index_buffer: 457 + index_buffer: 459 + index_buffer: 438 + index_buffer: 237 + index_buffer: 218 + index_buffer: 239 + index_buffer: 309 + index_buffer: 438 + index_buffer: 459 + index_buffer: 79 + index_buffer: 239 + index_buffer: 218 + index_buffer: 352 + index_buffer: 376 + index_buffer: 366 + index_buffer: 123 + index_buffer: 137 + index_buffer: 147 + index_buffer: 401 + index_buffer: 366 + index_buffer: 376 + index_buffer: 177 + index_buffer: 147 + index_buffer: 137 + index_buffer: 4 + index_buffer: 1 + index_buffer: 275 + index_buffer: 4 + index_buffer: 45 + index_buffer: 1 + index_buffer: 274 + index_buffer: 275 + index_buffer: 1 + index_buffer: 44 + index_buffer: 1 + index_buffer: 45 + index_buffer: 428 + index_buffer: 262 + index_buffer: 421 + index_buffer: 208 + index_buffer: 201 + index_buffer: 32 + index_buffer: 418 + index_buffer: 421 + index_buffer: 262 + index_buffer: 194 + index_buffer: 32 + index_buffer: 201 + index_buffer: 327 + index_buffer: 358 + index_buffer: 294 + index_buffer: 98 + index_buffer: 64 + index_buffer: 129 + index_buffer: 331 + index_buffer: 294 + index_buffer: 358 + index_buffer: 102 + index_buffer: 129 + index_buffer: 64 + index_buffer: 367 + index_buffer: 435 + index_buffer: 416 + index_buffer: 138 + index_buffer: 192 + index_buffer: 215 + index_buffer: 433 + index_buffer: 416 + index_buffer: 435 + index_buffer: 213 + index_buffer: 215 + index_buffer: 192 + index_buffer: 455 + index_buffer: 439 + index_buffer: 289 + index_buffer: 235 + index_buffer: 59 + index_buffer: 219 + index_buffer: 392 + index_buffer: 289 + index_buffer: 439 + index_buffer: 166 + index_buffer: 219 + index_buffer: 59 + index_buffer: 328 + index_buffer: 462 + index_buffer: 326 + index_buffer: 99 + index_buffer: 97 + index_buffer: 242 + index_buffer: 370 + index_buffer: 326 + index_buffer: 462 + index_buffer: 141 + index_buffer: 242 + index_buffer: 97 + index_buffer: 326 + index_buffer: 370 + index_buffer: 2 + index_buffer: 97 + index_buffer: 2 + index_buffer: 141 + index_buffer: 94 + index_buffer: 2 + index_buffer: 370 + index_buffer: 94 + index_buffer: 141 + index_buffer: 2 + index_buffer: 460 + index_buffer: 455 + index_buffer: 305 + index_buffer: 240 + index_buffer: 75 + index_buffer: 235 + index_buffer: 289 + index_buffer: 305 + index_buffer: 455 + index_buffer: 59 + index_buffer: 235 + index_buffer: 75 + index_buffer: 448 + index_buffer: 339 + index_buffer: 449 + index_buffer: 228 + index_buffer: 229 + index_buffer: 110 + index_buffer: 254 + index_buffer: 449 + index_buffer: 339 + index_buffer: 24 + index_buffer: 110 + index_buffer: 229 + index_buffer: 261 + index_buffer: 446 + index_buffer: 255 + index_buffer: 31 + index_buffer: 25 + index_buffer: 226 + index_buffer: 359 + index_buffer: 255 + index_buffer: 446 + index_buffer: 130 + index_buffer: 226 + index_buffer: 25 + index_buffer: 449 + index_buffer: 254 + index_buffer: 450 + index_buffer: 229 + index_buffer: 230 + index_buffer: 24 + index_buffer: 253 + index_buffer: 450 + index_buffer: 254 + index_buffer: 23 + index_buffer: 24 + index_buffer: 230 + index_buffer: 450 + index_buffer: 253 + index_buffer: 451 + index_buffer: 230 + index_buffer: 231 + index_buffer: 23 + index_buffer: 252 + index_buffer: 451 + index_buffer: 253 + index_buffer: 22 + index_buffer: 23 + index_buffer: 231 + index_buffer: 451 + index_buffer: 252 + index_buffer: 452 + index_buffer: 231 + index_buffer: 232 + index_buffer: 22 + index_buffer: 256 + index_buffer: 452 + index_buffer: 252 + index_buffer: 26 + index_buffer: 22 + index_buffer: 232 + index_buffer: 256 + index_buffer: 341 + index_buffer: 452 + index_buffer: 26 + index_buffer: 232 + index_buffer: 112 + index_buffer: 453 + index_buffer: 452 + index_buffer: 341 + index_buffer: 233 + index_buffer: 112 + index_buffer: 232 + index_buffer: 413 + index_buffer: 464 + index_buffer: 414 + index_buffer: 189 + index_buffer: 190 + index_buffer: 244 + index_buffer: 463 + index_buffer: 414 + index_buffer: 464 + index_buffer: 243 + index_buffer: 244 + index_buffer: 190 + index_buffer: 441 + index_buffer: 413 + index_buffer: 286 + index_buffer: 221 + index_buffer: 56 + index_buffer: 189 + index_buffer: 414 + index_buffer: 286 + index_buffer: 413 + index_buffer: 190 + index_buffer: 189 + index_buffer: 56 + index_buffer: 441 + index_buffer: 286 + index_buffer: 442 + index_buffer: 221 + index_buffer: 222 + index_buffer: 56 + index_buffer: 258 + index_buffer: 442 + index_buffer: 286 + index_buffer: 28 + index_buffer: 56 + index_buffer: 222 + index_buffer: 442 + index_buffer: 258 + index_buffer: 443 + index_buffer: 222 + index_buffer: 223 + index_buffer: 28 + index_buffer: 257 + index_buffer: 443 + index_buffer: 258 + index_buffer: 27 + index_buffer: 28 + index_buffer: 223 + index_buffer: 444 + index_buffer: 443 + index_buffer: 259 + index_buffer: 224 + index_buffer: 29 + index_buffer: 223 + index_buffer: 257 + index_buffer: 259 + index_buffer: 443 + index_buffer: 27 + index_buffer: 223 + index_buffer: 29 + index_buffer: 259 + index_buffer: 260 + index_buffer: 444 + index_buffer: 29 + index_buffer: 224 + index_buffer: 30 + index_buffer: 445 + index_buffer: 444 + index_buffer: 260 + index_buffer: 225 + index_buffer: 30 + index_buffer: 224 + index_buffer: 260 + index_buffer: 467 + index_buffer: 445 + index_buffer: 30 + index_buffer: 225 + index_buffer: 247 + index_buffer: 342 + index_buffer: 445 + index_buffer: 467 + index_buffer: 113 + index_buffer: 247 + index_buffer: 225 + index_buffer: 250 + index_buffer: 309 + index_buffer: 458 + index_buffer: 20 + index_buffer: 238 + index_buffer: 79 + index_buffer: 459 + index_buffer: 458 + index_buffer: 309 + index_buffer: 239 + index_buffer: 79 + index_buffer: 238 + index_buffer: 290 + index_buffer: 305 + index_buffer: 392 + index_buffer: 60 + index_buffer: 166 + index_buffer: 75 + index_buffer: 289 + index_buffer: 392 + index_buffer: 305 + index_buffer: 59 + index_buffer: 75 + index_buffer: 166 + index_buffer: 460 + index_buffer: 305 + index_buffer: 328 + index_buffer: 240 + index_buffer: 99 + index_buffer: 75 + index_buffer: 290 + index_buffer: 328 + index_buffer: 305 + index_buffer: 60 + index_buffer: 75 + index_buffer: 99 + index_buffer: 376 + index_buffer: 433 + index_buffer: 401 + index_buffer: 147 + index_buffer: 177 + index_buffer: 213 + index_buffer: 435 + index_buffer: 401 + index_buffer: 433 + index_buffer: 215 + index_buffer: 213 + index_buffer: 177 + index_buffer: 250 + index_buffer: 290 + index_buffer: 309 + index_buffer: 20 + index_buffer: 79 + index_buffer: 60 + index_buffer: 392 + index_buffer: 309 + index_buffer: 290 + index_buffer: 166 + index_buffer: 60 + index_buffer: 79 + index_buffer: 411 + index_buffer: 416 + index_buffer: 376 + index_buffer: 187 + index_buffer: 147 + index_buffer: 192 + index_buffer: 433 + index_buffer: 376 + index_buffer: 416 + index_buffer: 213 + index_buffer: 192 + index_buffer: 147 + index_buffer: 341 + index_buffer: 463 + index_buffer: 453 + index_buffer: 112 + index_buffer: 233 + index_buffer: 243 + index_buffer: 464 + index_buffer: 453 + index_buffer: 463 + index_buffer: 244 + index_buffer: 243 + index_buffer: 233 + index_buffer: 453 + index_buffer: 464 + index_buffer: 357 + index_buffer: 233 + index_buffer: 128 + index_buffer: 244 + index_buffer: 465 + index_buffer: 357 + index_buffer: 464 + index_buffer: 245 + index_buffer: 244 + index_buffer: 128 + index_buffer: 412 + index_buffer: 343 + index_buffer: 465 + index_buffer: 188 + index_buffer: 245 + index_buffer: 114 + index_buffer: 357 + index_buffer: 465 + index_buffer: 343 + index_buffer: 128 + index_buffer: 114 + index_buffer: 245 + index_buffer: 437 + index_buffer: 343 + index_buffer: 399 + index_buffer: 217 + index_buffer: 174 + index_buffer: 114 + index_buffer: 412 + index_buffer: 399 + index_buffer: 343 + index_buffer: 188 + index_buffer: 114 + index_buffer: 174 + index_buffer: 363 + index_buffer: 440 + index_buffer: 360 + index_buffer: 134 + index_buffer: 131 + index_buffer: 220 + index_buffer: 344 + index_buffer: 360 + index_buffer: 440 + index_buffer: 115 + index_buffer: 220 + index_buffer: 131 + index_buffer: 456 + index_buffer: 420 + index_buffer: 399 + index_buffer: 236 + index_buffer: 174 + index_buffer: 198 + index_buffer: 437 + index_buffer: 399 + index_buffer: 420 + index_buffer: 217 + index_buffer: 198 + index_buffer: 174 + index_buffer: 456 + index_buffer: 363 + index_buffer: 420 + index_buffer: 236 + index_buffer: 198 + index_buffer: 134 + index_buffer: 360 + index_buffer: 420 + index_buffer: 363 + index_buffer: 131 + index_buffer: 134 + index_buffer: 198 + index_buffer: 361 + index_buffer: 401 + index_buffer: 288 + index_buffer: 132 + index_buffer: 58 + index_buffer: 177 + index_buffer: 435 + index_buffer: 288 + index_buffer: 401 + index_buffer: 215 + index_buffer: 177 + index_buffer: 58 + index_buffer: 353 + index_buffer: 265 + index_buffer: 383 + index_buffer: 124 + index_buffer: 156 + index_buffer: 35 + index_buffer: 372 + index_buffer: 383 + index_buffer: 265 + index_buffer: 143 + index_buffer: 35 + index_buffer: 156 + index_buffer: 255 + index_buffer: 249 + index_buffer: 339 + index_buffer: 25 + index_buffer: 110 + index_buffer: 7 + index_buffer: 390 + index_buffer: 339 + index_buffer: 249 + index_buffer: 163 + index_buffer: 7 + index_buffer: 110 + index_buffer: 261 + index_buffer: 255 + index_buffer: 448 + index_buffer: 31 + index_buffer: 228 + index_buffer: 25 + index_buffer: 339 + index_buffer: 448 + index_buffer: 255 + index_buffer: 110 + index_buffer: 25 + index_buffer: 228 + index_buffer: 14 + index_buffer: 317 + index_buffer: 13 + index_buffer: 14 + index_buffer: 13 + index_buffer: 87 + index_buffer: 312 + index_buffer: 13 + index_buffer: 317 + index_buffer: 82 + index_buffer: 87 + index_buffer: 13 + index_buffer: 317 + index_buffer: 402 + index_buffer: 312 + index_buffer: 87 + index_buffer: 82 + index_buffer: 178 + index_buffer: 311 + index_buffer: 312 + index_buffer: 402 + index_buffer: 81 + index_buffer: 178 + index_buffer: 82 + index_buffer: 402 + index_buffer: 318 + index_buffer: 311 + index_buffer: 178 + index_buffer: 81 + index_buffer: 88 + index_buffer: 310 + index_buffer: 311 + index_buffer: 318 + index_buffer: 80 + index_buffer: 88 + index_buffer: 81 + index_buffer: 318 + index_buffer: 324 + index_buffer: 310 + index_buffer: 88 + index_buffer: 80 + index_buffer: 95 + index_buffer: 415 + index_buffer: 310 + index_buffer: 324 + index_buffer: 191 + index_buffer: 95 + index_buffer: 80 + index_buffer: 468 + index_buffer: 471 + index_buffer: 472 + index_buffer: 469 + index_buffer: 468 + index_buffer: 472 + index_buffer: 470 + index_buffer: 468 + index_buffer: 469 + index_buffer: 470 + index_buffer: 471 + index_buffer: 468 + index_buffer: 473 + index_buffer: 476 + index_buffer: 477 + index_buffer: 474 + index_buffer: 473 + index_buffer: 477 + index_buffer: 475 + index_buffer: 473 + index_buffer: 474 + index_buffer: 475 + index_buffer: 476 + index_buffer: 473 +} From 418680936d0e29bd1ef4f1392892affc37d7e557 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 10 Nov 2023 21:53:32 -0800 Subject: [PATCH 102/157] No public description PiperOrigin-RevId: 581450685 --- .../cc/components/processors/proto/BUILD | 11 ---- .../processors/proto/llm_params.proto | 41 ------------ .../processors/proto/transformer_params.proto | 64 ------------------- 3 files changed, 116 deletions(-) delete mode 100644 mediapipe/tasks/cc/components/processors/proto/llm_params.proto delete mode 100644 mediapipe/tasks/cc/components/processors/proto/transformer_params.proto diff --git a/mediapipe/tasks/cc/components/processors/proto/BUILD b/mediapipe/tasks/cc/components/processors/proto/BUILD index 55cf3fca1..82d4ea21b 100644 --- a/mediapipe/tasks/cc/components/processors/proto/BUILD +++ b/mediapipe/tasks/cc/components/processors/proto/BUILD @@ -93,14 +93,3 @@ mediapipe_proto_library( "//mediapipe/framework:calculator_proto", ], ) - -mediapipe_proto_library( - name = "transformer_params_proto", - srcs = ["transformer_params.proto"], -) - -mediapipe_proto_library( - name = "llm_params_proto", - srcs = ["llm_params.proto"], - deps = [":transformer_params_proto"], -) diff --git a/mediapipe/tasks/cc/components/processors/proto/llm_params.proto b/mediapipe/tasks/cc/components/processors/proto/llm_params.proto deleted file mode 100644 index b0c253598..000000000 --- a/mediapipe/tasks/cc/components/processors/proto/llm_params.proto +++ /dev/null @@ -1,41 +0,0 @@ -/* Copyright 2023 The MediaPipe Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -syntax = "proto3"; - -package mediapipe.tasks.components.processors.proto; - -import "mediapipe/tasks/cc/components/processors/proto/transformer_params.proto"; - -option java_package = "com.google.mediapipe.tasks.components.processors.proto"; -option java_outer_classname = "LLMParametersProto"; - -// Parameters for Large Language Models (LLM). -message LLMParameters { - TransformerParameters transformer_parameters = 1; - - // Size of vocabulary. - int32 vocab_size = 2; - - // Whether or not to disable KV cache, which is also referred as state - // somewhere else. - bool disable_kv_cache = 3; - - // Id of the start token. - int32 start_token_id = 4; - - // Token to determine the end of output stream. - string stop_token = 5; -} diff --git a/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto b/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto deleted file mode 100644 index a04aa9571..000000000 --- a/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto +++ /dev/null @@ -1,64 +0,0 @@ -/* Copyright 2023 The MediaPipe Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -syntax = "proto3"; - -package mediapipe.tasks.components.processors.proto; - -option java_package = "com.google.mediapipe.tasks.components.processors.proto"; -option java_outer_classname = "TransformerParametersProto"; - -// The parameters of transformer (https://arxiv.org/pdf/1706.03762.pdf) -message TransformerParameters { - // Batch size of tensors. - int32 batch_size = 1; - - // Maximum sequence length of the input/output tensor. - int32 max_seq_length = 2; - - // Embedding dimension (or model dimension), `d_model` in the paper. - // `d_k` == `d_v` == `d_model`/`h`. - int32 embedding_dim = 3; - - // Hidden dimension used in the feedforward layer, `d_ff` in the paper. - int32 hidden_dimension = 4; - - // Head dimension, `d_k` or `d_v` in the paper. - int32 head_dimension = 5; - - // Number of heads, `h` in the paper. - int32 num_heads = 6; - - // Number of stacked transformers, `N` in the paper. - int32 num_stacks = 7; - - // Deprecated: bool use_mqa. Use num_kv_heads below. - reserved 8; - - // Number of kv heads. 0 means Multi-Head-Attention (MHA), key and value have - // same number of heads as query; 1 means Multi-Query-Attention (MQA), key and - // value have one head; otherwise, this specifies the number of heads for key - // and value, and Grouped-Query-Attention (GQA) will be used. See - // https://arxiv.org/pdf/2305.13245.pdf for details. - int32 num_kv_heads = 9; - - // Different types of attention mask type. - enum AttentionMaskType { - UNSPECIFIED = 0; - CAUSAL = 1; - PREFIX = 2; - } - AttentionMaskType attention_mask_type = 10; -} From ad4da8c9ccf6f1419f03c580779d75ad7da30a13 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 10 Nov 2023 23:51:47 -0800 Subject: [PATCH 103/157] No public description PiperOrigin-RevId: 581468467 --- .../cc/components/processors/proto/BUILD | 11 ++++ .../processors/proto/llm_params.proto | 41 ++++++++++++ .../processors/proto/transformer_params.proto | 64 +++++++++++++++++++ 3 files changed, 116 insertions(+) create mode 100644 mediapipe/tasks/cc/components/processors/proto/llm_params.proto create mode 100644 mediapipe/tasks/cc/components/processors/proto/transformer_params.proto diff --git a/mediapipe/tasks/cc/components/processors/proto/BUILD b/mediapipe/tasks/cc/components/processors/proto/BUILD index 82d4ea21b..55cf3fca1 100644 --- a/mediapipe/tasks/cc/components/processors/proto/BUILD +++ b/mediapipe/tasks/cc/components/processors/proto/BUILD @@ -93,3 +93,14 @@ mediapipe_proto_library( "//mediapipe/framework:calculator_proto", ], ) + +mediapipe_proto_library( + name = "transformer_params_proto", + srcs = ["transformer_params.proto"], +) + +mediapipe_proto_library( + name = "llm_params_proto", + srcs = ["llm_params.proto"], + deps = [":transformer_params_proto"], +) diff --git a/mediapipe/tasks/cc/components/processors/proto/llm_params.proto b/mediapipe/tasks/cc/components/processors/proto/llm_params.proto new file mode 100644 index 000000000..b0c253598 --- /dev/null +++ b/mediapipe/tasks/cc/components/processors/proto/llm_params.proto @@ -0,0 +1,41 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +syntax = "proto3"; + +package mediapipe.tasks.components.processors.proto; + +import "mediapipe/tasks/cc/components/processors/proto/transformer_params.proto"; + +option java_package = "com.google.mediapipe.tasks.components.processors.proto"; +option java_outer_classname = "LLMParametersProto"; + +// Parameters for Large Language Models (LLM). +message LLMParameters { + TransformerParameters transformer_parameters = 1; + + // Size of vocabulary. + int32 vocab_size = 2; + + // Whether or not to disable KV cache, which is also referred as state + // somewhere else. + bool disable_kv_cache = 3; + + // Id of the start token. + int32 start_token_id = 4; + + // Token to determine the end of output stream. + string stop_token = 5; +} diff --git a/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto b/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto new file mode 100644 index 000000000..a04aa9571 --- /dev/null +++ b/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto @@ -0,0 +1,64 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +syntax = "proto3"; + +package mediapipe.tasks.components.processors.proto; + +option java_package = "com.google.mediapipe.tasks.components.processors.proto"; +option java_outer_classname = "TransformerParametersProto"; + +// The parameters of transformer (https://arxiv.org/pdf/1706.03762.pdf) +message TransformerParameters { + // Batch size of tensors. + int32 batch_size = 1; + + // Maximum sequence length of the input/output tensor. + int32 max_seq_length = 2; + + // Embedding dimension (or model dimension), `d_model` in the paper. + // `d_k` == `d_v` == `d_model`/`h`. + int32 embedding_dim = 3; + + // Hidden dimension used in the feedforward layer, `d_ff` in the paper. + int32 hidden_dimension = 4; + + // Head dimension, `d_k` or `d_v` in the paper. + int32 head_dimension = 5; + + // Number of heads, `h` in the paper. + int32 num_heads = 6; + + // Number of stacked transformers, `N` in the paper. + int32 num_stacks = 7; + + // Deprecated: bool use_mqa. Use num_kv_heads below. + reserved 8; + + // Number of kv heads. 0 means Multi-Head-Attention (MHA), key and value have + // same number of heads as query; 1 means Multi-Query-Attention (MQA), key and + // value have one head; otherwise, this specifies the number of heads for key + // and value, and Grouped-Query-Attention (GQA) will be used. See + // https://arxiv.org/pdf/2305.13245.pdf for details. + int32 num_kv_heads = 9; + + // Different types of attention mask type. + enum AttentionMaskType { + UNSPECIFIED = 0; + CAUSAL = 1; + PREFIX = 2; + } + AttentionMaskType attention_mask_type = 10; +} From 939a9c2a37ba36d5ef109adb9d28294f713d0dc3 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 10 Nov 2023 23:57:18 -0800 Subject: [PATCH 104/157] No public description PiperOrigin-RevId: 581469194 --- .../cc/components/processors/proto/BUILD | 11 ---- .../processors/proto/llm_params.proto | 41 ------------ .../processors/proto/transformer_params.proto | 64 ------------------- 3 files changed, 116 deletions(-) delete mode 100644 mediapipe/tasks/cc/components/processors/proto/llm_params.proto delete mode 100644 mediapipe/tasks/cc/components/processors/proto/transformer_params.proto diff --git a/mediapipe/tasks/cc/components/processors/proto/BUILD b/mediapipe/tasks/cc/components/processors/proto/BUILD index 55cf3fca1..82d4ea21b 100644 --- a/mediapipe/tasks/cc/components/processors/proto/BUILD +++ b/mediapipe/tasks/cc/components/processors/proto/BUILD @@ -93,14 +93,3 @@ mediapipe_proto_library( "//mediapipe/framework:calculator_proto", ], ) - -mediapipe_proto_library( - name = "transformer_params_proto", - srcs = ["transformer_params.proto"], -) - -mediapipe_proto_library( - name = "llm_params_proto", - srcs = ["llm_params.proto"], - deps = [":transformer_params_proto"], -) diff --git a/mediapipe/tasks/cc/components/processors/proto/llm_params.proto b/mediapipe/tasks/cc/components/processors/proto/llm_params.proto deleted file mode 100644 index b0c253598..000000000 --- a/mediapipe/tasks/cc/components/processors/proto/llm_params.proto +++ /dev/null @@ -1,41 +0,0 @@ -/* Copyright 2023 The MediaPipe Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -syntax = "proto3"; - -package mediapipe.tasks.components.processors.proto; - -import "mediapipe/tasks/cc/components/processors/proto/transformer_params.proto"; - -option java_package = "com.google.mediapipe.tasks.components.processors.proto"; -option java_outer_classname = "LLMParametersProto"; - -// Parameters for Large Language Models (LLM). -message LLMParameters { - TransformerParameters transformer_parameters = 1; - - // Size of vocabulary. - int32 vocab_size = 2; - - // Whether or not to disable KV cache, which is also referred as state - // somewhere else. - bool disable_kv_cache = 3; - - // Id of the start token. - int32 start_token_id = 4; - - // Token to determine the end of output stream. - string stop_token = 5; -} diff --git a/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto b/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto deleted file mode 100644 index a04aa9571..000000000 --- a/mediapipe/tasks/cc/components/processors/proto/transformer_params.proto +++ /dev/null @@ -1,64 +0,0 @@ -/* Copyright 2023 The MediaPipe Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -syntax = "proto3"; - -package mediapipe.tasks.components.processors.proto; - -option java_package = "com.google.mediapipe.tasks.components.processors.proto"; -option java_outer_classname = "TransformerParametersProto"; - -// The parameters of transformer (https://arxiv.org/pdf/1706.03762.pdf) -message TransformerParameters { - // Batch size of tensors. - int32 batch_size = 1; - - // Maximum sequence length of the input/output tensor. - int32 max_seq_length = 2; - - // Embedding dimension (or model dimension), `d_model` in the paper. - // `d_k` == `d_v` == `d_model`/`h`. - int32 embedding_dim = 3; - - // Hidden dimension used in the feedforward layer, `d_ff` in the paper. - int32 hidden_dimension = 4; - - // Head dimension, `d_k` or `d_v` in the paper. - int32 head_dimension = 5; - - // Number of heads, `h` in the paper. - int32 num_heads = 6; - - // Number of stacked transformers, `N` in the paper. - int32 num_stacks = 7; - - // Deprecated: bool use_mqa. Use num_kv_heads below. - reserved 8; - - // Number of kv heads. 0 means Multi-Head-Attention (MHA), key and value have - // same number of heads as query; 1 means Multi-Query-Attention (MQA), key and - // value have one head; otherwise, this specifies the number of heads for key - // and value, and Grouped-Query-Attention (GQA) will be used. See - // https://arxiv.org/pdf/2305.13245.pdf for details. - int32 num_kv_heads = 9; - - // Different types of attention mask type. - enum AttentionMaskType { - UNSPECIFIED = 0; - CAUSAL = 1; - PREFIX = 2; - } - AttentionMaskType attention_mask_type = 10; -} From 35f2f36733f6f80618a4ff4b8c57e43ee1fb4e9d Mon Sep 17 00:00:00 2001 From: Kinar Date: Sat, 11 Nov 2023 03:25:36 -0800 Subject: [PATCH 105/157] Added image classifier benchmark --- mediapipe/tasks/python/benchmark/__init__.py | 13 ++++ .../tasks/python/benchmark/vision/__init__.py | 13 ++++ .../vision/image_classifier/README.md | 34 +++++++++ .../image_classifier_benchmark.py | 74 +++++++++++++++++++ .../vision/image_classifier/setup.sh | 6 ++ 5 files changed, 140 insertions(+) create mode 100644 mediapipe/tasks/python/benchmark/__init__.py create mode 100644 mediapipe/tasks/python/benchmark/vision/__init__.py create mode 100644 mediapipe/tasks/python/benchmark/vision/image_classifier/README.md create mode 100644 mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py create mode 100644 mediapipe/tasks/python/benchmark/vision/image_classifier/setup.sh diff --git a/mediapipe/tasks/python/benchmark/__init__.py b/mediapipe/tasks/python/benchmark/__init__.py new file mode 100644 index 000000000..3fbc1e8a7 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/mediapipe/tasks/python/benchmark/vision/__init__.py b/mediapipe/tasks/python/benchmark/vision/__init__.py new file mode 100644 index 000000000..3fbc1e8a7 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md new file mode 100644 index 000000000..0c3e23432 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md @@ -0,0 +1,34 @@ +# MediaPipe Image Classifier Benchmark + +## Download the repository + +First, clone this Git repo. + +Run this script to install the required dependencies and download the TFLite models: + +``` +cd mediapipe/tasks/python/benchmark/vision/image_classifier +sh setup.sh +``` + +## Run the benchmark +``` +python3 image_classifier_benchmark.py +``` +* You can optionally specify the `model` parameter to set the TensorFlow Lite + model to be used: + * The default value is `classifier.tflite` + * TensorFlow Lite image classification models **with metadata** + * Models from [TensorFlow Hub](https://tfhub.dev/tensorflow/collections/lite/task-library/image-classifier/1) + * Models from [MediaPipe Models](https://developers.google.com/mediapipe/solutions/vision/image_classifier/index#models) + * Models trained with [MediaPipe Model Maker](https://developers.google.com/mediapipe/solutions/customization/image_classifier) are supported. +* You can optionally specify the `iterations` parameter to limit the number of + iterations for benchmarking: + * Supported value: A positive integer. + * Default value: `100` +* Example usage: + ``` + python3 image_classifier_benchmark.py \ + --model classifier.tflite \ + --iterations 200 + ``` diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py new file mode 100644 index 000000000..c6a80da11 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -0,0 +1,74 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Benchmark for the image classifier task.""" +import argparse +import time +import numpy as np +import mediapipe as mp +from mediapipe.tasks import python +from mediapipe.tasks.python import vision + +_IMAGE_FILE = 'burger.jpg' + + +def run(model: str, n_iterations: int, delegate: python.BaseOptions.Delegate): + """Run asynchronous inference on images and benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + """ + inference_times = [] + + # Initialize the image classifier + base_options = python.BaseOptions(model_asset_path=model, delegate=delegate) + options = vision.ImageClassifierOptions( + base_options=base_options, running_mode=vision.RunningMode.IMAGE, + max_results=1) + classifier = vision.ImageClassifier.create_from_options(options) + mp_image = mp.Image.create_from_file(_IMAGE_FILE) + + for _ in range(n_iterations): + start_time_ns = time.time_ns() + classifier.classify(mp_image) + end_time_ns = time.time_ns() + # Convert to milliseconds + inference_times.append((end_time_ns - start_time_ns) / 1_000_000) + + classifier.close() + return np.percentile(inference_times, 95) + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + '--model', help='Path to image classification model.', required=True) + parser.add_argument( + '--iterations', help='Number of iterations for benchmarking.', type=int, + default=100) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.CPU) + print(f"95th Percentile Inference Time on CPU: {cpu_time:.6f} milliseconds") + + # Run benchmark on GPU + gpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.GPU) + print(f"95th Percentile Inference Time on GPU: {gpu_time:.6f} milliseconds") + + +if __name__ == '__main__': + main() diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/setup.sh b/mediapipe/tasks/python/benchmark/vision/image_classifier/setup.sh new file mode 100644 index 000000000..79e35f447 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/setup.sh @@ -0,0 +1,6 @@ +# Install Python dependencies. +python3 -m pip install pip --upgrade +python3 -m pip install mediapipe + +wget -O classifier.tflite -q https://storage.googleapis.com/mediapipe-models/image_classifier/efficientnet_lite0/float32/1/efficientnet_lite0.tflite +wget -O burger.jpg https://storage.googleapis.com/mediapipe-assets/burger.jpg From 021c7edde74c0096372091d526db8dcf9c48bb14 Mon Sep 17 00:00:00 2001 From: Kinar Date: Sat, 11 Nov 2023 03:32:45 -0800 Subject: [PATCH 106/157] Updated README and script --- .../tasks/python/benchmark/vision/image_classifier/README.md | 2 +- .../vision/image_classifier/image_classifier_benchmark.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md index 0c3e23432..d7db3559d 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md @@ -7,7 +7,7 @@ First, clone this Git repo. Run this script to install the required dependencies and download the TFLite models: ``` -cd mediapipe/tasks/python/benchmark/vision/image_classifier +cd mediapipe/mediapipe/tasks/python/benchmark/vision/image_classifier sh setup.sh ``` diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py index c6a80da11..3fc7c8a2e 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -55,7 +55,8 @@ def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - '--model', help='Path to image classification model.', required=True) + '--model', help='Path to image classification model.', required=True, + default='classifier.tflite') parser.add_argument( '--iterations', help='Number of iterations for benchmarking.', type=int, default=100) From 99c8b9ee3c79ab5d5b367811bd30a80807283b3b Mon Sep 17 00:00:00 2001 From: Kinar Date: Sat, 11 Nov 2023 03:34:26 -0800 Subject: [PATCH 107/157] Updated copyright --- mediapipe/tasks/python/benchmark/__init__.py | 2 +- mediapipe/tasks/python/benchmark/vision/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mediapipe/tasks/python/benchmark/__init__.py b/mediapipe/tasks/python/benchmark/__init__.py index 3fbc1e8a7..2eb077987 100644 --- a/mediapipe/tasks/python/benchmark/__init__.py +++ b/mediapipe/tasks/python/benchmark/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. +# Copyright 2023 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/mediapipe/tasks/python/benchmark/vision/__init__.py b/mediapipe/tasks/python/benchmark/vision/__init__.py index 3fbc1e8a7..2eb077987 100644 --- a/mediapipe/tasks/python/benchmark/vision/__init__.py +++ b/mediapipe/tasks/python/benchmark/vision/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. +# Copyright 2023 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 38737849e6407fccc771d88f34bbaf6a5d5f5a16 Mon Sep 17 00:00:00 2001 From: Kinar Date: Sat, 11 Nov 2023 03:34:57 -0800 Subject: [PATCH 108/157] Updated copyright --- .../vision/image_classifier/image_classifier_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py index 3fc7c8a2e..502441879 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -1,4 +1,4 @@ -# Copyright 2022 The MediaPipe Authors. +# Copyright 2023 The MediaPipe Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From d504d3bf2202d7ebb087a348b123f26f7bbad976 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 13 Nov 2023 08:20:21 -0800 Subject: [PATCH 109/157] Create shared utilities to construct landmark lists PiperOrigin-RevId: 581970043 --- .../tasks/components/containers/BUILD | 2 + .../tasks/components/containers/Landmark.java | 30 ++++++++ .../containers/NormalizedLandmark.java | 31 ++++++++ .../facelandmarker/FaceLandmarkerResult.java | 22 ++---- .../handlandmarker/HandLandmarkerResult.java | 74 ++++++------------- .../poselandmarker/PoseLandmarkerResult.java | 43 +++-------- .../components/containers/LandmarkTest.java | 62 ++++++++++++++++ .../containers/NormalizedLandmarkTest.java | 62 ++++++++++++++++ 8 files changed, 226 insertions(+), 100 deletions(-) create mode 100644 mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/LandmarkTest.java create mode 100644 mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/NormalizedLandmarkTest.java diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD index bcdc0e5e5..1149ea036 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/BUILD @@ -96,6 +96,7 @@ android_library( "-Xep:AndroidJdkLibsChecker:OFF", ], deps = [ + "//mediapipe/framework/formats:landmark_java_proto_lite", "//third_party:autovalue", "@maven//:com_google_guava_guava", ], @@ -108,6 +109,7 @@ android_library( "-Xep:AndroidJdkLibsChecker:OFF", ], deps = [ + "//mediapipe/framework/formats:landmark_java_proto_lite", "//third_party:autovalue", "@maven//:com_google_guava_guava", ], diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java index e23d9115d..b3bb2d52e 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/Landmark.java @@ -14,7 +14,11 @@ package com.google.mediapipe.tasks.components.containers; +import android.annotation.TargetApi; import com.google.auto.value.AutoValue; +import com.google.mediapipe.formats.proto.LandmarkProto; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -24,18 +28,44 @@ import java.util.Optional; * is to the camera. */ @AutoValue +@TargetApi(31) public abstract class Landmark { private static final float TOLERANCE = 1e-6f; + /** Creates a landmark from x, y, z coordinates. */ public static Landmark create(float x, float y, float z) { return new AutoValue_Landmark(x, y, z, Optional.empty(), Optional.empty()); } + /** + * Creates a normalized landmark from x, y, z coordinates with optional visibility and presence. + */ public static Landmark create( float x, float y, float z, Optional visibility, Optional presence) { return new AutoValue_Landmark(x, y, z, visibility, presence); } + /** Creates a landmark from a landmark proto. */ + public static Landmark createFromProto(LandmarkProto.Landmark landmarkProto) { + return Landmark.create( + landmarkProto.getX(), + landmarkProto.getY(), + landmarkProto.getZ(), + landmarkProto.hasVisibility() + ? Optional.of(landmarkProto.getVisibility()) + : Optional.empty(), + landmarkProto.hasPresence() ? Optional.of(landmarkProto.getPresence()) : Optional.empty()); + } + + /** Creates a list of landmarks from a {@link LandmarkList}. */ + public static List createListFromProto(LandmarkProto.LandmarkList landmarkListProto) { + List landmarkList = new ArrayList<>(); + for (LandmarkProto.Landmark landmarkProto : landmarkListProto.getLandmarkList()) { + landmarkList.add(createFromProto(landmarkProto)); + } + return landmarkList; + } + // The x coordinates of the landmark. public abstract float x(); diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedLandmark.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedLandmark.java index 50a95d565..d6fa618a3 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedLandmark.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers/NormalizedLandmark.java @@ -14,7 +14,11 @@ package com.google.mediapipe.tasks.components.containers; +import android.annotation.TargetApi; import com.google.auto.value.AutoValue; +import com.google.mediapipe.formats.proto.LandmarkProto; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -25,18 +29,45 @@ import java.util.Optional; * uses roughly the same scale as x. */ @AutoValue +@TargetApi(31) public abstract class NormalizedLandmark { private static final float TOLERANCE = 1e-6f; + /** Creates a normalized landmark from x, y, z coordinates. */ public static NormalizedLandmark create(float x, float y, float z) { return new AutoValue_NormalizedLandmark(x, y, z, Optional.empty(), Optional.empty()); } + /** + * Creates a normalized landmark from x, y, z coordinates with optional visibility and presence. + */ public static NormalizedLandmark create( float x, float y, float z, Optional visibility, Optional presence) { return new AutoValue_NormalizedLandmark(x, y, z, visibility, presence); } + /** Creates a normalized landmark from a normalized landmark proto. */ + public static NormalizedLandmark createFromProto(LandmarkProto.NormalizedLandmark landmarkProto) { + return NormalizedLandmark.create( + landmarkProto.getX(), + landmarkProto.getY(), + landmarkProto.getZ(), + landmarkProto.hasVisibility() + ? Optional.of(landmarkProto.getVisibility()) + : Optional.empty(), + landmarkProto.hasPresence() ? Optional.of(landmarkProto.getPresence()) : Optional.empty()); + } + + /** Creates a list of normalized landmarks from a {@link NormalizedLandmarkList}. */ + public static List createListFromProto( + LandmarkProto.NormalizedLandmarkList landmarkListProto) { + List landmarkList = new ArrayList<>(); + for (LandmarkProto.NormalizedLandmark landmarkProto : landmarkListProto.getLandmarkList()) { + landmarkList.add(createFromProto(landmarkProto)); + } + return landmarkList; + } + // The x coordinates of the normalized landmark. public abstract float x(); diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java index 98fb9376f..78bc7efb9 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/facelandmarker/FaceLandmarkerResult.java @@ -46,23 +46,11 @@ public abstract class FaceLandmarkerResult implements TaskResult { long timestampMs) { List> multiFaceLandmarks = new ArrayList<>(); for (LandmarkProto.NormalizedLandmarkList faceLandmarksProto : multiFaceLandmarksProto) { - List faceLandmarks = new ArrayList<>(); - multiFaceLandmarks.add(faceLandmarks); - for (LandmarkProto.NormalizedLandmark faceLandmarkProto : - faceLandmarksProto.getLandmarkList()) { - faceLandmarks.add( - NormalizedLandmark.create( - faceLandmarkProto.getX(), - faceLandmarkProto.getY(), - faceLandmarkProto.getZ(), - faceLandmarkProto.hasVisibility() - ? Optional.of(faceLandmarkProto.getVisibility()) - : Optional.empty(), - faceLandmarkProto.hasPresence() - ? Optional.of(faceLandmarkProto.getPresence()) - : Optional.empty())); - } + List faceLandmarks = + NormalizedLandmark.createListFromProto(faceLandmarksProto); + multiFaceLandmarks.add(Collections.unmodifiableList(faceLandmarks)); } + Optional>> multiFaceBlendshapes = Optional.empty(); if (multiFaceBendshapesProto.isPresent()) { List> blendshapes = new ArrayList<>(); @@ -72,6 +60,7 @@ public abstract class FaceLandmarkerResult implements TaskResult { } multiFaceBlendshapes = Optional.of(Collections.unmodifiableList(blendshapes)); } + Optional> multiFaceTransformationMatrixes = Optional.empty(); if (multiFaceTransformationMatrixesProto.isPresent()) { List matrixes = new ArrayList<>(); @@ -90,6 +79,7 @@ public abstract class FaceLandmarkerResult implements TaskResult { } multiFaceTransformationMatrixes = Optional.of(Collections.unmodifiableList(matrixes)); } + return new AutoValue_FaceLandmarkerResult( timestampMs, Collections.unmodifiableList(multiFaceLandmarks), diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java index 54ed04848..5a1661a52 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/handlandmarker/HandLandmarkerResult.java @@ -24,7 +24,6 @@ import com.google.mediapipe.tasks.core.TaskResult; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Optional; /** Represents the hand landmarks deection results generated by {@link HandLandmarker}. */ @AutoValue @@ -34,63 +33,38 @@ public abstract class HandLandmarkerResult implements TaskResult { * Creates a {@link HandLandmarkerResult} instance from the lists of landmarks and handedness * protobuf messages. * - * @param landmarksProto a List of {@link NormalizedLandmarkList} - * @param worldLandmarksProto a List of {@link LandmarkList} - * @param handednessesProto a List of {@link ClassificationList} + * @param landmarksProtos a List of {@link NormalizedLandmarkList} + * @param worldLandmarksProtos a List of {@link LandmarkList} + * @param handednessesProtos a List of {@link ClassificationList} */ static HandLandmarkerResult create( - List landmarksProto, - List worldLandmarksProto, - List handednessesProto, + List landmarksProtos, + List worldLandmarksProtos, + List handednessesProtos, long timestampMs) { - List> multiHandLandmarks = new ArrayList<>(); - List> multiHandWorldLandmarks = new ArrayList<>(); - List> multiHandHandednesses = new ArrayList<>(); - for (LandmarkProto.NormalizedLandmarkList handLandmarksProto : landmarksProto) { - List handLandmarks = new ArrayList<>(); - multiHandLandmarks.add(handLandmarks); - for (LandmarkProto.NormalizedLandmark handLandmarkProto : - handLandmarksProto.getLandmarkList()) { - handLandmarks.add( - NormalizedLandmark.create( - handLandmarkProto.getX(), - handLandmarkProto.getY(), - handLandmarkProto.getZ(), - handLandmarkProto.hasVisibility() - ? Optional.of(handLandmarkProto.getVisibility()) - : Optional.empty(), - handLandmarkProto.hasPresence() - ? Optional.of(handLandmarkProto.getPresence()) - : Optional.empty())); - } + List> handLandmarks = new ArrayList<>(); + for (LandmarkProto.NormalizedLandmarkList handLandmarksProto : landmarksProtos) { + handLandmarks.add( + Collections.unmodifiableList(NormalizedLandmark.createListFromProto(handLandmarksProto))); } - for (LandmarkProto.LandmarkList handWorldLandmarksProto : worldLandmarksProto) { - List handWorldLandmarks = new ArrayList<>(); - multiHandWorldLandmarks.add(handWorldLandmarks); - for (LandmarkProto.Landmark handWorldLandmarkProto : - handWorldLandmarksProto.getLandmarkList()) { - handWorldLandmarks.add( - com.google.mediapipe.tasks.components.containers.Landmark.create( - handWorldLandmarkProto.getX(), - handWorldLandmarkProto.getY(), - handWorldLandmarkProto.getZ(), - handWorldLandmarkProto.hasVisibility() - ? Optional.of(handWorldLandmarkProto.getVisibility()) - : Optional.empty(), - handWorldLandmarkProto.hasPresence() - ? Optional.of(handWorldLandmarkProto.getPresence()) - : Optional.empty())); - } + + List> handWorldLandmarks = new ArrayList<>(); + for (LandmarkProto.LandmarkList handWorldLandmarksProto : worldLandmarksProtos) { + handWorldLandmarks.add( + Collections.unmodifiableList(Landmark.createListFromProto(handWorldLandmarksProto))); } - for (ClassificationList handednessProto : handednessesProto) { - List handedness = Category.createListFromProto(handednessProto); - multiHandHandednesses.add(Collections.unmodifiableList(handedness)); + + List> handHandednesses = new ArrayList<>(); + for (ClassificationList handednessProto : handednessesProtos) { + handHandednesses.add( + Collections.unmodifiableList(Category.createListFromProto(handednessProto))); } + return new AutoValue_HandLandmarkerResult( timestampMs, - Collections.unmodifiableList(multiHandLandmarks), - Collections.unmodifiableList(multiHandWorldLandmarks), - Collections.unmodifiableList(multiHandHandednesses)); + Collections.unmodifiableList(handLandmarks), + Collections.unmodifiableList(handWorldLandmarks), + Collections.unmodifiableList(handHandednesses)); } @Override diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerResult.java index e693c7b88..792d7407d 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerResult.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/poselandmarker/PoseLandmarkerResult.java @@ -50,43 +50,18 @@ public abstract class PoseLandmarkerResult implements TaskResult { } List> multiPoseLandmarks = new ArrayList<>(); + for (LandmarkProto.NormalizedLandmarkList handLandmarksProto : landmarksProto) { + List poseLandmarks = + NormalizedLandmark.createListFromProto(handLandmarksProto); + multiPoseLandmarks.add(Collections.unmodifiableList(poseLandmarks)); + } + List> multiPoseWorldLandmarks = new ArrayList<>(); - for (LandmarkProto.NormalizedLandmarkList poseLandmarksProto : landmarksProto) { - List poseLandmarks = new ArrayList<>(); - multiPoseLandmarks.add(poseLandmarks); - for (LandmarkProto.NormalizedLandmark poseLandmarkProto : - poseLandmarksProto.getLandmarkList()) { - poseLandmarks.add( - NormalizedLandmark.create( - poseLandmarkProto.getX(), - poseLandmarkProto.getY(), - poseLandmarkProto.getZ(), - poseLandmarkProto.hasVisibility() - ? Optional.of(poseLandmarkProto.getVisibility()) - : Optional.empty(), - poseLandmarkProto.hasPresence() - ? Optional.of(poseLandmarkProto.getPresence()) - : Optional.empty())); - } - } for (LandmarkProto.LandmarkList poseWorldLandmarksProto : worldLandmarksProto) { - List poseWorldLandmarks = new ArrayList<>(); - multiPoseWorldLandmarks.add(poseWorldLandmarks); - for (LandmarkProto.Landmark poseWorldLandmarkProto : - poseWorldLandmarksProto.getLandmarkList()) { - poseWorldLandmarks.add( - Landmark.create( - poseWorldLandmarkProto.getX(), - poseWorldLandmarkProto.getY(), - poseWorldLandmarkProto.getZ(), - poseWorldLandmarkProto.hasVisibility() - ? Optional.of(poseWorldLandmarkProto.getVisibility()) - : Optional.empty(), - poseWorldLandmarkProto.hasPresence() - ? Optional.of(poseWorldLandmarkProto.getPresence()) - : Optional.empty())); - } + List poseWorldLandmarks = Landmark.createListFromProto(poseWorldLandmarksProto); + multiPoseWorldLandmarks.add(Collections.unmodifiableList(poseWorldLandmarks)); } + return new AutoValue_PoseLandmarkerResult( timestampMs, Collections.unmodifiableList(multiPoseLandmarks), diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/LandmarkTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/LandmarkTest.java new file mode 100644 index 000000000..b5ff0564a --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/LandmarkTest.java @@ -0,0 +1,62 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.components.containers; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.mediapipe.formats.proto.LandmarkProto; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(AndroidJUnit4.class) +public final class LandmarkTest { + + @Test + public void createFromProto_succeedsWithCoordinates() { + LandmarkProto.Landmark input = + LandmarkProto.Landmark.newBuilder().setX(1.0f).setY(2.0f).setZ(3.0f).build(); + Landmark output = Landmark.createFromProto(input); + assertThat(output.x()).isEqualTo(1.0f); + assertThat(output.y()).isEqualTo(2.0f); + assertThat(output.z()).isEqualTo(3.0f); + assertFalse(output.visibility().isPresent()); + assertFalse(output.presence().isPresent()); + } + + @Test + public void createFromProto_succeedsWithVisibility() { + LandmarkProto.Landmark input = + LandmarkProto.Landmark.newBuilder().setVisibility(0.4f).setPresence(0.5f).build(); + Landmark output = Landmark.createFromProto(input); + assertTrue(output.visibility().isPresent()); + assertThat(output.visibility().get()).isEqualTo(0.4f); + assertTrue(output.presence().isPresent()); + assertThat(output.presence().get()).isEqualTo(0.5f); + } + + @Test + public void createListFromProto_succeeds() { + LandmarkProto.Landmark element = + LandmarkProto.Landmark.newBuilder().setX(1.0f).setY(2.0f).setZ(3.0f).build(); + LandmarkProto.LandmarkList input = + LandmarkProto.LandmarkList.newBuilder().addLandmark(element).build(); + List output = Landmark.createListFromProto(input); + assertThat(output).hasSize(1); + } +} diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/NormalizedLandmarkTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/NormalizedLandmarkTest.java new file mode 100644 index 000000000..64b61d263 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/components/containers/NormalizedLandmarkTest.java @@ -0,0 +1,62 @@ +// Copyright 2022 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.components.containers; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.mediapipe.formats.proto.LandmarkProto; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(AndroidJUnit4.class) +public final class NormalizedLandmarkTest { + + @Test + public void createFromProto_succeedsWithCoordinates() { + LandmarkProto.NormalizedLandmark input = + LandmarkProto.NormalizedLandmark.newBuilder().setX(0.1f).setY(0.2f).setZ(0.3f).build(); + NormalizedLandmark output = NormalizedLandmark.createFromProto(input); + assertThat(output.x()).isEqualTo(0.1f); + assertThat(output.y()).isEqualTo(0.2f); + assertThat(output.z()).isEqualTo(0.3f); + assertFalse(output.visibility().isPresent()); + assertFalse(output.presence().isPresent()); + } + + @Test + public void createFromProto_succeedsWithVisibility() { + LandmarkProto.NormalizedLandmark input = + LandmarkProto.NormalizedLandmark.newBuilder().setVisibility(0.4f).setPresence(0.5f).build(); + NormalizedLandmark output = NormalizedLandmark.createFromProto(input); + assertTrue(output.visibility().isPresent()); + assertThat(output.visibility().get()).isEqualTo(0.4f); + assertTrue(output.presence().isPresent()); + assertThat(output.presence().get()).isEqualTo(0.5f); + } + + @Test + public void createListFromProto_succeeds() { + LandmarkProto.NormalizedLandmark element = + LandmarkProto.NormalizedLandmark.newBuilder().setX(0.1f).setY(0.2f).setZ(0.3f).build(); + LandmarkProto.NormalizedLandmarkList input = + LandmarkProto.NormalizedLandmarkList.newBuilder().addLandmark(element).build(); + List output = NormalizedLandmark.createListFromProto(input); + assertThat(output).hasSize(1); + } +} From 1c860cace655c02b55bfe95eee5d24d2da1a50bf Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 13 Nov 2023 09:53:37 -0800 Subject: [PATCH 110/157] Added files for the Object Detector C Tasks API --- mediapipe/tasks/c/components/containers/BUILD | 87 ++++++ .../components/containers/detection_result.h | 63 ++++ .../containers/detection_result_converter.cc | 80 +++++ .../containers/detection_result_converter.h | 38 +++ .../detection_result_converter_test.cc | 28 ++ .../tasks/c/components/containers/keypoint.h | 48 +++ .../containers/keypoint_converter.cc | 45 +++ .../containers/keypoint_converter.h | 32 ++ .../containers/keypoint_converter_test.cc | 35 +++ .../tasks/c/components/containers/rect.h | 46 +++ .../c/components/containers/rect_converter.cc | 43 +++ .../c/components/containers/rect_converter.h | 32 ++ .../containers/rect_converter_test.cc | 51 ++++ .../tasks/c/vision/object_detector/BUILD | 64 ++++ .../vision/object_detector/object_detector.cc | 288 ++++++++++++++++++ .../vision/object_detector/object_detector.h | 157 ++++++++++ .../object_detector/object_detector_test.cc | 253 +++++++++++++++ 17 files changed, 1390 insertions(+) create mode 100644 mediapipe/tasks/c/components/containers/detection_result.h create mode 100644 mediapipe/tasks/c/components/containers/detection_result_converter.cc create mode 100644 mediapipe/tasks/c/components/containers/detection_result_converter.h create mode 100644 mediapipe/tasks/c/components/containers/detection_result_converter_test.cc create mode 100644 mediapipe/tasks/c/components/containers/keypoint.h create mode 100644 mediapipe/tasks/c/components/containers/keypoint_converter.cc create mode 100644 mediapipe/tasks/c/components/containers/keypoint_converter.h create mode 100644 mediapipe/tasks/c/components/containers/keypoint_converter_test.cc create mode 100644 mediapipe/tasks/c/components/containers/rect.h create mode 100644 mediapipe/tasks/c/components/containers/rect_converter.cc create mode 100644 mediapipe/tasks/c/components/containers/rect_converter.h create mode 100644 mediapipe/tasks/c/components/containers/rect_converter_test.cc create mode 100644 mediapipe/tasks/c/vision/object_detector/BUILD create mode 100644 mediapipe/tasks/c/vision/object_detector/object_detector.cc create mode 100644 mediapipe/tasks/c/vision/object_detector/object_detector.h create mode 100644 mediapipe/tasks/c/vision/object_detector/object_detector_test.cc diff --git a/mediapipe/tasks/c/components/containers/BUILD b/mediapipe/tasks/c/components/containers/BUILD index 4bb580873..d015ccb00 100644 --- a/mediapipe/tasks/c/components/containers/BUILD +++ b/mediapipe/tasks/c/components/containers/BUILD @@ -43,6 +43,60 @@ cc_test( ], ) +cc_library( + name = "rect", + hdrs = ["rect.h"], +) + +cc_library( + name = "rect_converter", + srcs = ["rect_converter.cc"], + hdrs = ["rect_converter.h"], + deps = [ + ":rect", + "//mediapipe/tasks/cc/components/containers:rect", + ], +) + +cc_test( + name = "rect_converter_test", + srcs = ["rect_converter_test.cc"], + deps = [ + ":rect", + ":rect_converter", + "//mediapipe/framework/port:gtest", + "//mediapipe/tasks/cc/components/containers:rect", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "keypoint", + hdrs = ["keypoint.h"], +) + +cc_library( + name = "keypoint_converter", + srcs = ["keypoint_converter.cc"], + hdrs = ["keypoint_converter.h"], + deps = [ + ":keypoint", + "//mediapipe/tasks/cc/components/containers:keypoint", + ], +) + +cc_test( + name = "keypoint_converter_test", + srcs = ["keypoint_converter_test.cc"], + deps = [ + ":keypoint", + ":keypoint_converter", + "//mediapipe/framework/port:gtest", + "//mediapipe/tasks/cc/components/containers:keypoint", + "@com_google_googletest//:gtest_main", + ], +) + cc_library( name = "classification_result", hdrs = ["classification_result.h"], @@ -72,6 +126,39 @@ cc_test( ], ) +cc_library( + name = "detection_result", + hdrs = ["detection_result.h"], +) + +cc_library( + name = "detection_result_converter", + srcs = ["detection_result_converter.cc"], + hdrs = ["detection_result_converter.h"], + deps = [ + ":rect", + ":rect_converter", + ":category", + ":category_converter", + ":keypoint", + ":keypoint_converter", + ":detection_result", + "//mediapipe/tasks/cc/components/containers:detection_result", + ], +) + +cc_test( + name = "detection_result_converter_test", + srcs = ["detection_result_converter_test.cc"], + deps = [ + ":detection_result", + ":detection_result_converter", + "//mediapipe/framework/port:gtest", + "//mediapipe/tasks/cc/components/containers:detection_result", + "@com_google_googletest//:gtest_main", + ], +) + cc_library( name = "embedding_result", hdrs = ["embedding_result.h"], diff --git a/mediapipe/tasks/c/components/containers/detection_result.h b/mediapipe/tasks/c/components/containers/detection_result.h new file mode 100644 index 000000000..48ce200f0 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/detection_result.h @@ -0,0 +1,63 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_DETECTION_RESULT_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_DETECTION_RESULT_H_ + +#include + +#include "mediapipe/tasks/c/components/containers/rect.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// Detection for a single bounding box. +struct DetectionC { + // An array of detected categories. + struct Category* categories; + + // The number of elements in the categories array. + uint32_t categories_count; + + // The bounding box location. + struct Rect bounding_box; + + // Optional list of keypoints associated with the detection. Keypoints + // represent interesting points related to the detection. For example, the + // keypoints represent the eye, ear and mouth from face detection model. Or + // in the template matching detection, e.g. KNIFT, they can represent the + // feature points for template matching. + // `nullptr` if keypoints is not present. + struct NormalizedKeypoint* keypoints; + + // The number of elements in the keypoints array. 0 if keypoints do not exist. + uint32_t keypoints_count; +}; + +// Detection results of a model. +struct DetectionResult { + // An array of Detections. + struct DetectionC* detections; + + // The number of detections in the detections array. + uint32_t detections_count; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_DETECTION_RESULT_H_ diff --git a/mediapipe/tasks/c/components/containers/detection_result_converter.cc b/mediapipe/tasks/c/components/containers/detection_result_converter.cc new file mode 100644 index 000000000..a752e50fe --- /dev/null +++ b/mediapipe/tasks/c/components/containers/detection_result_converter.cc @@ -0,0 +1,80 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/detection_result_converter.h" + +#include + +#include "mediapipe/tasks/c/components/containers/category.h" +#include "mediapipe/tasks/c/components/containers/category_converter.h" +#include "mediapipe/tasks/c/components/containers/detection_result.h" +#include "mediapipe/tasks/c/components/containers/keypoint.h" +#include "mediapipe/tasks/c/components/containers/keypoint_converter.h" +#include "mediapipe/tasks/c/components/containers/rect_converter.h" +#include "mediapipe/tasks/cc/components/containers/detection_result.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToDetection( + const mediapipe::tasks::components::containers::Detection& in, + DetectionC* out) { + out->categories_count = in.categories.size(); + out->categories = new Category[out->categories_count]; + for (size_t i = 0; i < out->categories_count; ++i) { + CppConvertToCategory(in.categories[i], &out->categories[i]); + } + + CppConvertToRect(in.bounding_box, &out->bounding_box); + + if (in.keypoints.has_value()) { + auto& keypoints = in.keypoints.value(); + out->keypoints_count = keypoints.size(); + out->keypoints = new NormalizedKeypoint[out->keypoints_count]; + for (size_t i = 0; i < out->keypoints_count; ++i) { + CppConvertToNormalizedKeypoint(keypoints[i], &out->keypoints[i]); + } + } else { + out->keypoints = nullptr; + out->keypoints_count = 0; + } +} + +void CppConvertToDetectionResult( + const mediapipe::tasks::components::containers::DetectionResult& in, + DetectionResult* out) { + out->detections_count = in.detections.size(); + out->detections = new DetectionC[out->detections_count]; + for (size_t i = 0; i < out->detections_count; ++i) { + CppConvertToDetection(in.detections[i], &out->detections[i]); + } +} + +// Functions to free the memory of C structures. +void CppCloseDetection(DetectionC* in) { + delete[] in->categories; + in->categories = nullptr; + delete[] in->keypoints; + in->keypoints = nullptr; +} + +void CppCloseDetectionResult(DetectionResult* in) { + for (size_t i = 0; i < in->detections_count; ++i) { + CppCloseDetection(&in->detections[i]); + } + delete[] in->detections; + in->detections = nullptr; +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/detection_result_converter.h b/mediapipe/tasks/c/components/containers/detection_result_converter.h new file mode 100644 index 000000000..e338e47e9 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/detection_result_converter.h @@ -0,0 +1,38 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_DETECTION_RESULT_CONVERTER_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_DETECTION_RESULT_CONVERTER_H_ + +#include "mediapipe/tasks/c/components/containers/detection_result.h" +#include "mediapipe/tasks/cc/components/containers/detection_result.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToDetection( + const mediapipe::tasks::components::containers::Detection& in, + Detection* out); + +void CppConvertToDetectionResult( + const mediapipe::tasks::components::containers::DetectionResult& in, + DetectionResult* out); + +void CppCloseDetection(Detection* in); + +void CppCloseDetectionResult(DetectionResult* in); + +} // namespace mediapipe::tasks::c::components::containers + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_DETECTION_RESULT_CONVERTER_H_ diff --git a/mediapipe/tasks/c/components/containers/detection_result_converter_test.cc b/mediapipe/tasks/c/components/containers/detection_result_converter_test.cc new file mode 100644 index 000000000..884481f29 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/detection_result_converter_test.cc @@ -0,0 +1,28 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/detection_result_converter.h" + +#include +#include +#include + +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/components/containers/detection_result.h" +#include "mediapipe/tasks/cc/components/containers/detection_result.h" + +namespace mediapipe::tasks::c::components::containers { + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/keypoint.h b/mediapipe/tasks/c/components/containers/keypoint.h new file mode 100644 index 000000000..8381b1850 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/keypoint.h @@ -0,0 +1,48 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_KEYPOINT_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_KEYPOINT_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// A keypoint, defined by the coordinates (x, y), normalized by the image +// dimensions. +struct NormalizedKeypoint { + // x in normalized image coordinates. + float x; + + // y in normalized image coordinates. + float y; + + // Optional label of the keypoint. + char* label; // `nullptr` if the label is not present. + + // Optional score of the keypoint. + float score; + + // Indicates if score is valid + bool has_score; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_KEYPOINT_H_ diff --git a/mediapipe/tasks/c/components/containers/keypoint_converter.cc b/mediapipe/tasks/c/components/containers/keypoint_converter.cc new file mode 100644 index 000000000..53e8a5da1 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/keypoint_converter.cc @@ -0,0 +1,45 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/keypoint_converter.h" + +#include +#include +#include + +#include "mediapipe/tasks/c/components/containers/keypoint.h" +#include "mediapipe/tasks/cc/components/containers/keypoint.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToNormalizedKeypoint( + const mediapipe::tasks::components::containers::NormalizedKeypoint& in, + NormalizedKeypoint* out) { + out->x = in.x; + out->y = in.y; + + out->label = in.label.has_value() ? strdup(in.label->c_str()) : nullptr; + out->has_score = in.score.has_value(); + out->score = out->has_score ? in.score.value() : 0; +} + +void CppCloseNormalizedKeypoint(NormalizedKeypoint* keypoint) { + if (keypoint && keypoint->label) { + free(keypoint->label); + keypoint->label = NULL; + } +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/keypoint_converter.h b/mediapipe/tasks/c/components/containers/keypoint_converter.h new file mode 100644 index 000000000..a4bd725f2 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/keypoint_converter.h @@ -0,0 +1,32 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_KEYPOINT_CONVERTER_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_KEYPOINT_CONVERTER_H_ + +#include "mediapipe/tasks/c/components/containers/keypoint.h" +#include "mediapipe/tasks/cc/components/containers/keypoint.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToNormalizedKeypoint( + const mediapipe::tasks::components::containers::NormalizedKeypoint& in, + NormalizedKeypoint* out); + +void CppCloseNormalizedKeypoint(NormalizedKeypoint* keypoint); + +} // namespace mediapipe::tasks::c::components::containers + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_KEYPOINT_CONVERTER_H_ diff --git a/mediapipe/tasks/c/components/containers/keypoint_converter_test.cc b/mediapipe/tasks/c/components/containers/keypoint_converter_test.cc new file mode 100644 index 000000000..ca09154c3 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/keypoint_converter_test.cc @@ -0,0 +1,35 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/keypoint_converter.h" + +#include +#include +#include + +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/components/containers/keypoint.h" +#include "mediapipe/tasks/cc/components/containers/keypoint.h" + +namespace mediapipe::tasks::c::components::containers { + +TEST(RectConverterTest, ConvertsRectCustomValues) { + mediapipe::tasks::components::containers::Rect cpp_rect = {0, 0, 0, 0}; + + Rect c_rect; + CppConvertToRect(cpp_rect, &c_rect); +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/rect.h b/mediapipe/tasks/c/components/containers/rect.h new file mode 100644 index 000000000..ae6c7a3ee --- /dev/null +++ b/mediapipe/tasks/c/components/containers/rect.h @@ -0,0 +1,46 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_RECT_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_RECT_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// Defines a rectangle, used e.g. as part of detection results or as input +// region-of-interest. +struct Rect { + int left; + int top; + int right; + int bottom; +}; + +// The coordinates are normalized wrt the image dimensions, i.e. generally in +// [0,1] but they may exceed these bounds if describing a region overlapping the +// image. The origin is on the top-left corner of the image. +struct RectF { + float left; + float top; + float right; + float bottom; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_RECT_H_ diff --git a/mediapipe/tasks/c/components/containers/rect_converter.cc b/mediapipe/tasks/c/components/containers/rect_converter.cc new file mode 100644 index 000000000..ff700acee --- /dev/null +++ b/mediapipe/tasks/c/components/containers/rect_converter.cc @@ -0,0 +1,43 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/rect_converter.h" + +#include + +#include "mediapipe/tasks/c/components/containers/rect.h" +#include "mediapipe/tasks/cc/components/containers/rect.h" + +namespace mediapipe::tasks::c::components::containers { + +// Converts a C++ Rect to a C Rect. +void CppConvertToRect(const mediapipe::tasks::components::containers::Rect& in, + struct Rect* out) { + out->left = in.left; + out->top = in.top; + out->right = in.right; + out->bottom = in.bottom; +} + +// Converts a C++ RectF to a C RectF. +void CppConvertToRectF( + const mediapipe::tasks::components::containers::RectF& in, RectF* out) { + out->left = in.left; + out->top = in.top; + out->right = in.right; + out->bottom = in.bottom; +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/rect_converter.h b/mediapipe/tasks/c/components/containers/rect_converter.h new file mode 100644 index 000000000..75e21ff52 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/rect_converter.h @@ -0,0 +1,32 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_RECT_CONVERTER_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_RECT_CONVERTER_H_ + +#include "mediapipe/tasks/c/components/containers/rect.h" +#include "mediapipe/tasks/cc/components/containers/rect.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToRect(const mediapipe::tasks::components::containers::Rect& in, + Rect* out); + +void CppConvertToRectF( + const mediapipe::tasks::components::containers::RectF& in, RectF* out); + +} // namespace mediapipe::tasks::c::components::containers + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_RECT_CONVERTER_H_ diff --git a/mediapipe/tasks/c/components/containers/rect_converter_test.cc b/mediapipe/tasks/c/components/containers/rect_converter_test.cc new file mode 100644 index 000000000..3e8848094 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/rect_converter_test.cc @@ -0,0 +1,51 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/rect_converter.h" + +#include +#include +#include + +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/components/containers/rect.h" +#include "mediapipe/tasks/cc/components/containers/rect.h" + +namespace mediapipe::tasks::c::components::containers { + +TEST(RectConverterTest, ConvertsRectCustomValues) { + mediapipe::tasks::components::containers::Rect cpp_rect = {0, 0, 0, 0}; + + Rect c_rect; + CppConvertToRect(cpp_rect, &c_rect); + EXPECT_EQ(c_rect.left, 0); + EXPECT_EQ(c_rect.right, 0); + EXPECT_EQ(c_rect.top, 0); + EXPECT_EQ(c_rect.bottom, 0); +} + +TEST(RectFConverterTest, ConvertsRectFCustomValues) { + mediapipe::tasks::components::containers::RectF cpp_rect = {0.1, 0.1, 0.1, + 0.1}; + + RectF c_rect; + CppConvertToRect(cpp_rect, &c_rect); + EXPECT_FLOAT_EQ(c_rect.left, 0.1); + EXPECT_FLOAT_EQ(c_rect.right, 0.1); + EXPECT_FLOAT_EQ(c_rect.top, 0.1); + EXPECT_FLOAT_EQ(c_rect.bottom, 0.1); +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/vision/object_detector/BUILD b/mediapipe/tasks/c/vision/object_detector/BUILD new file mode 100644 index 000000000..ee405b6df --- /dev/null +++ b/mediapipe/tasks/c/vision/object_detector/BUILD @@ -0,0 +1,64 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +cc_library( + name = "object_detector_lib", + srcs = ["object_detector.cc"], + hdrs = ["object_detector.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/tasks/c/components/containers:detection_result", + "//mediapipe/tasks/c/components/containers:detection_result_converter", + "//mediapipe/tasks/c/core:base_options", + "//mediapipe/tasks/c/core:base_options_converter", + "//mediapipe/tasks/c/vision/core:common", + "//mediapipe/tasks/cc/vision/core:running_mode", + "//mediapipe/tasks/cc/vision/object_detector", + "//mediapipe/tasks/cc/vision/utils:image_utils", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], + alwayslink = 1, +) + +cc_test( + name = "object_detector_test", + srcs = ["object_detector_test.cc"], + data = [ + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + linkstatic = 1, + deps = [ + ":object_detector_lib", + "//mediapipe/framework/deps:file_path", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/port:gtest", + "//mediapipe/tasks/c/components/containers:category", + "//mediapipe/tasks/cc/vision/utils:image_utils", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/mediapipe/tasks/c/vision/object_detector/object_detector.cc b/mediapipe/tasks/c/vision/object_detector/object_detector.cc new file mode 100644 index 000000000..9fdf77821 --- /dev/null +++ b/mediapipe/tasks/c/vision/object_detector/object_detector.cc @@ -0,0 +1,288 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/object_detector/object_detector.h" + +#include +#include +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/tasks/c/components/containers/detection_result_converter.h" +#include "mediapipe/tasks/c/core/base_options_converter.h" +#include "mediapipe/tasks/cc/vision/core/running_mode.h" +#include "mediapipe/tasks/cc/vision/object_detector/object_detector.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace mediapipe::tasks::c::vision::object_detector { + +namespace { + +using ::mediapipe::tasks::c::components::containers::CppCloseDetectionResult; +using ::mediapipe::tasks::c::components::containers:: + CppConvertToDetectionResult; +using ::mediapipe::tasks::c::core::CppConvertToBaseOptions; +using ::mediapipe::tasks::vision::CreateImageFromBuffer; +using ::mediapipe::tasks::vision::ObjectDetector; +using ::mediapipe::tasks::vision::core::RunningMode; +typedef ::mediapipe::tasks::vision::ObjectDetectorResult + CppObjectDetectorResult; + +int CppProcessError(absl::Status status, char** error_msg) { + if (error_msg) { + *error_msg = strdup(status.ToString().c_str()); + } + return status.raw_code(); +} + +} // namespace + +void CppConvertToDetectorOptions( + const ObjectDetectorOptions& in, + mediapipe::tasks::vision::ObjectDetectorOptions* out) { + out->display_names_locale = + in.display_names_locale ? std::string(in.display_names_locale) : "en"; + out->max_results = in.max_results; + out->score_threshold = in.score_threshold; + out->category_allowlist = + std::vector(in.category_allowlist_count); + for (uint32_t i = 0; i < in.category_allowlist_count; ++i) { + out->category_allowlist[i] = in.category_allowlist[i]; + } + out->category_denylist = std::vector(in.category_denylist_count); + for (uint32_t i = 0; i < in.category_denylist_count; ++i) { + out->category_denylist[i] = in.category_denylist[i]; + } +} + +ObjectDetector* CppObjectDetectorCreate(const ObjectDetectorOptions& options, + char** error_msg) { + auto cpp_options = + std::make_unique<::mediapipe::tasks::vision::ObjectDetectorOptions>(); + + CppConvertToBaseOptions(options.base_options, &cpp_options->base_options); + CppConvertToDetectorOptions(options, cpp_options.get()); + cpp_options->running_mode = static_cast(options.running_mode); + + // Enable callback for processing live stream data when the running mode is + // set to RunningMode::LIVE_STREAM. + if (cpp_options->running_mode == RunningMode::LIVE_STREAM) { + if (options.result_callback == nullptr) { + const absl::Status status = absl::InvalidArgumentError( + "Provided null pointer to callback function."); + ABSL_LOG(ERROR) << "Failed to create ObjectDetector: " << status; + CppProcessError(status, error_msg); + return nullptr; + } + + ObjectDetectorOptions::result_callback_fn result_callback = + options.result_callback; + cpp_options->result_callback = + [result_callback](absl::StatusOr cpp_result, + const Image& image, int64_t timestamp) { + char* error_msg = nullptr; + + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status(); + CppProcessError(cpp_result.status(), &error_msg); + result_callback(nullptr, MpImage(), timestamp, error_msg); + free(error_msg); + return; + } + + // Result is valid for the lifetime of the callback function. + ObjectDetectorResult result; + CppConvertToDetectionResult(*cpp_result, &result); + + const auto& image_frame = image.GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = { + .format = static_cast<::ImageFormat>(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + result_callback(&result, mp_image, timestamp, + /* error_msg= */ nullptr); + + CppCloseDetectionResult(&result); + }; + } + + auto detector = ObjectDetector::Create(std::move(cpp_options)); + if (!detector.ok()) { + ABSL_LOG(ERROR) << "Failed to create ObjectDetector: " << detector.status(); + CppProcessError(detector.status(), error_msg); + return nullptr; + } + return detector->release(); +} + +int CppObjectDetectorDetect(void* detector, const MpImage* image, + ObjectDetectorResult* result, char** error_msg) { + if (image->type == MpImage::GPU_BUFFER) { + const absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet."); + + ABSL_LOG(ERROR) << "Detection failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image->image_frame.format), + image->image_frame.image_buffer, image->image_frame.width, + image->image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_detector = static_cast(detector); + auto cpp_result = cpp_detector->Detect(*img); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status(); + return CppProcessError(cpp_result.status(), error_msg); + } + CppConvertToDetectionResult(*cpp_result, result); + return 0; +} + +int CppObjectDetectorDetectForVideo(void* detector, const MpImage* image, + int64_t timestamp_ms, + ObjectDetectorResult* result, + char** error_msg) { + if (image->type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Detection failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image->image_frame.format), + image->image_frame.image_buffer, image->image_frame.width, + image->image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_detector = static_cast(detector); + auto cpp_result = cpp_detector->DetectForVideo(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status(); + return CppProcessError(cpp_result.status(), error_msg); + } + CppConvertToDetectionResult(*cpp_result, result); + return 0; +} + +int CppObjectDetectorDetectAsync(void* detector, const MpImage* image, + int64_t timestamp_ms, char** error_msg) { + if (image->type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Detection failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image->image_frame.format), + image->image_frame.image_buffer, image->image_frame.width, + image->image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_detector = static_cast(detector); + auto cpp_result = cpp_detector->DetectAsync(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Data preparation for the object detection failed: " + << cpp_result; + return CppProcessError(cpp_result, error_msg); + } + return 0; +} + +void CppObjectDetectorCloseResult(ObjectDetectorResult* result) { + CppCloseDetectionResult(result); +} + +int CppObjectDetectorClose(void* detector, char** error_msg) { + auto cpp_detector = static_cast(detector); + auto result = cpp_detector->Close(); + if (!result.ok()) { + ABSL_LOG(ERROR) << "Failed to close ObjectDetector: " << result; + return CppProcessError(result, error_msg); + } + delete cpp_detector; + return 0; +} + +} // namespace mediapipe::tasks::c::vision::object_detector + +extern "C" { + +void* object_detector_create(struct ObjectDetectorOptions* options, + char** error_msg) { + return mediapipe::tasks::c::vision::object_detector::CppObjectDetectorCreate( + *options, error_msg); +} + +int object_detector_detect_image(void* detector, const MpImage* image, + ObjectDetectorResult* result, + char** error_msg) { + return mediapipe::tasks::c::vision::object_detector::CppObjectDetectorDetect( + detector, image, result, error_msg); +} + +int object_detector_detect_for_video(void* detector, const MpImage* image, + int64_t timestamp_ms, + ObjectDetectorResult* result, + char** error_msg) { + return mediapipe::tasks::c::vision::object_detector:: + CppObjectDetectorDetectForVideo(detector, image, timestamp_ms, result, + error_msg); +} + +int object_detector_detect_async(void* detector, const MpImage* image, + int64_t timestamp_ms, char** error_msg) { + return mediapipe::tasks::c::vision::object_detector:: + CppObjectDetectorDetectAsync(detector, image, timestamp_ms, error_msg); +} + +void object_detector_close_result(ObjectDetectorResult* result) { + mediapipe::tasks::c::vision::object_detector::CppObjectDetectorCloseResult( + result); +} + +int object_detector_close(void* detector, char** error_ms) { + return mediapipe::tasks::c::vision::object_detector::CppObjectDetectorClose( + detector, error_ms); +} + +} // extern "C" diff --git a/mediapipe/tasks/c/vision/object_detector/object_detector.h b/mediapipe/tasks/c/vision/object_detector/object_detector.h new file mode 100644 index 000000000..16ec32477 --- /dev/null +++ b/mediapipe/tasks/c/vision/object_detector/object_detector.h @@ -0,0 +1,157 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_VISION_OBJECT_DETECTOR_OBJECT_DETECTOR_H_ +#define MEDIAPIPE_TASKS_C_VISION_OBJECT_DETECTOR_OBJECT_DETECTOR_H_ + +#include "mediapipe/tasks/c/components/containers/detection_result.h" +#include "mediapipe/tasks/c/core/base_options.h" +#include "mediapipe/tasks/c/vision/core/common.h" + +#ifndef MP_EXPORT +#define MP_EXPORT __attribute__((visibility("default"))) +#endif // MP_EXPORT + +#ifdef __cplusplus +extern "C" { +#endif + +typedef DetectionResult ObjectDetectorResult; + +// The options for configuring a MediaPipe object detector task. +struct ObjectDetectorOptions { + // Base options for configuring MediaPipe Tasks, such as specifying the model + // file with metadata, accelerator options, op resolver, etc. + struct BaseOptions base_options; + + // The running mode of the task. Default to the image mode. + // Object detector has three running modes: + // 1) The image mode for detecting objects on single image inputs. + // 2) The video mode for detecting objects on the decoded frames of a video. + // 3) The live stream mode for detecting objects on the live stream of input + // data, such as from camera. In this mode, the "result_callback" below must + // be specified to receive the detection results asynchronously. + RunningMode running_mode; + + // The locale to use for display names specified through the TFLite Model + // Metadata, if any. Defaults to English. + const char* display_names_locale; + + // The maximum number of top-scored detection results to return. If < 0, + // all available results will be returned. If 0, an invalid argument error is + // returned. + int max_results; + + // Score threshold to override the one provided in the model metadata (if + // any). Results below this value are rejected. + float score_threshold; + + // The allowlist of category names. If non-empty, detection results whose + // category name is not in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_denylist. + const char** category_allowlist; + // The number of elements in the category allowlist. + uint32_t category_allowlist_count; + + // The denylist of category names. If non-empty, detection results whose + // category name is in this set will be filtered out. Duplicate or unknown + // category names are ignored. Mutually exclusive with category_allowlist. + const char** category_denylist; + // The number of elements in the category denylist. + uint32_t category_denylist_count; + + // The user-defined result callback for processing live stream data. + // The result callback should only be specified when the running mode is set + // to RunningMode::LIVE_STREAM. Arguments of the callback function include: + // the pointer to detection result, the image that result was obtained + // on, the timestamp relevant to detection results and pointer to error + // message in case of any failure. The validity of the passed arguments is + // true for the lifetime of the callback function. + // + // A caller is responsible for closing object detector result. + typedef void (*result_callback_fn)(ObjectDetectorResult* result, + const MpImage image, int64_t timestamp_ms, + char* error_msg); + result_callback_fn result_callback; +}; + +// Creates an ObjectDetector from provided `options`. +// Returns a pointer to the image detector on success. +// If an error occurs, returns `nullptr` and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT void* object_detector_create(struct ObjectDetectorOptions* options, + char** error_msg); + +// Performs image detection on the input `image`. Returns `0` on success. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int object_detector_detect_image(void* detector, const MpImage* image, + ObjectDetectorResult* result, + char** error_msg); + +// Performs image detection on the provided video frame. +// Only use this method when the ObjectDetector is created with the video +// running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide the video frame's timestamp (in milliseconds). The input timestamps +// must be monotonically increasing. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int object_detector_detect_for_video(void* detector, + const MpImage* image, + int64_t timestamp_ms, + ObjectDetectorResult* result, + char** error_msg); + +// Sends live image data to image detection, and the results will be +// available via the `result_callback` provided in the ObjectDetectorOptions. +// Only use this method when the ObjectDetector is created with the live +// stream running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide a timestamp (in milliseconds) to indicate when the input image is +// sent to the object detector. The input timestamps must be monotonically +// increasing. +// The `result_callback` provides: +// - The detection results as an ObjectDetectorResult object. +// - The const reference to the corresponding input image that the image +// detector runs on. Note that the const reference to the image will no +// longer be valid when the callback returns. To access the image data +// outside of the callback, callers need to make a copy of the image. +// - The input timestamp in milliseconds. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int object_detector_detect_async(void* detector, const MpImage* image, + int64_t timestamp_ms, + char** error_msg); + +// Frees the memory allocated inside a ObjectDetectorResult result. +// Does not free the result pointer itself. +MP_EXPORT void object_detector_close_result(ObjectDetectorResult* result); + +// Frees image detector. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int object_detector_close(void* detector, char** error_msg); + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_VISION_OBJECT_DETECTOR_OBJECT_DETECTOR_H_ diff --git a/mediapipe/tasks/c/vision/object_detector/object_detector_test.cc b/mediapipe/tasks/c/vision/object_detector/object_detector_test.cc new file mode 100644 index 000000000..ac404b0e7 --- /dev/null +++ b/mediapipe/tasks/c/vision/object_detector/object_detector_test.cc @@ -0,0 +1,253 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/object_detector/object_detector.h" + +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/strings/string_view.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/components/containers/category.h" +#include "mediapipe/tasks/c/components/containers/rect.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace { + +using ::mediapipe::file::JoinPath; +using ::mediapipe::tasks::vision::DecodeImageFromFile; +using testing::HasSubstr; + +constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; +constexpr char kImageFile[] = "cats_and_dogs.jpg"; +constexpr char kModelName[] = + "coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.tflite"; +constexpr float kPrecision = 1e-4; +constexpr int kIterations = 100; + +std::string GetFullPath(absl::string_view file_name) { + return JoinPath("./", kTestDataDirectory, file_name); +} + +TEST(ObjectDetectorTest, ImageModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + ObjectDetectorOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0, + }; + + void* detector = object_detector_create(&options, /* error_msg */ nullptr); + EXPECT_NE(detector, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + ObjectDetectorResult result; + object_detector_detect_image(detector, &mp_image, &result, + /* error_msg */ nullptr); + EXPECT_EQ(result.detections_count, 10); + EXPECT_EQ(result.detections[0].categories_count, 1); + EXPECT_EQ(std::string{result.detections[0].categories[0].category_name}, + "cat"); + EXPECT_NEAR(result.detections[0].categories[0].score, 0.6992f, kPrecision); + object_detector_close_result(&result); + object_detector_close(detector, /* error_msg */ nullptr); +} + +TEST(ObjectDetectorTest, VideoModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + ObjectDetectorOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::VIDEO, + /* display_names_locale= */ nullptr, + /* max_results= */ 3, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0, + }; + + void* detector = object_detector_create(&options, /* error_msg */ nullptr); + EXPECT_NE(detector, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + ObjectDetectorResult result; + object_detector_detect_for_video(detector, &mp_image, i, &result, + /* error_msg */ nullptr); + EXPECT_EQ(result.detections_count, 3); + EXPECT_EQ(result.detections[0].categories_count, 1); + EXPECT_EQ(std::string{result.detections[0].categories[0].category_name}, + "cat"); + EXPECT_NEAR(result.detections[0].categories[0].score, 0.6992f, kPrecision); + object_detector_close_result(&result); + } + object_detector_close(detector, /* error_msg */ nullptr); +} + +// A structure to support LiveStreamModeTest below. This structure holds a +// static method `Fn` for a callback function of C API. A `static` qualifier +// allows to take an address of the method to follow API style. Another static +// struct member is `last_timestamp` that is used to verify that current +// timestamp is greater than the previous one. +struct LiveStreamModeCallback { + static int64_t last_timestamp; + static void Fn(ObjectDetectorResult* detector_result, const MpImage image, + int64_t timestamp, char* error_msg) { + ASSERT_NE(detector_result, nullptr); + ASSERT_EQ(error_msg, nullptr); + EXPECT_EQ(detector_result->detections_count, 3); + EXPECT_EQ(detector_result->detections[0].categories_count, 1); + EXPECT_EQ( + std::string{detector_result->detections[0].categories[0].category_name}, + "cat"); + EXPECT_NEAR(detector_result->detections[0].categories[0].score, 0.6992f, + kPrecision); + EXPECT_GT(image.image_frame.width, 0); + EXPECT_GT(image.image_frame.height, 0); + EXPECT_GT(timestamp, last_timestamp); + last_timestamp++; + } +}; +int64_t LiveStreamModeCallback::last_timestamp = -1; + +TEST(ObjectDetectorTest, LiveStreamModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + + ObjectDetectorOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::LIVE_STREAM, + /* display_names_locale= */ nullptr, + /* max_results= */ 3, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0, + /* result_callback= */ LiveStreamModeCallback::Fn, + }; + + void* detector = object_detector_create(&options, /* error_msg */ + nullptr); + EXPECT_NE(detector, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + EXPECT_GE(object_detector_detect_async(detector, &mp_image, i, + /* error_msg */ nullptr), + 0); + } + object_detector_close(detector, /* error_msg */ nullptr); + + // Due to the flow limiter, the total of outputs might be smaller than the + // number of iterations. + EXPECT_LE(LiveStreamModeCallback::last_timestamp, kIterations); + EXPECT_GT(LiveStreamModeCallback::last_timestamp, 0); +} + +TEST(ObjectDetectorTest, InvalidArgumentHandling) { + // It is an error to set neither the asset buffer nor the path. + ObjectDetectorOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ nullptr}, + }; + + char* error_msg; + void* detector = object_detector_create(&options, &error_msg); + EXPECT_EQ(detector, nullptr); + + EXPECT_THAT(error_msg, HasSubstr("ExternalFile must specify")); + + free(error_msg); +} + +TEST(ObjectDetectorTest, FailedDetectionHandling) { + const std::string model_path = GetFullPath(kModelName); + ObjectDetectorOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0, + }; + + void* detector = object_detector_create(&options, /* error_msg */ + nullptr); + EXPECT_NE(detector, nullptr); + + const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}}; + ObjectDetectorResult result; + char* error_msg; + object_detector_detect_image(detector, &mp_image, &result, &error_msg); + EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet")); + free(error_msg); + object_detector_close(detector, /* error_msg */ nullptr); +} + +} // namespace From 71e9929f60c4fda6b9717d69cb245f921e626a98 Mon Sep 17 00:00:00 2001 From: Youchuan Hu Date: Mon, 13 Nov 2023 10:31:06 -0800 Subject: [PATCH 111/157] Refactor OpenCV path out of TensorsToSegmentationCalculator main file. ProcessCpu() is changed into an OpenCV converter that is owned by the calculator. The calculator should call converter.Convert() to get the conversion result. PiperOrigin-RevId: 582010350 --- mediapipe/calculators/tensor/BUILD | 61 ++++- .../tensors_to_segmentation_calculator.cc | 225 +++++------------- .../tensors_to_segmentation_converter.h | 43 ++++ ...ensors_to_segmentation_converter_opencv.cc | 157 ++++++++++++ ...tensors_to_segmentation_converter_opencv.h | 31 +++ .../tensor/tensors_to_segmentation_utils.cc | 52 ++++ .../tensor/tensors_to_segmentation_utils.h | 34 +++ .../tensors_to_segmentation_utils_test.cc | 63 +++++ 8 files changed, 495 insertions(+), 171 deletions(-) create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter.h create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils.h create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index ac2ced837..e95398a9d 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -1414,6 +1414,8 @@ cc_library( }), deps = [ ":tensors_to_segmentation_calculator_cc_proto", + ":tensors_to_segmentation_converter", + ":tensors_to_segmentation_utils", "//mediapipe/framework:calculator_context", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:port", @@ -1421,9 +1423,11 @@ cc_library( "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/util:resource_util", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/types:span", @@ -1434,6 +1438,7 @@ cc_library( "//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gl_simple_shaders", "//mediapipe/gpu:gpu_buffer", + "//mediapipe/gpu:gpu_buffer_format", "//mediapipe/gpu:shader_util", ], }) + selects.with_or({ @@ -1453,13 +1458,65 @@ cc_library( }) + select({ "//mediapipe/framework/port:disable_opencv": [], "//conditions:default": [ - "//mediapipe/framework/formats:image_opencv", - "//mediapipe/framework/port:opencv_imgproc", + ":tensors_to_segmentation_converter_opencv", ], }), alwayslink = 1, ) +cc_library( + name = "tensors_to_segmentation_utils", + srcs = ["tensors_to_segmentation_utils.cc"], + hdrs = ["tensors_to_segmentation_utils.h"], + deps = [ + "//mediapipe/framework:port", + "//mediapipe/framework/port:ret_check", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_test( + name = "tensors_to_segmentation_utils_test", + srcs = ["tensors_to_segmentation_utils_test.cc"], + deps = [ + ":tensors_to_segmentation_utils", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:status_matchers", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_library( + name = "tensors_to_segmentation_converter", + hdrs = ["tensors_to_segmentation_converter.h"], + deps = [ + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:tensor", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_library( + name = "tensors_to_segmentation_converter_opencv", + srcs = ["tensors_to_segmentation_converter_opencv.cc"], + hdrs = ["tensors_to_segmentation_converter_opencv.h"], + deps = [ + ":tensors_to_segmentation_calculator_cc_proto", + ":tensors_to_segmentation_converter", + ":tensors_to_segmentation_utils", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_opencv", + "//mediapipe/framework/formats:tensor", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], +) + cc_test( name = "tensors_to_segmentation_calculator_test", srcs = ["tensors_to_segmentation_calculator_test.cc"], diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc index 24fd1bd52..6164c7b0a 100644 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc @@ -12,32 +12,35 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include +#include #include -#include "absl/strings/str_format.h" -#include "absl/types/span.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" #include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port.h" #include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/statusor.h" +#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/gpu/gpu_origin.pb.h" -#include "mediapipe/util/resource_util.h" -#include "tensorflow/lite/interpreter.h" #if !MEDIAPIPE_DISABLE_GPU #include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gl_simple_shaders.h" -#include "mediapipe/gpu/gpu_buffer.h" +#include "mediapipe/gpu/gpu_buffer_format.h" #include "mediapipe/gpu/shader_util.h" #endif // !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_OPENCV -#include "mediapipe/framework/formats/image_opencv.h" -#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h" #endif // !MEDIAPIPE_DISABLE_OPENCV #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 @@ -62,37 +65,9 @@ namespace { constexpr int kWorkgroupSize = 8; // Block size for GPU shader. enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; -// Commonly used to compute the number of blocks to launch in a kernel. -int NumGroups(const int size, const int group_size) { // NOLINT - return (size + group_size - 1) / group_size; -} - -bool CanUseGpu() { -#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED - // TODO: Configure GPU usage policy in individual calculators. - constexpr bool kAllowGpuProcessing = true; - return kAllowGpuProcessing; -#else - return false; -#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED -} - constexpr char kTensorsTag[] = "TENSORS"; constexpr char kOutputSizeTag[] = "OUTPUT_SIZE"; constexpr char kMaskTag[] = "MASK"; - -absl::StatusOr> GetHwcFromDims( - const std::vector& dims) { - if (dims.size() == 3) { - return std::make_tuple(dims[0], dims[1], dims[2]); - } else if (dims.size() == 4) { - // BHWC format check B == 1 - RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap"; - return std::make_tuple(dims[1], dims[2], dims[3]); - } else { - RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size(); - } -} } // namespace namespace mediapipe { @@ -156,19 +131,28 @@ class TensorsToSegmentationCalculator : public CalculatorBase { private: absl::Status LoadOptions(CalculatorContext* cc); absl::Status InitGpu(CalculatorContext* cc); - absl::Status ProcessGpu(CalculatorContext* cc); - absl::Status ProcessCpu(CalculatorContext* cc); + absl::Status ProcessGpu(CalculatorContext* cc, + const std::vector& input_tensors, + std::tuple hwc, int output_width, + int output_height); void GlRender(); bool DoesGpuTextureStartAtBottom() { return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT; } - + absl::Status InitConverterIfNecessary() { #if !MEDIAPIPE_DISABLE_OPENCV - template - absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); + if (!cpu_converter_) { + MP_ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(options_)); + } +#else + RET_CHECK_FAIL() << "OpenCV processing disabled."; #endif // !MEDIAPIPE_DISABLE_OPENCV - ::mediapipe::TensorsToSegmentationCalculatorOptions options_; + return absl::OkStatus(); + } + + mediapipe::TensorsToSegmentationCalculatorOptions options_; + std::unique_ptr cpu_converter_; #if !MEDIAPIPE_DISABLE_GPU mediapipe::GlCalculatorHelper gpu_helper_; @@ -261,7 +245,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); int tensor_channels = std::get<2>(hwc); - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; + using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; switch (options_.activation()) { case Options::NONE: RET_CHECK_EQ(tensor_channels, 1); @@ -275,6 +259,17 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { } } + // Get dimensions. + MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); + auto [tensor_height, tensor_width, tensor_channels] = hwc; + int output_width = tensor_width, output_height = tensor_height; + if (cc->Inputs().HasTag(kOutputSizeTag)) { + const auto& size = + cc->Inputs().Tag(kOutputSizeTag).Get>(); + output_width = size.first; + output_height = size.second; + } + if (use_gpu) { #if !MEDIAPIPE_DISABLE_GPU if (!gpu_initialized_) { @@ -286,16 +281,25 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { #endif // !MEDIAPIPE_DISABLE_GPU #if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { - MP_RETURN_IF_ERROR(ProcessGpu(cc)); - return absl::OkStatus(); - })); + MP_RETURN_IF_ERROR( + gpu_helper_.RunInGlContext([this, cc, &input_tensors, output_width, + output_height, hwc]() -> absl::Status { + MP_RETURN_IF_ERROR( + ProcessGpu(cc, input_tensors, hwc, output_width, output_height)); + return absl::OkStatus(); + })); #else RET_CHECK_FAIL() << "GPU processing disabled."; #endif // !MEDIAPIPE_DISABLE_GPU } else { #if !MEDIAPIPE_DISABLE_OPENCV - MP_RETURN_IF_ERROR(ProcessCpu(cc)); + // Lazily initialize converter. + MP_RETURN_IF_ERROR(InitConverterIfNecessary()); + MP_ASSIGN_OR_RETURN( + std::unique_ptr output_mask, + cpu_converter_->Convert(input_tensors, output_width, output_height)); + cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), + cc->InputTimestamp()); #else RET_CHECK_FAIL() << "OpenCV processing disabled."; #endif // !MEDIAPIPE_DISABLE_OPENCV @@ -329,132 +333,15 @@ absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) { return absl::OkStatus(); } -absl::Status TensorsToSegmentationCalculator::ProcessCpu( - CalculatorContext* cc) { -#if !MEDIAPIPE_DISABLE_OPENCV - // Get input streams, and dimensions. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); - auto [tensor_height, tensor_width, tensor_channels] = hwc; - int output_width = tensor_width, output_height = tensor_height; - if (cc->Inputs().HasTag(kOutputSizeTag)) { - const auto& size = - cc->Inputs().Tag(kOutputSizeTag).Get>(); - output_width = size.first; - output_height = size.second; - } - - // Create initial working mask. - cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1); - - // Wrap input tensor. - auto raw_input_tensor = &input_tensors[0]; - auto raw_input_view = raw_input_tensor->GetCpuReadView(); - const float* raw_input_data = raw_input_view.buffer(); - cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height), - CV_MAKETYPE(CV_32F, tensor_channels), - const_cast(raw_input_data)); - - // Process mask tensor and apply activation function. - if (tensor_channels == 2) { - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else if (tensor_channels == 1) { - RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX != - options_.activation()); // Requires 2 channels. - if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE == - options_.activation()) // Pass-through optimization. - tensor_mat.copyTo(small_mask_mat); - else - MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); - } else { - RET_CHECK_FAIL() << "Unsupported number of tensor channels " - << tensor_channels; - } - - // Send out image as CPU packet. - std::shared_ptr mask_frame = std::make_shared( - ImageFormat::VEC32F1, output_width, output_height); - std::unique_ptr output_mask = absl::make_unique(mask_frame); - auto output_mat = formats::MatView(output_mask.get()); - // Upsample small mask into output. - cv::resize(small_mask_mat, *output_mat, - cv::Size(output_width, output_height)); - cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp()); -#endif // !MEDIAPIPE_DISABLE_OPENCV - - return absl::OkStatus(); -} - -#if !MEDIAPIPE_DISABLE_OPENCV -template -absl::Status TensorsToSegmentationCalculator::ApplyActivation( - cv::Mat& tensor_mat, cv::Mat* small_mask_mat) { - // Configure activation function. - const int output_layer_index = options_.output_layer_index(); - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; - const auto activation_fn = [&](const cv::Vec2f& mask_value) { - float new_mask_value = 0; - // TODO consider moving switch out of the loop, - // and also avoid float/Vec2f casting. - switch (options_.activation()) { - case Options::NONE: { - new_mask_value = mask_value[0]; - break; - } - case Options::SIGMOID: { - const float pixel0 = mask_value[0]; - new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0); - break; - } - case Options::SOFTMAX: { - const float pixel0 = mask_value[0]; - const float pixel1 = mask_value[1]; - const float max_pixel = std::max(pixel0, pixel1); - const float min_pixel = std::min(pixel0, pixel1); - const float softmax_denom = - /*exp(max_pixel - max_pixel)=*/1.0f + - std::exp(min_pixel - max_pixel); - new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) / - softmax_denom; - break; - } - } - return new_mask_value; - }; - - // Process mask tensor. - for (int i = 0; i < tensor_mat.rows; ++i) { - for (int j = 0; j < tensor_mat.cols; ++j) { - const T& input_pix = tensor_mat.at(i, j); - const float mask_value = activation_fn(input_pix); - small_mask_mat->at(i, j) = mask_value; - } - } - - return absl::OkStatus(); -} -#endif // !MEDIAPIPE_DISABLE_OPENCV - // Steps: // 1. receive tensor // 2. process segmentation tensor into small mask // 3. upsample small mask into output mask to be same size as input image absl::Status TensorsToSegmentationCalculator::ProcessGpu( - CalculatorContext* cc) { + CalculatorContext* cc, const std::vector& input_tensors, + std::tuple hwc, int output_width, int output_height) { #if !MEDIAPIPE_DISABLE_GPU - // Get input streams, and dimensions. - const auto& input_tensors = - cc->Inputs().Tag(kTensorsTag).Get>(); - MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); auto [tensor_height, tensor_width, tensor_channels] = hwc; - int output_width = tensor_width, output_height = tensor_height; - if (cc->Inputs().HasTag(kOutputSizeTag)) { - const auto& size = - cc->Inputs().Tag(kOutputSizeTag).Get>(); - output_width = size.first; - output_height = size.second; - } // Create initial working mask texture. #if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31) @@ -632,7 +519,7 @@ void TensorsToSegmentationCalculator::GlRender() { absl::Status TensorsToSegmentationCalculator::LoadOptions( CalculatorContext* cc) { // Get calculator options specified in the graph. - options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>(); + options_ = cc->Options(); return absl::OkStatus(); } @@ -826,7 +713,7 @@ void main() { #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 // Shader defines. - typedef mediapipe::TensorsToSegmentationCalculatorOptions Options; + using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; const std::string output_layer_index = "\n#define OUTPUT_LAYER_INDEX int(" + std::to_string(options_.output_layer_index()) + ")"; diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h b/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h new file mode 100644 index 000000000..61d95dfe0 --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_converter.h @@ -0,0 +1,43 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ + +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/tensor.h" + +namespace mediapipe { + +class TensorsToSegmentationConverter { + public: + virtual ~TensorsToSegmentationConverter() = default; + + // Converts tensors to image mask. + // Returns a unique pointer containing the converted image. + // @input_tensors contains the tensors needed to be processed. + // @output_width/height describes output dimensions to reshape the output mask + // into. + virtual absl::StatusOr> Convert( + const std::vector& input_tensors, int output_width, + int output_height) = 0; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc new file mode 100644 index 000000000..1ee2e172b --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.cc @@ -0,0 +1,157 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_opencv.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/opencv_core_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_macros.h" + +namespace mediapipe { +namespace { + +class OpenCvProcessor : public TensorsToSegmentationConverter { + public: + absl::Status Init(const TensorsToSegmentationCalculatorOptions& options) { + options_ = options; + return absl::OkStatus(); + } + + absl::StatusOr> Convert( + const std::vector& input_tensors, int output_width, + int output_height) override; + + private: + template + absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat); + + TensorsToSegmentationCalculatorOptions options_; +}; + +absl::StatusOr> OpenCvProcessor::Convert( + const std::vector& input_tensors, int output_width, + int output_height) { + MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims)); + auto [tensor_height, tensor_width, tensor_channels] = hwc; + // Create initial working mask. + cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1); + + // Wrap input tensor. + auto raw_input_tensor = &input_tensors[0]; + auto raw_input_view = raw_input_tensor->GetCpuReadView(); + const float* raw_input_data = raw_input_view.buffer(); + cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height), + CV_MAKETYPE(CV_32F, tensor_channels), + const_cast(raw_input_data)); + + // Process mask tensor and apply activation function. + if (tensor_channels == 2) { + MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); + } else if (tensor_channels == 1) { + RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX != + options_.activation()); // Requires 2 channels. + if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE == + options_.activation()) // Pass-through optimization. + tensor_mat.copyTo(small_mask_mat); + else + MP_RETURN_IF_ERROR(ApplyActivation(tensor_mat, &small_mask_mat)); + } else { + RET_CHECK_FAIL() << "Unsupported number of tensor channels " + << tensor_channels; + } + + // Send out image as CPU packet. + std::shared_ptr mask_frame = std::make_shared( + ImageFormat::VEC32F1, output_width, output_height); + auto output_mask = std::make_unique(mask_frame); + auto output_mat = formats::MatView(output_mask.get()); + // Upsample small mask into output. + cv::resize(small_mask_mat, *output_mat, + cv::Size(output_width, output_height)); + return output_mask; +} + +template +absl::Status OpenCvProcessor::ApplyActivation(cv::Mat& tensor_mat, + cv::Mat* small_mask_mat) { + // Configure activation function. + const int output_layer_index = options_.output_layer_index(); + using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; + const auto activation_fn = [&](const cv::Vec2f& mask_value) { + float new_mask_value = 0; + // TODO consider moving switch out of the loop, + // and also avoid float/Vec2f casting. + switch (options_.activation()) { + case Options::NONE: { + new_mask_value = mask_value[0]; + break; + } + case Options::SIGMOID: { + const float pixel0 = mask_value[0]; + new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0); + break; + } + case Options::SOFTMAX: { + const float pixel0 = mask_value[0]; + const float pixel1 = mask_value[1]; + const float max_pixel = std::max(pixel0, pixel1); + const float min_pixel = std::min(pixel0, pixel1); + const float softmax_denom = + /*exp(max_pixel - max_pixel)=*/1.0f + + std::exp(min_pixel - max_pixel); + new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) / + softmax_denom; + break; + } + } + return new_mask_value; + }; + + // Process mask tensor. + for (int i = 0; i < tensor_mat.rows; ++i) { + for (int j = 0; j < tensor_mat.cols; ++j) { + const T& input_pix = tensor_mat.at(i, j); + const float mask_value = activation_fn(input_pix); + small_mask_mat->at(i, j) = mask_value; + } + } + + return absl::OkStatus(); +} + +} // namespace + +absl::StatusOr> +CreateOpenCvConverter(const TensorsToSegmentationCalculatorOptions& options) { + auto converter = std::make_unique(); + MP_RETURN_IF_ERROR(converter->Init(options)); + return converter; +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h new file mode 100644 index 000000000..3ae41b5e0 --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h @@ -0,0 +1,31 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ + +#include + +#include "absl/status/statusor.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h" + +namespace mediapipe { +// Creates OpenCV tensors-to-segmentation converter. +absl::StatusOr> +CreateOpenCvConverter( + const mediapipe::TensorsToSegmentationCalculatorOptions& options); +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc new file mode 100644 index 000000000..ab1e9c139 --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.cc @@ -0,0 +1,52 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" + +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/port.h" +#include "mediapipe/framework/port/ret_check.h" + +namespace mediapipe { + +int NumGroups(int size, int group_size) { + return (size + group_size - 1) / group_size; +} + +bool CanUseGpu() { +#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED + // TODO: Configure GPU usage policy in individual calculators. + constexpr bool kAllowGpuProcessing = true; + return kAllowGpuProcessing; +#else + return false; +#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED +} + +absl::StatusOr> GetHwcFromDims( + const std::vector& dims) { + if (dims.size() == 3) { + return std::make_tuple(dims[0], dims[1], dims[2]); + } else if (dims.size() == 4) { + // BHWC format check B == 1 + RET_CHECK_EQ(dims[0], 1) << "Expected batch to be 1 for BHWC heatmap"; + return std::make_tuple(dims[1], dims[2], dims[3]); + } else { + RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size(); + } +} +} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h new file mode 100644 index 000000000..44893073b --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_utils.h @@ -0,0 +1,34 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ + +#include +#include + +#include "absl/status/statusor.h" + +namespace mediapipe { + +// Commonly used to compute the number of blocks to launch in a kernel. +int NumGroups(const int size, const int group_size); // NOLINT + +bool CanUseGpu(); + +absl::StatusOr> GetHwcFromDims( + const std::vector& dims); +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc new file mode 100644 index 000000000..5535d159d --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_utils_test.cc @@ -0,0 +1,63 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h" + +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/status_matchers.h" + +namespace mediapipe { +namespace { + +using ::testing::HasSubstr; + +TEST(TensorsToSegmentationUtilsTest, NumGroupsWorksProperly) { + EXPECT_EQ(NumGroups(13, 4), 4); + EXPECT_EQ(NumGroups(4, 13), 1); +} + +TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsWorksProperly) { + std::vector dims_3 = {2, 3, 4}; + absl::StatusOr> result_1 = GetHwcFromDims(dims_3); + MP_ASSERT_OK(result_1); + EXPECT_EQ(result_1.value(), (std::make_tuple(2, 3, 4))); + std::vector dims_4 = {1, 3, 4, 5}; + absl::StatusOr> result_2 = GetHwcFromDims(dims_4); + MP_ASSERT_OK(result_2); + EXPECT_EQ(result_2.value(), (std::make_tuple(3, 4, 5))); +} + +TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsBatchCheckFail) { + std::vector dims_4 = {2, 3, 4, 5}; + absl::StatusOr> result = GetHwcFromDims(dims_4); + EXPECT_FALSE(result.ok()); + EXPECT_THAT(result.status().message(), + HasSubstr("Expected batch to be 1 for BHWC heatmap")); +} + +TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsInvalidShape) { + std::vector dims_5 = {1, 2, 3, 4, 5}; + absl::StatusOr> result = GetHwcFromDims(dims_5); + EXPECT_FALSE(result.ok()); + EXPECT_THAT(result.status().message(), + HasSubstr("Invalid shape for segmentation tensor")); +} + +} // namespace +} // namespace mediapipe From a38467bae0355fb3737f66da2743f975624e05e5 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 13 Nov 2023 15:13:32 -0800 Subject: [PATCH 112/157] Internal PiperOrigin-RevId: 582098762 --- third_party/external_files.bzl | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/third_party/external_files.bzl b/third_party/external_files.bzl index 41abd6270..a3893b5f7 100644 --- a/third_party/external_files.bzl +++ b/third_party/external_files.bzl @@ -28,6 +28,12 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/associated_file_meta.json?generation=1665422792304395"], ) + http_file( + name = "com_google_mediapipe_average_word_classifier_tflite", + sha256 = "13bf6f7f35964f1e85d6cc762ee7b1952009b532b233baa5bdb4bf7441097f34", + urls = ["https://storage.googleapis.com/mediapipe-assets/average_word_classifier.tflite?generation=1699635086044360"], + ) + http_file( name = "com_google_mediapipe_bert_text_classifier_no_metadata_tflite", sha256 = "9b4554f6e28a72a3f40511964eed1ccf4e74cc074f81543cacca4faf169a173e", @@ -562,6 +568,12 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/holistic_hand_tracking_left_hand_graph.pbtxt?generation=1697732440362430"], ) + http_file( + name = "com_google_mediapipe_holistic_landmarker_task", + sha256 = "e2dab61191e2dcd0a15f943d8e3ed1dce13c82dfa597b9dd39f562975a50c3f8", + urls = ["https://storage.googleapis.com/mediapipe-assets/holistic_landmarker.task?generation=1699635090585884"], + ) + http_file( name = "com_google_mediapipe_holistic_pose_tracking_graph_pbtxt", sha256 = "1d36d014d38c09fea73042471d5d1a616f3cc9f22c8ca625deabc38efd63f6aa", @@ -700,6 +712,12 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/libimagegenerator_gpu.so?generation=1694488131511338"], ) + http_file( + name = "com_google_mediapipe_living_room_jpg", + sha256 = "8d74535dfe58e7d62dee99df5ab7741ad373a456797cf4d99048dfd17ccb0d6c", + urls = ["https://storage.googleapis.com/mediapipe-assets/living_room.jpg?generation=1699635092884512"], + ) + http_file( name = "com_google_mediapipe_male_full_height_hands_jpg", sha256 = "8a7fe5be8b90d6078b09913ca28f7e5d342f8d3cde856ab4e3327d2970b887f8", @@ -708,8 +726,8 @@ def external_files(): http_file( name = "com_google_mediapipe_male_full_height_hands_result_cpu_pbtxt", - sha256 = "f7bea74f60386baa05716341844876c89f43aaee64c110d259a824c4696ed451", - urls = ["https://storage.googleapis.com/mediapipe-assets/male_full_height_hands_result_cpu.pbtxt?generation=1692651587930202"], + sha256 = "f4a53dec51b621abeb1dbd854087dd0b02a3d40472f5fdbc5e836315bcb704f3", + urls = ["https://storage.googleapis.com/mediapipe-assets/male_full_height_hands_result_cpu.pbtxt?generation=1699635094875663"], ) http_file( @@ -862,6 +880,12 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/mobile_object_labeler_v1.tflite?generation=1666144701839813"], ) + http_file( + name = "com_google_mediapipe_mobile_raid_one_stage_v2_1_uint8_tflite", + sha256 = "2e397b750d8f270e3f41731c1ec1f5b7811f93bc3a39fb81a4c47dd5e9055915", + urls = ["https://storage.googleapis.com/mediapipe-assets/mobile_raid_one_stage_v2_1_uint8.tflite?generation=1699635098658451"], + ) + http_file( name = "com_google_mediapipe_model_without_metadata_tflite", sha256 = "05c5aea7ae00aeed0053a85f2b2e896b4ea272c5219052d32c06b655fbf5cc9b", @@ -892,6 +916,12 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/mozart_square.jpg?generation=1661875853838871"], ) + http_file( + name = "com_google_mediapipe_mraid_v25_max_boxes_40_max_classes_per_box_5_tflite", + sha256 = "1c4667b1a3caf90cc6971517df2525c0f8b6807d6598cf4554a9ca224faf42c5", + urls = ["https://storage.googleapis.com/mediapipe-assets/mraid_v25_max_boxes_40_max_classes_per_box_5.tflite?generation=1699635101036991"], + ) + http_file( name = "com_google_mediapipe_multi_objects_jpg", sha256 = "ada6e36b40519cf0a4fbdf1b535de7fa7d0c472f2c0a08ada8ee5728e16c0c68", From f8add5ad420798ff0ae023afed5bc0a8b8abed83 Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 13 Nov 2023 21:17:28 -0800 Subject: [PATCH 113/157] Documented the return value and added percentile to argparser --- .../image_classifier_benchmark.py | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py index 502441879..d505a8fe8 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -1,17 +1,3 @@ -# Copyright 2023 The MediaPipe Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Benchmark for the image classifier task.""" import argparse import time import numpy as np @@ -22,16 +8,21 @@ from mediapipe.tasks.python import vision _IMAGE_FILE = 'burger.jpg' -def run(model: str, n_iterations: int, delegate: python.BaseOptions.Delegate): +def run(model: str, n_iterations: int, delegate: python.BaseOptions.Delegate, + percentile: float): """Run asynchronous inference on images and benchmark. Args: model: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + Returns: + The n-th percentile of the inference times. """ inference_times = [] - + # Initialize the image classifier base_options = python.BaseOptions(model_asset_path=model, delegate=delegate) options = vision.ImageClassifierOptions( @@ -48,7 +39,7 @@ def run(model: str, n_iterations: int, delegate: python.BaseOptions.Delegate): inference_times.append((end_time_ns - start_time_ns) / 1_000_000) classifier.close() - return np.percentile(inference_times, 95) + return np.percentile(inference_times, percentile) def main(): @@ -60,15 +51,22 @@ def main(): parser.add_argument( '--iterations', help='Number of iterations for benchmarking.', type=int, default=100) + parser.add_argument( + '--percentile', help='Percentile for benchmarking statistics.', + type=float, default=95.0) args = parser.parse_args() # Run benchmark on CPU - cpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.CPU) - print(f"95th Percentile Inference Time on CPU: {cpu_time:.6f} milliseconds") + cpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.CPU, + args.percentile) + print(f"{args.percentile}th Percentile Inference Time on CPU: " + f"{cpu_time:.6f} milliseconds") # Run benchmark on GPU - gpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.GPU) - print(f"95th Percentile Inference Time on GPU: {gpu_time:.6f} milliseconds") + gpu_time = run(args.model, args.iterations, python.BaseOptions.Delegate.GPU, + args.percentile) + print(f"{args.percentile}th Percentile Inference Time on GPU: " + f"{gpu_time:.6f} milliseconds") if __name__ == '__main__': From 252cca72e77c13e5f284404a6b4c2b52e08369ca Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 13 Nov 2023 21:44:27 -0800 Subject: [PATCH 114/157] Allowed a default value for the model argument --- .../vision/image_classifier/image_classifier_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py index d505a8fe8..65a629bab 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -46,7 +46,7 @@ def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - '--model', help='Path to image classification model.', required=True, + '--model', help='Path to image classification model.', required=False, default='classifier.tflite') parser.add_argument( '--iterations', help='Number of iterations for benchmarking.', type=int, From e440a4da56d75d1400feedc4fd63b33f96abdc9a Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 15 Nov 2023 02:16:29 -0800 Subject: [PATCH 115/157] Explicitly delete some copy operations to improve compile errors. PiperOrigin-RevId: 582595026 --- mediapipe/framework/api2/builder.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mediapipe/framework/api2/builder.h b/mediapipe/framework/api2/builder.h index fde281121..c82998040 100644 --- a/mediapipe/framework/api2/builder.h +++ b/mediapipe/framework/api2/builder.h @@ -330,6 +330,14 @@ using MultiSideDestination = MultiPort>; class NodeBase { public: + NodeBase() = default; + ~NodeBase() = default; + NodeBase(NodeBase&&) = default; + NodeBase& operator=(NodeBase&&) = default; + // Explicitly delete copies to improve error messages. + NodeBase(const NodeBase&) = delete; + NodeBase& operator=(const NodeBase&) = delete; + // TODO: right now access to an indexed port is made directly by // specifying both a tag and an index. It would be better to represent this // as a two-step lookup, first getting a multi-port, and then accessing one @@ -585,6 +593,14 @@ class PacketGenerator { class Graph { public: + Graph() = default; + ~Graph() = default; + Graph(Graph&&) = default; + Graph& operator=(Graph&&) = default; + // Explicitly delete copies to improve error messages. + Graph(const Graph&) = delete; + Graph& operator=(const Graph&) = delete; + void SetType(std::string type) { type_ = std::move(type); } // Creates a node of a specific type. Should be used for calculators whose From 47e217896c53e47d420830daaa38449afa247a8b Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 15 Nov 2023 06:10:05 -0800 Subject: [PATCH 116/157] Add drawConfidenceMask() to our public API PiperOrigin-RevId: 582647409 --- mediapipe/tasks/web/vision/core/BUILD | 6 +- .../web/vision/core/drawing_utils.test.ts | 94 +++++++++++++ .../tasks/web/vision/core/drawing_utils.ts | 75 +++++++++++ .../core/drawing_utils_confidence_mask.ts | 125 ++++++++++++++++++ mediapipe/tasks/web/vision/core/image.ts | 31 +---- mediapipe/tasks/web/vision/core/mask.ts | 31 ++--- .../tasks/web/vision/core/render_utils.ts | 31 ----- 7 files changed, 309 insertions(+), 84 deletions(-) create mode 100644 mediapipe/tasks/web/vision/core/drawing_utils_confidence_mask.ts delete mode 100644 mediapipe/tasks/web/vision/core/render_utils.ts diff --git a/mediapipe/tasks/web/vision/core/BUILD b/mediapipe/tasks/web/vision/core/BUILD index 31bad937d..db9c27e0f 100644 --- a/mediapipe/tasks/web/vision/core/BUILD +++ b/mediapipe/tasks/web/vision/core/BUILD @@ -34,6 +34,7 @@ mediapipe_ts_library( srcs = [ "drawing_utils.ts", "drawing_utils_category_mask.ts", + "drawing_utils_confidence_mask.ts", ], deps = [ ":image", @@ -149,11 +150,6 @@ mediapipe_ts_library( ], ) -mediapipe_ts_library( - name = "render_utils", - srcs = ["render_utils.ts"], -) - jasmine_node_test( name = "vision_task_runner_test", deps = [":vision_task_runner_test_lib"], diff --git a/mediapipe/tasks/web/vision/core/drawing_utils.test.ts b/mediapipe/tasks/web/vision/core/drawing_utils.test.ts index b5ba8e9a4..aaef42bbf 100644 --- a/mediapipe/tasks/web/vision/core/drawing_utils.test.ts +++ b/mediapipe/tasks/web/vision/core/drawing_utils.test.ts @@ -59,6 +59,100 @@ if (skip) { drawingUtilsWebGL.close(); }); + describe( + 'drawConfidenceMask() blends background with foreground color', () => { + const foreground = new ImageData( + new Uint8ClampedArray( + [0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255]), + WIDTH, HEIGHT); + const background = [255, 255, 255, 255]; + const expectedResult = new Uint8Array([ + 255, 255, 255, 255, 178, 178, 178, 255, 102, 102, 102, 255, 0, 0, 0, + 255 + ]); + + it('on 2D canvas', () => { + const confidenceMask = new MPMask( + [new Float32Array([0.0, 0.3, 0.6, 1.0])], + /* ownsWebGLTexture= */ false, canvas2D, shaderContext, WIDTH, + HEIGHT); + + drawingUtils2D.drawConfidenceMask( + confidenceMask, background, foreground); + + const actualResult = context2D.getImageData(0, 0, WIDTH, HEIGHT).data; + expect(actualResult) + .toEqual(new Uint8ClampedArray(expectedResult.buffer)); + }); + + it('on WebGL canvas', () => { + const confidenceMask = new MPMask( + [new Float32Array( + [0.6, 1.0, 0.0, 0.3])], // Note: Vertically flipped + /* ownsWebGLTexture= */ false, canvasWebGL, shaderContext, WIDTH, + HEIGHT); + + drawingUtilsWebGL.drawConfidenceMask( + confidenceMask, background, foreground); + + const actualResult = new Uint8Array(WIDTH * HEIGHT * 4); + contextWebGL.readPixels( + 0, 0, WIDTH, HEIGHT, contextWebGL.RGBA, + contextWebGL.UNSIGNED_BYTE, actualResult); + expect(actualResult).toEqual(expectedResult); + }); + }); + + + describe( + 'drawConfidenceMask() blends background with foreground image', () => { + const foreground = new ImageData( + new Uint8ClampedArray( + [0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255]), + WIDTH, HEIGHT); + const background = new ImageData( + new Uint8ClampedArray([ + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255 + ]), + WIDTH, HEIGHT); + const expectedResult = new Uint8Array([ + 255, 255, 255, 255, 178, 178, 178, 255, 102, 102, 102, 255, 0, 0, 0, + 255 + ]); + + it('on 2D canvas', () => { + const confidenceMask = new MPMask( + [new Float32Array([0.0, 0.3, 0.6, 1.0])], + /* ownsWebGLTexture= */ false, canvas2D, shaderContext, WIDTH, + HEIGHT); + + drawingUtils2D.drawConfidenceMask( + confidenceMask, background, foreground); + + const actualResult = context2D.getImageData(0, 0, WIDTH, HEIGHT).data; + expect(actualResult) + .toEqual(new Uint8ClampedArray(expectedResult.buffer)); + }); + + it('on WebGL canvas', () => { + const confidenceMask = new MPMask( + [new Float32Array( + [0.6, 1.0, 0.0, 0.3])], // Note: Vertically flipped + /* ownsWebGLTexture= */ false, canvasWebGL, shaderContext, WIDTH, + HEIGHT); + + drawingUtilsWebGL.drawConfidenceMask( + confidenceMask, background, foreground); + + const actualResult = new Uint8Array(WIDTH * HEIGHT * 4); + contextWebGL.readPixels( + 0, 0, WIDTH, HEIGHT, contextWebGL.RGBA, + contextWebGL.UNSIGNED_BYTE, actualResult); + expect(actualResult).toEqual(expectedResult); + }); + }); + describe('drawCategoryMask() ', () => { const colors = [ [0, 0, 0, 255], diff --git a/mediapipe/tasks/web/vision/core/drawing_utils.ts b/mediapipe/tasks/web/vision/core/drawing_utils.ts index 796d7dcb6..154420f6b 100644 --- a/mediapipe/tasks/web/vision/core/drawing_utils.ts +++ b/mediapipe/tasks/web/vision/core/drawing_utils.ts @@ -17,6 +17,7 @@ import {BoundingBox} from '../../../../tasks/web/components/containers/bounding_box'; import {NormalizedLandmark} from '../../../../tasks/web/components/containers/landmark'; import {CategoryMaskShaderContext, CategoryToColorMap, RGBAColor} from '../../../../tasks/web/vision/core/drawing_utils_category_mask'; +import {ConfidenceMaskShaderContext} from '../../../../tasks/web/vision/core/drawing_utils_confidence_mask'; import {MPImageShaderContext} from '../../../../tasks/web/vision/core/image_shader_context'; import {MPMask} from '../../../../tasks/web/vision/core/mask'; import {Connection} from '../../../../tasks/web/vision/core/types'; @@ -115,6 +116,7 @@ export {RGBAColor, CategoryToColorMap}; /** Helper class to visualize the result of a MediaPipe Vision task. */ export class DrawingUtils { private categoryMaskShaderContext?: CategoryMaskShaderContext; + private confidenceMaskShaderContext?: ConfidenceMaskShaderContext; private convertToWebGLTextureShaderContext?: MPImageShaderContext; private readonly context2d?: CanvasRenderingContext2D| OffscreenCanvasRenderingContext2D; @@ -213,6 +215,13 @@ export class DrawingUtils { return this.categoryMaskShaderContext; } + private getConfidenceMaskShaderContext(): ConfidenceMaskShaderContext { + if (!this.confidenceMaskShaderContext) { + this.confidenceMaskShaderContext = new ConfidenceMaskShaderContext(); + } + return this.confidenceMaskShaderContext; + } + /** * Draws circles onto the provided landmarks. * @@ -422,6 +431,70 @@ export class DrawingUtils { callback(mask.getAsWebGLTexture()); } } + + /** Draws a confidence mask on a WebGL2RenderingContext2D. */ + private drawConfidenceMaskWebGL( + maskTexture: WebGLTexture, defaultTexture: RGBAColor|ImageSource, + overlayTexture: RGBAColor|ImageSource): void { + const gl = this.getWebGLRenderingContext(); + const shaderContext = this.getConfidenceMaskShaderContext(); + const defaultImage = Array.isArray(defaultTexture) ? + new ImageData(new Uint8ClampedArray(defaultTexture), 1, 1) : + defaultTexture; + const overlayImage = Array.isArray(overlayTexture) ? + new ImageData(new Uint8ClampedArray(overlayTexture), 1, 1) : + overlayTexture; + + shaderContext.run(gl, /* flipTexturesVertically= */ true, () => { + shaderContext.bindAndUploadTextures( + defaultImage, overlayImage, maskTexture); + gl.clearColor(0, 0, 0, 0); + gl.clear(gl.COLOR_BUFFER_BIT); + gl.drawArrays(gl.TRIANGLE_FAN, 0, 4); + gl.bindTexture(gl.TEXTURE_2D, null); + shaderContext.unbindTextures(); + }); + } + + /** Draws a confidence mask on a CanvasRenderingContext2D. */ + private drawConfidenceMask2D( + mask: MPMask, defaultTexture: RGBAColor|ImageSource, + overlayTexture: RGBAColor|ImageSource): void { + // Use the WebGL renderer to draw result on our internal canvas. + const gl = this.getWebGLRenderingContext(); + this.runWithWebGLTexture(mask, texture => { + this.drawConfidenceMaskWebGL(texture, defaultTexture, overlayTexture); + // Draw the result on the user canvas. + const ctx = this.getCanvasRenderingContext(); + ctx.drawImage(gl.canvas, 0, 0, ctx.canvas.width, ctx.canvas.height); + }); + } + + /** + * Blends two images using the provided confidence mask. + * + * If you are using an `ImageData` or `HTMLImageElement` as your data source + * and drawing the result onto a `WebGL2RenderingContext`, this method uploads + * the image data to the GPU. For still image input that gets re-used every + * frame, you can reduce the cost of re-uploading these images by passing a + * `HTMLCanvasElement` instead. + * + * @param mask A confidence mask that was returned from a segmentation task. + * @param defaultTexture An image or a four-channel color that will be used + * when confidence values are low. + * @param overlayTexture An image or four-channel color that will be used when + * confidence values are high. + */ + drawConfidenceMask( + mask: MPMask, defaultTexture: RGBAColor|ImageSource, + overlayTexture: RGBAColor|ImageSource): void { + if (this.context2d) { + this.drawConfidenceMask2D(mask, defaultTexture, overlayTexture); + } else { + this.drawConfidenceMaskWebGL( + mask.getAsWebGLTexture(), defaultTexture, overlayTexture); + } + } /** * Frees all WebGL resources held by this class. * @export @@ -429,6 +502,8 @@ export class DrawingUtils { close(): void { this.categoryMaskShaderContext?.close(); this.categoryMaskShaderContext = undefined; + this.confidenceMaskShaderContext?.close(); + this.confidenceMaskShaderContext = undefined; this.convertToWebGLTextureShaderContext?.close(); this.convertToWebGLTextureShaderContext = undefined; } diff --git a/mediapipe/tasks/web/vision/core/drawing_utils_confidence_mask.ts b/mediapipe/tasks/web/vision/core/drawing_utils_confidence_mask.ts new file mode 100644 index 000000000..c8d30c9ee --- /dev/null +++ b/mediapipe/tasks/web/vision/core/drawing_utils_confidence_mask.ts @@ -0,0 +1,125 @@ +/** + * Copyright 2023 The MediaPipe Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {assertNotNull, MPImageShaderContext} from '../../../../tasks/web/vision/core/image_shader_context'; +import {ImageSource} from '../../../../web/graph_runner/graph_runner'; + +/** + * A fragment shader that blends a default image and overlay texture based on an + * input texture that contains confidence values. + */ +const FRAGMENT_SHADER = ` + precision mediump float; + uniform sampler2D maskTexture; + uniform sampler2D defaultTexture; + uniform sampler2D overlayTexture; + varying vec2 vTex; + void main() { + float confidence = texture2D(maskTexture, vTex).r; + vec4 defaultColor = texture2D(defaultTexture, vTex); + vec4 overlayColor = texture2D(overlayTexture, vTex); + // Apply the alpha from the overlay and merge in the default color + overlayColor = mix(defaultColor, overlayColor, overlayColor.a); + gl_FragColor = mix(defaultColor, overlayColor, confidence); + } + `; + +/** A drawing util class for confidence masks. */ +export class ConfidenceMaskShaderContext extends MPImageShaderContext { + defaultTexture?: WebGLTexture; + overlayTexture?: WebGLTexture; + defaultTextureUniform?: WebGLUniformLocation; + overlayTextureUniform?: WebGLUniformLocation; + maskTextureUniform?: WebGLUniformLocation; + + protected override getFragmentShader(): string { + return FRAGMENT_SHADER; + } + + protected override setupTextures(): void { + const gl = this.gl!; + gl.activeTexture(gl.TEXTURE0); + this.defaultTexture = this.createTexture(gl); + gl.activeTexture(gl.TEXTURE1); + this.overlayTexture = this.createTexture(gl); + } + + protected override setupShaders(): void { + super.setupShaders(); + const gl = this.gl!; + this.defaultTextureUniform = assertNotNull( + gl.getUniformLocation(this.program!, 'defaultTexture'), + 'Uniform location'); + this.overlayTextureUniform = assertNotNull( + gl.getUniformLocation(this.program!, 'overlayTexture'), + 'Uniform location'); + this.maskTextureUniform = assertNotNull( + gl.getUniformLocation(this.program!, 'maskTexture'), + 'Uniform location'); + } + + protected override configureUniforms(): void { + super.configureUniforms(); + const gl = this.gl!; + gl.uniform1i(this.defaultTextureUniform!, 0); + gl.uniform1i(this.overlayTextureUniform!, 1); + gl.uniform1i(this.maskTextureUniform!, 2); + } + + bindAndUploadTextures( + defaultImage: ImageSource, overlayImage: ImageSource, + confidenceMask: WebGLTexture) { + // TODO: We should avoid uploading textures from CPU to GPU + // if the textures haven't changed. This can lead to drastic performance + // slowdowns (~50ms per frame). Users can reduce the penalty by passing a + // canvas object instead of ImageData/HTMLImageElement. + const gl = this.gl!; + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this.defaultTexture!); + gl.texImage2D( + gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, defaultImage); + + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, this.overlayTexture!); + gl.texImage2D( + gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, overlayImage); + + gl.activeTexture(gl.TEXTURE2); + gl.bindTexture(gl.TEXTURE_2D, confidenceMask); + } + + unbindTextures() { + const gl = this.gl!; + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, null); + + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, null); + + gl.activeTexture(gl.TEXTURE2); + gl.bindTexture(gl.TEXTURE_2D, null); + } + + override close(): void { + if (this.defaultTexture) { + this.gl!.deleteTexture(this.defaultTexture); + } + if (this.overlayTexture) { + this.gl!.deleteTexture(this.overlayTexture); + } + super.close(); + } +} diff --git a/mediapipe/tasks/web/vision/core/image.ts b/mediapipe/tasks/web/vision/core/image.ts index 570d32318..bb88c0ee1 100644 --- a/mediapipe/tasks/web/vision/core/image.ts +++ b/mediapipe/tasks/web/vision/core/image.ts @@ -198,10 +198,8 @@ export class MPImage { // Create a new texture and use it to back a framebuffer gl.activeTexture(gl.TEXTURE1); - destinationContainer = - assertNotNull(gl.createTexture(), 'Failed to create texture'); + destinationContainer = shaderContext.createTexture(gl); gl.bindTexture(gl.TEXTURE_2D, destinationContainer); - this.configureTextureParams(); gl.texImage2D( gl.TEXTURE_2D, 0, gl.RGBA, this.width, this.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); @@ -252,7 +250,7 @@ export class MPImage { } if (!this.gl) { this.gl = assertNotNull( - this.canvas.getContext('webgl2') as WebGL2RenderingContext | null, + this.canvas.getContext('webgl2'), 'You cannot use a canvas that is already bound to a different ' + 'type of rendering context.'); } @@ -317,20 +315,6 @@ export class MPImage { return webGLTexture; } - /** Sets texture params for the currently bound texture. */ - private configureTextureParams() { - const gl = this.getGL(); - // `gl.LINEAR` might break rendering for some textures, but it allows us to - // do smooth resizing. Ideally, this would be user-configurable, but for now - // we hard-code the value here to `gl.LINEAR` (versus `gl.NEAREST` for - // `MPMask` where we do not want to interpolate mask values, especially for - // category masks). - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); - } - /** * Binds the backing texture to the canvas. If the texture does not yet * exist, creates it first. @@ -343,16 +327,13 @@ export class MPImage { let webGLTexture = this.getContainer(MPImageType.WEBGL_TEXTURE); if (!webGLTexture) { - webGLTexture = - assertNotNull(gl.createTexture(), 'Failed to create texture'); + const shaderContext = this.getShaderContext(); + webGLTexture = shaderContext.createTexture(gl); this.containers.push(webGLTexture); this.ownsWebGLTexture = true; - - gl.bindTexture(gl.TEXTURE_2D, webGLTexture); - this.configureTextureParams(); - } else { - gl.bindTexture(gl.TEXTURE_2D, webGLTexture); } + + gl.bindTexture(gl.TEXTURE_2D, webGLTexture); return webGLTexture; } diff --git a/mediapipe/tasks/web/vision/core/mask.ts b/mediapipe/tasks/web/vision/core/mask.ts index 6ef852508..b463589e4 100644 --- a/mediapipe/tasks/web/vision/core/mask.ts +++ b/mediapipe/tasks/web/vision/core/mask.ts @@ -215,10 +215,8 @@ export class MPMask { // Create a new texture and use it to back a framebuffer gl.activeTexture(gl.TEXTURE1); - destinationContainer = - assertNotNull(gl.createTexture(), 'Failed to create texture'); + destinationContainer = shaderContext.createTexture(gl, gl.NEAREST); gl.bindTexture(gl.TEXTURE_2D, destinationContainer); - this.configureTextureParams(); const format = this.getTexImage2DFormat(); gl.texImage2D( gl.TEXTURE_2D, 0, format, this.width, this.height, 0, gl.RED, @@ -339,19 +337,6 @@ export class MPMask { return webGLTexture; } - /** Sets texture params for the currently bound texture. */ - private configureTextureParams() { - const gl = this.getGL(); - // `gl.NEAREST` ensures that we do not get interpolated values for - // masks. In some cases, the user might want interpolation (e.g. for - // confidence masks), so we might want to make this user-configurable. - // Note that `MPImage` uses `gl.LINEAR`. - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); - } - /** * Binds the backing texture to the canvas. If the texture does not yet * exist, creates it first. @@ -364,17 +349,17 @@ export class MPMask { let webGLTexture = this.getContainer(MPMaskType.WEBGL_TEXTURE); if (!webGLTexture) { - webGLTexture = - assertNotNull(gl.createTexture(), 'Failed to create texture'); + const shaderContext = this.getShaderContext(); + // `gl.NEAREST` ensures that we do not get interpolated values for + // masks. In some cases, the user might want interpolation (e.g. for + // confidence masks), so we might want to make this user-configurable. + // Note that `MPImage` uses `gl.LINEAR`. + webGLTexture = shaderContext.createTexture(gl, gl.NEAREST); this.containers.push(webGLTexture); this.ownsWebGLTexture = true; - - gl.bindTexture(gl.TEXTURE_2D, webGLTexture); - this.configureTextureParams(); - } else { - gl.bindTexture(gl.TEXTURE_2D, webGLTexture); } + gl.bindTexture(gl.TEXTURE_2D, webGLTexture); return webGLTexture; } diff --git a/mediapipe/tasks/web/vision/core/render_utils.ts b/mediapipe/tasks/web/vision/core/render_utils.ts deleted file mode 100644 index 3ee981bab..000000000 --- a/mediapipe/tasks/web/vision/core/render_utils.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** @fileoverview Utility functions used in the vision demos. */ - -/** - * Copyright 2023 The MediaPipe Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** Helper function to draw a confidence mask */ -export function drawConfidenceMask( - ctx: CanvasRenderingContext2D, image: Float32Array, width: number, - height: number): void { - const uint8Array = new Uint8ClampedArray(width * height * 4); - for (let i = 0; i < image.length; i++) { - uint8Array[4 * i] = 128; - uint8Array[4 * i + 1] = 0; - uint8Array[4 * i + 2] = 0; - uint8Array[4 * i + 3] = image[i] * 255; - } - ctx.putImageData(new ImageData(uint8Array, width, height), 0, 0); -} From 12340a8e82cec65d6086326155e0c39d0f4caa58 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Wed, 15 Nov 2023 13:00:08 -0800 Subject: [PATCH 117/157] Use gl.LINEAR interpolation for confidence masks PiperOrigin-RevId: 582777383 --- .../web/vision/core/drawing_utils.test.ts | 51 +++++++++++-------- .../tasks/web/vision/core/drawing_utils.ts | 1 + .../core/drawing_utils_category_mask.ts | 23 +++++---- .../core/drawing_utils_confidence_mask.ts | 18 +++---- mediapipe/tasks/web/vision/core/mask.test.ts | 6 +-- mediapipe/tasks/web/vision/core/mask.ts | 34 +++++++++---- .../web/vision/core/vision_task_runner.ts | 7 +-- .../vision/image_segmenter/image_segmenter.ts | 10 +++- .../interactive_segmenter.ts | 8 ++- .../vision/pose_landmarker/pose_landmarker.ts | 3 +- 10 files changed, 98 insertions(+), 63 deletions(-) diff --git a/mediapipe/tasks/web/vision/core/drawing_utils.test.ts b/mediapipe/tasks/web/vision/core/drawing_utils.test.ts index aaef42bbf..c32a5fc56 100644 --- a/mediapipe/tasks/web/vision/core/drawing_utils.test.ts +++ b/mediapipe/tasks/web/vision/core/drawing_utils.test.ts @@ -30,25 +30,20 @@ if (skip) { (skip ? xdescribe : describe)('DrawingUtils', () => { let shaderContext = new MPImageShaderContext(); - let canvas2D: HTMLCanvasElement; - let context2D: CanvasRenderingContext2D; + let canvas2D: OffscreenCanvas; + let context2D: OffscreenCanvasRenderingContext2D; let drawingUtils2D: DrawingUtils; - let canvasWebGL: HTMLCanvasElement; + let canvasWebGL: OffscreenCanvas; let contextWebGL: WebGL2RenderingContext; let drawingUtilsWebGL: DrawingUtils; beforeEach(() => { - shaderContext = new MPImageShaderContext(); + canvas2D = canvas2D ?? new OffscreenCanvas(WIDTH, HEIGHT); + canvasWebGL = canvasWebGL ?? new OffscreenCanvas(WIDTH, HEIGHT); - canvasWebGL = document.createElement('canvas'); - canvasWebGL.width = WIDTH; - canvasWebGL.height = HEIGHT; + shaderContext = new MPImageShaderContext(); contextWebGL = canvasWebGL.getContext('webgl2')!; drawingUtilsWebGL = new DrawingUtils(contextWebGL); - - canvas2D = document.createElement('canvas'); - canvas2D.width = WIDTH; - canvas2D.height = HEIGHT; context2D = canvas2D.getContext('2d')!; drawingUtils2D = new DrawingUtils(context2D, contextWebGL); }); @@ -61,11 +56,11 @@ if (skip) { describe( 'drawConfidenceMask() blends background with foreground color', () => { - const foreground = new ImageData( + const defaultColor = [255, 255, 255, 255]; + const overlayImage = new ImageData( new Uint8ClampedArray( [0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255]), WIDTH, HEIGHT); - const background = [255, 255, 255, 255]; const expectedResult = new Uint8Array([ 255, 255, 255, 255, 178, 178, 178, 255, 102, 102, 102, 255, 0, 0, 0, 255 @@ -74,48 +69,52 @@ if (skip) { it('on 2D canvas', () => { const confidenceMask = new MPMask( [new Float32Array([0.0, 0.3, 0.6, 1.0])], + /* interpolateValues= */ true, /* ownsWebGLTexture= */ false, canvas2D, shaderContext, WIDTH, HEIGHT); drawingUtils2D.drawConfidenceMask( - confidenceMask, background, foreground); + confidenceMask, defaultColor, overlayImage); const actualResult = context2D.getImageData(0, 0, WIDTH, HEIGHT).data; expect(actualResult) .toEqual(new Uint8ClampedArray(expectedResult.buffer)); + confidenceMask.close(); }); it('on WebGL canvas', () => { const confidenceMask = new MPMask( [new Float32Array( [0.6, 1.0, 0.0, 0.3])], // Note: Vertically flipped + /* interpolateValues= */ true, /* ownsWebGLTexture= */ false, canvasWebGL, shaderContext, WIDTH, HEIGHT); drawingUtilsWebGL.drawConfidenceMask( - confidenceMask, background, foreground); + confidenceMask, defaultColor, overlayImage); const actualResult = new Uint8Array(WIDTH * HEIGHT * 4); contextWebGL.readPixels( 0, 0, WIDTH, HEIGHT, contextWebGL.RGBA, contextWebGL.UNSIGNED_BYTE, actualResult); expect(actualResult).toEqual(expectedResult); + confidenceMask.close(); }); }); describe( 'drawConfidenceMask() blends background with foreground image', () => { - const foreground = new ImageData( - new Uint8ClampedArray( - [0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255]), - WIDTH, HEIGHT); - const background = new ImageData( + const defaultImage = new ImageData( new Uint8ClampedArray([ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 ]), WIDTH, HEIGHT); + const overlayImage = new ImageData( + new Uint8ClampedArray( + [0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255, 0, 0, 0, 255]), + WIDTH, HEIGHT); const expectedResult = new Uint8Array([ 255, 255, 255, 255, 178, 178, 178, 255, 102, 102, 102, 255, 0, 0, 0, 255 @@ -124,32 +123,36 @@ if (skip) { it('on 2D canvas', () => { const confidenceMask = new MPMask( [new Float32Array([0.0, 0.3, 0.6, 1.0])], + /* interpolateValues= */ true, /* ownsWebGLTexture= */ false, canvas2D, shaderContext, WIDTH, HEIGHT); drawingUtils2D.drawConfidenceMask( - confidenceMask, background, foreground); + confidenceMask, defaultImage, overlayImage); const actualResult = context2D.getImageData(0, 0, WIDTH, HEIGHT).data; expect(actualResult) .toEqual(new Uint8ClampedArray(expectedResult.buffer)); + confidenceMask.close(); }); it('on WebGL canvas', () => { const confidenceMask = new MPMask( [new Float32Array( [0.6, 1.0, 0.0, 0.3])], // Note: Vertically flipped + /* interpolateValues= */ true, /* ownsWebGLTexture= */ false, canvasWebGL, shaderContext, WIDTH, HEIGHT); drawingUtilsWebGL.drawConfidenceMask( - confidenceMask, background, foreground); + confidenceMask, defaultImage, overlayImage); const actualResult = new Uint8Array(WIDTH * HEIGHT * 4); contextWebGL.readPixels( 0, 0, WIDTH, HEIGHT, contextWebGL.RGBA, contextWebGL.UNSIGNED_BYTE, actualResult); expect(actualResult).toEqual(expectedResult); + confidenceMask.close(); }); }); @@ -167,6 +170,7 @@ if (skip) { it('on 2D canvas', () => { const categoryMask = new MPMask( [new Uint8Array([0, 1, 2, 3])], + /* interpolateValues= */ false, /* ownsWebGLTexture= */ false, canvas2D, shaderContext, WIDTH, HEIGHT); @@ -175,11 +179,13 @@ if (skip) { const actualResult = context2D.getImageData(0, 0, WIDTH, HEIGHT).data; expect(actualResult) .toEqual(new Uint8ClampedArray(expectedResult.buffer)); + categoryMask.close(); }); it('on WebGL canvas', () => { const categoryMask = new MPMask( [new Uint8Array([2, 3, 0, 1])], // Note: Vertically flipped + /* interpolateValues= */ false, /* ownsWebGLTexture= */ false, canvasWebGL, shaderContext, WIDTH, HEIGHT); @@ -190,6 +196,7 @@ if (skip) { 0, 0, WIDTH, HEIGHT, contextWebGL.RGBA, contextWebGL.UNSIGNED_BYTE, actualResult); expect(actualResult).toEqual(expectedResult); + categoryMask.close(); }); }); diff --git a/mediapipe/tasks/web/vision/core/drawing_utils.ts b/mediapipe/tasks/web/vision/core/drawing_utils.ts index 154420f6b..520f9e2b3 100644 --- a/mediapipe/tasks/web/vision/core/drawing_utils.ts +++ b/mediapipe/tasks/web/vision/core/drawing_utils.ts @@ -419,6 +419,7 @@ export class DrawingUtils { const convertedMask = new MPMask( [data], + mask.interpolateValues, /* ownsWebGlTexture= */ false, gl.canvas, this.convertToWebGLTextureShaderContext, diff --git a/mediapipe/tasks/web/vision/core/drawing_utils_category_mask.ts b/mediapipe/tasks/web/vision/core/drawing_utils_category_mask.ts index d7706075f..3b7cc0b47 100644 --- a/mediapipe/tasks/web/vision/core/drawing_utils_category_mask.ts +++ b/mediapipe/tasks/web/vision/core/drawing_utils_category_mask.ts @@ -92,11 +92,15 @@ export class CategoryMaskShaderContext extends MPImageShaderContext { colorMap: Map|number[][]) { const gl = this.gl!; + // Bind category mask + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, categoryMask); + // TODO: We should avoid uploading textures from CPU to GPU // if the textures haven't changed. This can lead to drastic performance // slowdowns (~50ms per frame). Users can reduce the penalty by passing a // canvas object instead of ImageData/HTMLImageElement. - gl.activeTexture(gl.TEXTURE0); + gl.activeTexture(gl.TEXTURE1); gl.bindTexture(gl.TEXTURE_2D, this.backgroundTexture!); gl.texImage2D( gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, background); @@ -117,19 +121,15 @@ export class CategoryMaskShaderContext extends MPImageShaderContext { pixels[index * 4 + 2] = rgba[2]; pixels[index * 4 + 3] = rgba[3]; }); - gl.activeTexture(gl.TEXTURE1); + gl.activeTexture(gl.TEXTURE2); gl.bindTexture(gl.TEXTURE_2D, this.colorMappingTexture!); gl.texImage2D( gl.TEXTURE_2D, 0, gl.RGBA, 256, 1, 0, gl.RGBA, gl.UNSIGNED_BYTE, new Uint8Array(pixels)); } else { - gl.activeTexture(gl.TEXTURE1); + gl.activeTexture(gl.TEXTURE2); gl.bindTexture(gl.TEXTURE_2D, this.colorMappingTexture!); } - - // Bind category mask - gl.activeTexture(gl.TEXTURE2); - gl.bindTexture(gl.TEXTURE_2D, categoryMask); } unbindTextures() { @@ -148,10 +148,11 @@ export class CategoryMaskShaderContext extends MPImageShaderContext { protected override setupTextures(): void { const gl = this.gl!; - gl.activeTexture(gl.TEXTURE0); + gl.activeTexture(gl.TEXTURE1); this.backgroundTexture = this.createTexture(gl, gl.LINEAR); // Use `gl.NEAREST` to prevent interpolating values in our category to // color map. + gl.activeTexture(gl.TEXTURE2); this.colorMappingTexture = this.createTexture(gl, gl.NEAREST); } @@ -172,9 +173,9 @@ export class CategoryMaskShaderContext extends MPImageShaderContext { protected override configureUniforms(): void { super.configureUniforms(); const gl = this.gl!; - gl.uniform1i(this.backgroundTextureUniform!, 0); - gl.uniform1i(this.colorMappingTextureUniform!, 1); - gl.uniform1i(this.maskTextureUniform!, 2); + gl.uniform1i(this.maskTextureUniform!, 0); + gl.uniform1i(this.backgroundTextureUniform!, 1); + gl.uniform1i(this.colorMappingTextureUniform!, 2); } override close(): void { diff --git a/mediapipe/tasks/web/vision/core/drawing_utils_confidence_mask.ts b/mediapipe/tasks/web/vision/core/drawing_utils_confidence_mask.ts index c8d30c9ee..953911f01 100644 --- a/mediapipe/tasks/web/vision/core/drawing_utils_confidence_mask.ts +++ b/mediapipe/tasks/web/vision/core/drawing_utils_confidence_mask.ts @@ -51,9 +51,9 @@ export class ConfidenceMaskShaderContext extends MPImageShaderContext { protected override setupTextures(): void { const gl = this.gl!; - gl.activeTexture(gl.TEXTURE0); - this.defaultTexture = this.createTexture(gl); gl.activeTexture(gl.TEXTURE1); + this.defaultTexture = this.createTexture(gl); + gl.activeTexture(gl.TEXTURE2); this.overlayTexture = this.createTexture(gl); } @@ -74,9 +74,9 @@ export class ConfidenceMaskShaderContext extends MPImageShaderContext { protected override configureUniforms(): void { super.configureUniforms(); const gl = this.gl!; - gl.uniform1i(this.defaultTextureUniform!, 0); - gl.uniform1i(this.overlayTextureUniform!, 1); - gl.uniform1i(this.maskTextureUniform!, 2); + gl.uniform1i(this.maskTextureUniform!, 0); + gl.uniform1i(this.defaultTextureUniform!, 1); + gl.uniform1i(this.overlayTextureUniform!, 2); } bindAndUploadTextures( @@ -88,17 +88,17 @@ export class ConfidenceMaskShaderContext extends MPImageShaderContext { // canvas object instead of ImageData/HTMLImageElement. const gl = this.gl!; gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, confidenceMask); + + gl.activeTexture(gl.TEXTURE1); gl.bindTexture(gl.TEXTURE_2D, this.defaultTexture!); gl.texImage2D( gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, defaultImage); - gl.activeTexture(gl.TEXTURE1); + gl.activeTexture(gl.TEXTURE2); gl.bindTexture(gl.TEXTURE_2D, this.overlayTexture!); gl.texImage2D( gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, overlayImage); - - gl.activeTexture(gl.TEXTURE2); - gl.bindTexture(gl.TEXTURE_2D, confidenceMask); } unbindTextures() { diff --git a/mediapipe/tasks/web/vision/core/mask.test.ts b/mediapipe/tasks/web/vision/core/mask.test.ts index d2f5ddb09..29ed5ea02 100644 --- a/mediapipe/tasks/web/vision/core/mask.test.ts +++ b/mediapipe/tasks/web/vision/core/mask.test.ts @@ -136,7 +136,7 @@ class MPMaskTestContext { shaderContext: MPImageShaderContext, input: MaskType, width: number, height: number): MPMask { return new MPMask( - [input], + [input], /* interpolateValues= */ false, /* ownsWebGLTexture= */ false, context.canvas, shaderContext, width, height); } @@ -182,7 +182,7 @@ class MPMaskTestContext { const shaderContext = new MPImageShaderContext(); const mask = new MPMask( - [context.webGLTexture], + [context.webGLTexture], /* interpolateValues= */ false, /* ownsWebGLTexture= */ false, context.canvas, shaderContext, WIDTH, HEIGHT); @@ -196,7 +196,7 @@ class MPMaskTestContext { const shaderContext = new MPImageShaderContext(); const mask = new MPMask( - [context.webGLTexture], + [context.webGLTexture], /* interpolateValues= */ false, /* ownsWebGLTexture= */ false, context.canvas, shaderContext, WIDTH, HEIGHT); diff --git a/mediapipe/tasks/web/vision/core/mask.ts b/mediapipe/tasks/web/vision/core/mask.ts index b463589e4..d08145a2d 100644 --- a/mediapipe/tasks/web/vision/core/mask.ts +++ b/mediapipe/tasks/web/vision/core/mask.ts @@ -62,9 +62,25 @@ export class MPMask { /** The format used to write pixel values from textures. */ private static texImage2DFormat?: GLenum; - /** @hideconstructor */ + /** + * @param containers The data source for this mask as a `WebGLTexture`, + * `Unit8Array` or `Float32Array`. Multiple sources of the same data can + * be provided to reduce conversions. + * @param interpolateValues If enabled, uses `gl.LINEAR` instead of + * `gl.NEAREST` to interpolate between mask values. + * @param ownsWebGLTexture Whether the MPMask should take ownership of the + * `WebGLTexture` and free it when closed. + * @param canvas The canvas to use for rendering and conversion. Must be the + * same canvas for any WebGL resources. + * @param shaderContext A shader context that is shared between all masks from + * a single task. + * @param width The width of the mask. + * @param height The height of the mask. + * @hideconstructor + */ constructor( private readonly containers: MPMaskContainer[], + readonly interpolateValues: boolean, private ownsWebGLTexture: boolean, /** Returns the canvas element that the mask is bound to. */ readonly canvas: HTMLCanvasElement|OffscreenCanvas|undefined, @@ -215,7 +231,8 @@ export class MPMask { // Create a new texture and use it to back a framebuffer gl.activeTexture(gl.TEXTURE1); - destinationContainer = shaderContext.createTexture(gl, gl.NEAREST); + destinationContainer = shaderContext.createTexture( + gl, this.interpolateValues ? gl.LINEAR : gl.NEAREST); gl.bindTexture(gl.TEXTURE_2D, destinationContainer); const format = this.getTexImage2DFormat(); gl.texImage2D( @@ -242,8 +259,8 @@ export class MPMask { } return new MPMask( - destinationContainers, this.hasWebGLTexture(), this.canvas, - this.shaderContext, this.width, this.height); + destinationContainers, this.interpolateValues, this.hasWebGLTexture(), + this.canvas, this.shaderContext, this.width, this.height); } private getGL(): WebGL2RenderingContext { @@ -254,7 +271,7 @@ export class MPMask { } if (!this.gl) { this.gl = assertNotNull( - this.canvas.getContext('webgl2') as WebGL2RenderingContext | null, + this.canvas.getContext('webgl2'), 'You cannot use a canvas that is already bound to a different ' + 'type of rendering context.'); } @@ -350,11 +367,8 @@ export class MPMask { let webGLTexture = this.getContainer(MPMaskType.WEBGL_TEXTURE); if (!webGLTexture) { const shaderContext = this.getShaderContext(); - // `gl.NEAREST` ensures that we do not get interpolated values for - // masks. In some cases, the user might want interpolation (e.g. for - // confidence masks), so we might want to make this user-configurable. - // Note that `MPImage` uses `gl.LINEAR`. - webGLTexture = shaderContext.createTexture(gl, gl.NEAREST); + webGLTexture = shaderContext.createTexture( + gl, this.interpolateValues ? gl.LINEAR : gl.NEAREST); this.containers.push(webGLTexture); this.ownsWebGLTexture = true; } diff --git a/mediapipe/tasks/web/vision/core/vision_task_runner.ts b/mediapipe/tasks/web/vision/core/vision_task_runner.ts index b9aa5e352..292a37eec 100644 --- a/mediapipe/tasks/web/vision/core/vision_task_runner.ts +++ b/mediapipe/tasks/web/vision/core/vision_task_runner.ts @@ -274,8 +274,9 @@ export abstract class VisionTaskRunner extends TaskRunner { } /** Converts a WasmImage to an MPMask. */ - protected convertToMPMask(wasmImage: WasmImage, shouldCopyData: boolean): - MPMask { + protected convertToMPMask( + wasmImage: WasmImage, interpolateValues: boolean, + shouldCopyData: boolean): MPMask { const {data, width, height} = wasmImage; const pixels = width * height; @@ -291,7 +292,7 @@ export abstract class VisionTaskRunner extends TaskRunner { } const mask = new MPMask( - [container], + [container], interpolateValues, /* ownsWebGLTexture= */ false, this.graphRunner.wasmModule.canvas!, this.shaderContext, width, height); return shouldCopyData ? mask.clone() : mask; diff --git a/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts b/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts index d8751b9e3..cbd20450b 100644 --- a/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts +++ b/mediapipe/tasks/web/vision/image_segmenter/image_segmenter.ts @@ -424,7 +424,10 @@ export class ImageSegmenter extends VisionTaskRunner { CONFIDENCE_MASKS_STREAM, (masks, timestamp) => { this.confidenceMasks = masks.map( wasmImage => this.convertToMPMask( - wasmImage, /* shouldCopyData= */ !this.userCallback)); + wasmImage, + /* interpolateValues= */ true, + /* shouldCopyData= */ !this.userCallback, + )); this.setLatestOutputTimestamp(timestamp); }); this.graphRunner.attachEmptyPacketListener( @@ -442,7 +445,10 @@ export class ImageSegmenter extends VisionTaskRunner { this.graphRunner.attachImageListener( CATEGORY_MASK_STREAM, (mask, timestamp) => { this.categoryMask = this.convertToMPMask( - mask, /* shouldCopyData= */ !this.userCallback); + mask, + /* interpolateValues= */ false, + /* shouldCopyData= */ !this.userCallback, + ); this.setLatestOutputTimestamp(timestamp); }); this.graphRunner.attachEmptyPacketListener( diff --git a/mediapipe/tasks/web/vision/interactive_segmenter/interactive_segmenter.ts b/mediapipe/tasks/web/vision/interactive_segmenter/interactive_segmenter.ts index 887f55839..5a37b9ff0 100644 --- a/mediapipe/tasks/web/vision/interactive_segmenter/interactive_segmenter.ts +++ b/mediapipe/tasks/web/vision/interactive_segmenter/interactive_segmenter.ts @@ -341,7 +341,10 @@ export class InteractiveSegmenter extends VisionTaskRunner { CONFIDENCE_MASKS_STREAM, (masks, timestamp) => { this.confidenceMasks = masks.map( wasmImage => this.convertToMPMask( - wasmImage, /* shouldCopyData= */ !this.userCallback)); + wasmImage, + /* interpolateValues= */ true, + /* shouldCopyData= */ !this.userCallback, + )); this.setLatestOutputTimestamp(timestamp); }); this.graphRunner.attachEmptyPacketListener( @@ -359,7 +362,8 @@ export class InteractiveSegmenter extends VisionTaskRunner { this.graphRunner.attachImageListener( CATEGORY_MASK_STREAM, (mask, timestamp) => { this.categoryMask = this.convertToMPMask( - mask, /* shouldCopyData= */ !this.userCallback); + mask, /* interpolateValues= */ false, + /* shouldCopyData= */ !this.userCallback); this.setLatestOutputTimestamp(timestamp); }); this.graphRunner.attachEmptyPacketListener( diff --git a/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts b/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts index 8f6531827..262966d72 100644 --- a/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts +++ b/mediapipe/tasks/web/vision/pose_landmarker/pose_landmarker.ts @@ -470,7 +470,8 @@ export class PoseLandmarker extends VisionTaskRunner { SEGMENTATION_MASK_STREAM, (masks, timestamp) => { this.segmentationMasks = masks.map( wasmImage => this.convertToMPMask( - wasmImage, /* shouldCopyData= */ !this.userCallback)); + wasmImage, /* interpolateValues= */ true, + /* shouldCopyData= */ !this.userCallback)); this.setLatestOutputTimestamp(timestamp); }); this.graphRunner.attachEmptyPacketListener( From f13c6974ee046791d6b5a4af4e77ab81ea8588c0 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 16 Nov 2023 02:31:58 -0800 Subject: [PATCH 118/157] Extract CPU conversion methods into a separate library & add test PiperOrigin-RevId: 582966041 --- mediapipe/calculators/tensor/BUILD | 40 +++- .../tensor/tensor_converter_calculator.cc | 120 ++---------- .../tensor/tensor_converter_cpu.cc | 145 +++++++++++++++ .../calculators/tensor/tensor_converter_cpu.h | 61 ++++++ .../tensor/tensor_converter_cpu_test.cc | 175 ++++++++++++++++++ mediapipe/util/image_test_utils.cc | 38 ++++ mediapipe/util/image_test_utils.h | 7 + 7 files changed, 478 insertions(+), 108 deletions(-) create mode 100644 mediapipe/calculators/tensor/tensor_converter_cpu.cc create mode 100644 mediapipe/calculators/tensor/tensor_converter_cpu.h create mode 100644 mediapipe/calculators/tensor/tensor_converter_cpu_test.cc diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index e95398a9d..96a29089e 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -657,6 +657,7 @@ cc_library( }), deps = [ ":tensor_converter_calculator_cc_proto", + ":tensor_converter_cpu", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:port", "//mediapipe/framework/formats:image_frame", @@ -665,6 +666,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", + "//mediapipe/gpu:gpu_buffer", "//mediapipe/gpu:gpu_buffer_format", "//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/util:resource_util", @@ -674,10 +676,17 @@ cc_library( "@com_google_absl//absl/log:check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ] + select({ "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": ["tensor_converter_calculator_gpu_deps"], + "//conditions:default": [ + "tensor_converter_calculator_gpu_deps", + "//mediapipe/gpu:gl_base", + "//mediapipe/gpu:gl_calculator_helper", + "//mediapipe/gpu:gl_simple_shaders", + "//mediapipe/gpu:shader_util", + ], }) + select({ "//mediapipe:apple": [ "//third_party/apple_frameworks:MetalKit", @@ -687,6 +696,35 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "tensor_converter_cpu", + srcs = ["tensor_converter_cpu.cc"], + hdrs = ["tensor_converter_cpu.h"], + deps = [ + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:matrix", + "//mediapipe/framework/formats:tensor", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_test( + name = "tensor_converter_cpu_test", + srcs = ["tensor_converter_cpu_test.cc"], + deps = [ + ":tensor_converter_cpu", + "//mediapipe/framework/formats:matrix", + "//mediapipe/framework/formats:tensor", + "//mediapipe/framework/port:gtest", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:status_matchers", + "//mediapipe/util:image_test_utils", + ], +) + cc_library( name = "tensor_converter_calculator_gpu_deps", visibility = ["//visibility:private"], diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator.cc b/mediapipe/calculators/tensor/tensor_converter_calculator.cc index b42cb0b17..80cac63c7 100644 --- a/mediapipe/calculators/tensor/tensor_converter_calculator.cc +++ b/mediapipe/calculators/tensor/tensor_converter_calculator.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include "absl/log/absl_check.h" @@ -21,17 +22,21 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_format.h" +#include "absl/strings/substitute.h" #include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensor_converter_cpu.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port.h" #include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/gpu/gpu_buffer_format.h" #include "mediapipe/gpu/gpu_origin.pb.h" #if !MEDIAPIPE_DISABLE_GPU +#include "mediapipe/gpu/gl_base.h" #include "mediapipe/gpu/gpu_buffer.h" #if MEDIAPIPE_METAL_ENABLED #import @@ -94,11 +99,6 @@ absl::StatusOr ShouldFlipVertically( } } -typedef Eigen::Matrix - RowMajorMatrixXf; -typedef Eigen::Matrix - ColMajorMatrixXf; - constexpr char kImageFrameTag[] = "IMAGE"; constexpr char kGpuBufferTag[] = "IMAGE_GPU"; constexpr char kTensorsTag[] = "TENSORS"; @@ -156,10 +156,6 @@ class TensorConverterCalculator : public CalculatorBase { private: absl::Status InitGpu(CalculatorContext* cc); absl::Status LoadOptions(CalculatorContext* cc, bool use_gpu); - template - absl::Status NormalizeImage(const ImageFrame& image_frame, - bool flip_vertically, float* tensor_ptr); - absl::Status CopyMatrixToTensor(const Matrix& matrix, float* tensor_ptr); absl::Status ProcessCPU(CalculatorContext* cc); absl::Status ProcessGPU(CalculatorContext* cc); @@ -279,46 +275,19 @@ absl::Status TensorConverterCalculator::ProcessCPU(CalculatorContext* cc) { } const auto& image_frame = cc->Inputs().Tag(kImageFrameTag).Get(); - const int height = image_frame.Height(); - const int width = image_frame.Width(); - const int channels = image_frame.NumberOfChannels(); - const int channels_preserved = std::min(channels, max_num_channels_); - const mediapipe::ImageFormat::Format format = image_frame.Format(); - - if (!(format == mediapipe::ImageFormat::SRGBA || - format == mediapipe::ImageFormat::SRGB || - format == mediapipe::ImageFormat::GRAY8 || - format == mediapipe::ImageFormat::VEC32F1)) - RET_CHECK_FAIL() << "Unsupported CPU input format."; - - output_tensors->emplace_back( - Tensor::ElementType::kFloat32, - Tensor::Shape{1, height, width, channels_preserved}); - auto cpu_view = output_tensors->back().GetCpuWriteView(); - - // Copy image data into tensor. - if (image_frame.ByteDepth() == 1) { - MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, - cpu_view.buffer())); - } else if (image_frame.ByteDepth() == 4) { - MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, - cpu_view.buffer())); - } else { - return absl::InternalError( - "Only byte-based (8 bit) and float (32 bit) images supported."); - } + MP_ASSIGN_OR_RETURN( + Tensor output, + ConvertImageFrameToTensorOnCpu(image_frame, *output_range_, + flip_vertically_, max_num_channels_)); + output_tensors->emplace_back(std::move(output)); } else if (cc->Inputs().HasTag(kMatrixTag)) { if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) { return absl::OkStatus(); } const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get(); - const int height = matrix.rows(); - const int width = matrix.cols(); - const int channels = 1; - output_tensors->emplace_back(Tensor::ElementType::kFloat32, - Tensor::Shape{1, height, width, channels}); - MP_RETURN_IF_ERROR(CopyMatrixToTensor( - matrix, output_tensors->back().GetCpuWriteView().buffer())); + MP_ASSIGN_OR_RETURN(Tensor output, + ConvertMatrixToTensorOnCpu(matrix, row_major_matrix_)); + output_tensors->emplace_back(std::move(output)); } else { return absl::OkStatus(); } @@ -669,67 +638,4 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc, return absl::OkStatus(); } -template -absl::Status TensorConverterCalculator::NormalizeImage( - const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) { - const int height = image_frame.Height(); - const int width = image_frame.Width(); - const int channels = image_frame.NumberOfChannels(); - const int channels_preserved = std::min(channels, max_num_channels_); - const int channels_ignored = channels - channels_preserved; - - if (output_range_.has_value()) { - // If the output float range is set and we are not using custom - // normalization, normalize the pixel values from [0, 255] to the specified - // output range. - RET_CHECK_NE(output_range_->first, output_range_->second); - const float scale = (output_range_->second - output_range_->first) / 255.0f; - const float bias = output_range_->first; - - for (int i = 0; i < height; ++i) { - const T* image_ptr = reinterpret_cast( - image_frame.PixelData() + - (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); - for (int j = 0; j < width; ++j) { - for (int c = 0; c < channels_preserved; ++c) { - *tensor_ptr++ = *image_ptr++ * scale + bias; - } - image_ptr += channels_ignored; - } - } - } else { - // [0,1], scale only (bias == 0) - // Verified that there are no precision issues with 1.0f / 255.0f expression - const float scale = 1.0f / 255.0f; - for (int i = 0; i < height; ++i) { - const T* image_ptr = reinterpret_cast( - image_frame.PixelData() + - (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); - for (int j = 0; j < width; ++j) { - for (int c = 0; c < channels_preserved; ++c) { - *tensor_ptr++ = *image_ptr++ * scale; - } - image_ptr += channels_ignored; - } - } - } - - return absl::OkStatus(); -} - -absl::Status TensorConverterCalculator::CopyMatrixToTensor(const Matrix& matrix, - float* tensor_ptr) { - if (row_major_matrix_) { - auto matrix_map = - Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } else { - auto matrix_map = - Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } - - return absl::OkStatus(); -} - } // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensor_converter_cpu.cc b/mediapipe/calculators/tensor/tensor_converter_cpu.cc new file mode 100644 index 000000000..f72a24c31 --- /dev/null +++ b/mediapipe/calculators/tensor/tensor_converter_cpu.cc @@ -0,0 +1,145 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensor_converter_cpu.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_macros.h" + +namespace mediapipe { +namespace { + +typedef Eigen::Matrix + RowMajorMatrixXf; +typedef Eigen::Matrix + ColMajorMatrixXf; + +template +absl::Status NormalizeImage(const ImageFrame& image_frame, bool flip_vertically, + const std::pair& output_range, + int max_num_channels, float* tensor_ptr) { + const int height = image_frame.Height(); + const int width = image_frame.Width(); + const int channels = image_frame.NumberOfChannels(); + const int channels_preserved = std::min(channels, max_num_channels); + const int channels_ignored = channels - channels_preserved; + + RET_CHECK_NE(output_range.first, output_range.second); + const float scale = (output_range.second - output_range.first) / 255.0f; + const float bias = output_range.first; + + for (int i = 0; i < height; ++i) { + const T* image_ptr = reinterpret_cast( + image_frame.PixelData() + + (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); + for (int j = 0; j < width; ++j) { + for (int c = 0; c < channels_preserved; ++c) { + *tensor_ptr++ = *image_ptr++ * scale + bias; + } + image_ptr += channels_ignored; + } + } + return absl::OkStatus(); +} + +} // namespace + +absl::Status NormalizeUInt8Image(const ImageFrame& image_frame, + bool flip_vertically, + const std::pair& output_range, + int max_num_channels, float* tensor_ptr) { + return NormalizeImage(image_frame, flip_vertically, output_range, + max_num_channels, tensor_ptr); +} + +absl::Status NormalizeFloatImage(const ImageFrame& image_frame, + bool flip_vertically, + const std::pair& output_range, + int max_num_channels, float* tensor_ptr) { + return NormalizeImage(image_frame, flip_vertically, output_range, + max_num_channels, tensor_ptr); +} + +absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix, + float* tensor_ptr) { + if (is_row_major_matrix) { + auto matrix_map = + Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); + matrix_map = matrix; + } else { + auto matrix_map = + Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); + matrix_map = matrix; + } + return absl::OkStatus(); +} + +absl::StatusOr ConvertImageFrameToTensorOnCpu( + const ImageFrame& image_frame, const std::pair& output_range, + bool flip_vertically, int max_num_channels) { + const int height = image_frame.Height(); + const int width = image_frame.Width(); + const int channels = image_frame.NumberOfChannels(); + const int channels_preserved = std::min(channels, max_num_channels); + const mediapipe::ImageFormat::Format format = image_frame.Format(); + + if (!(format == mediapipe::ImageFormat::SRGBA || + format == mediapipe::ImageFormat::SRGB || + format == mediapipe::ImageFormat::GRAY8 || + format == mediapipe::ImageFormat::VEC32F1)) + RET_CHECK_FAIL() << "Unsupported CPU input format."; + + Tensor output_tensor(Tensor::ElementType::kFloat32, + Tensor::Shape{1, height, width, channels_preserved}); + auto cpu_view = output_tensor.GetCpuWriteView(); + + // Copy image data into tensor. + if (image_frame.ByteDepth() == 1) { + MP_RETURN_IF_ERROR(NormalizeUInt8Image(image_frame, flip_vertically, + output_range, max_num_channels, + cpu_view.buffer())); + } else if (image_frame.ByteDepth() == 4) { + MP_RETURN_IF_ERROR(NormalizeFloatImage(image_frame, flip_vertically, + output_range, max_num_channels, + cpu_view.buffer())); + } else { + return absl::InternalError( + "Only byte-based (8 bit) and float (32 bit) images supported."); + } + return output_tensor; +} + +absl::StatusOr ConvertMatrixToTensorOnCpu(const Matrix& matrix, + bool row_major_matrix) { + const int height = matrix.rows(); + const int width = matrix.cols(); + const int channels = 1; + Tensor output_tensor(Tensor::ElementType::kFloat32, + Tensor::Shape{1, height, width, channels}); + MP_RETURN_IF_ERROR( + CopyMatrixToTensor(matrix, row_major_matrix, + output_tensor.GetCpuWriteView().buffer())); + return output_tensor; +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensor_converter_cpu.h b/mediapipe/calculators/tensor/tensor_converter_cpu.h new file mode 100644 index 000000000..784bade80 --- /dev/null +++ b/mediapipe/calculators/tensor/tensor_converter_cpu.h @@ -0,0 +1,61 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_ + +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/tensor.h" + +namespace mediapipe { + +// Converts an ImageFrame to a vector of Tensors. +// @flip_vertically enables to flip the image during conversion. +// @max_num_channels can be used to reserve extra channels in the output +// tensors. +// Returns output Tensor. +absl::StatusOr ConvertImageFrameToTensorOnCpu( + const ImageFrame& image_frame, const std::pair& output_range, + bool flip_vertically, int max_num_channels); + +// Converts a Matrix to a vector of Tensors. +// @row_major_matrix defines the ordering in the input matrix. +// @max_num_channels can be used to reserve extra channels in the output +// tensors. +// Returns output Tensor. +absl::StatusOr ConvertMatrixToTensorOnCpu(const Matrix& matrix, + bool row_major_matrix); + +// For testing only below. +absl::Status NormalizeUInt8Image(const ImageFrame& image_frame, + bool flip_vertically, + const std::pair& output_range, + int max_num_channels, float* tensor_ptr); + +absl::Status NormalizeFloatImage(const ImageFrame& image_frame, + bool flip_vertically, + const std::pair& output_range, + int max_num_channels, float* tensor_ptr); + +absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix, + float* tensor_ptr); + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_ diff --git a/mediapipe/calculators/tensor/tensor_converter_cpu_test.cc b/mediapipe/calculators/tensor/tensor_converter_cpu_test.cc new file mode 100644 index 000000000..478a9c6dc --- /dev/null +++ b/mediapipe/calculators/tensor/tensor_converter_cpu_test.cc @@ -0,0 +1,175 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensor_converter_cpu.h" + +#include +#include +#include + +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/util/image_test_utils.h" + +namespace mediapipe { +namespace { + +Matrix CreateTestMatrix(int num_rows, int num_columns) { + Matrix matrix(num_rows, num_columns); + for (int r = 0; r < num_rows; ++r) { + for (int c = 0; c < num_columns; ++c) { + matrix(r, c) = r * num_columns + c; + } + } + return matrix; +} + +TEST(TensorConverterCpuTest, ShouldCopyMatrixInRowMajorFormatToTensor) { + auto test_matrix = CreateTestMatrix(/* num_rows=*/3, /*num_columns=*/4); + std::vector tensor_data(test_matrix.size(), 0.0f); + + MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/true, + tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + const int row = i / test_matrix.cols(); + const int column = i % test_matrix.cols(); + EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column)); + } +} + +TEST(TensorConverterCpuTest, ShouldCopyMatrixInColumnMajorFormatToTensor) { + auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4); + std::vector tensor_data(test_matrix.size(), 0.0f); + + MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/false, + tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + const int row = i % test_matrix.rows(); + const int column = i / test_matrix.rows(); + EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column)); + } +} + +TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithDefaultRange) { + auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); + std::vector tensor_data( + grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f); + + MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, + {0.0f, 1.0f}, /*num_tensor_channels=*/1, + tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + EXPECT_FLOAT_EQ( + tensor_data[i], + static_cast(grey8_image_frame.PixelData()[i]) / 255.0f); + } +} + +TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithSpecifiedRange) { + auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); + std::vector tensor_data( + grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f); + const auto range = std::make_pair(2.0f, 3.0f); + + MP_EXPECT_OK( + NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, range, + /*num_tensor_channels=*/1, tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + EXPECT_FLOAT_EQ(tensor_data[i], + static_cast(grey8_image_frame.PixelData()[i]) / + 255.0f * (range.second - range.first) + + range.first); + } +} + +TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageFlipped) { + auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); + std::vector tensor_data( + grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f); + + MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/true, + {0.0f, 1.0f}, /*num_tensor_channels=*/1, + tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + const int x = i % grey8_image_frame.Width(); + const int y = i / grey8_image_frame.Width(); + const int flipped_y = grey8_image_frame.Height() - y - 1; + + const int index = flipped_y * grey8_image_frame.Width() + x; + EXPECT_FLOAT_EQ( + tensor_data[index], + static_cast(grey8_image_frame.PixelData()[i]) / 255.0f); + } +} + +TEST(TensorConverterCpuTest, ShouldNormalizeFloatImageWithDefaultRange) { + auto float_image_frame = + CreateTestFloat32ImageFrame(/*width=*/3, /*height=*/4); + std::vector tensor_data( + float_image_frame.Width() * float_image_frame.Height(), 0.0f); + + MP_EXPECT_OK(NormalizeFloatImage(float_image_frame, /*flip_vertically=*/false, + {0.0f, 1.0f}, /*num_tensor_channels=*/1, + tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + EXPECT_FLOAT_EQ(tensor_data[i], reinterpret_cast( + float_image_frame.PixelData())[i] / + 255.0f); + } +} + +TEST(TensorConverterCpuTest, ConvertImageFrameToTensorOnCpu) { + auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); + + MP_ASSERT_OK_AND_ASSIGN(Tensor output, ConvertImageFrameToTensorOnCpu( + grey8_image_frame, {0.0f, 1.0f}, + /*flip_vertically=*/false, + /*max_num_channels=*/1)); + + const auto cpu_read_view = output.GetCpuReadView(); + const float* tensor_ptr = cpu_read_view.buffer(); + for (int i = 0; i < grey8_image_frame.Width() * grey8_image_frame.Height(); + ++i) { + EXPECT_FLOAT_EQ( + tensor_ptr[i], + static_cast(grey8_image_frame.PixelData()[i]) / 255.0); + } +} + +TEST(TensorConverterCpuTest, ConvertMatrixToTensorOnCpu) { + auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4); + + MP_ASSERT_OK_AND_ASSIGN( + Tensor output, ConvertMatrixToTensorOnCpu(test_matrix, + /*row_major_matrix=*/false)); + + const auto cpu_read_view = output.GetCpuReadView(); + const float* tensor_ptr = cpu_read_view.buffer(); + for (int i = 0; i < test_matrix.size(); ++i) { + EXPECT_FLOAT_EQ(tensor_ptr[i], test_matrix.data()[i]); + } +} + +} // namespace + +} // namespace mediapipe diff --git a/mediapipe/util/image_test_utils.cc b/mediapipe/util/image_test_utils.cc index 9e10f40c1..325b308f1 100644 --- a/mediapipe/util/image_test_utils.cc +++ b/mediapipe/util/image_test_utils.cc @@ -17,6 +17,34 @@ namespace mediapipe { +namespace { + +template +ImageFrame CreateTestImageFrame(int width, int height, DataType max_value) { + ImageFrame image_frame(Format, width, height, + /*alignment_boundary=*/1); + const int num_channels = image_frame.NumberOfChannels(); + const float num_values = width * height * num_channels; + uint8_t* const data_ptr = + reinterpret_cast(image_frame.MutablePixelData()); + for (int y = 0; y < height; ++y) { + uint8_t* const row = data_ptr + image_frame.WidthStep() * y; + for (int x = 0; x < width; ++x) { + DataType* pixel = reinterpret_cast(row) + x * num_channels; + for (int c = 0; c < num_channels; ++c) { + // Fill pixel channel with a value in [0:max_value] range. + pixel[c] = + static_cast(static_cast(y * width * num_channels + + x * num_channels + c) / + num_values * max_value); + } + } + } + return image_frame; +} + +} // namespace + cv::Mat GetRgb(const std::string& path) { cv::Mat bgr = cv::imread(path); cv::Mat rgb; @@ -71,4 +99,14 @@ cv::Mat RgbaToBgr(cv::Mat rgba) { return bgra; } +ImageFrame CreateTestFloat32ImageFrame(int width, int height) { + return CreateTestImageFrame(width, height, + /*max_value=*/1.0f); +} + +ImageFrame CreateTestGrey8ImageFrame(int width, int height) { + return CreateTestImageFrame(width, height, + /*max_value=*/255); +} + } // namespace mediapipe diff --git a/mediapipe/util/image_test_utils.h b/mediapipe/util/image_test_utils.h index 15a21c5b1..49943382f 100644 --- a/mediapipe/util/image_test_utils.h +++ b/mediapipe/util/image_test_utils.h @@ -4,6 +4,7 @@ #include #include "mediapipe/framework/formats/image_format.pb.h" +#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/packet.h" #include "mediapipe/framework/port/opencv_core_inc.h" @@ -30,6 +31,12 @@ Packet MakeImagePacket(cv::Mat input, int timestamp = 0); // Converts RGBA Mat to BGR. cv::Mat RgbaToBgr(cv::Mat rgba); +// Generates single-channel float32 ImageFrame with increasing [0,1] values. +ImageFrame CreateTestFloat32ImageFrame(int width, int height); + +// Generates single-channel uint8 ImageFrame with increasing [0,255] values. +ImageFrame CreateTestGrey8ImageFrame(int width, int height); + } // namespace mediapipe #endif // MEDIAPIPE_UTIL_IMAGE_TEST_UTILS_H_ From b879e3a2041b71c8a5264a650a8f3fed6fb1437f Mon Sep 17 00:00:00 2001 From: Kinar Date: Thu, 16 Nov 2023 10:05:34 -0800 Subject: [PATCH 119/157] Updated components and their tests in the C Tasks API --- .../detection_result_converter_test.cc | 50 +++++++++++++++++++ .../containers/keypoint_converter.cc | 3 +- .../containers/keypoint_converter_test.cc | 26 ++++++++-- .../c/components/containers/rect_converter.cc | 2 - .../containers/rect_converter_test.cc | 2 +- 5 files changed, 74 insertions(+), 9 deletions(-) diff --git a/mediapipe/tasks/c/components/containers/detection_result_converter_test.cc b/mediapipe/tasks/c/components/containers/detection_result_converter_test.cc index 884481f29..2fd85bf31 100644 --- a/mediapipe/tasks/c/components/containers/detection_result_converter_test.cc +++ b/mediapipe/tasks/c/components/containers/detection_result_converter_test.cc @@ -25,4 +25,54 @@ limitations under the License. namespace mediapipe::tasks::c::components::containers { +TEST(DetectionResultConverterTest, ConvertsDetectionResultCustomCategory) { + mediapipe::tasks::components::containers::DetectionResult + cpp_detection_result = {/* detections= */ { + {/* categories= */ {{/* index= */ 1, /* score= */ 0.1, + /* category_name= */ "cat", + /* display_name= */ "cat"}}, + /* bounding_box= */ {10, 10, 10, 10}, + {/* keypoints */ {{0.1, 0.1, "foo", 0.5}}}}}}; + + DetectionResult c_detection_result; + CppConvertToDetectionResult(cpp_detection_result, &c_detection_result); + EXPECT_NE(c_detection_result.detections, nullptr); + EXPECT_EQ(c_detection_result.detections_count, 1); + EXPECT_NE(c_detection_result.detections[0].categories, nullptr); + EXPECT_EQ(c_detection_result.detections[0].categories_count, 1); + EXPECT_EQ(c_detection_result.detections[0].bounding_box.left, 10); + EXPECT_EQ(c_detection_result.detections[0].bounding_box.top, 10); + EXPECT_EQ(c_detection_result.detections[0].bounding_box.right, 10); + EXPECT_EQ(c_detection_result.detections[0].bounding_box.bottom, 10); + EXPECT_NE(c_detection_result.detections[0].keypoints, nullptr); + + CppCloseDetectionResult(&c_detection_result); +} + +TEST(DetectionResultConverterTest, ConvertsDetectionResultNoCategory) { + mediapipe::tasks::components::containers::DetectionResult + cpp_detection_result = {/* detections= */ {/* categories= */ {}}}; + + DetectionResult c_detection_result; + CppConvertToDetectionResult(cpp_detection_result, &c_detection_result); + EXPECT_NE(c_detection_result.detections, nullptr); + EXPECT_EQ(c_detection_result.detections_count, 1); + EXPECT_NE(c_detection_result.detections[0].categories, nullptr); + EXPECT_EQ(c_detection_result.detections[0].categories_count, 0); + + CppCloseDetectionResult(&c_detection_result); +} + +TEST(DetectionResultConverterTest, FreesMemory) { + mediapipe::tasks::components::containers::DetectionResult + cpp_detection_result = {/* detections= */ {{/* categories= */ {}}}}; + + DetectionResult c_detection_result; + CppConvertToDetectionResult(cpp_detection_result, &c_detection_result); + EXPECT_NE(c_detection_result.detections, nullptr); + + CppCloseDetectionResult(&c_detection_result); + EXPECT_EQ(c_detection_result.detections, nullptr); +} + } // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/keypoint_converter.cc b/mediapipe/tasks/c/components/containers/keypoint_converter.cc index 53e8a5da1..2d64e8063 100644 --- a/mediapipe/tasks/c/components/containers/keypoint_converter.cc +++ b/mediapipe/tasks/c/components/containers/keypoint_converter.cc @@ -15,7 +15,6 @@ limitations under the License. #include "mediapipe/tasks/c/components/containers/keypoint_converter.h" -#include #include #include @@ -38,7 +37,7 @@ void CppConvertToNormalizedKeypoint( void CppCloseNormalizedKeypoint(NormalizedKeypoint* keypoint) { if (keypoint && keypoint->label) { free(keypoint->label); - keypoint->label = NULL; + keypoint->label = nullptr; } } diff --git a/mediapipe/tasks/c/components/containers/keypoint_converter_test.cc b/mediapipe/tasks/c/components/containers/keypoint_converter_test.cc index ca09154c3..38bf1e3c6 100644 --- a/mediapipe/tasks/c/components/containers/keypoint_converter_test.cc +++ b/mediapipe/tasks/c/components/containers/keypoint_converter_test.cc @@ -25,11 +25,29 @@ limitations under the License. namespace mediapipe::tasks::c::components::containers { -TEST(RectConverterTest, ConvertsRectCustomValues) { - mediapipe::tasks::components::containers::Rect cpp_rect = {0, 0, 0, 0}; +constexpr float kPrecision = 1e-6; - Rect c_rect; - CppConvertToRect(cpp_rect, &c_rect); +TEST(KeypointConverterTest, ConvertsKeypointCustomValues) { + mediapipe::tasks::components::containers::NormalizedKeypoint cpp_keypoint = { + 0.1, 0.1, "foo", 0.5}; + + NormalizedKeypoint c_keypoint; + CppConvertToNormalizedKeypoint(cpp_keypoint, &c_keypoint); + EXPECT_NEAR(c_keypoint.x, 0.1f, kPrecision); + EXPECT_NEAR(c_keypoint.x, 0.1f, kPrecision); + EXPECT_EQ(std::string(c_keypoint.label), "foo"); + EXPECT_NEAR(c_keypoint.score, 0.5f, kPrecision); +} + +TEST(KeypointConverterTest, FreesMemory) { + mediapipe::tasks::components::containers::NormalizedKeypoint cpp_keypoint = { + 0.1, 0.1, "foo", 0.5}; + + NormalizedKeypoint c_keypoint; + CppConvertToNormalizedKeypoint(cpp_keypoint, &c_keypoint); + EXPECT_NE(c_keypoint.label, nullptr); + CppCloseNormalizedKeypoint(&c_keypoint); + EXPECT_EQ(c_keypoint.label, nullptr); } } // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/rect_converter.cc b/mediapipe/tasks/c/components/containers/rect_converter.cc index ff700acee..9f30bec4e 100644 --- a/mediapipe/tasks/c/components/containers/rect_converter.cc +++ b/mediapipe/tasks/c/components/containers/rect_converter.cc @@ -15,8 +15,6 @@ limitations under the License. #include "mediapipe/tasks/c/components/containers/rect_converter.h" -#include - #include "mediapipe/tasks/c/components/containers/rect.h" #include "mediapipe/tasks/cc/components/containers/rect.h" diff --git a/mediapipe/tasks/c/components/containers/rect_converter_test.cc b/mediapipe/tasks/c/components/containers/rect_converter_test.cc index 3e8848094..eb2107240 100644 --- a/mediapipe/tasks/c/components/containers/rect_converter_test.cc +++ b/mediapipe/tasks/c/components/containers/rect_converter_test.cc @@ -41,7 +41,7 @@ TEST(RectFConverterTest, ConvertsRectFCustomValues) { 0.1}; RectF c_rect; - CppConvertToRect(cpp_rect, &c_rect); + CppConvertToRectF(cpp_rect, &c_rect); EXPECT_FLOAT_EQ(c_rect.left, 0.1); EXPECT_FLOAT_EQ(c_rect.right, 0.1); EXPECT_FLOAT_EQ(c_rect.top, 0.1); From 8f32fda6d82aa66c85ce9f5bde1017c4fcdd0079 Mon Sep 17 00:00:00 2001 From: Kinar Date: Thu, 16 Nov 2023 12:53:36 -0800 Subject: [PATCH 120/157] Added more benchmark scripts for the Tasks Python API --- mediapipe/tasks/python/benchmark/BUILD | 24 ++++ .../tasks/python/benchmark/benchmark_utils.py | 58 ++++++++ .../tasks/python/benchmark/vision/core/BUILD | 22 +++ .../python/benchmark/vision/core/__init__.py | 14 ++ .../vision/core/base_vision_benchmark_api.py | 44 ++++++ .../benchmark/vision/face_aligner/BUILD | 34 +++++ .../face_aligner/face_aligner_benchmark.py | 120 ++++++++++++++++ .../benchmark/vision/face_detector/BUILD | 34 +++++ .../face_detector/face_detector_benchmark.py | 120 ++++++++++++++++ .../benchmark/vision/face_landmarker/BUILD | 34 +++++ .../face_landmarker_benchmark.py | 120 ++++++++++++++++ .../benchmark/vision/hand_landmarker/BUILD | 34 +++++ .../hand_landmarker_benchmark.py | 120 ++++++++++++++++ .../benchmark/vision/image_classifier/BUILD | 9 +- .../image_classifier_benchmark.py | 38 +++--- .../benchmark/vision/image_embedder/BUILD | 34 +++++ .../image_embedder_benchmark.py | 120 ++++++++++++++++ .../benchmark/vision/image_segmenter/BUILD | 34 +++++ .../image_segmenter_benchmark.py | 121 +++++++++++++++++ .../vision/interactive_segmenter/BUILD | 34 +++++ .../interactive_segmenter_benchmark.py | 128 ++++++++++++++++++ .../benchmark/vision/object_detector/BUILD | 34 +++++ .../object_detector_benchmark.py | 120 ++++++++++++++++ .../benchmark/vision/pose_landmarker/BUILD | 34 +++++ .../pose_landmarker_benchmark.py | 120 ++++++++++++++++ 25 files changed, 1586 insertions(+), 18 deletions(-) create mode 100644 mediapipe/tasks/python/benchmark/BUILD create mode 100644 mediapipe/tasks/python/benchmark/benchmark_utils.py create mode 100644 mediapipe/tasks/python/benchmark/vision/core/BUILD create mode 100644 mediapipe/tasks/python/benchmark/vision/core/__init__.py create mode 100644 mediapipe/tasks/python/benchmark/vision/core/base_vision_benchmark_api.py create mode 100644 mediapipe/tasks/python/benchmark/vision/face_aligner/BUILD create mode 100644 mediapipe/tasks/python/benchmark/vision/face_aligner/face_aligner_benchmark.py create mode 100644 mediapipe/tasks/python/benchmark/vision/face_detector/BUILD create mode 100644 mediapipe/tasks/python/benchmark/vision/face_detector/face_detector_benchmark.py create mode 100644 mediapipe/tasks/python/benchmark/vision/face_landmarker/BUILD create mode 100644 mediapipe/tasks/python/benchmark/vision/face_landmarker/face_landmarker_benchmark.py create mode 100644 mediapipe/tasks/python/benchmark/vision/hand_landmarker/BUILD create mode 100644 mediapipe/tasks/python/benchmark/vision/hand_landmarker/hand_landmarker_benchmark.py create mode 100644 mediapipe/tasks/python/benchmark/vision/image_embedder/BUILD create mode 100644 mediapipe/tasks/python/benchmark/vision/image_embedder/image_embedder_benchmark.py create mode 100644 mediapipe/tasks/python/benchmark/vision/image_segmenter/BUILD create mode 100644 mediapipe/tasks/python/benchmark/vision/image_segmenter/image_segmenter_benchmark.py create mode 100644 mediapipe/tasks/python/benchmark/vision/interactive_segmenter/BUILD create mode 100644 mediapipe/tasks/python/benchmark/vision/interactive_segmenter/interactive_segmenter_benchmark.py create mode 100644 mediapipe/tasks/python/benchmark/vision/object_detector/BUILD create mode 100644 mediapipe/tasks/python/benchmark/vision/object_detector/object_detector_benchmark.py create mode 100644 mediapipe/tasks/python/benchmark/vision/pose_landmarker/BUILD create mode 100644 mediapipe/tasks/python/benchmark/vision/pose_landmarker/pose_landmarker_benchmark.py diff --git a/mediapipe/tasks/python/benchmark/BUILD b/mediapipe/tasks/python/benchmark/BUILD new file mode 100644 index 000000000..fc5836da5 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/BUILD @@ -0,0 +1,24 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +py_library( + name = "benchmark_utils", + srcs = ["benchmark_utils.py"] +) diff --git a/mediapipe/tasks/python/benchmark/benchmark_utils.py b/mediapipe/tasks/python/benchmark/benchmark_utils.py new file mode 100644 index 000000000..5c1b102a2 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/benchmark_utils.py @@ -0,0 +1,58 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Benchmark utils for MediaPipe Tasks.""" + +import os + + +def get_test_data_path(test_srcdir, file_or_dirname_path: str) -> str: + """Determine the test data path. + + Args: + test_srcdir: The path to the test source directory. + file_or_dirname_path: The path to the file or directory. + + Returns: + The full test data path. + """ + """Returns full test data path.""" + for directory, subdirs, files in os.walk(test_srcdir): + for f in subdirs + files: + path = os.path.join(directory, f) + if path.endswith(file_or_dirname_path): + return path + raise ValueError( + "No %s in test directory: %s." % (file_or_dirname_path, test_srcdir) + ) + + +def get_model_path(custom_model, default_model_path): + """Determine the model path based on the existence of the custom model. + + Args: + custom_model: The path to the custom model provided by the user. + default_model_path: The path to the default model. + + Returns: + The path to the model to be used. + """ + if custom_model is not None and os.path.exists(custom_model): + print(f"Using provided model: {custom_model}") + return custom_model + else: + if custom_model is not None: + print(f"Warning: Provided model '{custom_model}' not found. " + f"Using default model instead.") + print(f"Using default model: {default_model_path}") + return default_model_path diff --git a/mediapipe/tasks/python/benchmark/vision/core/BUILD b/mediapipe/tasks/python/benchmark/vision/core/BUILD new file mode 100644 index 000000000..48bfc6522 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/core/BUILD @@ -0,0 +1,22 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "base_vision_benchmark_api", + srcs = ["base_vision_benchmark_api.py"] +) diff --git a/mediapipe/tasks/python/benchmark/vision/core/__init__.py b/mediapipe/tasks/python/benchmark/vision/core/__init__.py new file mode 100644 index 000000000..b87aebd51 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/core/__init__.py @@ -0,0 +1,14 @@ +"""Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" diff --git a/mediapipe/tasks/python/benchmark/vision/core/base_vision_benchmark_api.py b/mediapipe/tasks/python/benchmark/vision/core/base_vision_benchmark_api.py new file mode 100644 index 000000000..65c460613 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/core/base_vision_benchmark_api.py @@ -0,0 +1,44 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe vision benchmark base api.""" +import os +import time +import numpy as np + +VISION_TEST_DATA_DIR = 'mediapipe/tasks/testdata/vision' + + +def nth_percentile(func, image, n_iterations, percentile): + """Run a nth percentile benchmark for a given task using the function. + + Args: + func: The method associated with a given task used for benchmarking. + image: The input MediaPipe Image. + n_iterations: Number of iterations to run the benchmark. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + + Returns: + The n-th percentile of the inference times in milliseconds. + """ + inference_times = [] + + for _ in range(n_iterations): + start_time_ns = time.time_ns() + # Run the method for the task (e.g., classify) + func(image) + end_time_ns = time.time_ns() + inference_times.append((end_time_ns - start_time_ns) / 1_000_000) + + return np.percentile(inference_times, percentile) diff --git a/mediapipe/tasks/python/benchmark/vision/face_aligner/BUILD b/mediapipe/tasks/python/benchmark/vision/face_aligner/BUILD new file mode 100644 index 000000000..61791070f --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/face_aligner/BUILD @@ -0,0 +1,34 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "face_aligner_benchmark", + main = "face_aligner_benchmark.py", + srcs = ["face_aligner_benchmark.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/vision:face_aligner", + "//mediapipe/tasks/python/benchmark:benchmark_utils", + "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + ], +) diff --git a/mediapipe/tasks/python/benchmark/vision/face_aligner/face_aligner_benchmark.py b/mediapipe/tasks/python/benchmark/vision/face_aligner/face_aligner_benchmark.py new file mode 100644 index 000000000..e51ae056a --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/face_aligner/face_aligner_benchmark.py @@ -0,0 +1,120 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe face aligner benchmark.""" + +import argparse + +from mediapipe.python._framework_bindings import image +from mediapipe.tasks.python.core import base_options +from mediapipe.tasks.python.vision import face_aligner +from mediapipe.tasks.python.benchmark import benchmark_utils +from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api + +_MODEL_FILE = 'face_landmarker_v2.task' +_IMAGE_FILE = 'portrait.jpg' + + +def run( + model: str, + n_iterations: int, + delegate: base_options.BaseOptions.Delegate, + percentile: float, +): + """Run an face aligner benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + + Returns: + The n-th percentile of the inference times. + """ + # Initialize the face aligner + default_model_path = benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE + ) + model_path = benchmark_utils.get_model_path(model, default_model_path) + options = face_aligner.FaceAlignerOptions( + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) + ) + + with face_aligner.FaceAligner.create_from_options(options) as aligner: + mp_image = image.Image.create_from_file( + benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE + ) + ) + # Run the benchmark and return the nth percentile of the inference times + nth_percentile = base_vision_benchmark_api.nth_percentile( + aligner.align, mp_image, n_iterations, percentile + ) + return nth_percentile + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + '--model', + help='Path to face aligner task.', + required=False, + default=None, + ) + parser.add_argument( + '--iterations', + help='Number of iterations for benchmarking.', + type=int, + default=100, + ) + parser.add_argument( + '--percentile', + help='Percentile for benchmarking statistics.', + type=float, + default=95.0, + ) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.CPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on CPU: ' + f'{cpu_time:.6f} milliseconds' + ) + + # Run benchmark on GPU + gpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.GPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on GPU: ' + f'{gpu_time:.6f} milliseconds' + ) + + +if __name__ == '__main__': + main() diff --git a/mediapipe/tasks/python/benchmark/vision/face_detector/BUILD b/mediapipe/tasks/python/benchmark/vision/face_detector/BUILD new file mode 100644 index 000000000..9eb67c19f --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/face_detector/BUILD @@ -0,0 +1,34 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "face_detector_benchmark", + main = "face_detector_benchmark.py", + srcs = ["face_detector_benchmark.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/vision:face_detector", + "//mediapipe/tasks/python/benchmark:benchmark_utils", + "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + ], +) diff --git a/mediapipe/tasks/python/benchmark/vision/face_detector/face_detector_benchmark.py b/mediapipe/tasks/python/benchmark/vision/face_detector/face_detector_benchmark.py new file mode 100644 index 000000000..c4cd75100 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/face_detector/face_detector_benchmark.py @@ -0,0 +1,120 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe face detector benchmark.""" + +import argparse + +from mediapipe.python._framework_bindings import image +from mediapipe.tasks.python.core import base_options +from mediapipe.tasks.python.vision import face_detector +from mediapipe.tasks.python.benchmark import benchmark_utils +from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api + +_MODEL_FILE = 'face_detection_short_range.tflite' +_IMAGE_FILE = 'portrait.jpg' + + +def run( + model: str, + n_iterations: int, + delegate: base_options.BaseOptions.Delegate, + percentile: float, +): + """Run an face detector benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + + Returns: + The n-th percentile of the inference times. + """ + # Initialize the face detector + default_model_path = benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE + ) + model_path = benchmark_utils.get_model_path(model, default_model_path) + options = face_detector.FaceDetectorOptions( + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) + ) + + with face_detector.FaceDetector.create_from_options(options) as detector: + mp_image = image.Image.create_from_file( + benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE + ) + ) + # Run the benchmark and return the nth percentile of the inference times + nth_percentile = base_vision_benchmark_api.nth_percentile( + detector.detect, mp_image, n_iterations, percentile + ) + return nth_percentile + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + '--model', + help='Path to face detector task.', + required=False, + default=None, + ) + parser.add_argument( + '--iterations', + help='Number of iterations for benchmarking.', + type=int, + default=100, + ) + parser.add_argument( + '--percentile', + help='Percentile for benchmarking statistics.', + type=float, + default=95.0, + ) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.CPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on CPU: ' + f'{cpu_time:.6f} milliseconds' + ) + + # Run benchmark on GPU + gpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.GPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on GPU: ' + f'{gpu_time:.6f} milliseconds' + ) + + +if __name__ == '__main__': + main() diff --git a/mediapipe/tasks/python/benchmark/vision/face_landmarker/BUILD b/mediapipe/tasks/python/benchmark/vision/face_landmarker/BUILD new file mode 100644 index 000000000..70dd311e4 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/face_landmarker/BUILD @@ -0,0 +1,34 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "face_landmarker_benchmark", + main = "face_landmarker_benchmark.py", + srcs = ["face_landmarker_benchmark.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/vision:face_landmarker", + "//mediapipe/tasks/python/benchmark:benchmark_utils", + "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + ], +) diff --git a/mediapipe/tasks/python/benchmark/vision/face_landmarker/face_landmarker_benchmark.py b/mediapipe/tasks/python/benchmark/vision/face_landmarker/face_landmarker_benchmark.py new file mode 100644 index 000000000..7a1f3f817 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/face_landmarker/face_landmarker_benchmark.py @@ -0,0 +1,120 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe face landmarker benchmark.""" + +import argparse + +from mediapipe.python._framework_bindings import image +from mediapipe.tasks.python.core import base_options +from mediapipe.tasks.python.vision import face_landmarker +from mediapipe.tasks.python.benchmark import benchmark_utils +from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api + +_MODEL_FILE = 'face_landmarker_v2.task' +_IMAGE_FILE = 'portrait.jpg' + + +def run( + model: str, + n_iterations: int, + delegate: base_options.BaseOptions.Delegate, + percentile: float, +): + """Run an face landmarker benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + + Returns: + The n-th percentile of the inference times. + """ + # Initialize the face landmarker + default_model_path = benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE + ) + model_path = benchmark_utils.get_model_path(model, default_model_path) + options = face_landmarker.FaceLandmarkerOptions( + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) + ) + + with face_landmarker.FaceLandmarker.create_from_options(options) as landmarker: + mp_image = image.Image.create_from_file( + benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE + ) + ) + # Run the benchmark and return the nth percentile of the inference times + nth_percentile = base_vision_benchmark_api.nth_percentile( + landmarker.detect, mp_image, n_iterations, percentile + ) + return nth_percentile + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + '--model', + help='Path to face landmarker task.', + required=False, + default=None, + ) + parser.add_argument( + '--iterations', + help='Number of iterations for benchmarking.', + type=int, + default=100, + ) + parser.add_argument( + '--percentile', + help='Percentile for benchmarking statistics.', + type=float, + default=95.0, + ) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.CPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on CPU: ' + f'{cpu_time:.6f} milliseconds' + ) + + # Run benchmark on GPU + gpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.GPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on GPU: ' + f'{gpu_time:.6f} milliseconds' + ) + + +if __name__ == '__main__': + main() diff --git a/mediapipe/tasks/python/benchmark/vision/hand_landmarker/BUILD b/mediapipe/tasks/python/benchmark/vision/hand_landmarker/BUILD new file mode 100644 index 000000000..693b42faf --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/hand_landmarker/BUILD @@ -0,0 +1,34 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "hand_landmarker_benchmark", + main = "hand_landmarker_benchmark.py", + srcs = ["hand_landmarker_benchmark.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/vision:hand_landmarker", + "//mediapipe/tasks/python/benchmark:benchmark_utils", + "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + ], +) diff --git a/mediapipe/tasks/python/benchmark/vision/hand_landmarker/hand_landmarker_benchmark.py b/mediapipe/tasks/python/benchmark/vision/hand_landmarker/hand_landmarker_benchmark.py new file mode 100644 index 000000000..8fd8fc210 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/hand_landmarker/hand_landmarker_benchmark.py @@ -0,0 +1,120 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe hand landmarker benchmark.""" + +import argparse + +from mediapipe.python._framework_bindings import image +from mediapipe.tasks.python.core import base_options +from mediapipe.tasks.python.vision import hand_landmarker +from mediapipe.tasks.python.benchmark import benchmark_utils +from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api + +_MODEL_FILE = 'hand_landmarker.task' +_IMAGE_FILE = 'thumb_up.jpg' + + +def run( + model: str, + n_iterations: int, + delegate: base_options.BaseOptions.Delegate, + percentile: float, +): + """Run an hand landmarker benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + + Returns: + The n-th percentile of the inference times. + """ + # Initialize the hand landmarker + default_model_path = benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE + ) + model_path = benchmark_utils.get_model_path(model, default_model_path) + options = hand_landmarker.HandLandmarkerOptions( + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) + ) + + with hand_landmarker.HandLandmarker.create_from_options(options) as landmarker: + mp_image = image.Image.create_from_file( + benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE + ) + ) + # Run the benchmark and return the nth percentile of the inference times + nth_percentile = base_vision_benchmark_api.nth_percentile( + landmarker.detect, mp_image, n_iterations, percentile + ) + return nth_percentile + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + '--model', + help='Path to hand landmarker task.', + required=False, + default=None, + ) + parser.add_argument( + '--iterations', + help='Number of iterations for benchmarking.', + type=int, + default=100, + ) + parser.add_argument( + '--percentile', + help='Percentile for benchmarking statistics.', + type=float, + default=95.0, + ) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.CPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on CPU: ' + f'{cpu_time:.6f} milliseconds' + ) + + # Run benchmark on GPU + gpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.GPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on GPU: ' + f'{gpu_time:.6f} milliseconds' + ) + + +if __name__ == '__main__': + main() diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/BUILD b/mediapipe/tasks/python/benchmark/vision/image_classifier/BUILD index aec17ea9d..738b9738a 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/BUILD +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/BUILD @@ -16,12 +16,19 @@ package(default_visibility = ["//visibility:public"]) -py_library( +py_binary( name = "image_classifier_benchmark", + main = "image_classifier_benchmark.py", srcs = ["image_classifier_benchmark.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], deps = [ "//mediapipe/python:_framework_bindings", "//mediapipe/tasks/python/core:base_options", "//mediapipe/tasks/python/vision:image_classifier", + "//mediapipe/tasks/python/benchmark:benchmark_utils", + "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", ], ) diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py index dffd616fd..f24b7dcd5 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -14,12 +14,14 @@ """MediaPipe image classsifier benchmark.""" import argparse -import time -import numpy as np + from mediapipe.python._framework_bindings import image from mediapipe.tasks.python.core import base_options from mediapipe.tasks.python.vision import image_classifier +from mediapipe.tasks.python.benchmark import benchmark_utils +from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api +_MODEL_FILE = 'mobilenet_v2_1.0_224.tflite' _IMAGE_FILE = 'burger.jpg' @@ -41,27 +43,29 @@ def run( Returns: The n-th percentile of the inference times. """ - inference_times = [] - # Initialize the image classifier + default_model_path = benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE + ) + model_path = benchmark_utils.get_model_path(model, default_model_path) options = image_classifier.ImageClassifierOptions( base_options=base_options.BaseOptions( - model_asset_path=model, delegate=delegate + model_asset_path=model_path, delegate=delegate ), max_results=1, ) - classifier = image_classifier.ImageClassifier.create_from_options(options) - mp_image = image.Image.create_from_file(_IMAGE_FILE) - for _ in range(n_iterations): - start_time_ns = time.time_ns() - classifier.classify(mp_image) - end_time_ns = time.time_ns() - # Convert to milliseconds - inference_times.append((end_time_ns - start_time_ns) / 1_000_000) - - classifier.close() - return np.percentile(inference_times, percentile) + with image_classifier.ImageClassifier.create_from_options(options) as classifier: + mp_image = image.Image.create_from_file( + benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE + ) + ) + # Run the benchmark and return the nth percentile of the inference times + nth_percentile = base_vision_benchmark_api.nth_percentile( + classifier.classify, mp_image, n_iterations, percentile + ) + return nth_percentile def main(): @@ -72,7 +76,7 @@ def main(): '--model', help='Path to image classification model.', required=False, - default='classifier.tflite', + default=None, ) parser.add_argument( '--iterations', diff --git a/mediapipe/tasks/python/benchmark/vision/image_embedder/BUILD b/mediapipe/tasks/python/benchmark/vision/image_embedder/BUILD new file mode 100644 index 000000000..059bdd095 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_embedder/BUILD @@ -0,0 +1,34 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "image_embedder_benchmark", + main = "image_embedder_benchmark.py", + srcs = ["image_embedder_benchmark.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/vision:image_embedder", + "//mediapipe/tasks/python/benchmark:benchmark_utils", + "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + ], +) diff --git a/mediapipe/tasks/python/benchmark/vision/image_embedder/image_embedder_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_embedder/image_embedder_benchmark.py new file mode 100644 index 000000000..1da3b8e89 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_embedder/image_embedder_benchmark.py @@ -0,0 +1,120 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe image embedder benchmark.""" + +import argparse + +from mediapipe.python._framework_bindings import image +from mediapipe.tasks.python.core import base_options +from mediapipe.tasks.python.vision import image_embedder +from mediapipe.tasks.python.benchmark import benchmark_utils +from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api + +_MODEL_FILE = 'mobilenet_v3_small_100_224_embedder.tflite' +_IMAGE_FILE = 'burger.jpg' + + +def run( + model: str, + n_iterations: int, + delegate: base_options.BaseOptions.Delegate, + percentile: float, +): + """Run an image embedding extraction benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + + Returns: + The n-th percentile of the inference times. + """ + # Initialize the image embedder + default_model_path = benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE + ) + model_path = benchmark_utils.get_model_path(model, default_model_path) + options = image_embedder.ImageEmbedderOptions( + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) + ) + + with image_embedder.ImageEmbedder.create_from_options(options) as embedder: + mp_image = image.Image.create_from_file( + benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE + ) + ) + # Run the benchmark and return the nth percentile of the inference times + nth_percentile = base_vision_benchmark_api.nth_percentile( + embedder.embed, mp_image, n_iterations, percentile + ) + return nth_percentile + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + '--model', + help='Path to image embedding extraction model.', + required=False, + default=None, + ) + parser.add_argument( + '--iterations', + help='Number of iterations for benchmarking.', + type=int, + default=100, + ) + parser.add_argument( + '--percentile', + help='Percentile for benchmarking statistics.', + type=float, + default=95.0, + ) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.CPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on CPU: ' + f'{cpu_time:.6f} milliseconds' + ) + + # Run benchmark on GPU + gpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.GPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on GPU: ' + f'{gpu_time:.6f} milliseconds' + ) + + +if __name__ == '__main__': + main() diff --git a/mediapipe/tasks/python/benchmark/vision/image_segmenter/BUILD b/mediapipe/tasks/python/benchmark/vision/image_segmenter/BUILD new file mode 100644 index 000000000..bfb2ee763 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_segmenter/BUILD @@ -0,0 +1,34 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "image_segmenter_benchmark", + main = "image_segmenter_benchmark.py", + srcs = ["image_segmenter_benchmark.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/vision:image_segmenter", + "//mediapipe/tasks/python/benchmark:benchmark_utils", + "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + ], +) diff --git a/mediapipe/tasks/python/benchmark/vision/image_segmenter/image_segmenter_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_segmenter/image_segmenter_benchmark.py new file mode 100644 index 000000000..ef885b11c --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/image_segmenter/image_segmenter_benchmark.py @@ -0,0 +1,121 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe image segmenter benchmark.""" + +import argparse + +from mediapipe.python._framework_bindings import image +from mediapipe.tasks.python.core import base_options +from mediapipe.tasks.python.vision import image_segmenter +from mediapipe.tasks.python.benchmark import benchmark_utils +from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api + +_MODEL_FILE = 'deeplabv3.tflite' +_IMAGE_FILE = 'segmentation_input_rotation0.jpg' + + +def run( + model: str, + n_iterations: int, + delegate: base_options.BaseOptions.Delegate, + percentile: float, +): + """Run an image segmentation benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + + Returns: + The n-th percentile of the inference times. + """ + # Initialize the image segmenter + default_model_path = benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE + ) + model_path = benchmark_utils.get_model_path(model, default_model_path) + options = image_segmenter.ImageSegmenterOptions( + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ), + output_confidence_masks=True, output_category_mask=True + ) + + with image_segmenter.ImageSegmenter.create_from_options(options) as segmenter: + mp_image = image.Image.create_from_file( + benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE + ) + ) + # Run the benchmark and return the nth percentile of the inference times + nth_percentile = base_vision_benchmark_api.nth_percentile( + segmenter.segment, mp_image, n_iterations, percentile + ) + return nth_percentile + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + '--model', + help='Path to image segmentation model.', + required=False, + default=None, + ) + parser.add_argument( + '--iterations', + help='Number of iterations for benchmarking.', + type=int, + default=100, + ) + parser.add_argument( + '--percentile', + help='Percentile for benchmarking statistics.', + type=float, + default=95.0, + ) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.CPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on CPU: ' + f'{cpu_time:.6f} milliseconds' + ) + + # Run benchmark on GPU + gpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.GPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on GPU: ' + f'{gpu_time:.6f} milliseconds' + ) + + +if __name__ == '__main__': + main() diff --git a/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/BUILD b/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/BUILD new file mode 100644 index 000000000..1a0bab418 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/BUILD @@ -0,0 +1,34 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "interactive_segmenter_benchmark", + main = "interactive_segmenter_benchmark.py", + srcs = ["interactive_segmenter_benchmark.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/vision:interactive_segmenter", + "//mediapipe/tasks/python/benchmark:benchmark_utils", + "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + ], +) diff --git a/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/interactive_segmenter_benchmark.py b/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/interactive_segmenter_benchmark.py new file mode 100644 index 000000000..3ecc7f661 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/interactive_segmenter_benchmark.py @@ -0,0 +1,128 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe interactive segmenter benchmark.""" + +from functools import partial +import argparse + +from mediapipe.python._framework_bindings import image +from mediapipe.tasks.python.core import base_options +from mediapipe.tasks.python.components.containers import keypoint +from mediapipe.tasks.python.vision import interactive_segmenter +from mediapipe.tasks.python.benchmark import benchmark_utils +from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api + +_MODEL_FILE = 'deeplabv3.tflite' +_IMAGE_FILE = 'segmentation_input_rotation0.jpg' + + +def run( + model: str, + n_iterations: int, + delegate: base_options.BaseOptions.Delegate, + percentile: float, +): + """Run an interactive segmentation benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + + Returns: + The n-th percentile of the inference times. + """ + # Initialize the interactive segmenter + default_model_path = benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE + ) + model_path = benchmark_utils.get_model_path(model, default_model_path) + + options = interactive_segmenter.InteractiveSegmenterOptions( + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ), + output_category_mask=True, output_confidence_masks=False + ) + roi = interactive_segmenter.RegionOfInterest( + format=interactive_segmenter.RegionOfInterest.Format.KEYPOINT, + keypoint=keypoint.NormalizedKeypoint(0.44, 0.7) + ) + + with interactive_segmenter.InteractiveSegmenter.create_from_options(options) as segmenter: + mp_image = image.Image.create_from_file( + benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE + ) + ) + # Run the benchmark and return the nth percentile of the inference times + nth_percentile = base_vision_benchmark_api.nth_percentile( + partial(segmenter.segment, roi=roi), mp_image, n_iterations, percentile + ) + return nth_percentile + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + '--model', + help='Path to interactive segmentation model.', + required=False, + default=None, + ) + parser.add_argument( + '--iterations', + help='Number of iterations for benchmarking.', + type=int, + default=100, + ) + parser.add_argument( + '--percentile', + help='Percentile for benchmarking statistics.', + type=float, + default=95.0, + ) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.CPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on CPU: ' + f'{cpu_time:.6f} milliseconds' + ) + + # Run benchmark on GPU + gpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.GPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on GPU: ' + f'{gpu_time:.6f} milliseconds' + ) + + +if __name__ == '__main__': + main() diff --git a/mediapipe/tasks/python/benchmark/vision/object_detector/BUILD b/mediapipe/tasks/python/benchmark/vision/object_detector/BUILD new file mode 100644 index 000000000..2db19db5e --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/object_detector/BUILD @@ -0,0 +1,34 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "object_detector_benchmark", + main = "object_detector_benchmark.py", + srcs = ["object_detector_benchmark.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/vision:object_detector", + "//mediapipe/tasks/python/benchmark:benchmark_utils", + "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + ], +) diff --git a/mediapipe/tasks/python/benchmark/vision/object_detector/object_detector_benchmark.py b/mediapipe/tasks/python/benchmark/vision/object_detector/object_detector_benchmark.py new file mode 100644 index 000000000..80f8302a2 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/object_detector/object_detector_benchmark.py @@ -0,0 +1,120 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe object detector benchmark.""" + +import argparse + +from mediapipe.python._framework_bindings import image +from mediapipe.tasks.python.core import base_options +from mediapipe.tasks.python.vision import object_detector +from mediapipe.tasks.python.benchmark import benchmark_utils +from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api + +_MODEL_FILE = 'coco_efficientdet_lite0_v1_1.0_quant_2021_09_06.tflite' +_IMAGE_FILE = 'cats_and_dogs.jpg' + + +def run( + model: str, + n_iterations: int, + delegate: base_options.BaseOptions.Delegate, + percentile: float, +): + """Run an object detector benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + + Returns: + The n-th percentile of the inference times. + """ + # Initialize the object detector + default_model_path = benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE + ) + model_path = benchmark_utils.get_model_path(model, default_model_path) + options = object_detector.ObjectDetectorOptions( + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) + ) + + with object_detector.ObjectDetector.create_from_options(options) as detector: + mp_image = image.Image.create_from_file( + benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE + ) + ) + # Run the benchmark and return the nth percentile of the inference times + nth_percentile = base_vision_benchmark_api.nth_percentile( + detector.detect, mp_image, n_iterations, percentile + ) + return nth_percentile + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + '--model', + help='Path to object detector model.', + required=False, + default=None, + ) + parser.add_argument( + '--iterations', + help='Number of iterations for benchmarking.', + type=int, + default=100, + ) + parser.add_argument( + '--percentile', + help='Percentile for benchmarking statistics.', + type=float, + default=95.0, + ) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.CPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on CPU: ' + f'{cpu_time:.6f} milliseconds' + ) + + # Run benchmark on GPU + gpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.GPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on GPU: ' + f'{gpu_time:.6f} milliseconds' + ) + + +if __name__ == '__main__': + main() diff --git a/mediapipe/tasks/python/benchmark/vision/pose_landmarker/BUILD b/mediapipe/tasks/python/benchmark/vision/pose_landmarker/BUILD new file mode 100644 index 000000000..38c778c00 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/pose_landmarker/BUILD @@ -0,0 +1,34 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "pose_landmarker_benchmark", + main = "pose_landmarker_benchmark.py", + srcs = ["pose_landmarker_benchmark.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/vision:pose_landmarker", + "//mediapipe/tasks/python/benchmark:benchmark_utils", + "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + ], +) diff --git a/mediapipe/tasks/python/benchmark/vision/pose_landmarker/pose_landmarker_benchmark.py b/mediapipe/tasks/python/benchmark/vision/pose_landmarker/pose_landmarker_benchmark.py new file mode 100644 index 000000000..a1c55f0b8 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/pose_landmarker/pose_landmarker_benchmark.py @@ -0,0 +1,120 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe pose landmarker benchmark.""" + +import argparse + +from mediapipe.python._framework_bindings import image +from mediapipe.tasks.python.core import base_options +from mediapipe.tasks.python.vision import pose_landmarker +from mediapipe.tasks.python.benchmark import benchmark_utils +from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api + +_MODEL_FILE = 'pose_landmarker.task' +_IMAGE_FILE = 'pose.jpg' + + +def run( + model: str, + n_iterations: int, + delegate: base_options.BaseOptions.Delegate, + percentile: float, +): + """Run an pose landmarker benchmark. + + Args: + model: Path to the TFLite model. + n_iterations: Number of iterations to run the benchmark. + delegate: CPU or GPU delegate for inference. + percentile: Percentage for the percentiles to compute. Values must be + between 0 and 100 inclusive. + + Returns: + The n-th percentile of the inference times. + """ + # Initialize the pose landmarker + default_model_path = benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE + ) + model_path = benchmark_utils.get_model_path(model, default_model_path) + options = pose_landmarker.PoseLandmarkerOptions( + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) + ) + + with pose_landmarker.PoseLandmarker.create_from_options(options) as landmarker: + mp_image = image.Image.create_from_file( + benchmark_utils.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE + ) + ) + # Run the benchmark and return the nth percentile of the inference times + nth_percentile = base_vision_benchmark_api.nth_percentile( + landmarker.detect, mp_image, n_iterations, percentile + ) + return nth_percentile + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + '--model', + help='Path to pose landmarker task.', + required=False, + default=None, + ) + parser.add_argument( + '--iterations', + help='Number of iterations for benchmarking.', + type=int, + default=100, + ) + parser.add_argument( + '--percentile', + help='Percentile for benchmarking statistics.', + type=float, + default=95.0, + ) + args = parser.parse_args() + + # Run benchmark on CPU + cpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.CPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on CPU: ' + f'{cpu_time:.6f} milliseconds' + ) + + # Run benchmark on GPU + gpu_time = run( + args.model, + args.iterations, + base_options.BaseOptions.Delegate.GPU, + args.percentile, + ) + print( + f'{args.percentile}th Percentile Inference Time on GPU: ' + f'{gpu_time:.6f} milliseconds' + ) + + +if __name__ == '__main__': + main() From e7c7638833d35517e4a89b0cc4c2e5ad0c875431 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 16 Nov 2023 15:36:20 -0800 Subject: [PATCH 121/157] No public description PiperOrigin-RevId: 583186277 --- mediapipe/calculators/tensor/BUILD | 40 +--- .../tensor/tensor_converter_calculator.cc | 120 ++++++++++-- .../tensor/tensor_converter_cpu.cc | 145 --------------- .../calculators/tensor/tensor_converter_cpu.h | 61 ------ .../tensor/tensor_converter_cpu_test.cc | 175 ------------------ mediapipe/util/image_test_utils.cc | 38 ---- mediapipe/util/image_test_utils.h | 7 - 7 files changed, 108 insertions(+), 478 deletions(-) delete mode 100644 mediapipe/calculators/tensor/tensor_converter_cpu.cc delete mode 100644 mediapipe/calculators/tensor/tensor_converter_cpu.h delete mode 100644 mediapipe/calculators/tensor/tensor_converter_cpu_test.cc diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index 96a29089e..e95398a9d 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -657,7 +657,6 @@ cc_library( }), deps = [ ":tensor_converter_calculator_cc_proto", - ":tensor_converter_cpu", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:port", "//mediapipe/framework/formats:image_frame", @@ -666,7 +665,6 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", - "//mediapipe/gpu:gpu_buffer", "//mediapipe/gpu:gpu_buffer_format", "//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/util:resource_util", @@ -676,17 +674,10 @@ cc_library( "@com_google_absl//absl/log:check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ] + select({ "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": [ - "tensor_converter_calculator_gpu_deps", - "//mediapipe/gpu:gl_base", - "//mediapipe/gpu:gl_calculator_helper", - "//mediapipe/gpu:gl_simple_shaders", - "//mediapipe/gpu:shader_util", - ], + "//conditions:default": ["tensor_converter_calculator_gpu_deps"], }) + select({ "//mediapipe:apple": [ "//third_party/apple_frameworks:MetalKit", @@ -696,35 +687,6 @@ cc_library( alwayslink = 1, ) -cc_library( - name = "tensor_converter_cpu", - srcs = ["tensor_converter_cpu.cc"], - hdrs = ["tensor_converter_cpu.h"], - deps = [ - "//mediapipe/framework/formats:image_frame", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:ret_check", - "//mediapipe/framework/port:status", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - ], -) - -cc_test( - name = "tensor_converter_cpu_test", - srcs = ["tensor_converter_cpu_test.cc"], - deps = [ - ":tensor_converter_cpu", - "//mediapipe/framework/formats:matrix", - "//mediapipe/framework/formats:tensor", - "//mediapipe/framework/port:gtest", - "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:status_matchers", - "//mediapipe/util:image_test_utils", - ], -) - cc_library( name = "tensor_converter_calculator_gpu_deps", visibility = ["//visibility:private"], diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator.cc b/mediapipe/calculators/tensor/tensor_converter_calculator.cc index 80cac63c7..b42cb0b17 100644 --- a/mediapipe/calculators/tensor/tensor_converter_calculator.cc +++ b/mediapipe/calculators/tensor/tensor_converter_calculator.cc @@ -14,7 +14,6 @@ #include #include -#include #include #include "absl/log/absl_check.h" @@ -22,21 +21,17 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_format.h" -#include "absl/strings/substitute.h" #include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h" -#include "mediapipe/calculators/tensor/tensor_converter_cpu.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port.h" #include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/gpu/gpu_buffer_format.h" #include "mediapipe/gpu/gpu_origin.pb.h" #if !MEDIAPIPE_DISABLE_GPU -#include "mediapipe/gpu/gl_base.h" #include "mediapipe/gpu/gpu_buffer.h" #if MEDIAPIPE_METAL_ENABLED #import @@ -99,6 +94,11 @@ absl::StatusOr ShouldFlipVertically( } } +typedef Eigen::Matrix + RowMajorMatrixXf; +typedef Eigen::Matrix + ColMajorMatrixXf; + constexpr char kImageFrameTag[] = "IMAGE"; constexpr char kGpuBufferTag[] = "IMAGE_GPU"; constexpr char kTensorsTag[] = "TENSORS"; @@ -156,6 +156,10 @@ class TensorConverterCalculator : public CalculatorBase { private: absl::Status InitGpu(CalculatorContext* cc); absl::Status LoadOptions(CalculatorContext* cc, bool use_gpu); + template + absl::Status NormalizeImage(const ImageFrame& image_frame, + bool flip_vertically, float* tensor_ptr); + absl::Status CopyMatrixToTensor(const Matrix& matrix, float* tensor_ptr); absl::Status ProcessCPU(CalculatorContext* cc); absl::Status ProcessGPU(CalculatorContext* cc); @@ -275,19 +279,46 @@ absl::Status TensorConverterCalculator::ProcessCPU(CalculatorContext* cc) { } const auto& image_frame = cc->Inputs().Tag(kImageFrameTag).Get(); - MP_ASSIGN_OR_RETURN( - Tensor output, - ConvertImageFrameToTensorOnCpu(image_frame, *output_range_, - flip_vertically_, max_num_channels_)); - output_tensors->emplace_back(std::move(output)); + const int height = image_frame.Height(); + const int width = image_frame.Width(); + const int channels = image_frame.NumberOfChannels(); + const int channels_preserved = std::min(channels, max_num_channels_); + const mediapipe::ImageFormat::Format format = image_frame.Format(); + + if (!(format == mediapipe::ImageFormat::SRGBA || + format == mediapipe::ImageFormat::SRGB || + format == mediapipe::ImageFormat::GRAY8 || + format == mediapipe::ImageFormat::VEC32F1)) + RET_CHECK_FAIL() << "Unsupported CPU input format."; + + output_tensors->emplace_back( + Tensor::ElementType::kFloat32, + Tensor::Shape{1, height, width, channels_preserved}); + auto cpu_view = output_tensors->back().GetCpuWriteView(); + + // Copy image data into tensor. + if (image_frame.ByteDepth() == 1) { + MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, + cpu_view.buffer())); + } else if (image_frame.ByteDepth() == 4) { + MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, + cpu_view.buffer())); + } else { + return absl::InternalError( + "Only byte-based (8 bit) and float (32 bit) images supported."); + } } else if (cc->Inputs().HasTag(kMatrixTag)) { if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) { return absl::OkStatus(); } const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get(); - MP_ASSIGN_OR_RETURN(Tensor output, - ConvertMatrixToTensorOnCpu(matrix, row_major_matrix_)); - output_tensors->emplace_back(std::move(output)); + const int height = matrix.rows(); + const int width = matrix.cols(); + const int channels = 1; + output_tensors->emplace_back(Tensor::ElementType::kFloat32, + Tensor::Shape{1, height, width, channels}); + MP_RETURN_IF_ERROR(CopyMatrixToTensor( + matrix, output_tensors->back().GetCpuWriteView().buffer())); } else { return absl::OkStatus(); } @@ -638,4 +669,67 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc, return absl::OkStatus(); } +template +absl::Status TensorConverterCalculator::NormalizeImage( + const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) { + const int height = image_frame.Height(); + const int width = image_frame.Width(); + const int channels = image_frame.NumberOfChannels(); + const int channels_preserved = std::min(channels, max_num_channels_); + const int channels_ignored = channels - channels_preserved; + + if (output_range_.has_value()) { + // If the output float range is set and we are not using custom + // normalization, normalize the pixel values from [0, 255] to the specified + // output range. + RET_CHECK_NE(output_range_->first, output_range_->second); + const float scale = (output_range_->second - output_range_->first) / 255.0f; + const float bias = output_range_->first; + + for (int i = 0; i < height; ++i) { + const T* image_ptr = reinterpret_cast( + image_frame.PixelData() + + (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); + for (int j = 0; j < width; ++j) { + for (int c = 0; c < channels_preserved; ++c) { + *tensor_ptr++ = *image_ptr++ * scale + bias; + } + image_ptr += channels_ignored; + } + } + } else { + // [0,1], scale only (bias == 0) + // Verified that there are no precision issues with 1.0f / 255.0f expression + const float scale = 1.0f / 255.0f; + for (int i = 0; i < height; ++i) { + const T* image_ptr = reinterpret_cast( + image_frame.PixelData() + + (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); + for (int j = 0; j < width; ++j) { + for (int c = 0; c < channels_preserved; ++c) { + *tensor_ptr++ = *image_ptr++ * scale; + } + image_ptr += channels_ignored; + } + } + } + + return absl::OkStatus(); +} + +absl::Status TensorConverterCalculator::CopyMatrixToTensor(const Matrix& matrix, + float* tensor_ptr) { + if (row_major_matrix_) { + auto matrix_map = + Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); + matrix_map = matrix; + } else { + auto matrix_map = + Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); + matrix_map = matrix; + } + + return absl::OkStatus(); +} + } // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensor_converter_cpu.cc b/mediapipe/calculators/tensor/tensor_converter_cpu.cc deleted file mode 100644 index f72a24c31..000000000 --- a/mediapipe/calculators/tensor/tensor_converter_cpu.cc +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/tensor_converter_cpu.h" - -#include -#include -#include - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/ret_check.h" -#include "mediapipe/framework/port/status_macros.h" - -namespace mediapipe { -namespace { - -typedef Eigen::Matrix - RowMajorMatrixXf; -typedef Eigen::Matrix - ColMajorMatrixXf; - -template -absl::Status NormalizeImage(const ImageFrame& image_frame, bool flip_vertically, - const std::pair& output_range, - int max_num_channels, float* tensor_ptr) { - const int height = image_frame.Height(); - const int width = image_frame.Width(); - const int channels = image_frame.NumberOfChannels(); - const int channels_preserved = std::min(channels, max_num_channels); - const int channels_ignored = channels - channels_preserved; - - RET_CHECK_NE(output_range.first, output_range.second); - const float scale = (output_range.second - output_range.first) / 255.0f; - const float bias = output_range.first; - - for (int i = 0; i < height; ++i) { - const T* image_ptr = reinterpret_cast( - image_frame.PixelData() + - (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); - for (int j = 0; j < width; ++j) { - for (int c = 0; c < channels_preserved; ++c) { - *tensor_ptr++ = *image_ptr++ * scale + bias; - } - image_ptr += channels_ignored; - } - } - return absl::OkStatus(); -} - -} // namespace - -absl::Status NormalizeUInt8Image(const ImageFrame& image_frame, - bool flip_vertically, - const std::pair& output_range, - int max_num_channels, float* tensor_ptr) { - return NormalizeImage(image_frame, flip_vertically, output_range, - max_num_channels, tensor_ptr); -} - -absl::Status NormalizeFloatImage(const ImageFrame& image_frame, - bool flip_vertically, - const std::pair& output_range, - int max_num_channels, float* tensor_ptr) { - return NormalizeImage(image_frame, flip_vertically, output_range, - max_num_channels, tensor_ptr); -} - -absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix, - float* tensor_ptr) { - if (is_row_major_matrix) { - auto matrix_map = - Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } else { - auto matrix_map = - Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } - return absl::OkStatus(); -} - -absl::StatusOr ConvertImageFrameToTensorOnCpu( - const ImageFrame& image_frame, const std::pair& output_range, - bool flip_vertically, int max_num_channels) { - const int height = image_frame.Height(); - const int width = image_frame.Width(); - const int channels = image_frame.NumberOfChannels(); - const int channels_preserved = std::min(channels, max_num_channels); - const mediapipe::ImageFormat::Format format = image_frame.Format(); - - if (!(format == mediapipe::ImageFormat::SRGBA || - format == mediapipe::ImageFormat::SRGB || - format == mediapipe::ImageFormat::GRAY8 || - format == mediapipe::ImageFormat::VEC32F1)) - RET_CHECK_FAIL() << "Unsupported CPU input format."; - - Tensor output_tensor(Tensor::ElementType::kFloat32, - Tensor::Shape{1, height, width, channels_preserved}); - auto cpu_view = output_tensor.GetCpuWriteView(); - - // Copy image data into tensor. - if (image_frame.ByteDepth() == 1) { - MP_RETURN_IF_ERROR(NormalizeUInt8Image(image_frame, flip_vertically, - output_range, max_num_channels, - cpu_view.buffer())); - } else if (image_frame.ByteDepth() == 4) { - MP_RETURN_IF_ERROR(NormalizeFloatImage(image_frame, flip_vertically, - output_range, max_num_channels, - cpu_view.buffer())); - } else { - return absl::InternalError( - "Only byte-based (8 bit) and float (32 bit) images supported."); - } - return output_tensor; -} - -absl::StatusOr ConvertMatrixToTensorOnCpu(const Matrix& matrix, - bool row_major_matrix) { - const int height = matrix.rows(); - const int width = matrix.cols(); - const int channels = 1; - Tensor output_tensor(Tensor::ElementType::kFloat32, - Tensor::Shape{1, height, width, channels}); - MP_RETURN_IF_ERROR( - CopyMatrixToTensor(matrix, row_major_matrix, - output_tensor.GetCpuWriteView().buffer())); - return output_tensor; -} - -} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensor_converter_cpu.h b/mediapipe/calculators/tensor/tensor_converter_cpu.h deleted file mode 100644 index 784bade80..000000000 --- a/mediapipe/calculators/tensor/tensor_converter_cpu.h +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_ -#define MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_ - -#include - -#include "absl/status/status.h" -#include "absl/status/statusor.h" -#include "mediapipe/framework/formats/image_frame.h" -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/tensor.h" - -namespace mediapipe { - -// Converts an ImageFrame to a vector of Tensors. -// @flip_vertically enables to flip the image during conversion. -// @max_num_channels can be used to reserve extra channels in the output -// tensors. -// Returns output Tensor. -absl::StatusOr ConvertImageFrameToTensorOnCpu( - const ImageFrame& image_frame, const std::pair& output_range, - bool flip_vertically, int max_num_channels); - -// Converts a Matrix to a vector of Tensors. -// @row_major_matrix defines the ordering in the input matrix. -// @max_num_channels can be used to reserve extra channels in the output -// tensors. -// Returns output Tensor. -absl::StatusOr ConvertMatrixToTensorOnCpu(const Matrix& matrix, - bool row_major_matrix); - -// For testing only below. -absl::Status NormalizeUInt8Image(const ImageFrame& image_frame, - bool flip_vertically, - const std::pair& output_range, - int max_num_channels, float* tensor_ptr); - -absl::Status NormalizeFloatImage(const ImageFrame& image_frame, - bool flip_vertically, - const std::pair& output_range, - int max_num_channels, float* tensor_ptr); - -absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix, - float* tensor_ptr); - -} // namespace mediapipe - -#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_ diff --git a/mediapipe/calculators/tensor/tensor_converter_cpu_test.cc b/mediapipe/calculators/tensor/tensor_converter_cpu_test.cc deleted file mode 100644 index 478a9c6dc..000000000 --- a/mediapipe/calculators/tensor/tensor_converter_cpu_test.cc +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright 2023 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "mediapipe/calculators/tensor/tensor_converter_cpu.h" - -#include -#include -#include - -#include "mediapipe/framework/formats/matrix.h" -#include "mediapipe/framework/formats/tensor.h" -#include "mediapipe/framework/port/gmock.h" -#include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/status_matchers.h" -#include "mediapipe/util/image_test_utils.h" - -namespace mediapipe { -namespace { - -Matrix CreateTestMatrix(int num_rows, int num_columns) { - Matrix matrix(num_rows, num_columns); - for (int r = 0; r < num_rows; ++r) { - for (int c = 0; c < num_columns; ++c) { - matrix(r, c) = r * num_columns + c; - } - } - return matrix; -} - -TEST(TensorConverterCpuTest, ShouldCopyMatrixInRowMajorFormatToTensor) { - auto test_matrix = CreateTestMatrix(/* num_rows=*/3, /*num_columns=*/4); - std::vector tensor_data(test_matrix.size(), 0.0f); - - MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/true, - tensor_data.data())); - - for (int i = 0; i < tensor_data.size(); ++i) { - const int row = i / test_matrix.cols(); - const int column = i % test_matrix.cols(); - EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column)); - } -} - -TEST(TensorConverterCpuTest, ShouldCopyMatrixInColumnMajorFormatToTensor) { - auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4); - std::vector tensor_data(test_matrix.size(), 0.0f); - - MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/false, - tensor_data.data())); - - for (int i = 0; i < tensor_data.size(); ++i) { - const int row = i % test_matrix.rows(); - const int column = i / test_matrix.rows(); - EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column)); - } -} - -TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithDefaultRange) { - auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); - std::vector tensor_data( - grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f); - - MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, - {0.0f, 1.0f}, /*num_tensor_channels=*/1, - tensor_data.data())); - - for (int i = 0; i < tensor_data.size(); ++i) { - EXPECT_FLOAT_EQ( - tensor_data[i], - static_cast(grey8_image_frame.PixelData()[i]) / 255.0f); - } -} - -TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithSpecifiedRange) { - auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); - std::vector tensor_data( - grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f); - const auto range = std::make_pair(2.0f, 3.0f); - - MP_EXPECT_OK( - NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, range, - /*num_tensor_channels=*/1, tensor_data.data())); - - for (int i = 0; i < tensor_data.size(); ++i) { - EXPECT_FLOAT_EQ(tensor_data[i], - static_cast(grey8_image_frame.PixelData()[i]) / - 255.0f * (range.second - range.first) + - range.first); - } -} - -TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageFlipped) { - auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); - std::vector tensor_data( - grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f); - - MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/true, - {0.0f, 1.0f}, /*num_tensor_channels=*/1, - tensor_data.data())); - - for (int i = 0; i < tensor_data.size(); ++i) { - const int x = i % grey8_image_frame.Width(); - const int y = i / grey8_image_frame.Width(); - const int flipped_y = grey8_image_frame.Height() - y - 1; - - const int index = flipped_y * grey8_image_frame.Width() + x; - EXPECT_FLOAT_EQ( - tensor_data[index], - static_cast(grey8_image_frame.PixelData()[i]) / 255.0f); - } -} - -TEST(TensorConverterCpuTest, ShouldNormalizeFloatImageWithDefaultRange) { - auto float_image_frame = - CreateTestFloat32ImageFrame(/*width=*/3, /*height=*/4); - std::vector tensor_data( - float_image_frame.Width() * float_image_frame.Height(), 0.0f); - - MP_EXPECT_OK(NormalizeFloatImage(float_image_frame, /*flip_vertically=*/false, - {0.0f, 1.0f}, /*num_tensor_channels=*/1, - tensor_data.data())); - - for (int i = 0; i < tensor_data.size(); ++i) { - EXPECT_FLOAT_EQ(tensor_data[i], reinterpret_cast( - float_image_frame.PixelData())[i] / - 255.0f); - } -} - -TEST(TensorConverterCpuTest, ConvertImageFrameToTensorOnCpu) { - auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); - - MP_ASSERT_OK_AND_ASSIGN(Tensor output, ConvertImageFrameToTensorOnCpu( - grey8_image_frame, {0.0f, 1.0f}, - /*flip_vertically=*/false, - /*max_num_channels=*/1)); - - const auto cpu_read_view = output.GetCpuReadView(); - const float* tensor_ptr = cpu_read_view.buffer(); - for (int i = 0; i < grey8_image_frame.Width() * grey8_image_frame.Height(); - ++i) { - EXPECT_FLOAT_EQ( - tensor_ptr[i], - static_cast(grey8_image_frame.PixelData()[i]) / 255.0); - } -} - -TEST(TensorConverterCpuTest, ConvertMatrixToTensorOnCpu) { - auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4); - - MP_ASSERT_OK_AND_ASSIGN( - Tensor output, ConvertMatrixToTensorOnCpu(test_matrix, - /*row_major_matrix=*/false)); - - const auto cpu_read_view = output.GetCpuReadView(); - const float* tensor_ptr = cpu_read_view.buffer(); - for (int i = 0; i < test_matrix.size(); ++i) { - EXPECT_FLOAT_EQ(tensor_ptr[i], test_matrix.data()[i]); - } -} - -} // namespace - -} // namespace mediapipe diff --git a/mediapipe/util/image_test_utils.cc b/mediapipe/util/image_test_utils.cc index 325b308f1..9e10f40c1 100644 --- a/mediapipe/util/image_test_utils.cc +++ b/mediapipe/util/image_test_utils.cc @@ -17,34 +17,6 @@ namespace mediapipe { -namespace { - -template -ImageFrame CreateTestImageFrame(int width, int height, DataType max_value) { - ImageFrame image_frame(Format, width, height, - /*alignment_boundary=*/1); - const int num_channels = image_frame.NumberOfChannels(); - const float num_values = width * height * num_channels; - uint8_t* const data_ptr = - reinterpret_cast(image_frame.MutablePixelData()); - for (int y = 0; y < height; ++y) { - uint8_t* const row = data_ptr + image_frame.WidthStep() * y; - for (int x = 0; x < width; ++x) { - DataType* pixel = reinterpret_cast(row) + x * num_channels; - for (int c = 0; c < num_channels; ++c) { - // Fill pixel channel with a value in [0:max_value] range. - pixel[c] = - static_cast(static_cast(y * width * num_channels + - x * num_channels + c) / - num_values * max_value); - } - } - } - return image_frame; -} - -} // namespace - cv::Mat GetRgb(const std::string& path) { cv::Mat bgr = cv::imread(path); cv::Mat rgb; @@ -99,14 +71,4 @@ cv::Mat RgbaToBgr(cv::Mat rgba) { return bgra; } -ImageFrame CreateTestFloat32ImageFrame(int width, int height) { - return CreateTestImageFrame(width, height, - /*max_value=*/1.0f); -} - -ImageFrame CreateTestGrey8ImageFrame(int width, int height) { - return CreateTestImageFrame(width, height, - /*max_value=*/255); -} - } // namespace mediapipe diff --git a/mediapipe/util/image_test_utils.h b/mediapipe/util/image_test_utils.h index 49943382f..15a21c5b1 100644 --- a/mediapipe/util/image_test_utils.h +++ b/mediapipe/util/image_test_utils.h @@ -4,7 +4,6 @@ #include #include "mediapipe/framework/formats/image_format.pb.h" -#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/packet.h" #include "mediapipe/framework/port/opencv_core_inc.h" @@ -31,12 +30,6 @@ Packet MakeImagePacket(cv::Mat input, int timestamp = 0); // Converts RGBA Mat to BGR. cv::Mat RgbaToBgr(cv::Mat rgba); -// Generates single-channel float32 ImageFrame with increasing [0,1] values. -ImageFrame CreateTestFloat32ImageFrame(int width, int height); - -// Generates single-channel uint8 ImageFrame with increasing [0,255] values. -ImageFrame CreateTestGrey8ImageFrame(int width, int height); - } // namespace mediapipe #endif // MEDIAPIPE_UTIL_IMAGE_TEST_UTILS_H_ From 46c6c9403c661c555229e75a2eb647304cc714dd Mon Sep 17 00:00:00 2001 From: Kinar Date: Thu, 16 Nov 2023 16:26:29 -0800 Subject: [PATCH 122/157] Code cleanup and revised benchmarking API --- .../tasks/python/benchmark/benchmark_utils.py | 11 ++ mediapipe/tasks/python/benchmark/vision/BUILD | 33 ++++++ .../python/benchmark/vision/benchmark.py | 104 ++++++++++++++++++ .../vision/core/base_vision_benchmark_api.py | 14 +-- .../benchmark/vision/face_aligner/BUILD | 3 +- .../face_aligner/face_aligner_benchmark.py | 87 ++------------- .../benchmark/vision/face_detector/BUILD | 3 +- .../face_detector/face_detector_benchmark.py | 88 +++------------ .../benchmark/vision/face_landmarker/BUILD | 3 +- .../face_landmarker_benchmark.py | 80 ++------------ .../benchmark/vision/hand_landmarker/BUILD | 3 +- .../hand_landmarker_benchmark.py | 86 ++------------- .../benchmark/vision/image_classifier/BUILD | 3 +- .../vision/image_classifier/README.md | 5 +- .../image_classifier_benchmark.py | 90 +++------------ .../benchmark/vision/image_embedder/BUILD | 3 +- .../image_embedder_benchmark.py | 80 ++------------ .../benchmark/vision/image_segmenter/BUILD | 1 + .../image_segmenter_benchmark.py | 80 ++------------ .../vision/interactive_segmenter/BUILD | 3 +- .../interactive_segmenter_benchmark.py | 89 +++------------ .../benchmark/vision/object_detector/BUILD | 3 +- .../object_detector_benchmark.py | 84 ++------------ .../benchmark/vision/pose_landmarker/BUILD | 3 +- .../pose_landmarker_benchmark.py | 84 ++------------ mediapipe/tasks/testdata/vision/BUILD | 2 + 26 files changed, 289 insertions(+), 756 deletions(-) create mode 100644 mediapipe/tasks/python/benchmark/vision/BUILD create mode 100644 mediapipe/tasks/python/benchmark/vision/benchmark.py diff --git a/mediapipe/tasks/python/benchmark/benchmark_utils.py b/mediapipe/tasks/python/benchmark/benchmark_utils.py index 5c1b102a2..338bd0ea7 100644 --- a/mediapipe/tasks/python/benchmark/benchmark_utils.py +++ b/mediapipe/tasks/python/benchmark/benchmark_utils.py @@ -14,6 +14,17 @@ """Benchmark utils for MediaPipe Tasks.""" import os +import numpy as np + + +def nth_percentile(inference_times, percentile): + """Calculate the nth percentile of the inference times.""" + return np.percentile(inference_times, percentile) + + +def average(inference_times): + """Calculate the average of the inference times.""" + return np.mean(inference_times) def get_test_data_path(test_srcdir, file_or_dirname_path: str) -> str: diff --git a/mediapipe/tasks/python/benchmark/vision/BUILD b/mediapipe/tasks/python/benchmark/vision/BUILD new file mode 100644 index 000000000..591ebb2d2 --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/BUILD @@ -0,0 +1,33 @@ +# Copyright 2022 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Placeholder for internal Python strict library and test compatibility macro. + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "benchmark", + main = "benchmark.py", + srcs = ["benchmark.py"], + data = [ + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + deps = [ + "//mediapipe/python:_framework_bindings", + "//mediapipe/tasks/python/core:base_options", + "//mediapipe/tasks/python/benchmark:benchmark_utils", + "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + ], +) diff --git a/mediapipe/tasks/python/benchmark/vision/benchmark.py b/mediapipe/tasks/python/benchmark/vision/benchmark.py new file mode 100644 index 000000000..273668c3c --- /dev/null +++ b/mediapipe/tasks/python/benchmark/vision/benchmark.py @@ -0,0 +1,104 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""MediaPipe vision benchmarker.""" + +import argparse + +from mediapipe.tasks.python.core import base_options +from mediapipe.tasks.python.benchmark import benchmark_utils as bu +from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api + + +def benchmarker(benchmark_function, default_model_name, default_image_name): + """Executes a benchmarking process using a specified function and + a default model (or a specified model) and reports the benchmarking + statistics. + + Args: + benchmark_function: A callable function to be executed for benchmarking. + This function should contain the logic of the task to be benchmarked and + should be capable of utilizing a model specified by its name. + default_model_name: The name or path of the default model to be used in + the benchmarking process. This is useful when the benchmarking function + requires a model and no other model is explicitly specified. + """ + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + + parser.add_argument( + '--mode', + help='Benchmarking mode (e.g., "nth_percentile").', + required=False, + default='nth_percentile' + ) + parser.add_argument( + '--model', + help='Path to the model.', + default=None + ) + parser.add_argument( + '--iterations', + help='Number of iterations for benchmarking.', + type=int, + default=100 + ) + parser.add_argument( + '--percentile', + help='Percentile for benchmarking statistics.', + type=float, + default=95.0 + ) + + args = parser.parse_args() + + # Get the model path + default_model_path = bu.get_test_data_path( + base_vision_benchmark_api.VISION_TEST_DATA_DIR, + default_model_name + ) + model_path = bu.get_model_path(args.model, default_model_path) + + # Define a mapping of modes to their respective function argument lists + mode_args_mapping = { + 'nth_percentile': {'percentile': args.percentile}, + 'average': {} + # Add other modes and their arguments here + } + + # Check if the mode is supported and get the argument dictionary + if args.mode not in mode_args_mapping: + raise ValueError(f"Unsupported benchmarking mode: {args.mode}") + + mode_args = mode_args_mapping[args.mode] + + # Run the benchmark for both CPU and GPU and calculate results based on mode + results = {} + for delegate_type in [ + base_options.BaseOptions.Delegate.CPU, + base_options.BaseOptions.Delegate.GPU + ]: + inference_times = benchmark_function(model_path, args.iterations, + delegate_type) + + # Calculate the benchmark result based on the mode + if args.mode == 'nth_percentile': + results[delegate_type] = bu.nth_percentile(inference_times, **mode_args) + elif args.mode == 'average': + results[delegate_type] = bu.average(inference_times) + + # Report benchmarking results + for delegate_type, result in results.items(): + print(f'Inference time {delegate_type} {mode_args_mapping[args.mode]}: ' + f'{result:.6f} milliseconds') diff --git a/mediapipe/tasks/python/benchmark/vision/core/base_vision_benchmark_api.py b/mediapipe/tasks/python/benchmark/vision/core/base_vision_benchmark_api.py index 65c460613..ed3a0c9b7 100644 --- a/mediapipe/tasks/python/benchmark/vision/core/base_vision_benchmark_api.py +++ b/mediapipe/tasks/python/benchmark/vision/core/base_vision_benchmark_api.py @@ -12,25 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. """MediaPipe vision benchmark base api.""" -import os import time -import numpy as np VISION_TEST_DATA_DIR = 'mediapipe/tasks/testdata/vision' -def nth_percentile(func, image, n_iterations, percentile): - """Run a nth percentile benchmark for a given task using the function. +def benchmark_task(func, image, n_iterations): + """Collect inference times for a given task after benchmarking Args: - func: The method associated with a given task used for benchmarking. + func: The task function used for benchmarking. image: The input MediaPipe Image. n_iterations: Number of iterations to run the benchmark. - percentile: Percentage for the percentiles to compute. Values must be - between 0 and 100 inclusive. Returns: - The n-th percentile of the inference times in milliseconds. + List of inference times in milliseconds. """ inference_times = [] @@ -41,4 +37,4 @@ def nth_percentile(func, image, n_iterations, percentile): end_time_ns = time.time_ns() inference_times.append((end_time_ns - start_time_ns) / 1_000_000) - return np.percentile(inference_times, percentile) + return inference_times diff --git a/mediapipe/tasks/python/benchmark/vision/face_aligner/BUILD b/mediapipe/tasks/python/benchmark/vision/face_aligner/BUILD index 61791070f..95ccaeba6 100644 --- a/mediapipe/tasks/python/benchmark/vision/face_aligner/BUILD +++ b/mediapipe/tasks/python/benchmark/vision/face_aligner/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary compatibility macro. package(default_visibility = ["//visibility:public"]) @@ -30,5 +30,6 @@ py_binary( "//mediapipe/tasks/python/vision:face_aligner", "//mediapipe/tasks/python/benchmark:benchmark_utils", "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + "//mediapipe/tasks/python/benchmark/vision:benchmark", ], ) diff --git a/mediapipe/tasks/python/benchmark/vision/face_aligner/face_aligner_benchmark.py b/mediapipe/tasks/python/benchmark/vision/face_aligner/face_aligner_benchmark.py index e51ae056a..83ead70eb 100644 --- a/mediapipe/tasks/python/benchmark/vision/face_aligner/face_aligner_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/face_aligner/face_aligner_benchmark.py @@ -13,45 +13,32 @@ # limitations under the License. """MediaPipe face aligner benchmark.""" -import argparse - from mediapipe.python._framework_bindings import image from mediapipe.tasks.python.core import base_options from mediapipe.tasks.python.vision import face_aligner from mediapipe.tasks.python.benchmark import benchmark_utils from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api +from mediapipe.tasks.python.benchmark.vision import benchmark _MODEL_FILE = 'face_landmarker_v2.task' _IMAGE_FILE = 'portrait.jpg' -def run( - model: str, - n_iterations: int, - delegate: base_options.BaseOptions.Delegate, - percentile: float, -): - """Run an face aligner benchmark. - +def run(model_path, n_iterations, delegate): + """Run a face aligner benchmark. Args: - model: Path to the TFLite model. + model_path: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. - percentile: Percentage for the percentiles to compute. Values must be - between 0 and 100 inclusive. Returns: - The n-th percentile of the inference times. + List of inference times. """ # Initialize the face aligner - default_model_path = benchmark_utils.get_test_data_path( - base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE - ) - model_path = benchmark_utils.get_model_path(model, default_model_path) options = face_aligner.FaceAlignerOptions( - base_options=base_options.BaseOptions( - model_asset_path=model_path, delegate=delegate - ) + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) ) with face_aligner.FaceAligner.create_from_options(options) as aligner: @@ -60,61 +47,11 @@ def run( base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE ) ) - # Run the benchmark and return the nth percentile of the inference times - nth_percentile = base_vision_benchmark_api.nth_percentile( - aligner.align, mp_image, n_iterations, percentile + inference_times = base_vision_benchmark_api.benchmark_task( + aligner.align, mp_image, n_iterations ) - return nth_percentile - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model', - help='Path to face aligner task.', - required=False, - default=None, - ) - parser.add_argument( - '--iterations', - help='Number of iterations for benchmarking.', - type=int, - default=100, - ) - parser.add_argument( - '--percentile', - help='Percentile for benchmarking statistics.', - type=float, - default=95.0, - ) - args = parser.parse_args() - - # Run benchmark on CPU - cpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.CPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on CPU: ' - f'{cpu_time:.6f} milliseconds' - ) - - # Run benchmark on GPU - gpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.GPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on GPU: ' - f'{gpu_time:.6f} milliseconds' - ) + return inference_times if __name__ == '__main__': - main() + benchmark.benchmarker(run, _MODEL_FILE) diff --git a/mediapipe/tasks/python/benchmark/vision/face_detector/BUILD b/mediapipe/tasks/python/benchmark/vision/face_detector/BUILD index 9eb67c19f..9941e904b 100644 --- a/mediapipe/tasks/python/benchmark/vision/face_detector/BUILD +++ b/mediapipe/tasks/python/benchmark/vision/face_detector/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary compatibility macro. package(default_visibility = ["//visibility:public"]) @@ -30,5 +30,6 @@ py_binary( "//mediapipe/tasks/python/vision:face_detector", "//mediapipe/tasks/python/benchmark:benchmark_utils", "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + "//mediapipe/tasks/python/benchmark/vision:benchmark", ], ) diff --git a/mediapipe/tasks/python/benchmark/vision/face_detector/face_detector_benchmark.py b/mediapipe/tasks/python/benchmark/vision/face_detector/face_detector_benchmark.py index c4cd75100..cff31b035 100644 --- a/mediapipe/tasks/python/benchmark/vision/face_detector/face_detector_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/face_detector/face_detector_benchmark.py @@ -11,47 +11,35 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""MediaPipe face detector benchmark.""" - -import argparse +"""MediaPipe image embedder benchmark.""" from mediapipe.python._framework_bindings import image from mediapipe.tasks.python.core import base_options from mediapipe.tasks.python.vision import face_detector from mediapipe.tasks.python.benchmark import benchmark_utils from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api +from mediapipe.tasks.python.benchmark.vision import benchmark _MODEL_FILE = 'face_detection_short_range.tflite' _IMAGE_FILE = 'portrait.jpg' -def run( - model: str, - n_iterations: int, - delegate: base_options.BaseOptions.Delegate, - percentile: float, -): - """Run an face detector benchmark. +def run(model_path, n_iterations, delegate): + """Run a face detector benchmark. Args: - model: Path to the TFLite model. + model_path: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. - percentile: Percentage for the percentiles to compute. Values must be - between 0 and 100 inclusive. Returns: - The n-th percentile of the inference times. + List of inference times. """ # Initialize the face detector - default_model_path = benchmark_utils.get_test_data_path( - base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE - ) - model_path = benchmark_utils.get_model_path(model, default_model_path) options = face_detector.FaceDetectorOptions( - base_options=base_options.BaseOptions( - model_asset_path=model_path, delegate=delegate - ) + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) ) with face_detector.FaceDetector.create_from_options(options) as detector: @@ -60,61 +48,11 @@ def run( base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE ) ) - # Run the benchmark and return the nth percentile of the inference times - nth_percentile = base_vision_benchmark_api.nth_percentile( - detector.detect, mp_image, n_iterations, percentile + inference_times = base_vision_benchmark_api.benchmark_task( + detector.detect, mp_image, n_iterations ) - return nth_percentile - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model', - help='Path to face detector task.', - required=False, - default=None, - ) - parser.add_argument( - '--iterations', - help='Number of iterations for benchmarking.', - type=int, - default=100, - ) - parser.add_argument( - '--percentile', - help='Percentile for benchmarking statistics.', - type=float, - default=95.0, - ) - args = parser.parse_args() - - # Run benchmark on CPU - cpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.CPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on CPU: ' - f'{cpu_time:.6f} milliseconds' - ) - - # Run benchmark on GPU - gpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.GPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on GPU: ' - f'{gpu_time:.6f} milliseconds' - ) + return inference_times if __name__ == '__main__': - main() + benchmark.benchmarker(run, _MODEL_FILE) diff --git a/mediapipe/tasks/python/benchmark/vision/face_landmarker/BUILD b/mediapipe/tasks/python/benchmark/vision/face_landmarker/BUILD index 70dd311e4..5500c3683 100644 --- a/mediapipe/tasks/python/benchmark/vision/face_landmarker/BUILD +++ b/mediapipe/tasks/python/benchmark/vision/face_landmarker/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary compatibility macro. package(default_visibility = ["//visibility:public"]) @@ -30,5 +30,6 @@ py_binary( "//mediapipe/tasks/python/vision:face_landmarker", "//mediapipe/tasks/python/benchmark:benchmark_utils", "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + "//mediapipe/tasks/python/benchmark/vision:benchmark", ], ) diff --git a/mediapipe/tasks/python/benchmark/vision/face_landmarker/face_landmarker_benchmark.py b/mediapipe/tasks/python/benchmark/vision/face_landmarker/face_landmarker_benchmark.py index 7a1f3f817..dc584b2e3 100644 --- a/mediapipe/tasks/python/benchmark/vision/face_landmarker/face_landmarker_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/face_landmarker/face_landmarker_benchmark.py @@ -13,41 +13,29 @@ # limitations under the License. """MediaPipe face landmarker benchmark.""" -import argparse - from mediapipe.python._framework_bindings import image from mediapipe.tasks.python.core import base_options from mediapipe.tasks.python.vision import face_landmarker from mediapipe.tasks.python.benchmark import benchmark_utils from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api +from mediapipe.tasks.python.benchmark.vision import benchmark _MODEL_FILE = 'face_landmarker_v2.task' _IMAGE_FILE = 'portrait.jpg' -def run( - model: str, - n_iterations: int, - delegate: base_options.BaseOptions.Delegate, - percentile: float, -): - """Run an face landmarker benchmark. +def run(model_path, n_iterations, delegate): + """Run a face landmarker benchmark. Args: - model: Path to the TFLite model. + model_path: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. - percentile: Percentage for the percentiles to compute. Values must be - between 0 and 100 inclusive. Returns: - The n-th percentile of the inference times. + List of inference times. """ # Initialize the face landmarker - default_model_path = benchmark_utils.get_test_data_path( - base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE - ) - model_path = benchmark_utils.get_model_path(model, default_model_path) options = face_landmarker.FaceLandmarkerOptions( base_options=base_options.BaseOptions( model_asset_path=model_path, delegate=delegate @@ -60,61 +48,11 @@ def run( base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE ) ) - # Run the benchmark and return the nth percentile of the inference times - nth_percentile = base_vision_benchmark_api.nth_percentile( - landmarker.detect, mp_image, n_iterations, percentile + inference_times = base_vision_benchmark_api.benchmark_task( + landmarker.detect, mp_image, n_iterations ) - return nth_percentile - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model', - help='Path to face landmarker task.', - required=False, - default=None, - ) - parser.add_argument( - '--iterations', - help='Number of iterations for benchmarking.', - type=int, - default=100, - ) - parser.add_argument( - '--percentile', - help='Percentile for benchmarking statistics.', - type=float, - default=95.0, - ) - args = parser.parse_args() - - # Run benchmark on CPU - cpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.CPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on CPU: ' - f'{cpu_time:.6f} milliseconds' - ) - - # Run benchmark on GPU - gpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.GPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on GPU: ' - f'{gpu_time:.6f} milliseconds' - ) + return inference_times if __name__ == '__main__': - main() + benchmark.benchmarker(run, _MODEL_FILE) diff --git a/mediapipe/tasks/python/benchmark/vision/hand_landmarker/BUILD b/mediapipe/tasks/python/benchmark/vision/hand_landmarker/BUILD index 693b42faf..425df549f 100644 --- a/mediapipe/tasks/python/benchmark/vision/hand_landmarker/BUILD +++ b/mediapipe/tasks/python/benchmark/vision/hand_landmarker/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary compatibility macro. package(default_visibility = ["//visibility:public"]) @@ -30,5 +30,6 @@ py_binary( "//mediapipe/tasks/python/vision:hand_landmarker", "//mediapipe/tasks/python/benchmark:benchmark_utils", "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + "//mediapipe/tasks/python/benchmark/vision:benchmark", ], ) diff --git a/mediapipe/tasks/python/benchmark/vision/hand_landmarker/hand_landmarker_benchmark.py b/mediapipe/tasks/python/benchmark/vision/hand_landmarker/hand_landmarker_benchmark.py index 8fd8fc210..9669af715 100644 --- a/mediapipe/tasks/python/benchmark/vision/hand_landmarker/hand_landmarker_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/hand_landmarker/hand_landmarker_benchmark.py @@ -13,45 +13,33 @@ # limitations under the License. """MediaPipe hand landmarker benchmark.""" -import argparse - from mediapipe.python._framework_bindings import image from mediapipe.tasks.python.core import base_options from mediapipe.tasks.python.vision import hand_landmarker from mediapipe.tasks.python.benchmark import benchmark_utils from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api +from mediapipe.tasks.python.benchmark.vision import benchmark _MODEL_FILE = 'hand_landmarker.task' _IMAGE_FILE = 'thumb_up.jpg' -def run( - model: str, - n_iterations: int, - delegate: base_options.BaseOptions.Delegate, - percentile: float, -): - """Run an hand landmarker benchmark. +def run(model_path, n_iterations, delegate): + """Run a hand landmarker benchmark. Args: - model: Path to the TFLite model. + model_path: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. - percentile: Percentage for the percentiles to compute. Values must be - between 0 and 100 inclusive. Returns: - The n-th percentile of the inference times. + List of inference times. """ # Initialize the hand landmarker - default_model_path = benchmark_utils.get_test_data_path( - base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE - ) - model_path = benchmark_utils.get_model_path(model, default_model_path) options = hand_landmarker.HandLandmarkerOptions( - base_options=base_options.BaseOptions( - model_asset_path=model_path, delegate=delegate - ) + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) ) with hand_landmarker.HandLandmarker.create_from_options(options) as landmarker: @@ -60,61 +48,11 @@ def run( base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE ) ) - # Run the benchmark and return the nth percentile of the inference times - nth_percentile = base_vision_benchmark_api.nth_percentile( - landmarker.detect, mp_image, n_iterations, percentile + inference_times = base_vision_benchmark_api.benchmark_task( + landmarker.detect, mp_image, n_iterations ) - return nth_percentile - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model', - help='Path to hand landmarker task.', - required=False, - default=None, - ) - parser.add_argument( - '--iterations', - help='Number of iterations for benchmarking.', - type=int, - default=100, - ) - parser.add_argument( - '--percentile', - help='Percentile for benchmarking statistics.', - type=float, - default=95.0, - ) - args = parser.parse_args() - - # Run benchmark on CPU - cpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.CPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on CPU: ' - f'{cpu_time:.6f} milliseconds' - ) - - # Run benchmark on GPU - gpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.GPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on GPU: ' - f'{gpu_time:.6f} milliseconds' - ) + return inference_times if __name__ == '__main__': - main() + benchmark.benchmarker(run, _MODEL_FILE) diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/BUILD b/mediapipe/tasks/python/benchmark/vision/image_classifier/BUILD index 738b9738a..c4cbe55dd 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/BUILD +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary compatibility macro. package(default_visibility = ["//visibility:public"]) @@ -30,5 +30,6 @@ py_binary( "//mediapipe/tasks/python/vision:image_classifier", "//mediapipe/tasks/python/benchmark:benchmark_utils", "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + "//mediapipe/tasks/python/benchmark/vision:benchmark", ], ) diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md index b444fd5b8..67ab6350d 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/README.md @@ -9,7 +9,6 @@ Run this commands to download the TFLite models and image files: ``` cd mediapipe/mediapipe/tasks/python/benchmark/vision/image_classifier wget -O classifier.tflite -q https://storage.googleapis.com/mediapipe-models/image_classifier/efficientnet_lite0/float32/1/efficientnet_lite0.tflite -wget -O burger.jpg https://storage.googleapis.com/mediapipe-assets/burger.jpg ``` ## Run the benchmark @@ -18,7 +17,7 @@ bazel run -c opt //mediapipe/tasks/python/benchmark/vision/image_classifier:imag ``` * You can optionally specify the `model` parameter to set the TensorFlow Lite model to be used: - * The default value is `classifier.tflite` + * The default value is `mobilenet_v2_1.0_224.tflite` * TensorFlow Lite image classification models **with metadata** * Models from [TensorFlow Hub](https://tfhub.dev/tensorflow/collections/lite/task-library/image-classifier/1) * Models from [MediaPipe Models](https://developers.google.com/mediapipe/solutions/vision/image_classifier/index#models) @@ -29,7 +28,7 @@ bazel run -c opt //mediapipe/tasks/python/benchmark/vision/image_classifier:imag * Default value: `100` * Example usage: ``` - bazel run -c opt :image_classifier_benchmark \ + bazel run -c opt :image_classifier_benchmark -- \ --model classifier.tflite \ --iterations 200 ``` diff --git a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py index f24b7dcd5..a49ccf46e 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_classifier/image_classifier_benchmark.py @@ -11,48 +11,36 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""MediaPipe image classsifier benchmark.""" - -import argparse +"""MediaPipe image classifier benchmark.""" from mediapipe.python._framework_bindings import image from mediapipe.tasks.python.core import base_options from mediapipe.tasks.python.vision import image_classifier from mediapipe.tasks.python.benchmark import benchmark_utils from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api +from mediapipe.tasks.python.benchmark.vision import benchmark _MODEL_FILE = 'mobilenet_v2_1.0_224.tflite' _IMAGE_FILE = 'burger.jpg' -def run( - model: str, - n_iterations: int, - delegate: base_options.BaseOptions.Delegate, - percentile: float, -): - """Run an image classification benchmark. +def run(model_path, n_iterations, delegate): + """Run an image classifier benchmark. Args: - model: Path to the TFLite model. + model_path: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. - percentile: Percentage for the percentiles to compute. Values must be - between 0 and 100 inclusive. Returns: - The n-th percentile of the inference times. + List of inference times. """ # Initialize the image classifier - default_model_path = benchmark_utils.get_test_data_path( - base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE - ) - model_path = benchmark_utils.get_model_path(model, default_model_path) options = image_classifier.ImageClassifierOptions( - base_options=base_options.BaseOptions( - model_asset_path=model_path, delegate=delegate - ), - max_results=1, + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ), + max_results=1, ) with image_classifier.ImageClassifier.create_from_options(options) as classifier: @@ -61,61 +49,11 @@ def run( base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE ) ) - # Run the benchmark and return the nth percentile of the inference times - nth_percentile = base_vision_benchmark_api.nth_percentile( - classifier.classify, mp_image, n_iterations, percentile + inference_times = base_vision_benchmark_api.benchmark_task( + classifier.classify, mp_image, n_iterations ) - return nth_percentile - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model', - help='Path to image classification model.', - required=False, - default=None, - ) - parser.add_argument( - '--iterations', - help='Number of iterations for benchmarking.', - type=int, - default=100, - ) - parser.add_argument( - '--percentile', - help='Percentile for benchmarking statistics.', - type=float, - default=95.0, - ) - args = parser.parse_args() - - # Run benchmark on CPU - cpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.CPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on CPU: ' - f'{cpu_time:.6f} milliseconds' - ) - - # Run benchmark on GPU - gpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.GPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on GPU: ' - f'{gpu_time:.6f} milliseconds' - ) + return inference_times if __name__ == '__main__': - main() + benchmark.benchmarker(run, _MODEL_FILE) diff --git a/mediapipe/tasks/python/benchmark/vision/image_embedder/BUILD b/mediapipe/tasks/python/benchmark/vision/image_embedder/BUILD index 059bdd095..9e00ead3e 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_embedder/BUILD +++ b/mediapipe/tasks/python/benchmark/vision/image_embedder/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary compatibility macro. package(default_visibility = ["//visibility:public"]) @@ -30,5 +30,6 @@ py_binary( "//mediapipe/tasks/python/vision:image_embedder", "//mediapipe/tasks/python/benchmark:benchmark_utils", "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + "//mediapipe/tasks/python/benchmark/vision:benchmark", ], ) diff --git a/mediapipe/tasks/python/benchmark/vision/image_embedder/image_embedder_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_embedder/image_embedder_benchmark.py index 1da3b8e89..331cbb77b 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_embedder/image_embedder_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_embedder/image_embedder_benchmark.py @@ -13,41 +13,29 @@ # limitations under the License. """MediaPipe image embedder benchmark.""" -import argparse - from mediapipe.python._framework_bindings import image from mediapipe.tasks.python.core import base_options from mediapipe.tasks.python.vision import image_embedder from mediapipe.tasks.python.benchmark import benchmark_utils from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api +from mediapipe.tasks.python.benchmark.vision import benchmark _MODEL_FILE = 'mobilenet_v3_small_100_224_embedder.tflite' _IMAGE_FILE = 'burger.jpg' -def run( - model: str, - n_iterations: int, - delegate: base_options.BaseOptions.Delegate, - percentile: float, -): - """Run an image embedding extraction benchmark. +def run(model_path, n_iterations, delegate): + """Run an image embedding benchmark. Args: - model: Path to the TFLite model. + model_path: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. - percentile: Percentage for the percentiles to compute. Values must be - between 0 and 100 inclusive. Returns: - The n-th percentile of the inference times. + List of inference times. """ # Initialize the image embedder - default_model_path = benchmark_utils.get_test_data_path( - base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE - ) - model_path = benchmark_utils.get_model_path(model, default_model_path) options = image_embedder.ImageEmbedderOptions( base_options=base_options.BaseOptions( model_asset_path=model_path, delegate=delegate @@ -60,61 +48,11 @@ def run( base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE ) ) - # Run the benchmark and return the nth percentile of the inference times - nth_percentile = base_vision_benchmark_api.nth_percentile( - embedder.embed, mp_image, n_iterations, percentile + inference_times = base_vision_benchmark_api.benchmark_task( + embedder.embed, mp_image, n_iterations ) - return nth_percentile - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model', - help='Path to image embedding extraction model.', - required=False, - default=None, - ) - parser.add_argument( - '--iterations', - help='Number of iterations for benchmarking.', - type=int, - default=100, - ) - parser.add_argument( - '--percentile', - help='Percentile for benchmarking statistics.', - type=float, - default=95.0, - ) - args = parser.parse_args() - - # Run benchmark on CPU - cpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.CPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on CPU: ' - f'{cpu_time:.6f} milliseconds' - ) - - # Run benchmark on GPU - gpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.GPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on GPU: ' - f'{gpu_time:.6f} milliseconds' - ) + return inference_times if __name__ == '__main__': - main() + benchmark.benchmarker(run, _MODEL_FILE) diff --git a/mediapipe/tasks/python/benchmark/vision/image_segmenter/BUILD b/mediapipe/tasks/python/benchmark/vision/image_segmenter/BUILD index bfb2ee763..fc8d87239 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_segmenter/BUILD +++ b/mediapipe/tasks/python/benchmark/vision/image_segmenter/BUILD @@ -30,5 +30,6 @@ py_binary( "//mediapipe/tasks/python/vision:image_segmenter", "//mediapipe/tasks/python/benchmark:benchmark_utils", "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + "//mediapipe/tasks/python/benchmark/vision:benchmark", ], ) diff --git a/mediapipe/tasks/python/benchmark/vision/image_segmenter/image_segmenter_benchmark.py b/mediapipe/tasks/python/benchmark/vision/image_segmenter/image_segmenter_benchmark.py index ef885b11c..4cff3419a 100644 --- a/mediapipe/tasks/python/benchmark/vision/image_segmenter/image_segmenter_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/image_segmenter/image_segmenter_benchmark.py @@ -13,41 +13,29 @@ # limitations under the License. """MediaPipe image segmenter benchmark.""" -import argparse - from mediapipe.python._framework_bindings import image from mediapipe.tasks.python.core import base_options from mediapipe.tasks.python.vision import image_segmenter from mediapipe.tasks.python.benchmark import benchmark_utils from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api +from mediapipe.tasks.python.benchmark.vision import benchmark _MODEL_FILE = 'deeplabv3.tflite' _IMAGE_FILE = 'segmentation_input_rotation0.jpg' -def run( - model: str, - n_iterations: int, - delegate: base_options.BaseOptions.Delegate, - percentile: float, -): - """Run an image segmentation benchmark. +def run(model_path, n_iterations, delegate): + """Run an image segmenter benchmark. Args: - model: Path to the TFLite model. + model_path: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. - percentile: Percentage for the percentiles to compute. Values must be - between 0 and 100 inclusive. Returns: - The n-th percentile of the inference times. + List of inference times. """ # Initialize the image segmenter - default_model_path = benchmark_utils.get_test_data_path( - base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE - ) - model_path = benchmark_utils.get_model_path(model, default_model_path) options = image_segmenter.ImageSegmenterOptions( base_options=base_options.BaseOptions( model_asset_path=model_path, delegate=delegate @@ -61,61 +49,11 @@ def run( base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE ) ) - # Run the benchmark and return the nth percentile of the inference times - nth_percentile = base_vision_benchmark_api.nth_percentile( - segmenter.segment, mp_image, n_iterations, percentile + inference_times = base_vision_benchmark_api.benchmark_task( + segmenter.segment, mp_image, n_iterations ) - return nth_percentile - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model', - help='Path to image segmentation model.', - required=False, - default=None, - ) - parser.add_argument( - '--iterations', - help='Number of iterations for benchmarking.', - type=int, - default=100, - ) - parser.add_argument( - '--percentile', - help='Percentile for benchmarking statistics.', - type=float, - default=95.0, - ) - args = parser.parse_args() - - # Run benchmark on CPU - cpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.CPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on CPU: ' - f'{cpu_time:.6f} milliseconds' - ) - - # Run benchmark on GPU - gpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.GPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on GPU: ' - f'{gpu_time:.6f} milliseconds' - ) + return inference_times if __name__ == '__main__': - main() + benchmark.benchmarker(run, _MODEL_FILE) diff --git a/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/BUILD b/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/BUILD index 1a0bab418..d238529f1 100644 --- a/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/BUILD +++ b/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary compatibility macro. package(default_visibility = ["//visibility:public"]) @@ -30,5 +30,6 @@ py_binary( "//mediapipe/tasks/python/vision:interactive_segmenter", "//mediapipe/tasks/python/benchmark:benchmark_utils", "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + "//mediapipe/tasks/python/benchmark/vision:benchmark", ], ) diff --git a/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/interactive_segmenter_benchmark.py b/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/interactive_segmenter_benchmark.py index 3ecc7f661..e13fac456 100644 --- a/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/interactive_segmenter_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/interactive_segmenter/interactive_segmenter_benchmark.py @@ -12,45 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. """MediaPipe interactive segmenter benchmark.""" - from functools import partial -import argparse from mediapipe.python._framework_bindings import image from mediapipe.tasks.python.core import base_options -from mediapipe.tasks.python.components.containers import keypoint from mediapipe.tasks.python.vision import interactive_segmenter +from mediapipe.tasks.python.components.containers import keypoint from mediapipe.tasks.python.benchmark import benchmark_utils from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api +from mediapipe.tasks.python.benchmark.vision import benchmark -_MODEL_FILE = 'deeplabv3.tflite' -_IMAGE_FILE = 'segmentation_input_rotation0.jpg' +_MODEL_FILE = 'ptm_512_hdt_ptm_woid.tflite' +_IMAGE_FILE = 'cats_and_dogs.jpg' -def run( - model: str, - n_iterations: int, - delegate: base_options.BaseOptions.Delegate, - percentile: float, -): - """Run an interactive segmentation benchmark. +def run(model_path, n_iterations, delegate): + """Run an interactive segmenter benchmark. Args: - model: Path to the TFLite model. + model_path: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. - percentile: Percentage for the percentiles to compute. Values must be - between 0 and 100 inclusive. Returns: - The n-th percentile of the inference times. + List of inference times. """ - # Initialize the interactive segmenter - default_model_path = benchmark_utils.get_test_data_path( - base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE - ) - model_path = benchmark_utils.get_model_path(model, default_model_path) - + # Initialize the image segmenter options = interactive_segmenter.InteractiveSegmenterOptions( base_options=base_options.BaseOptions( model_asset_path=model_path, delegate=delegate @@ -68,61 +55,11 @@ def run( base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE ) ) - # Run the benchmark and return the nth percentile of the inference times - nth_percentile = base_vision_benchmark_api.nth_percentile( - partial(segmenter.segment, roi=roi), mp_image, n_iterations, percentile + inference_times = base_vision_benchmark_api.benchmark_task( + partial(segmenter.segment, roi=roi), mp_image, n_iterations ) - return nth_percentile - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model', - help='Path to interactive segmentation model.', - required=False, - default=None, - ) - parser.add_argument( - '--iterations', - help='Number of iterations for benchmarking.', - type=int, - default=100, - ) - parser.add_argument( - '--percentile', - help='Percentile for benchmarking statistics.', - type=float, - default=95.0, - ) - args = parser.parse_args() - - # Run benchmark on CPU - cpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.CPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on CPU: ' - f'{cpu_time:.6f} milliseconds' - ) - - # Run benchmark on GPU - gpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.GPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on GPU: ' - f'{gpu_time:.6f} milliseconds' - ) + return inference_times if __name__ == '__main__': - main() + benchmark.benchmarker(run, _MODEL_FILE) diff --git a/mediapipe/tasks/python/benchmark/vision/object_detector/BUILD b/mediapipe/tasks/python/benchmark/vision/object_detector/BUILD index 2db19db5e..f91bdb891 100644 --- a/mediapipe/tasks/python/benchmark/vision/object_detector/BUILD +++ b/mediapipe/tasks/python/benchmark/vision/object_detector/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary compatibility macro. package(default_visibility = ["//visibility:public"]) @@ -30,5 +30,6 @@ py_binary( "//mediapipe/tasks/python/vision:object_detector", "//mediapipe/tasks/python/benchmark:benchmark_utils", "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + "//mediapipe/tasks/python/benchmark/vision:benchmark", ], ) diff --git a/mediapipe/tasks/python/benchmark/vision/object_detector/object_detector_benchmark.py b/mediapipe/tasks/python/benchmark/vision/object_detector/object_detector_benchmark.py index 80f8302a2..8829a9410 100644 --- a/mediapipe/tasks/python/benchmark/vision/object_detector/object_detector_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/object_detector/object_detector_benchmark.py @@ -13,45 +13,33 @@ # limitations under the License. """MediaPipe object detector benchmark.""" -import argparse - from mediapipe.python._framework_bindings import image from mediapipe.tasks.python.core import base_options from mediapipe.tasks.python.vision import object_detector from mediapipe.tasks.python.benchmark import benchmark_utils from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api +from mediapipe.tasks.python.benchmark.vision import benchmark _MODEL_FILE = 'coco_efficientdet_lite0_v1_1.0_quant_2021_09_06.tflite' _IMAGE_FILE = 'cats_and_dogs.jpg' -def run( - model: str, - n_iterations: int, - delegate: base_options.BaseOptions.Delegate, - percentile: float, -): +def run(model_path, n_iterations, delegate): """Run an object detector benchmark. Args: - model: Path to the TFLite model. + model_path: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. - percentile: Percentage for the percentiles to compute. Values must be - between 0 and 100 inclusive. Returns: - The n-th percentile of the inference times. + List of inference times. """ # Initialize the object detector - default_model_path = benchmark_utils.get_test_data_path( - base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE - ) - model_path = benchmark_utils.get_model_path(model, default_model_path) options = object_detector.ObjectDetectorOptions( - base_options=base_options.BaseOptions( - model_asset_path=model_path, delegate=delegate - ) + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) ) with object_detector.ObjectDetector.create_from_options(options) as detector: @@ -60,61 +48,11 @@ def run( base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE ) ) - # Run the benchmark and return the nth percentile of the inference times - nth_percentile = base_vision_benchmark_api.nth_percentile( - detector.detect, mp_image, n_iterations, percentile + inference_times = base_vision_benchmark_api.benchmark_task( + detector.detect, mp_image, n_iterations ) - return nth_percentile - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model', - help='Path to object detector model.', - required=False, - default=None, - ) - parser.add_argument( - '--iterations', - help='Number of iterations for benchmarking.', - type=int, - default=100, - ) - parser.add_argument( - '--percentile', - help='Percentile for benchmarking statistics.', - type=float, - default=95.0, - ) - args = parser.parse_args() - - # Run benchmark on CPU - cpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.CPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on CPU: ' - f'{cpu_time:.6f} milliseconds' - ) - - # Run benchmark on GPU - gpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.GPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on GPU: ' - f'{gpu_time:.6f} milliseconds' - ) + return inference_times if __name__ == '__main__': - main() + benchmark.benchmarker(run, _MODEL_FILE) diff --git a/mediapipe/tasks/python/benchmark/vision/pose_landmarker/BUILD b/mediapipe/tasks/python/benchmark/vision/pose_landmarker/BUILD index 38c778c00..75e0dad18 100644 --- a/mediapipe/tasks/python/benchmark/vision/pose_landmarker/BUILD +++ b/mediapipe/tasks/python/benchmark/vision/pose_landmarker/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Placeholder for internal Python strict library and test compatibility macro. +# Placeholder for internal Python strict binary compatibility macro. package(default_visibility = ["//visibility:public"]) @@ -30,5 +30,6 @@ py_binary( "//mediapipe/tasks/python/vision:pose_landmarker", "//mediapipe/tasks/python/benchmark:benchmark_utils", "//mediapipe/tasks/python/benchmark/vision/core:base_vision_benchmark_api", + "//mediapipe/tasks/python/benchmark/vision:benchmark", ], ) diff --git a/mediapipe/tasks/python/benchmark/vision/pose_landmarker/pose_landmarker_benchmark.py b/mediapipe/tasks/python/benchmark/vision/pose_landmarker/pose_landmarker_benchmark.py index a1c55f0b8..38ac0e546 100644 --- a/mediapipe/tasks/python/benchmark/vision/pose_landmarker/pose_landmarker_benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/pose_landmarker/pose_landmarker_benchmark.py @@ -13,45 +13,33 @@ # limitations under the License. """MediaPipe pose landmarker benchmark.""" -import argparse - from mediapipe.python._framework_bindings import image from mediapipe.tasks.python.core import base_options from mediapipe.tasks.python.vision import pose_landmarker from mediapipe.tasks.python.benchmark import benchmark_utils from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api +from mediapipe.tasks.python.benchmark.vision import benchmark _MODEL_FILE = 'pose_landmarker.task' _IMAGE_FILE = 'pose.jpg' -def run( - model: str, - n_iterations: int, - delegate: base_options.BaseOptions.Delegate, - percentile: float, -): +def run(model_path, n_iterations, delegate): """Run an pose landmarker benchmark. Args: - model: Path to the TFLite model. + model_path: Path to the TFLite model. n_iterations: Number of iterations to run the benchmark. delegate: CPU or GPU delegate for inference. - percentile: Percentage for the percentiles to compute. Values must be - between 0 and 100 inclusive. Returns: - The n-th percentile of the inference times. + List of inference times. """ # Initialize the pose landmarker - default_model_path = benchmark_utils.get_test_data_path( - base_vision_benchmark_api.VISION_TEST_DATA_DIR, _MODEL_FILE - ) - model_path = benchmark_utils.get_model_path(model, default_model_path) options = pose_landmarker.PoseLandmarkerOptions( - base_options=base_options.BaseOptions( - model_asset_path=model_path, delegate=delegate - ) + base_options=base_options.BaseOptions( + model_asset_path=model_path, delegate=delegate + ) ) with pose_landmarker.PoseLandmarker.create_from_options(options) as landmarker: @@ -60,61 +48,11 @@ def run( base_vision_benchmark_api.VISION_TEST_DATA_DIR, _IMAGE_FILE ) ) - # Run the benchmark and return the nth percentile of the inference times - nth_percentile = base_vision_benchmark_api.nth_percentile( - landmarker.detect, mp_image, n_iterations, percentile + inference_times = base_vision_benchmark_api.benchmark_task( + landmarker.detect, mp_image, n_iterations ) - return nth_percentile - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument( - '--model', - help='Path to pose landmarker task.', - required=False, - default=None, - ) - parser.add_argument( - '--iterations', - help='Number of iterations for benchmarking.', - type=int, - default=100, - ) - parser.add_argument( - '--percentile', - help='Percentile for benchmarking statistics.', - type=float, - default=95.0, - ) - args = parser.parse_args() - - # Run benchmark on CPU - cpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.CPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on CPU: ' - f'{cpu_time:.6f} milliseconds' - ) - - # Run benchmark on GPU - gpu_time = run( - args.model, - args.iterations, - base_options.BaseOptions.Delegate.GPU, - args.percentile, - ) - print( - f'{args.percentile}th Percentile Inference Time on GPU: ' - f'{gpu_time:.6f} milliseconds' - ) + return inference_times if __name__ == '__main__': - main() + benchmark.benchmarker(run, _MODEL_FILE) diff --git a/mediapipe/tasks/testdata/vision/BUILD b/mediapipe/tasks/testdata/vision/BUILD index 3f83118b0..117da6eaa 100644 --- a/mediapipe/tasks/testdata/vision/BUILD +++ b/mediapipe/tasks/testdata/vision/BUILD @@ -90,6 +90,7 @@ mediapipe_files(srcs = [ "pose_landmark_lite.tflite", "pose_landmarker.task", "pose_segmentation_mask_golden.png", + "ptm_512_hdt_ptm_woid.tflite", "right_hands.jpg", "right_hands_rotated.jpg", "segmentation_golden_rotation0.png", @@ -202,6 +203,7 @@ filegroup( "pose_detection.tflite", "pose_landmark_lite.tflite", "pose_landmarker.task", + "ptm_512_hdt_ptm_woid.tflite", "selfie_segm_128_128_3.tflite", "selfie_segm_144_256_3.tflite", "selfie_segmentation.tflite", From 6bdc7ce016b70db9e1b0e8e58351d028b9a1c790 Mon Sep 17 00:00:00 2001 From: Kinar Date: Thu, 16 Nov 2023 16:39:21 -0800 Subject: [PATCH 123/157] Removed unused param --- mediapipe/tasks/python/benchmark/vision/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/python/benchmark/vision/benchmark.py b/mediapipe/tasks/python/benchmark/vision/benchmark.py index 273668c3c..66bee680b 100644 --- a/mediapipe/tasks/python/benchmark/vision/benchmark.py +++ b/mediapipe/tasks/python/benchmark/vision/benchmark.py @@ -20,7 +20,7 @@ from mediapipe.tasks.python.benchmark import benchmark_utils as bu from mediapipe.tasks.python.benchmark.vision.core import base_vision_benchmark_api -def benchmarker(benchmark_function, default_model_name, default_image_name): +def benchmarker(benchmark_function, default_model_name): """Executes a benchmarking process using a specified function and a default model (or a specified model) and reports the benchmarking statistics. From 9456c6483064d9b9907dc2fff9c07a16824cadfb Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 17 Nov 2023 09:57:05 -0800 Subject: [PATCH 124/157] No public description PiperOrigin-RevId: 583417701 --- .../com/google/mediapipe/framework/PacketGetter.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mediapipe/java/com/google/mediapipe/framework/PacketGetter.java b/mediapipe/java/com/google/mediapipe/framework/PacketGetter.java index 5ea12872a..3d6b16ce6 100644 --- a/mediapipe/java/com/google/mediapipe/framework/PacketGetter.java +++ b/mediapipe/java/com/google/mediapipe/framework/PacketGetter.java @@ -128,6 +128,16 @@ public final class PacketGetter { return ProtoUtil.unpack(result, defaultInstance); } + public static T getProto(final Packet packet, Parser messageParser) { + SerializedMessage result = new SerializedMessage(); + nativeGetProto(packet.getNativeHandle(), result); + try { + return messageParser.parseFrom(result.value); + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException(e); + } + } + /** * @deprecated {@link #getProto(Packet, MessageLite)} is safer to use in obfuscated builds. */ From d29ea119ffa7b8a97a0bdaca396d92babc7a603e Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 17 Nov 2023 10:26:32 -0800 Subject: [PATCH 125/157] Add the result class for the HolisticLandmarker Java API PiperOrigin-RevId: 583426528 --- .../com/google/mediapipe/tasks/vision/BUILD | 22 +++ .../HolisticLandmarkerResult.java | 141 ++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarkerResult.java diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD index 181b45dc8..fc56bfa27 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD @@ -426,6 +426,28 @@ filegroup( visibility = ["//mediapipe/tasks/java/com/google/mediapipe/tasks/vision:__subpackages__"], ) +android_library( + name = "holisticlandmarker", + srcs = [ + "holisticlandmarker/HolisticLandmarkerResult.java", + ], + javacopts = [ + "-Xep:AndroidJdkLibsChecker:OFF", + ], + manifest = "facedetector/AndroidManifest.xml", + deps = [ + ":core", + "//mediapipe/framework/formats:classification_java_proto_lite", + "//mediapipe/framework/formats:landmark_java_proto_lite", + "//mediapipe/java/com/google/mediapipe/framework/image", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:landmark", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:normalized_landmark", + "//third_party:autovalue", + "@maven//:com_google_guava_guava", + ], +) + load("//mediapipe/tasks/java/com/google/mediapipe/tasks:mediapipe_tasks_aar.bzl", "mediapipe_tasks_vision_aar") mediapipe_tasks_vision_aar( diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarkerResult.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarkerResult.java new file mode 100644 index 000000000..06a866771 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarkerResult.java @@ -0,0 +1,141 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.vision.holisticlandmarker; + +import com.google.auto.value.AutoValue; +import com.google.mediapipe.formats.proto.LandmarkProto.LandmarkList; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList; +import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList; +import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.tasks.components.containers.Category; +import com.google.mediapipe.tasks.components.containers.Landmark; +import com.google.mediapipe.tasks.components.containers.NormalizedLandmark; +import com.google.mediapipe.tasks.core.TaskResult; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +/** Represents the holistic landmarks detection results generated by {@link HolisticLandmarker}. */ +@AutoValue +@SuppressWarnings("AutoValueImmutableFields") // 3P API that does not depend on Guava +public abstract class HolisticLandmarkerResult implements TaskResult { + + /** + * Creates a {@link HolisticLandmarkerResult} instance from a list of proto and image inputs. + * + * @param faceLandmarkListProto the detected face landmarks in normalized image coordinates + * @param faceBlendshapeProtos the optional face blendshapes result + * @param poseLandmarkListProtos the detected pose landmarks in normalized image coordinates + * @param poseWorldLandmarkListProto the pose landmarks in world coordinates of detected poses + * @param segmentationMask the segmentation mask for the detected pose + * @param leftHandLandmarkListProto left hand landmarks of detected left hands + * @param leftHandWorldLandmarkListProto left hand landmarks in world coordinates of detected left + * hands + * @param rightHandLandmarkListProto right hand landmarks of detected left hands. + * @param rightHandWorldLandmarkListProto right hand landmarks in world coordinates of detected + * left hands + * @param timestampMs the time in milliseconds this result was created at + * @throws IllegalArgumentException if there was an error creating {@link + * HolisticLandmarkerResult} + */ + static HolisticLandmarkerResult create( + NormalizedLandmarkList faceLandmarkListProto, + Optional faceBlendshapeProtos, + NormalizedLandmarkList poseLandmarkListProtos, + LandmarkList poseWorldLandmarkListProto, + Optional segmentationMask, + NormalizedLandmarkList leftHandLandmarkListProto, + LandmarkList leftHandWorldLandmarkListProto, + NormalizedLandmarkList rightHandLandmarkListProto, + LandmarkList rightHandWorldLandmarkListProto, + long timestampMs) { + List faceLandmarks = + NormalizedLandmark.createListFromProto(faceLandmarkListProto); + Optional> faceBlendshapes = + faceBlendshapeProtos.map(Category::createListFromProto); + List poseLandmarks = + NormalizedLandmark.createListFromProto(poseLandmarkListProtos); + List poseWorldLandmarks = Landmark.createListFromProto(poseWorldLandmarkListProto); + List leftHandLandmarks = + NormalizedLandmark.createListFromProto(leftHandLandmarkListProto); + List leftHandWorldLandmarks = + Landmark.createListFromProto(leftHandWorldLandmarkListProto); + List rightHandLandmarks = + NormalizedLandmark.createListFromProto(rightHandLandmarkListProto); + List rightHandWorldLandmarks = + Landmark.createListFromProto(rightHandWorldLandmarkListProto); + + return new AutoValue_HolisticLandmarkerResult( + timestampMs, + Collections.unmodifiableList(faceLandmarks), + faceBlendshapes, + Collections.unmodifiableList(poseLandmarks), + Collections.unmodifiableList(poseWorldLandmarks), + segmentationMask, + Collections.unmodifiableList(leftHandLandmarks), + Collections.unmodifiableList(leftHandWorldLandmarks), + Collections.unmodifiableList(rightHandLandmarks), + Collections.unmodifiableList(rightHandWorldLandmarks)); + } + + /** + * Creates an empty {@link HolisticLandmarkerResult} instance. + * + * @param timestampMs the time in milliseconds this result was created at + */ + static HolisticLandmarkerResult createEmpty(long timestampMs) { + return new AutoValue_HolisticLandmarkerResult( + timestampMs, + Collections.emptyList(), + Optional.empty(), + Collections.emptyList(), + Collections.emptyList(), + Optional.empty(), + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyList()); + } + + @Override + public abstract long timestampMs(); + + /** Detected face landmarks in normalized image coordinates. */ + public abstract List faceLandmarks(); + + /** Optional face blendshapes. */ + public abstract Optional> faceBlendshapes(); + + /** Detected pose landmarks in normalized image coordinates. */ + public abstract List poseLandmarks(); + + /** Pose landmarks in world coordinates of the detected pose. */ + public abstract List poseWorldLandmarks(); + + /** Segmentation mask for the detected pose. */ + public abstract Optional segmentationMask(); + + /** Hand landmarks of detected left hands. */ + public abstract List leftHandLandmarks(); + + /** Hnd landmarks in world coordinates of detected left hands. */ + public abstract List leftHandWorldLandmarks(); + + /** Hand landmarks of detected right hands. */ + public abstract List rightHandLandmarks(); + + /** Hand landmarks in world coordinates of detected right hands. */ + public abstract List rightHandWorldLandmarks(); +} From 42d42a5ea199a20eb9176b712e46078cb737b076 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 17 Nov 2023 14:44:35 -0800 Subject: [PATCH 126/157] Ensure that releaseGl() is called if prepapreGl throws Without this logic, we might have resources created within prepareGl() leaking, since they will never be released. PiperOrigin-RevId: 583491569 --- mediapipe/java/com/google/mediapipe/glutil/GlThread.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mediapipe/java/com/google/mediapipe/glutil/GlThread.java b/mediapipe/java/com/google/mediapipe/glutil/GlThread.java index 24b8ca094..b8d4fa636 100644 --- a/mediapipe/java/com/google/mediapipe/glutil/GlThread.java +++ b/mediapipe/java/com/google/mediapipe/glutil/GlThread.java @@ -128,6 +128,10 @@ public class GlThread extends Thread { prepareGl(); startedSuccessfully = true; + } catch (RuntimeException e) { + releaseGl(); + eglManager.release(); + throw e; } finally { // Always stop waitUntilReady here, even if we got an exception. // Otherwise the main thread may be stuck waiting. From bd4be30b02f967b53377233e84848b582ac790ec Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 20 Nov 2023 00:56:40 -0800 Subject: [PATCH 127/157] No public description PiperOrigin-RevId: 583936442 --- mediapipe/calculators/tensor/BUILD | 40 +++- .../tensor/tensor_converter_calculator.cc | 125 ++----------- .../tensor_converter_calculator_test.cc | 55 ++++++ .../tensor/tensor_converter_cpu.cc | 145 +++++++++++++++ .../calculators/tensor/tensor_converter_cpu.h | 61 ++++++ .../tensor/tensor_converter_cpu_test.cc | 175 ++++++++++++++++++ mediapipe/util/image_test_utils.cc | 38 ++++ mediapipe/util/image_test_utils.h | 7 + 8 files changed, 538 insertions(+), 108 deletions(-) create mode 100644 mediapipe/calculators/tensor/tensor_converter_cpu.cc create mode 100644 mediapipe/calculators/tensor/tensor_converter_cpu.h create mode 100644 mediapipe/calculators/tensor/tensor_converter_cpu_test.cc diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index e95398a9d..96a29089e 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -657,6 +657,7 @@ cc_library( }), deps = [ ":tensor_converter_calculator_cc_proto", + ":tensor_converter_cpu", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:port", "//mediapipe/framework/formats:image_frame", @@ -665,6 +666,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", + "//mediapipe/gpu:gpu_buffer", "//mediapipe/gpu:gpu_buffer_format", "//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/util:resource_util", @@ -674,10 +676,17 @@ cc_library( "@com_google_absl//absl/log:check", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:str_format", ] + select({ "//mediapipe/gpu:disable_gpu": [], - "//conditions:default": ["tensor_converter_calculator_gpu_deps"], + "//conditions:default": [ + "tensor_converter_calculator_gpu_deps", + "//mediapipe/gpu:gl_base", + "//mediapipe/gpu:gl_calculator_helper", + "//mediapipe/gpu:gl_simple_shaders", + "//mediapipe/gpu:shader_util", + ], }) + select({ "//mediapipe:apple": [ "//third_party/apple_frameworks:MetalKit", @@ -687,6 +696,35 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "tensor_converter_cpu", + srcs = ["tensor_converter_cpu.cc"], + hdrs = ["tensor_converter_cpu.h"], + deps = [ + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:matrix", + "//mediapipe/framework/formats:tensor", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_test( + name = "tensor_converter_cpu_test", + srcs = ["tensor_converter_cpu_test.cc"], + deps = [ + ":tensor_converter_cpu", + "//mediapipe/framework/formats:matrix", + "//mediapipe/framework/formats:tensor", + "//mediapipe/framework/port:gtest", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:status_matchers", + "//mediapipe/util:image_test_utils", + ], +) + cc_library( name = "tensor_converter_calculator_gpu_deps", visibility = ["//visibility:private"], diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator.cc b/mediapipe/calculators/tensor/tensor_converter_calculator.cc index b42cb0b17..03eb2ff80 100644 --- a/mediapipe/calculators/tensor/tensor_converter_calculator.cc +++ b/mediapipe/calculators/tensor/tensor_converter_calculator.cc @@ -14,6 +14,7 @@ #include #include +#include #include #include "absl/log/absl_check.h" @@ -21,17 +22,22 @@ #include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_format.h" +#include "absl/strings/substitute.h" +#include "absl/types/optional.h" #include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensor_converter_cpu.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/matrix.h" #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/port.h" #include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/gpu/gpu_buffer_format.h" #include "mediapipe/gpu/gpu_origin.pb.h" #if !MEDIAPIPE_DISABLE_GPU +#include "mediapipe/gpu/gl_base.h" #include "mediapipe/gpu/gpu_buffer.h" #if MEDIAPIPE_METAL_ENABLED #import @@ -94,16 +100,13 @@ absl::StatusOr ShouldFlipVertically( } } -typedef Eigen::Matrix - RowMajorMatrixXf; -typedef Eigen::Matrix - ColMajorMatrixXf; - constexpr char kImageFrameTag[] = "IMAGE"; constexpr char kGpuBufferTag[] = "IMAGE_GPU"; constexpr char kTensorsTag[] = "TENSORS"; constexpr char kMatrixTag[] = "MATRIX"; +constexpr std::pair kDefaultOutputRange = {0.0f, 1.0f}; + } // namespace namespace mediapipe { @@ -156,10 +159,6 @@ class TensorConverterCalculator : public CalculatorBase { private: absl::Status InitGpu(CalculatorContext* cc); absl::Status LoadOptions(CalculatorContext* cc, bool use_gpu); - template - absl::Status NormalizeImage(const ImageFrame& image_frame, - bool flip_vertically, float* tensor_ptr); - absl::Status CopyMatrixToTensor(const Matrix& matrix, float* tensor_ptr); absl::Status ProcessCPU(CalculatorContext* cc); absl::Status ProcessGPU(CalculatorContext* cc); @@ -279,46 +278,21 @@ absl::Status TensorConverterCalculator::ProcessCPU(CalculatorContext* cc) { } const auto& image_frame = cc->Inputs().Tag(kImageFrameTag).Get(); - const int height = image_frame.Height(); - const int width = image_frame.Width(); - const int channels = image_frame.NumberOfChannels(); - const int channels_preserved = std::min(channels, max_num_channels_); - const mediapipe::ImageFormat::Format format = image_frame.Format(); - - if (!(format == mediapipe::ImageFormat::SRGBA || - format == mediapipe::ImageFormat::SRGB || - format == mediapipe::ImageFormat::GRAY8 || - format == mediapipe::ImageFormat::VEC32F1)) - RET_CHECK_FAIL() << "Unsupported CPU input format."; - - output_tensors->emplace_back( - Tensor::ElementType::kFloat32, - Tensor::Shape{1, height, width, channels_preserved}); - auto cpu_view = output_tensors->back().GetCpuWriteView(); - - // Copy image data into tensor. - if (image_frame.ByteDepth() == 1) { - MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, - cpu_view.buffer())); - } else if (image_frame.ByteDepth() == 4) { - MP_RETURN_IF_ERROR(NormalizeImage(image_frame, flip_vertically_, - cpu_view.buffer())); - } else { - return absl::InternalError( - "Only byte-based (8 bit) and float (32 bit) images supported."); - } + MP_ASSIGN_OR_RETURN(Tensor output, + ConvertImageFrameToTensorOnCpu( + image_frame, + output_range_.has_value() ? output_range_.value() + : kDefaultOutputRange, + flip_vertically_, max_num_channels_)); + output_tensors->emplace_back(std::move(output)); } else if (cc->Inputs().HasTag(kMatrixTag)) { if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) { return absl::OkStatus(); } const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get(); - const int height = matrix.rows(); - const int width = matrix.cols(); - const int channels = 1; - output_tensors->emplace_back(Tensor::ElementType::kFloat32, - Tensor::Shape{1, height, width, channels}); - MP_RETURN_IF_ERROR(CopyMatrixToTensor( - matrix, output_tensors->back().GetCpuWriteView().buffer())); + MP_ASSIGN_OR_RETURN(Tensor output, + ConvertMatrixToTensorOnCpu(matrix, row_major_matrix_)); + output_tensors->emplace_back(std::move(output)); } else { return absl::OkStatus(); } @@ -669,67 +643,4 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc, return absl::OkStatus(); } -template -absl::Status TensorConverterCalculator::NormalizeImage( - const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) { - const int height = image_frame.Height(); - const int width = image_frame.Width(); - const int channels = image_frame.NumberOfChannels(); - const int channels_preserved = std::min(channels, max_num_channels_); - const int channels_ignored = channels - channels_preserved; - - if (output_range_.has_value()) { - // If the output float range is set and we are not using custom - // normalization, normalize the pixel values from [0, 255] to the specified - // output range. - RET_CHECK_NE(output_range_->first, output_range_->second); - const float scale = (output_range_->second - output_range_->first) / 255.0f; - const float bias = output_range_->first; - - for (int i = 0; i < height; ++i) { - const T* image_ptr = reinterpret_cast( - image_frame.PixelData() + - (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); - for (int j = 0; j < width; ++j) { - for (int c = 0; c < channels_preserved; ++c) { - *tensor_ptr++ = *image_ptr++ * scale + bias; - } - image_ptr += channels_ignored; - } - } - } else { - // [0,1], scale only (bias == 0) - // Verified that there are no precision issues with 1.0f / 255.0f expression - const float scale = 1.0f / 255.0f; - for (int i = 0; i < height; ++i) { - const T* image_ptr = reinterpret_cast( - image_frame.PixelData() + - (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); - for (int j = 0; j < width; ++j) { - for (int c = 0; c < channels_preserved; ++c) { - *tensor_ptr++ = *image_ptr++ * scale; - } - image_ptr += channels_ignored; - } - } - } - - return absl::OkStatus(); -} - -absl::Status TensorConverterCalculator::CopyMatrixToTensor(const Matrix& matrix, - float* tensor_ptr) { - if (row_major_matrix_) { - auto matrix_map = - Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } else { - auto matrix_map = - Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); - matrix_map = matrix; - } - - return absl::OkStatus(); -} - } // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensor_converter_calculator_test.cc b/mediapipe/calculators/tensor/tensor_converter_calculator_test.cc index 0394daebd..3446ea301 100644 --- a/mediapipe/calculators/tensor/tensor_converter_calculator_test.cc +++ b/mediapipe/calculators/tensor/tensor_converter_calculator_test.cc @@ -321,6 +321,61 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) { } } +TEST_F(TensorConverterCalculatorTest, + ShouldConvertImageWithDefaultOutputRange) { + CalculatorGraph graph; + CalculatorGraphConfig graph_config = + mediapipe::ParseTextProtoOrDie( + R"pb( + input_stream: "input_image" + node { + calculator: "TensorConverterCalculator" + input_stream: "IMAGE:input_image" + output_stream: "TENSORS:tensor" + options { + [mediapipe.TensorConverterCalculatorOptions.ext] { + zero_center: false + } + } + } + )pb"); + std::vector output_packets; + tool::AddVectorSink("tensor", &graph_config, &output_packets); + + // Run the graph. + MP_ASSERT_OK(graph.Initialize(graph_config)); + MP_ASSERT_OK(graph.StartRun({})); + auto input_image = std::make_unique(ImageFormat::GRAY8, 1, 1); + cv::Mat mat = mediapipe::formats::MatView(input_image.get()); + mat.at(0, 0) = 200; + MP_ASSERT_OK(graph.AddPacketToInputStream( + "input_image", Adopt(input_image.release()).At(Timestamp(0)))); + + // Wait until the calculator finishes processing. + MP_ASSERT_OK(graph.WaitUntilIdle()); + ASSERT_EQ(output_packets.size(), 1); + + // Get and process results. + const std::vector& tensor_vec = + output_packets[0].Get>(); + ASSERT_EQ(tensor_vec.size(), 1); + + const Tensor* tensor = &tensor_vec[0]; + + // Calculate the expected normalized value: + float expected_value = 200.0 / 255.0; + + EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32); + auto view = tensor->GetCpuReadView(); + float actual_value = *view.buffer(); + EXPECT_FLOAT_EQ(actual_value, expected_value); + + // Fully close graph at end, otherwise calculator+tensors are destroyed + // after calling WaitUntilDone(). + MP_ASSERT_OK(graph.CloseInputStream("input_image")); + MP_ASSERT_OK(graph.WaitUntilDone()); +} + TEST_F(TensorConverterCalculatorTest, FlipVertically) { CalculatorGraph graph; CalculatorGraphConfig graph_config = diff --git a/mediapipe/calculators/tensor/tensor_converter_cpu.cc b/mediapipe/calculators/tensor/tensor_converter_cpu.cc new file mode 100644 index 000000000..f72a24c31 --- /dev/null +++ b/mediapipe/calculators/tensor/tensor_converter_cpu.cc @@ -0,0 +1,145 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensor_converter_cpu.h" + +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_macros.h" + +namespace mediapipe { +namespace { + +typedef Eigen::Matrix + RowMajorMatrixXf; +typedef Eigen::Matrix + ColMajorMatrixXf; + +template +absl::Status NormalizeImage(const ImageFrame& image_frame, bool flip_vertically, + const std::pair& output_range, + int max_num_channels, float* tensor_ptr) { + const int height = image_frame.Height(); + const int width = image_frame.Width(); + const int channels = image_frame.NumberOfChannels(); + const int channels_preserved = std::min(channels, max_num_channels); + const int channels_ignored = channels - channels_preserved; + + RET_CHECK_NE(output_range.first, output_range.second); + const float scale = (output_range.second - output_range.first) / 255.0f; + const float bias = output_range.first; + + for (int i = 0; i < height; ++i) { + const T* image_ptr = reinterpret_cast( + image_frame.PixelData() + + (flip_vertically ? height - 1 - i : i) * image_frame.WidthStep()); + for (int j = 0; j < width; ++j) { + for (int c = 0; c < channels_preserved; ++c) { + *tensor_ptr++ = *image_ptr++ * scale + bias; + } + image_ptr += channels_ignored; + } + } + return absl::OkStatus(); +} + +} // namespace + +absl::Status NormalizeUInt8Image(const ImageFrame& image_frame, + bool flip_vertically, + const std::pair& output_range, + int max_num_channels, float* tensor_ptr) { + return NormalizeImage(image_frame, flip_vertically, output_range, + max_num_channels, tensor_ptr); +} + +absl::Status NormalizeFloatImage(const ImageFrame& image_frame, + bool flip_vertically, + const std::pair& output_range, + int max_num_channels, float* tensor_ptr) { + return NormalizeImage(image_frame, flip_vertically, output_range, + max_num_channels, tensor_ptr); +} + +absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix, + float* tensor_ptr) { + if (is_row_major_matrix) { + auto matrix_map = + Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); + matrix_map = matrix; + } else { + auto matrix_map = + Eigen::Map(tensor_ptr, matrix.rows(), matrix.cols()); + matrix_map = matrix; + } + return absl::OkStatus(); +} + +absl::StatusOr ConvertImageFrameToTensorOnCpu( + const ImageFrame& image_frame, const std::pair& output_range, + bool flip_vertically, int max_num_channels) { + const int height = image_frame.Height(); + const int width = image_frame.Width(); + const int channels = image_frame.NumberOfChannels(); + const int channels_preserved = std::min(channels, max_num_channels); + const mediapipe::ImageFormat::Format format = image_frame.Format(); + + if (!(format == mediapipe::ImageFormat::SRGBA || + format == mediapipe::ImageFormat::SRGB || + format == mediapipe::ImageFormat::GRAY8 || + format == mediapipe::ImageFormat::VEC32F1)) + RET_CHECK_FAIL() << "Unsupported CPU input format."; + + Tensor output_tensor(Tensor::ElementType::kFloat32, + Tensor::Shape{1, height, width, channels_preserved}); + auto cpu_view = output_tensor.GetCpuWriteView(); + + // Copy image data into tensor. + if (image_frame.ByteDepth() == 1) { + MP_RETURN_IF_ERROR(NormalizeUInt8Image(image_frame, flip_vertically, + output_range, max_num_channels, + cpu_view.buffer())); + } else if (image_frame.ByteDepth() == 4) { + MP_RETURN_IF_ERROR(NormalizeFloatImage(image_frame, flip_vertically, + output_range, max_num_channels, + cpu_view.buffer())); + } else { + return absl::InternalError( + "Only byte-based (8 bit) and float (32 bit) images supported."); + } + return output_tensor; +} + +absl::StatusOr ConvertMatrixToTensorOnCpu(const Matrix& matrix, + bool row_major_matrix) { + const int height = matrix.rows(); + const int width = matrix.cols(); + const int channels = 1; + Tensor output_tensor(Tensor::ElementType::kFloat32, + Tensor::Shape{1, height, width, channels}); + MP_RETURN_IF_ERROR( + CopyMatrixToTensor(matrix, row_major_matrix, + output_tensor.GetCpuWriteView().buffer())); + return output_tensor; +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensor_converter_cpu.h b/mediapipe/calculators/tensor/tensor_converter_cpu.h new file mode 100644 index 000000000..784bade80 --- /dev/null +++ b/mediapipe/calculators/tensor/tensor_converter_cpu.h @@ -0,0 +1,61 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_ + +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/tensor.h" + +namespace mediapipe { + +// Converts an ImageFrame to a vector of Tensors. +// @flip_vertically enables to flip the image during conversion. +// @max_num_channels can be used to reserve extra channels in the output +// tensors. +// Returns output Tensor. +absl::StatusOr ConvertImageFrameToTensorOnCpu( + const ImageFrame& image_frame, const std::pair& output_range, + bool flip_vertically, int max_num_channels); + +// Converts a Matrix to a vector of Tensors. +// @row_major_matrix defines the ordering in the input matrix. +// @max_num_channels can be used to reserve extra channels in the output +// tensors. +// Returns output Tensor. +absl::StatusOr ConvertMatrixToTensorOnCpu(const Matrix& matrix, + bool row_major_matrix); + +// For testing only below. +absl::Status NormalizeUInt8Image(const ImageFrame& image_frame, + bool flip_vertically, + const std::pair& output_range, + int max_num_channels, float* tensor_ptr); + +absl::Status NormalizeFloatImage(const ImageFrame& image_frame, + bool flip_vertically, + const std::pair& output_range, + int max_num_channels, float* tensor_ptr); + +absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix, + float* tensor_ptr); + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_ diff --git a/mediapipe/calculators/tensor/tensor_converter_cpu_test.cc b/mediapipe/calculators/tensor/tensor_converter_cpu_test.cc new file mode 100644 index 000000000..478a9c6dc --- /dev/null +++ b/mediapipe/calculators/tensor/tensor_converter_cpu_test.cc @@ -0,0 +1,175 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensor_converter_cpu.h" + +#include +#include +#include + +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/util/image_test_utils.h" + +namespace mediapipe { +namespace { + +Matrix CreateTestMatrix(int num_rows, int num_columns) { + Matrix matrix(num_rows, num_columns); + for (int r = 0; r < num_rows; ++r) { + for (int c = 0; c < num_columns; ++c) { + matrix(r, c) = r * num_columns + c; + } + } + return matrix; +} + +TEST(TensorConverterCpuTest, ShouldCopyMatrixInRowMajorFormatToTensor) { + auto test_matrix = CreateTestMatrix(/* num_rows=*/3, /*num_columns=*/4); + std::vector tensor_data(test_matrix.size(), 0.0f); + + MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/true, + tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + const int row = i / test_matrix.cols(); + const int column = i % test_matrix.cols(); + EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column)); + } +} + +TEST(TensorConverterCpuTest, ShouldCopyMatrixInColumnMajorFormatToTensor) { + auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4); + std::vector tensor_data(test_matrix.size(), 0.0f); + + MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/false, + tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + const int row = i % test_matrix.rows(); + const int column = i / test_matrix.rows(); + EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column)); + } +} + +TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithDefaultRange) { + auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); + std::vector tensor_data( + grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f); + + MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, + {0.0f, 1.0f}, /*num_tensor_channels=*/1, + tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + EXPECT_FLOAT_EQ( + tensor_data[i], + static_cast(grey8_image_frame.PixelData()[i]) / 255.0f); + } +} + +TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithSpecifiedRange) { + auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); + std::vector tensor_data( + grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f); + const auto range = std::make_pair(2.0f, 3.0f); + + MP_EXPECT_OK( + NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, range, + /*num_tensor_channels=*/1, tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + EXPECT_FLOAT_EQ(tensor_data[i], + static_cast(grey8_image_frame.PixelData()[i]) / + 255.0f * (range.second - range.first) + + range.first); + } +} + +TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageFlipped) { + auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); + std::vector tensor_data( + grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f); + + MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/true, + {0.0f, 1.0f}, /*num_tensor_channels=*/1, + tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + const int x = i % grey8_image_frame.Width(); + const int y = i / grey8_image_frame.Width(); + const int flipped_y = grey8_image_frame.Height() - y - 1; + + const int index = flipped_y * grey8_image_frame.Width() + x; + EXPECT_FLOAT_EQ( + tensor_data[index], + static_cast(grey8_image_frame.PixelData()[i]) / 255.0f); + } +} + +TEST(TensorConverterCpuTest, ShouldNormalizeFloatImageWithDefaultRange) { + auto float_image_frame = + CreateTestFloat32ImageFrame(/*width=*/3, /*height=*/4); + std::vector tensor_data( + float_image_frame.Width() * float_image_frame.Height(), 0.0f); + + MP_EXPECT_OK(NormalizeFloatImage(float_image_frame, /*flip_vertically=*/false, + {0.0f, 1.0f}, /*num_tensor_channels=*/1, + tensor_data.data())); + + for (int i = 0; i < tensor_data.size(); ++i) { + EXPECT_FLOAT_EQ(tensor_data[i], reinterpret_cast( + float_image_frame.PixelData())[i] / + 255.0f); + } +} + +TEST(TensorConverterCpuTest, ConvertImageFrameToTensorOnCpu) { + auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4); + + MP_ASSERT_OK_AND_ASSIGN(Tensor output, ConvertImageFrameToTensorOnCpu( + grey8_image_frame, {0.0f, 1.0f}, + /*flip_vertically=*/false, + /*max_num_channels=*/1)); + + const auto cpu_read_view = output.GetCpuReadView(); + const float* tensor_ptr = cpu_read_view.buffer(); + for (int i = 0; i < grey8_image_frame.Width() * grey8_image_frame.Height(); + ++i) { + EXPECT_FLOAT_EQ( + tensor_ptr[i], + static_cast(grey8_image_frame.PixelData()[i]) / 255.0); + } +} + +TEST(TensorConverterCpuTest, ConvertMatrixToTensorOnCpu) { + auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4); + + MP_ASSERT_OK_AND_ASSIGN( + Tensor output, ConvertMatrixToTensorOnCpu(test_matrix, + /*row_major_matrix=*/false)); + + const auto cpu_read_view = output.GetCpuReadView(); + const float* tensor_ptr = cpu_read_view.buffer(); + for (int i = 0; i < test_matrix.size(); ++i) { + EXPECT_FLOAT_EQ(tensor_ptr[i], test_matrix.data()[i]); + } +} + +} // namespace + +} // namespace mediapipe diff --git a/mediapipe/util/image_test_utils.cc b/mediapipe/util/image_test_utils.cc index 9e10f40c1..325b308f1 100644 --- a/mediapipe/util/image_test_utils.cc +++ b/mediapipe/util/image_test_utils.cc @@ -17,6 +17,34 @@ namespace mediapipe { +namespace { + +template +ImageFrame CreateTestImageFrame(int width, int height, DataType max_value) { + ImageFrame image_frame(Format, width, height, + /*alignment_boundary=*/1); + const int num_channels = image_frame.NumberOfChannels(); + const float num_values = width * height * num_channels; + uint8_t* const data_ptr = + reinterpret_cast(image_frame.MutablePixelData()); + for (int y = 0; y < height; ++y) { + uint8_t* const row = data_ptr + image_frame.WidthStep() * y; + for (int x = 0; x < width; ++x) { + DataType* pixel = reinterpret_cast(row) + x * num_channels; + for (int c = 0; c < num_channels; ++c) { + // Fill pixel channel with a value in [0:max_value] range. + pixel[c] = + static_cast(static_cast(y * width * num_channels + + x * num_channels + c) / + num_values * max_value); + } + } + } + return image_frame; +} + +} // namespace + cv::Mat GetRgb(const std::string& path) { cv::Mat bgr = cv::imread(path); cv::Mat rgb; @@ -71,4 +99,14 @@ cv::Mat RgbaToBgr(cv::Mat rgba) { return bgra; } +ImageFrame CreateTestFloat32ImageFrame(int width, int height) { + return CreateTestImageFrame(width, height, + /*max_value=*/1.0f); +} + +ImageFrame CreateTestGrey8ImageFrame(int width, int height) { + return CreateTestImageFrame(width, height, + /*max_value=*/255); +} + } // namespace mediapipe diff --git a/mediapipe/util/image_test_utils.h b/mediapipe/util/image_test_utils.h index 15a21c5b1..49943382f 100644 --- a/mediapipe/util/image_test_utils.h +++ b/mediapipe/util/image_test_utils.h @@ -4,6 +4,7 @@ #include #include "mediapipe/framework/formats/image_format.pb.h" +#include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/packet.h" #include "mediapipe/framework/port/opencv_core_inc.h" @@ -30,6 +31,12 @@ Packet MakeImagePacket(cv::Mat input, int timestamp = 0); // Converts RGBA Mat to BGR. cv::Mat RgbaToBgr(cv::Mat rgba); +// Generates single-channel float32 ImageFrame with increasing [0,1] values. +ImageFrame CreateTestFloat32ImageFrame(int width, int height); + +// Generates single-channel uint8 ImageFrame with increasing [0,255] values. +ImageFrame CreateTestGrey8ImageFrame(int width, int height); + } // namespace mediapipe #endif // MEDIAPIPE_UTIL_IMAGE_TEST_UTILS_H_ From 5cd30374430fe1991e6f2b22a07f817a88b8a0bf Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 20 Nov 2023 03:00:09 -0800 Subject: [PATCH 128/157] Adding a GpuTestWithParamBase test class to support value parameterized tests PiperOrigin-RevId: 583967017 --- mediapipe/gpu/gpu_test_base.h | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/mediapipe/gpu/gpu_test_base.h b/mediapipe/gpu/gpu_test_base.h index 6ec53603b..94842c769 100644 --- a/mediapipe/gpu/gpu_test_base.h +++ b/mediapipe/gpu/gpu_test_base.h @@ -15,6 +15,9 @@ #ifndef MEDIAPIPE_GPU_GPU_TEST_BASE_H_ #define MEDIAPIPE_GPU_GPU_TEST_BASE_H_ +#include +#include + #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/gpu/gl_calculator_helper.h" @@ -22,9 +25,9 @@ namespace mediapipe { -class GpuTestBase : public ::testing::Test { +class GpuTestEnvironment { protected: - GpuTestBase() { helper_.InitializeForTest(gpu_resources_.get()); } + GpuTestEnvironment() { helper_.InitializeForTest(gpu_resources_.get()); } void RunInGlContext(std::function gl_func) { helper_.RunInGlContext(std::move(gl_func)); @@ -35,6 +38,12 @@ class GpuTestBase : public ::testing::Test { GlCalculatorHelper helper_; }; +class GpuTestBase : public testing::Test, public GpuTestEnvironment {}; + +template +class GpuTestWithParamBase : public testing::TestWithParam, + public GpuTestEnvironment {}; + } // namespace mediapipe #endif // MEDIAPIPE_GPU_GPU_TEST_BASE_H_ From d8fd986517a5be4abb10e44514bffd70c4d9a2f4 Mon Sep 17 00:00:00 2001 From: Matt Kreileder Date: Mon, 20 Nov 2023 03:29:57 -0800 Subject: [PATCH 129/157] No public description PiperOrigin-RevId: 583973946 --- .../tasks/cc/components/processors/BUILD | 10 +++++-- .../processors/image_preprocessing_graph.cc | 3 +- mediapipe/tasks/cc/core/BUILD | 30 ++++++++++--------- mediapipe/tasks/cc/core/model_task_graph.cc | 4 ++- mediapipe/tasks/cc/vision/core/BUILD | 22 +++++++++----- mediapipe/tasks/cc/vision/face_detector/BUILD | 24 +++++++++------ .../face_detector/face_detector_graph.cc | 10 +++++-- .../face_detector/face_detector_graph_test.cc | 27 ++++++++++------- ...ace_detector_graph_test_task_runner_gms.cc | 28 +++++++++++++++++ ...face_detector_graph_test_task_runner_gms.h | 27 +++++++++++++++++ 10 files changed, 136 insertions(+), 49 deletions(-) create mode 100644 mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test_task_runner_gms.cc create mode 100644 mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test_task_runner_gms.h diff --git a/mediapipe/tasks/cc/components/processors/BUILD b/mediapipe/tasks/cc/components/processors/BUILD index 1a1e75d41..f02c6cd04 100644 --- a/mediapipe/tasks/cc/components/processors/BUILD +++ b/mediapipe/tasks/cc/components/processors/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@org_tensorflow//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite") + package(default_visibility = ["//mediapipe/tasks:internal"]) licenses(["notice"]) @@ -99,10 +101,14 @@ cc_library( alwayslink = 1, ) -cc_library( +cc_library_with_tflite( name = "image_preprocessing_graph", srcs = ["image_preprocessing_graph.cc"], hdrs = ["image_preprocessing_graph.h"], + tflite_deps = [ + "//mediapipe/tasks/cc/core:model_resources", + "//mediapipe/tasks/cc/vision/utils:image_tensor_specs", + ], deps = [ "//mediapipe/calculators/core:pass_through_calculator", "//mediapipe/calculators/image:image_clone_calculator", @@ -120,10 +126,8 @@ cc_library( "//mediapipe/gpu:gpu_origin_cc_proto", "//mediapipe/tasks/cc:common", "//mediapipe/tasks/cc/components/processors/proto:image_preprocessing_graph_options_cc_proto", - "//mediapipe/tasks/cc/core:model_resources", "//mediapipe/tasks/cc/core/proto:acceleration_cc_proto", "//mediapipe/tasks/cc/core/proto:base_options_cc_proto", - "//mediapipe/tasks/cc/vision/utils:image_tensor_specs", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@org_tensorflow//tensorflow/lite/schema:schema_fbs", diff --git a/mediapipe/tasks/cc/components/processors/image_preprocessing_graph.cc b/mediapipe/tasks/cc/components/processors/image_preprocessing_graph.cc index 1604dfbd5..da9d66c71 100644 --- a/mediapipe/tasks/cc/components/processors/image_preprocessing_graph.cc +++ b/mediapipe/tasks/cc/components/processors/image_preprocessing_graph.cc @@ -271,8 +271,9 @@ class ImagePreprocessingGraph : public Subgraph { }; } }; + REGISTER_MEDIAPIPE_GRAPH( - ::mediapipe::tasks::components::processors::ImagePreprocessingGraph); + ::mediapipe::tasks::components::processors::ImagePreprocessingGraph) } // namespace processors } // namespace components diff --git a/mediapipe/tasks/cc/core/BUILD b/mediapipe/tasks/cc/core/BUILD index bb0d4b001..9185d0a97 100644 --- a/mediapipe/tasks/cc/core/BUILD +++ b/mediapipe/tasks/cc/core/BUILD @@ -91,16 +91,16 @@ cc_library( ], ) -# TODO: Switch to use cc_library_with_tflite after the MediaPipe InferenceCalculator -# supports TFLite-in-GMSCore. -cc_library( +cc_library_with_tflite( name = "model_task_graph", srcs = ["model_task_graph.cc"], hdrs = ["model_task_graph.h"], - deps = [ - ":model_asset_bundle_resources", + tflite_deps = [ ":model_resources", ":model_resources_cache", + ], + deps = [ + ":model_asset_bundle_resources", ":model_resources_calculator", "//mediapipe/calculators/tensor:inference_calculator_cc_proto", "//mediapipe/framework:calculator_cc_proto", @@ -224,19 +224,16 @@ cc_library_with_tflite( alwayslink = 1, ) -cc_test_with_tflite( +cc_test( name = "model_resources_calculator_test", srcs = ["model_resources_calculator_test.cc"], data = [ "//mediapipe/tasks/testdata/core:test_models", ], - tflite_deps = [ + deps = [ ":model_resources", ":model_resources_cache", ":model_resources_calculator", - "@org_tensorflow//tensorflow/lite:test_util", - ], - deps = [ "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/tasks/cc/core/proto:external_file_cc_proto", @@ -245,6 +242,7 @@ cc_test_with_tflite( "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", + "@org_tensorflow//tensorflow/lite:test_util", "@org_tensorflow//tensorflow/lite/core/api:op_resolver", ], ) @@ -303,22 +301,26 @@ cc_test_with_tflite( ], ) -cc_library( +cc_library_with_tflite( name = "base_task_api", hdrs = ["base_task_api.h"], - deps = [ + tflite_deps = [ ":task_runner", + ], + deps = [ "//mediapipe/calculators/core:flow_limiter_calculator", ], ) -cc_library( +cc_library_with_tflite( name = "task_api_factory", hdrs = ["task_api_factory.h"], - deps = [ + tflite_deps = [ ":base_task_api", ":model_resources", ":task_runner", + ], + deps = [ ":utils", "//mediapipe/framework:calculator_cc_proto", "//mediapipe/framework:executor", diff --git a/mediapipe/tasks/cc/core/model_task_graph.cc b/mediapipe/tasks/cc/core/model_task_graph.cc index b82a69718..57bb25bf8 100644 --- a/mediapipe/tasks/cc/core/model_task_graph.cc +++ b/mediapipe/tasks/cc/core/model_task_graph.cc @@ -31,6 +31,7 @@ limitations under the License. #include "mediapipe/framework/api2/builder.h" #include "mediapipe/framework/api2/port.h" #include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/tasks/cc/common.h" #include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h" #include "mediapipe/tasks/cc/core/model_resources.h" @@ -147,7 +148,8 @@ class InferenceSubgraph : public Subgraph { return delegate; } }; -REGISTER_MEDIAPIPE_GRAPH(::mediapipe::tasks::core::InferenceSubgraph); + +REGISTER_MEDIAPIPE_GRAPH(::mediapipe::tasks::core::InferenceSubgraph) absl::StatusOr ModelTaskGraph::GetConfig( SubgraphContext* sc) { diff --git a/mediapipe/tasks/cc/vision/core/BUILD b/mediapipe/tasks/cc/vision/core/BUILD index 6bcf2f5d6..d7ebfec68 100644 --- a/mediapipe/tasks/cc/vision/core/BUILD +++ b/mediapipe/tasks/cc/vision/core/BUILD @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@org_tensorflow//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite") + licenses(["notice"]) package(default_visibility = ["//mediapipe/tasks:internal"]) @@ -31,9 +33,15 @@ cc_library( ], ) -cc_library( +cc_library_with_tflite( name = "base_vision_task_api", hdrs = ["base_vision_task_api.h"], + tflite_deps = [ + "//mediapipe/tasks/cc/core:base_task_api", + "//mediapipe/tasks/cc/core:task_api_factory", + "//mediapipe/tasks/cc/core:task_runner", + "//mediapipe/tasks/cc/vision/utils:image_tensor_specs", + ], deps = [ ":image_processing_options", ":running_mode", @@ -42,24 +50,22 @@ cc_library( "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/tasks/cc/components/containers:rect", - "//mediapipe/tasks/cc/core:base_task_api", - "//mediapipe/tasks/cc/core:task_api_factory", - "//mediapipe/tasks/cc/core:task_runner", - "//mediapipe/tasks/cc/vision/utils:image_tensor_specs", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", ], ) -cc_library( +cc_library_with_tflite( name = "vision_task_api_factory", hdrs = ["vision_task_api_factory.h"], - deps = [ + tflite_deps = [ ":base_vision_task_api", + "//mediapipe/tasks/cc/core:task_api_factory", + ], + deps = [ "//mediapipe/calculators/core:flow_limiter_calculator", "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/tasks/cc/core:task_api_factory", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", "@com_google_absl//absl/strings", diff --git a/mediapipe/tasks/cc/vision/face_detector/BUILD b/mediapipe/tasks/cc/vision/face_detector/BUILD index fbfd94628..bdad3bd06 100644 --- a/mediapipe/tasks/cc/vision/face_detector/BUILD +++ b/mediapipe/tasks/cc/vision/face_detector/BUILD @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +load("@org_tensorflow//tensorflow/lite/core/shims:cc_library_with_tflite.bzl", "cc_library_with_tflite") package(default_visibility = [ "//mediapipe/tasks:internal", @@ -18,9 +19,15 @@ package(default_visibility = [ licenses(["notice"]) -cc_library( +cc_library_with_tflite( name = "face_detector_graph", srcs = ["face_detector_graph.cc"], + tflite_deps = [ + "//mediapipe/tasks/cc/components/processors:image_preprocessing_graph", + "//mediapipe/tasks/cc/core:model_task_graph", + "//mediapipe/tasks/cc/core:model_resources", + "//mediapipe/tasks/cc/vision/utils:image_tensor_specs", + ], deps = [ "//mediapipe/calculators/core:clip_vector_size_calculator", "//mediapipe/calculators/core:clip_vector_size_calculator_cc_proto", @@ -38,6 +45,7 @@ cc_library( "//mediapipe/calculators/util:non_max_suppression_calculator_cc_proto", "//mediapipe/calculators/util:rect_transformation_calculator", "//mediapipe/calculators/util:rect_transformation_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", "//mediapipe/framework/api2:builder", "//mediapipe/framework/api2:port", "//mediapipe/framework/formats:detection_cc_proto", @@ -45,36 +53,34 @@ cc_library( "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/framework/formats:tensor", "//mediapipe/tasks/cc:common", - "//mediapipe/tasks/cc/components/processors:image_preprocessing_graph", - "//mediapipe/tasks/cc/core:model_resources", - "//mediapipe/tasks/cc/core:model_task_graph", "//mediapipe/tasks/cc/core:utils", "//mediapipe/tasks/cc/core/proto:inference_subgraph_cc_proto", "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto", - "//mediapipe/tasks/cc/vision/utils:image_tensor_specs", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", ], alwayslink = 1, ) -cc_library( +cc_library_with_tflite( name = "face_detector", srcs = ["face_detector.cc"], hdrs = ["face_detector.h"], + tflite_deps = [ + ":face_detector_graph", + "//mediapipe/tasks/cc/vision/core:base_vision_task_api", + "//mediapipe/tasks/cc/vision/core:vision_task_api_factory", + ], visibility = ["//visibility:public"], deps = [ - ":face_detector_graph", "//mediapipe/framework/api2:builder", "//mediapipe/framework/formats:detection_cc_proto", "//mediapipe/framework/formats:image", "//mediapipe/tasks/cc/components/containers:detection_result", "//mediapipe/tasks/cc/core:base_options", "//mediapipe/tasks/cc/core:utils", - "//mediapipe/tasks/cc/vision/core:base_vision_task_api", "//mediapipe/tasks/cc/vision/core:image_processing_options", "//mediapipe/tasks/cc/vision/core:running_mode", - "//mediapipe/tasks/cc/vision/core:vision_task_api_factory", "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto", "@com_google_absl//absl/status", "@com_google_absl//absl/status:statusor", diff --git a/mediapipe/tasks/cc/vision/face_detector/face_detector_graph.cc b/mediapipe/tasks/cc/vision/face_detector/face_detector_graph.cc index 8586a7ebd..5a8a60101 100644 --- a/mediapipe/tasks/cc/vision/face_detector/face_detector_graph.cc +++ b/mediapipe/tasks/cc/vision/face_detector/face_detector_graph.cc @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ +#include #include #include "absl/status/status.h" @@ -26,6 +27,7 @@ limitations under the License. #include "mediapipe/calculators/util/rect_transformation_calculator.pb.h" #include "mediapipe/framework/api2/builder.h" #include "mediapipe/framework/api2/port.h" +#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/formats/detection.pb.h" #include "mediapipe/framework/formats/image.h" #include "mediapipe/framework/formats/rect.pb.h" @@ -213,14 +215,16 @@ class FaceDetectorGraph : public core::ModelTaskGraph { } private: + std::string GetImagePreprocessingGraphName() { + return "mediapipe.tasks.components.processors.ImagePreprocessingGraph"; + } absl::StatusOr BuildFaceDetectionSubgraph( const FaceDetectorGraphOptions& subgraph_options, const core::ModelResources& model_resources, Source image_in, Source norm_rect_in, Graph& graph) { // Image preprocessing subgraph to convert image to tensor for the tflite // model. - auto& preprocessing = graph.AddNode( - "mediapipe.tasks.components.processors.ImagePreprocessingGraph"); + auto& preprocessing = graph.AddNode(GetImagePreprocessingGraphName()); bool use_gpu = components::processors::DetermineImagePreprocessingGpuBackend( subgraph_options.base_options().acceleration()); @@ -337,7 +341,7 @@ class FaceDetectorGraph : public core::ModelTaskGraph { }; REGISTER_MEDIAPIPE_GRAPH( - ::mediapipe::tasks::vision::face_detector::FaceDetectorGraph); + ::mediapipe::tasks::vision::face_detector::FaceDetectorGraph) } // namespace face_detector } // namespace vision diff --git a/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test.cc b/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test.cc index 651ad722d..768b92cfd 100644 --- a/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test.cc +++ b/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test.cc @@ -23,6 +23,7 @@ limitations under the License. #include "absl/flags/flag.h" #include "absl/log/absl_check.h" #include "absl/status/statusor.h" +#include "absl/strings/match.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/api2/builder.h" @@ -92,11 +93,10 @@ constexpr float kFaceDetectionMaxDiff = 0.01; // Helper function to create a TaskRunner. absl::StatusOr> CreateTaskRunner( - absl::string_view model_name) { + absl::string_view model_name, std::string graph_name) { Graph graph; - auto& face_detector_graph = - graph.AddNode("mediapipe.tasks.vision.face_detector.FaceDetectorGraph"); + auto& face_detector_graph = graph.AddNode(graph_name); auto options = std::make_unique(); options->mutable_base_options()->mutable_model_asset()->set_file_name( @@ -136,6 +136,8 @@ struct TestParams { std::string test_image_name; // Expected face detection results. std::vector expected_result; + // The name of the mediapipe graph to run. + std::string graph_name; }; class FaceDetectorGraphTest : public testing::TestWithParam {}; @@ -149,8 +151,9 @@ TEST_P(FaceDetectorGraphTest, Succeed) { input_norm_rect.set_y_center(0.5); input_norm_rect.set_width(1.0); input_norm_rect.set_height(1.0); - MP_ASSERT_OK_AND_ASSIGN( - auto task_runner, CreateTaskRunner(GetParam().face_detection_model_name)); + MP_ASSERT_OK_AND_ASSIGN(auto task_runner, + CreateTaskRunner(GetParam().face_detection_model_name, + GetParam().graph_name)); auto output_packets = task_runner->Process( {{kImageName, MakePacket(std::move(image))}, {kNormRectName, @@ -165,11 +168,15 @@ TEST_P(FaceDetectorGraphTest, Succeed) { INSTANTIATE_TEST_SUITE_P( FaceDetectorGraphTest, FaceDetectorGraphTest, - Values(TestParams{.test_name = "ShortRange", - .face_detection_model_name = kShortRangeBlazeFaceModel, - .test_image_name = kPortraitImage, - .expected_result = {GetExpectedFaceDetectionResult( - kPortraitExpectedDetection)}}), + Values( + TestParams{ + .test_name = "ShortRange", + .face_detection_model_name = kShortRangeBlazeFaceModel, + .test_image_name = kPortraitImage, + .expected_result = {GetExpectedFaceDetectionResult( + kPortraitExpectedDetection)}, + .graph_name = + "mediapipe.tasks.vision.face_detector.FaceDetectorGraph"}, ), [](const TestParamInfo& info) { return info.param.test_name; }); diff --git a/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test_task_runner_gms.cc b/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test_task_runner_gms.cc new file mode 100644 index 000000000..a8b82a2ac --- /dev/null +++ b/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test_task_runner_gms.cc @@ -0,0 +1,28 @@ +#include "mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test_task_runner_gms.h" + +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/tasks/cc/core/task_runner.h" +#include "tensorflow/lite/core/api/op_resolver.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace face_detector { +namespace test_util { + +absl::StatusOr> +CreateTaskRunnerGms(mediapipe::CalculatorGraphConfig config, + std::unique_ptr op_resolver) { + return mediapipe::tasks::core::TaskRunner::Create(std::move(config), + std::move(op_resolver)); +} + +} // namespace test_util +} // namespace face_detector +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test_task_runner_gms.h b/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test_task_runner_gms.h new file mode 100644 index 000000000..c34464c9e --- /dev/null +++ b/mediapipe/tasks/cc/vision/face_detector/face_detector_graph_test_task_runner_gms.h @@ -0,0 +1,27 @@ +#ifndef MEDIAPIPE_TASKS_CC_VISION_FACE_DETECTOR_FACE_DETECTOR_GRAPH_TEST_TASK_RUNNER_GMS_H_ +#define MEDIAPIPE_TASKS_CC_VISION_FACE_DETECTOR_FACE_DETECTOR_GRAPH_TEST_TASK_RUNNER_GMS_H_ + +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/tasks/cc/core/task_runner.h" +#include "tensorflow/lite/core/api/op_resolver.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace face_detector { +namespace test_util { + +absl::StatusOr> +CreateTaskRunnerGms(mediapipe::CalculatorGraphConfig config, + std::unique_ptr op_resolver = nullptr); + +} // namespace test_util +} // namespace face_detector +} // namespace vision +} // namespace tasks +} // namespace mediapipe + +#endif // MEDIAPIPE_TASKS_CC_VISION_FACE_DETECTOR_FACE_DETECTOR_GRAPH_TEST_TASK_RUNNER_GMS_H_ From 972e3d81c008c11c974bede2be6cfbbdf1fff189 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 20 Nov 2023 22:33:58 +0530 Subject: [PATCH 130/157] Added iOS Objective C Pose Landmarker Tests --- .../ios/test/vision/pose_landmarker/BUILD | 63 +++ .../pose_landmarker/MPPPoseLandmarkerTests.mm | 478 ++++++++++++++++++ 2 files changed, 541 insertions(+) create mode 100644 mediapipe/tasks/ios/test/vision/pose_landmarker/BUILD create mode 100644 mediapipe/tasks/ios/test/vision/pose_landmarker/MPPPoseLandmarkerTests.mm diff --git a/mediapipe/tasks/ios/test/vision/pose_landmarker/BUILD b/mediapipe/tasks/ios/test/vision/pose_landmarker/BUILD new file mode 100644 index 000000000..4312f8265 --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/pose_landmarker/BUILD @@ -0,0 +1,63 @@ +load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") +load( + "//mediapipe/framework/tool:ios.bzl", + "MPP_TASK_MINIMUM_OS_VERSION", +) +load( + "@org_tensorflow//tensorflow/lite:special_rules.bzl", + "tflite_ios_lab_runner", +) + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# Default tags for filtering iOS targets. Targets are restricted to Apple platforms. +TFL_DEFAULT_TAGS = [ + "apple", +] + +# Following sanitizer tests are not supported by iOS test targets. +TFL_DISABLED_SANITIZER_TAGS = [ + "noasan", + "nomsan", + "notsan", +] + +objc_library( + name = "MPPPoseLandmarkerObjcTestLibrary", + testonly = 1, + srcs = ["MPPPoseLandmarkerTests.mm"], + copts = [ + "-ObjC++", + "-std=c++17", + "-x objective-c++", + ], + data = [ + "//mediapipe/tasks/testdata/vision:pose_landmarker.task", + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_protos", + ], + deps = [ + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/test/vision/pose_landmarker/utils:MPPPoseLandmarkerResultProtobufHelpers", + "//mediapipe/tasks/ios/test/vision/utils:MPPImageTestUtils", + "//mediapipe/tasks/ios/test/vision/utils:MPPMaskTestUtils", + "//mediapipe/tasks/ios/vision/pose_landmarker:MPPPoseLandmarker", + ] + select({ + "//third_party:opencv_ios_sim_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_arm64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//third_party:opencv_ios_x86_64_source_build": ["@ios_opencv_source//:opencv_xcframework"], + "//conditions:default": ["@ios_opencv//:OpencvFramework"], + }), +) + +ios_unit_test( + name = "MPPPoseLandmarkerObjcTest", + minimum_os_version = MPP_TASK_MINIMUM_OS_VERSION, + runner = tflite_ios_lab_runner("IOS_LATEST"), + tags = TFL_DEFAULT_TAGS + TFL_DISABLED_SANITIZER_TAGS, + deps = [ + ":MPPPoseLandmarkerObjcTestLibrary", + ], +) diff --git a/mediapipe/tasks/ios/test/vision/pose_landmarker/MPPPoseLandmarkerTests.mm b/mediapipe/tasks/ios/test/vision/pose_landmarker/MPPPoseLandmarkerTests.mm new file mode 100644 index 000000000..2f728fdef --- /dev/null +++ b/mediapipe/tasks/ios/test/vision/pose_landmarker/MPPPoseLandmarkerTests.mm @@ -0,0 +1,478 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" +#import "mediapipe/tasks/ios/test/vision/pose_landmarker/utils/sources/MPPPoseLandmarkerResult+ProtobufHelpers.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPImage+TestUtils.h" +#import "mediapipe/tasks/ios/test/vision/utils/sources/MPPMask+TestUtils.h" +#import "mediapipe/tasks/ios/vision/pose_landmarker/sources/MPPPoseLandmarker.h" + +static MPPFileInfo *const kPoseLandmarkerBundleAssetFileInfo = + [[MPPFileInfo alloc] initWithName:@"pose_landmarker" type:@"task"]; + +static MPPFileInfo *const kPoseImageFileInfo = [[MPPFileInfo alloc] initWithName:@"pose" + type:@"jpg"]; +static MPPFileInfo *const kNoPoseImageFileInfo = [[MPPFileInfo alloc] initWithName:@"burger" + type:@"jpg"]; + +static MPPFileInfo *const kExpectedPoseLandmarksFileInfo = + [[MPPFileInfo alloc] initWithName:@"pose_landmarks" type:@"pbtxt"]; + +static NSString *const kExpectedErrorDomain = @"com.google.mediapipe.tasks"; +static const float kLandmarksErrorTolerance = 0.03f; +static const float kVisibilityTolerance = 0.9f; +static const float kPresenceTolerance = 0.9f; + +static NSString *const kLiveStreamTestsDictPoseLandmarkerKey = @"pose_landmarker"; +static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; + +#define AssertEqualErrors(error, expectedError) \ + XCTAssertNotNil(error); \ + XCTAssertEqualObjects(error.domain, expectedError.domain); \ + XCTAssertEqual(error.code, expectedError.code); \ + XCTAssertEqualObjects(error.localizedDescription, expectedError.localizedDescription) + +#define AssertApproximatelyEqualLandmarks(landmark, expectedLandmark, poseIndex, landmarkIndex) \ + XCTAssertEqualWithAccuracy(landmark.x, expectedLandmark.x, kLandmarksErrorTolerance, \ + @"pose index = %d landmark index j = %d", poseIndex, landmarkIndex); \ + XCTAssertEqualWithAccuracy(landmark.y, expectedLandmark.y, kLandmarksErrorTolerance, \ + @"pose index = %d landmark index j = %d", poseIndex, landmarkIndex); + +@interface MPPPoseLandmarkerTests : XCTestCase { + NSDictionary *_liveStreamSucceedsTestDict; + NSDictionary *_outOfOrderTimestampTestDict; +} +@end + +@implementation MPPPoseLandmarkerTests + +#pragma mark General Tests + +- (void)testDetectWithModelPathSucceeds { + MPPPoseLandmarker *poseLandmarker = + [[MPPPoseLandmarker alloc] initWithModelPath:kPoseLandmarkerBundleAssetFileInfo.path + error:nil]; + XCTAssertNotNil(poseLandmarker); + + [self assertResultsOfDetectInImageWithFileInfo:kPoseImageFileInfo + usingPoseLandmarker:poseLandmarker + approximatelyEqualsPoseLandmarkerResult:[MPPPoseLandmarkerTests + expectedPoseLandmarkerResult]]; +} + +- (void)testDetectWithOptionsSucceeds { + MPPPoseLandmarkerOptions *options = [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + MPPPoseLandmarker *poseLandmarker = [self createPoseLandmarkerWithOptionsSucceeds:options]; + + [self assertResultsOfDetectInImageWithFileInfo:kPoseImageFileInfo + usingPoseLandmarker:poseLandmarker + approximatelyEqualsPoseLandmarkerResult:[MPPPoseLandmarkerTests + expectedPoseLandmarkerResult]]; +} + +- (void)testDetectWithEmptyResultsSucceeds { + MPPPoseLandmarkerOptions *options = [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + MPPPoseLandmarker *poseLandmarker = [self createPoseLandmarkerWithOptionsSucceeds:options]; + + [self assertResultsOfDetectInImageWithFileInfo:kNoPoseImageFileInfo + usingPoseLandmarker:poseLandmarker + approximatelyEqualsPoseLandmarkerResult:[MPPPoseLandmarkerTests + emptyPoseLandmarkerResult]]; +} + +- (void)testCreatePoseLandmarkerFailsWithDelegateInNonLiveStreamMode { + MPPRunningMode runningModesToTest[] = {MPPRunningModeImage, MPPRunningModeVideo}; + for (int i = 0; i < sizeof(runningModesToTest) / sizeof(runningModesToTest[0]); i++) { + MPPPoseLandmarkerOptions *options = + [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + + options.runningMode = runningModesToTest[i]; + options.poseLandmarkerLiveStreamDelegate = self; + + [self + assertCreatePoseLandmarkerWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in image or video mode. The " + @"delegate must not be set in the task's options." + }]]; + } +} + +#pragma mark Running Mode Tests + +- (void)testCreatePoseLandmarkerFailsWithMissingDelegateInLiveStreamMode { + MPPPoseLandmarkerOptions *options = + [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + + options.runningMode = MPPRunningModeLiveStream; + + [self assertCreatePoseLandmarkerWithOptions:options + failsWithExpectedError: + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"The vision task is in live stream mode. An " + @"object must be set as the delegate of the task " + @"in its options to ensure asynchronous delivery " + @"of results." + }]]; +} + +- (void)testDetectFailsWithCallingWrongApiInImageMode { + MPPPoseLandmarkerOptions *options = + [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + + MPPPoseLandmarker *poseLandmarker = [self createPoseLandmarkerWithOptionsSucceeds:options]; + + MPPImage *image = [MPPImage imageWithFileInfo:kPoseImageFileInfo]; + + NSError *liveStreamApiCallError; + XCTAssertFalse([poseLandmarker detectAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Image" + }]; + + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([poseLandmarker detectVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Image" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testDetectFailsWithCallingWrongApiInVideoMode { + MPPPoseLandmarkerOptions *options = + [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + options.runningMode = MPPRunningModeVideo; + + MPPPoseLandmarker *poseLandmarker = [self createPoseLandmarkerWithOptionsSucceeds:options]; + + MPPImage *image = [MPPImage imageWithFileInfo:kPoseImageFileInfo]; + + NSError *liveStreamApiCallError; + XCTAssertFalse([poseLandmarker detectAsyncImage:image + timestampInMilliseconds:0 + error:&liveStreamApiCallError]); + + NSError *expectedLiveStreamApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with live " + @"stream mode. Current Running Mode: Video" + }]; + + AssertEqualErrors(liveStreamApiCallError, expectedLiveStreamApiCallError); + + NSError *imageApiCallError; + XCTAssertFalse([poseLandmarker detectImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Video" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); +} + +- (void)testDetectFailsWithCallingWrongApiInLiveStreamMode { + MPPPoseLandmarkerOptions *options = + [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + options.runningMode = MPPRunningModeLiveStream; + options.poseLandmarkerLiveStreamDelegate = self; + + MPPPoseLandmarker *poseLandmarker = [self createPoseLandmarkerWithOptionsSucceeds:options]; + + MPPImage *image = [MPPImage imageWithFileInfo:kPoseImageFileInfo]; + + NSError *imageApiCallError; + XCTAssertFalse([poseLandmarker detectImage:image error:&imageApiCallError]); + + NSError *expectedImageApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"image mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(imageApiCallError, expectedImageApiCallError); + + NSError *videoApiCallError; + XCTAssertFalse([poseLandmarker detectVideoFrame:image + timestampInMilliseconds:0 + error:&videoApiCallError]); + + NSError *expectedVideoApiCallError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : @"The vision task is not initialized with " + @"video mode. Current Running Mode: Live Stream" + }]; + AssertEqualErrors(videoApiCallError, expectedVideoApiCallError); +} + +- (void)testDetectWithVideoModeSucceeds { + MPPPoseLandmarkerOptions *options = + [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + options.runningMode = MPPRunningModeVideo; + + MPPPoseLandmarker *poseLandmarker = [self createPoseLandmarkerWithOptionsSucceeds:options]; + + MPPImage *image = [MPPImage imageWithFileInfo:kPoseImageFileInfo]; + + for (int i = 0; i < 3; i++) { + MPPPoseLandmarkerResult *poseLandmarkerResult = [poseLandmarker detectVideoFrame:image + timestampInMilliseconds:i + error:nil]; + [self assertPoseLandmarkerResult:poseLandmarkerResult + isApproximatelyEqualToExpectedResult:[MPPPoseLandmarkerTests expectedPoseLandmarkerResult]]; + } +} + +- (void)testDetectWithOutOfOrderTimestampsAndLiveStreamModeFails { + MPPPoseLandmarkerOptions *options = + [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + options.runningMode = MPPRunningModeLiveStream; + options.poseLandmarkerLiveStreamDelegate = self; + + XCTestExpectation *expectation = [[XCTestExpectation alloc] + initWithDescription:@"detectWiththOutOfOrderTimestampsAndLiveStream"]; + + expectation.expectedFulfillmentCount = 1; + + MPPPoseLandmarker *poseLandmarker = [self createPoseLandmarkerWithOptionsSucceeds:options]; + + _outOfOrderTimestampTestDict = @{ + kLiveStreamTestsDictPoseLandmarkerKey : poseLandmarker, + kLiveStreamTestsDictExpectationKey : expectation + }; + + MPPImage *image = [MPPImage imageWithFileInfo:kPoseImageFileInfo]; + + XCTAssertTrue([poseLandmarker detectAsyncImage:image timestampInMilliseconds:1 error:nil]); + + NSError *error; + XCTAssertFalse([poseLandmarker detectAsyncImage:image timestampInMilliseconds:0 error:&error]); + + NSError *expectedError = + [NSError errorWithDomain:kExpectedErrorDomain + code:MPPTasksErrorCodeInvalidArgumentError + userInfo:@{ + NSLocalizedDescriptionKey : + @"INVALID_ARGUMENT: Input timestamp must be monotonically increasing." + }]; + AssertEqualErrors(error, expectedError); + + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; +} + +- (void)testDetectWithLiveStreamModeSucceeds { + MPPPoseLandmarkerOptions *options = + [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + options.runningMode = MPPRunningModeLiveStream; + options.poseLandmarkerLiveStreamDelegate = self; + + NSInteger iterationCount = 100; + + // Because of flow limiting, we cannot ensure that the callback will be invoked `iterationCount` + // times. An normal expectation will fail if expectation.fulfill() is not called + // `expectation.expectedFulfillmentCount` times. If `expectation.isInverted = true`, the test will + // only succeed if expectation is not fulfilled for the specified `expectedFulfillmentCount`. + // Since in our case we cannot predict how many times the expectation is supposed to be fulfilled + // setting, `expectation.expectedFulfillmentCount` = `iterationCount` + 1 and + // `expectation.isInverted = true` ensures that test succeeds if the expectation is fulfilled <= + // `iterationCount` times. + XCTestExpectation *expectation = + [[XCTestExpectation alloc] initWithDescription:@"detectWithLiveStream"]; + + expectation.expectedFulfillmentCount = iterationCount + 1; + expectation.inverted = YES; + + MPPPoseLandmarker *poseLandmarker = [self createPoseLandmarkerWithOptionsSucceeds:options]; + + _liveStreamSucceedsTestDict = @{ + kLiveStreamTestsDictPoseLandmarkerKey : poseLandmarker, + kLiveStreamTestsDictExpectationKey : expectation + }; + + // TODO: Mimic initialization from CMSampleBuffer as live stream mode is most likely to be used + // with the iOS camera. AVCaptureVideoDataOutput sample buffer delegates provide frames of type + // `CMSampleBuffer`. + MPPImage *image = [MPPImage imageWithFileInfo:kPoseImageFileInfo]; + + for (int i = 0; i < iterationCount; i++) { + XCTAssertTrue([poseLandmarker detectAsyncImage:image timestampInMilliseconds:i error:nil]); + } + + NSTimeInterval timeout = 0.5f; + [self waitForExpectations:@[ expectation ] timeout:timeout]; +} + +- (void)poseLandmarker:(MPPPoseLandmarker *)poseLandmarker + didFinishDetectionWithResult:(MPPPoseLandmarkerResult *)poseLandmarkerResult + timestampInMilliseconds:(NSInteger)timestampInMilliseconds + error:(NSError *)error { + [self assertPoseLandmarkerResult:poseLandmarkerResult + isApproximatelyEqualToExpectedResult:[MPPPoseLandmarkerTests expectedPoseLandmarkerResult]]; + + if (poseLandmarker == _outOfOrderTimestampTestDict[kLiveStreamTestsDictPoseLandmarkerKey]) { + [_outOfOrderTimestampTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } else if (poseLandmarker == _liveStreamSucceedsTestDict[kLiveStreamTestsDictPoseLandmarkerKey]) { + [_liveStreamSucceedsTestDict[kLiveStreamTestsDictExpectationKey] fulfill]; + } +} + +#pragma mark Pose Landmarker Initializers + +- (MPPPoseLandmarkerOptions *)poseLandmarkerOptionsWithModelFileInfo:(MPPFileInfo *)modelFileInfo { + MPPPoseLandmarkerOptions *poseLandmarkerOptions = [[MPPPoseLandmarkerOptions alloc] init]; + poseLandmarkerOptions.baseOptions.modelAssetPath = modelFileInfo.path; + + return poseLandmarkerOptions; +} + +- (MPPPoseLandmarker *)createPoseLandmarkerWithOptionsSucceeds: + (MPPPoseLandmarkerOptions *)poseLandmarkerOptions { + NSError *error; + MPPPoseLandmarker *poseLandmarker = + [[MPPPoseLandmarker alloc] initWithOptions:poseLandmarkerOptions error:&error]; + XCTAssertNotNil(poseLandmarker); + XCTAssertNil(error); + + return poseLandmarker; +} + +- (void)assertCreatePoseLandmarkerWithOptions:(MPPPoseLandmarkerOptions *)poseLandmarkerOptions + failsWithExpectedError:(NSError *)expectedError { + NSError *error = nil; + MPPPoseLandmarker *poseLandmarker = + [[MPPPoseLandmarker alloc] initWithOptions:poseLandmarkerOptions error:&error]; + + XCTAssertNil(poseLandmarker); + AssertEqualErrors(error, expectedError); +} + +#pragma mark Results + ++ (MPPPoseLandmarkerResult *)emptyPoseLandmarkerResult { + return [[MPPPoseLandmarkerResult alloc] initWithLandmarks:@[] + worldLandmarks:@[] + segmentationMasks:@[] + timestampInMilliseconds:0]; +} + ++ (MPPPoseLandmarkerResult *)expectedPoseLandmarkerResult { + return [MPPPoseLandmarkerResult + poseLandmarkerResultFromProtobufFileWithName:kExpectedPoseLandmarksFileInfo.path + shouldRemoveZPosition:YES]; +} + +- (void)assertResultsOfDetectInImageWithFileInfo:(MPPFileInfo *)fileInfo + usingPoseLandmarker:(MPPPoseLandmarker *)poseLandmarker + approximatelyEqualsPoseLandmarkerResult: + (MPPPoseLandmarkerResult *)expectedPoseLandmarkerResult { + MPPPoseLandmarkerResult *poseLandmarkerResult = [self detectImageWithFileInfo:fileInfo + usingPoseLandmarker:poseLandmarker]; + [self assertPoseLandmarkerResult:poseLandmarkerResult + isApproximatelyEqualToExpectedResult:expectedPoseLandmarkerResult]; +} + +- (MPPPoseLandmarkerResult *)detectImageWithFileInfo:(MPPFileInfo *)imageFileInfo + usingPoseLandmarker:(MPPPoseLandmarker *)poseLandmarker { + MPPImage *image = [MPPImage imageWithFileInfo:imageFileInfo]; + + MPPPoseLandmarkerResult *poseLandmarkerResult = [poseLandmarker detectImage:image error:nil]; + XCTAssertNotNil(poseLandmarkerResult); + + return poseLandmarkerResult; +} + +- (void)assertPoseLandmarkerResult:(MPPPoseLandmarkerResult *)poseLandmarkerResult + isApproximatelyEqualToExpectedResult:(MPPPoseLandmarkerResult *)expectedPoseLandmarkerResult { + // TODO: Add additional tests for auxiliary, world landmarks and segmentation masks. + // Expects to have the same number of poses detected. + [self assertMultiPoseLandmarks:poseLandmarkerResult.landmarks + areApproximatelyEqualToExpectedMultiPoseLandmarks:expectedPoseLandmarkerResult.landmarks]; + + + [self assertLandmarksAreVisibleAndPresentInPoseLandmarkerResult:poseLandmarkerResult]; +} + +- (void)assertMultiPoseLandmarks:(NSArray *> *)multiPoseLandmarks + areApproximatelyEqualToExpectedMultiPoseLandmarks: + (NSArray *> *)expectedMultiPoseLandmarks { + XCTAssertEqual(multiPoseLandmarks.count, expectedMultiPoseLandmarks.count); + + if (multiPoseLandmarks.count == 0) { + return; + } + + NSArray *topPoseLandmarks = multiPoseLandmarks[0]; + NSArray *expectedTopPoseLandmarks = expectedMultiPoseLandmarks[0]; + + XCTAssertEqual(topPoseLandmarks.count, expectedTopPoseLandmarks.count); + for (int i = 0; i < expectedTopPoseLandmarks.count; i++) { + MPPNormalizedLandmark *landmark = topPoseLandmarks[i]; + XCTAssertNotNil(landmark); + AssertApproximatelyEqualLandmarks(landmark, expectedTopPoseLandmarks[i], 0, i); + } +} + +- (void)assertLandmarksAreVisibleAndPresentInPoseLandmarkerResult: + (MPPPoseLandmarkerResult *)poseLandmarkerResult { + for (int i = 0; i < poseLandmarkerResult.landmarks.count; i++) { + NSArray *landmarks = poseLandmarkerResult.landmarks[i]; + for (int j = 0; j < landmarks.count; j++) { + MPPNormalizedLandmark *landmark = landmarks[i]; + XCTAssertGreaterThanOrEqual( + landmark.visibility.floatValue, kVisibilityTolerance, + @"multi pose landmark index i = %d landmark index j = %d visibility %f", i, j, + landmark.visibility.floatValue); + XCTAssertGreaterThanOrEqual( + landmark.presence.floatValue, kPresenceTolerance, + @"multi pose landmark index i = %d landmark index j = %d presence %f", i, j, + landmark.presence.floatValue); + } + } +} + +@end \ No newline at end of file From 447f9cc452cc8336e8d6be7ff8422e8e19be286f Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Mon, 20 Nov 2023 22:37:43 +0530 Subject: [PATCH 131/157] Fixed formatting of MPPPoseLandmarkerTests.mm --- .../pose_landmarker/MPPPoseLandmarkerTests.mm | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/mediapipe/tasks/ios/test/vision/pose_landmarker/MPPPoseLandmarkerTests.mm b/mediapipe/tasks/ios/test/vision/pose_landmarker/MPPPoseLandmarkerTests.mm index 2f728fdef..c22386f98 100644 --- a/mediapipe/tasks/ios/test/vision/pose_landmarker/MPPPoseLandmarkerTests.mm +++ b/mediapipe/tasks/ios/test/vision/pose_landmarker/MPPPoseLandmarkerTests.mm @@ -75,7 +75,8 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; } - (void)testDetectWithOptionsSucceeds { - MPPPoseLandmarkerOptions *options = [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + MPPPoseLandmarkerOptions *options = + [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; MPPPoseLandmarker *poseLandmarker = [self createPoseLandmarkerWithOptionsSucceeds:options]; [self assertResultsOfDetectInImageWithFileInfo:kPoseImageFileInfo @@ -85,13 +86,14 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; } - (void)testDetectWithEmptyResultsSucceeds { - MPPPoseLandmarkerOptions *options = [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; + MPPPoseLandmarkerOptions *options = + [self poseLandmarkerOptionsWithModelFileInfo:kPoseLandmarkerBundleAssetFileInfo]; MPPPoseLandmarker *poseLandmarker = [self createPoseLandmarkerWithOptionsSucceeds:options]; - [self assertResultsOfDetectInImageWithFileInfo:kNoPoseImageFileInfo - usingPoseLandmarker:poseLandmarker - approximatelyEqualsPoseLandmarkerResult:[MPPPoseLandmarkerTests - emptyPoseLandmarkerResult]]; + [self + assertResultsOfDetectInImageWithFileInfo:kNoPoseImageFileInfo + usingPoseLandmarker:poseLandmarker + approximatelyEqualsPoseLandmarkerResult:[MPPPoseLandmarkerTests emptyPoseLandmarkerResult]]; } - (void)testCreatePoseLandmarkerFailsWithDelegateInNonLiveStreamMode { @@ -432,7 +434,6 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; // Expects to have the same number of poses detected. [self assertMultiPoseLandmarks:poseLandmarkerResult.landmarks areApproximatelyEqualToExpectedMultiPoseLandmarks:expectedPoseLandmarkerResult.landmarks]; - [self assertLandmarksAreVisibleAndPresentInPoseLandmarkerResult:poseLandmarkerResult]; } @@ -440,8 +441,8 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; - (void)assertMultiPoseLandmarks:(NSArray *> *)multiPoseLandmarks areApproximatelyEqualToExpectedMultiPoseLandmarks: (NSArray *> *)expectedMultiPoseLandmarks { - XCTAssertEqual(multiPoseLandmarks.count, expectedMultiPoseLandmarks.count); - + XCTAssertEqual(multiPoseLandmarks.count, expectedMultiPoseLandmarks.count); + if (multiPoseLandmarks.count == 0) { return; } @@ -475,4 +476,4 @@ static NSString *const kLiveStreamTestsDictExpectationKey = @"expectation"; } } -@end \ No newline at end of file +@end From 0d298d7a6732866ab7d1592d531f0ad1bbf4f989 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 21 Nov 2023 09:35:20 -0800 Subject: [PATCH 132/157] No public description PiperOrigin-RevId: 584349220 --- mediapipe/gpu/BUILD | 2 + mediapipe/gpu/gpu_shared_data_internal.cc | 57 +++++++++++++++++------ mediapipe/gpu/gpu_shared_data_internal.h | 7 ++- 3 files changed, 52 insertions(+), 14 deletions(-) diff --git a/mediapipe/gpu/BUILD b/mediapipe/gpu/BUILD index 6ce4ec117..88864e894 100644 --- a/mediapipe/gpu/BUILD +++ b/mediapipe/gpu/BUILD @@ -669,6 +669,8 @@ cc_library( "//mediapipe/framework/port:ret_check", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/log:absl_check", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", ] + select({ "//conditions:default": [], "//mediapipe:apple": [ diff --git a/mediapipe/gpu/gpu_shared_data_internal.cc b/mediapipe/gpu/gpu_shared_data_internal.cc index 31bdf18b5..965a8b4c5 100644 --- a/mediapipe/gpu/gpu_shared_data_internal.cc +++ b/mediapipe/gpu/gpu_shared_data_internal.cc @@ -14,10 +14,14 @@ #include "mediapipe/gpu/gpu_shared_data_internal.h" +#include +#include + #include "absl/base/attributes.h" #include "absl/log/absl_check.h" +#include "absl/log/absl_log.h" +#include "absl/status/status.h" #include "mediapipe/framework/deps/no_destructor.h" -#include "mediapipe/framework/port/ret_check.h" #include "mediapipe/gpu/gl_context.h" #include "mediapipe/gpu/gl_context_options.pb.h" #include "mediapipe/gpu/graph_support.h" @@ -83,8 +87,25 @@ GpuResources::StatusOrGpuResources GpuResources::Create( } GpuResources::GpuResources(std::shared_ptr gl_context) + : gl_key_context_(new GlContextMapType(), + [](auto* map) { + // This flushes all pending jobs in all GL contexts, + // ensuring that all GL contexts not referenced + // elsewhere are destroyed as part of this destructor. + // Failure to do this may cause GL threads to outlast + // this destructor and execute jobs after the + // GpuResources object is destroyed. + for (auto& [key, context] : *map) { + const auto status = std::move(context)->Run( + []() { return absl::OkStatus(); }); + ABSL_LOG_IF(ERROR, !status.ok()) + << "Failed to flush GlContext jobs: " << status; + } + delete map; + }) #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER - : texture_caches_(std::make_shared()), + , + texture_caches_(std::make_shared()), gpu_buffer_pool_( [tc = texture_caches_](const internal::GpuBufferSpec& spec, const MultiPoolOptions& options) { @@ -92,7 +113,7 @@ GpuResources::GpuResources(std::shared_ptr gl_context) }) #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER { - gl_key_context_[SharedContextKey()] = gl_context; + gl_key_context_->insert({SharedContextKey(), gl_context}); named_executors_[kGpuExecutorName] = std::make_shared(gl_context.get()); #if __APPLE__ @@ -104,6 +125,15 @@ GpuResources::GpuResources(std::shared_ptr gl_context) } GpuResources::~GpuResources() { + // This flushes all pending jobs in all GL contexts, + // ensuring that all existing jobs, which may refer GpuResource and kept their + // gpu resources (e.g. GpuResources::gpu_buffer_pool_) through a raw pointer, + // have finished before kept gpu resources get deleted. + for (auto& [key, context] : *gl_key_context_) { + const auto status = context->Run([]() { return absl::OkStatus(); }); + ABSL_LOG_IF(ERROR, !status.ok()) + << "Failed to flush GlContext jobs: " << status; + } #if __APPLE__ // Note: on Apple platforms, this object contains Objective-C objects. // The destructor will release them, but ARC must be on. @@ -111,7 +141,7 @@ GpuResources::~GpuResources() { #error This file must be built with ARC. #endif #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER - for (auto& kv : gl_key_context_) { + for (auto& kv : *gl_key_context_) { texture_caches_->UnregisterTextureCache(kv.second->cv_texture_cache()); } #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER @@ -173,23 +203,24 @@ absl::Status GpuResources::PrepareGpuNode(CalculatorNode* node) { const std::shared_ptr& GpuResources::gl_context( CalculatorContext* cc) { if (cc) { - auto it = gl_key_context_.find(node_key_[cc->NodeName()]); - if (it != gl_key_context_.end()) { + auto it = gl_key_context_->find(node_key_[cc->NodeName()]); + if (it != gl_key_context_->end()) { return it->second; } } - return gl_key_context_[SharedContextKey()]; + return gl_key_context_->at(SharedContextKey()); } GlContext::StatusOrGlContext GpuResources::GetOrCreateGlContext( const std::string& key) { - auto it = gl_key_context_.find(key); - if (it == gl_key_context_.end()) { - MP_ASSIGN_OR_RETURN(std::shared_ptr new_context, - GlContext::Create(*gl_key_context_[SharedContextKey()], - kGlContextUseDedicatedThread)); - it = gl_key_context_.emplace(key, new_context).first; + auto it = gl_key_context_->find(key); + if (it == gl_key_context_->end()) { + MP_ASSIGN_OR_RETURN( + std::shared_ptr new_context, + GlContext::Create(*gl_key_context_->at(SharedContextKey()), + kGlContextUseDedicatedThread)); + it = gl_key_context_->emplace(key, new_context).first; #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER texture_caches_->RegisterTextureCache(it->second->cv_texture_cache()); #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER diff --git a/mediapipe/gpu/gpu_shared_data_internal.h b/mediapipe/gpu/gpu_shared_data_internal.h index 3f7c67e2e..5e8be45ab 100644 --- a/mediapipe/gpu/gpu_shared_data_internal.h +++ b/mediapipe/gpu/gpu_shared_data_internal.h @@ -21,6 +21,8 @@ #ifndef MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_ #define MEDIAPIPE_GPU_GPU_SHARED_DATA_INTERNAL_H_ +#include + #include "mediapipe/framework/calculator_context.h" #include "mediapipe/framework/calculator_node.h" #include "mediapipe/framework/executor.h" @@ -82,7 +84,10 @@ class GpuResources { const std::string& ContextKey(const std::string& canonical_node_name); std::map node_key_; - std::map> gl_key_context_; + + using GlContextMapType = std::map>; + std::unique_ptr + gl_key_context_; #ifdef MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER std::shared_ptr texture_caches_; From e7edd97effcc0dcdb714a45eef3590923397e6d0 Mon Sep 17 00:00:00 2001 From: Tony Allevato Date: Tue, 21 Nov 2023 12:31:50 -0800 Subject: [PATCH 133/157] Internal change. PiperOrigin-RevId: 584399894 --- mediapipe/gpu/metal.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/gpu/metal.bzl b/mediapipe/gpu/metal.bzl index 77129baf2..4ee6dd8f8 100644 --- a/mediapipe/gpu/metal.bzl +++ b/mediapipe/gpu/metal.bzl @@ -171,7 +171,7 @@ METAL_LIBRARY_ATTRS = dicts.add(apple_support.action_required_attrs(), { metal_library = rule( implementation = _metal_library_impl, attrs = METAL_LIBRARY_ATTRS, - fragments = ["apple", "objc", "swift"], + fragments = ["apple", "objc"], output_to_genfiles = True, ) """ From 5ca859f90b56ca3dffa54beab6b64e421ad04a09 Mon Sep 17 00:00:00 2001 From: Alex Macdonald-Smith Date: Tue, 21 Nov 2023 16:03:57 -0500 Subject: [PATCH 134/157] Updated mediapipe/mediapipe/tasks/web /vision/README.md There was a typo in the url referencing Gesture Recognizer ``` const gestureRecognizer = await GestureRecognizer.createFromModelPath(vision, "hhttps://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task" ); ``` changed to ``` const gestureRecognizer = await GestureRecognizer.createFromModelPath(vision, "https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task" ); ``` The extra 'h' was dropped. Let me know if there are anymore updates needed for this. --- mediapipe/tasks/web/vision/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/web/vision/README.md b/mediapipe/tasks/web/vision/README.md index 816ef9e4f..c603beaea 100644 --- a/mediapipe/tasks/web/vision/README.md +++ b/mediapipe/tasks/web/vision/README.md @@ -66,7 +66,7 @@ const vision = await FilesetResolver.forVisionTasks( "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm" ); const gestureRecognizer = await GestureRecognizer.createFromModelPath(vision, - "hhttps://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task" + "https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task" ); const image = document.getElementById("image") as HTMLImageElement; const recognitions = gestureRecognizer.recognize(image); From 17c0c960be5b13bffa8436fd91b37581b701459b Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 27 Nov 2023 04:51:32 -0800 Subject: [PATCH 135/157] Added Gesture Recognizer C API and tests --- mediapipe/tasks/c/components/containers/BUILD | 52 ++++ .../gesture_recognizer_result_converter.cc | 181 +++++++++++ .../gesture_recognizer_result_converter.h | 33 ++ ...esture_recognizer_result_converter_test.cc | 24 ++ .../tasks/c/components/containers/landmark.h | 90 ++++++ .../containers/landmark_converter.cc | 132 ++++++++ .../containers/landmark_converter.h | 51 +++ .../containers/landmark_converter_test.cc | 28 ++ .../tasks/c/vision/gesture_recognizer/BUILD | 76 +++++ .../gesture_recognizer/gesture_recognizer.cc | 294 ++++++++++++++++++ .../gesture_recognizer/gesture_recognizer.h | 156 ++++++++++ .../gesture_recognizer_result.h | 70 +++++ .../gesture_recognizer_test.cc | 123 ++++++++ mediapipe/tasks/testdata/vision/BUILD | 1 + 14 files changed, 1311 insertions(+) create mode 100644 mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc create mode 100644 mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.h create mode 100644 mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc create mode 100644 mediapipe/tasks/c/components/containers/landmark.h create mode 100644 mediapipe/tasks/c/components/containers/landmark_converter.cc create mode 100644 mediapipe/tasks/c/components/containers/landmark_converter.h create mode 100644 mediapipe/tasks/c/components/containers/landmark_converter_test.cc create mode 100644 mediapipe/tasks/c/vision/gesture_recognizer/BUILD create mode 100644 mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.cc create mode 100644 mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h create mode 100644 mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_result.h create mode 100644 mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc diff --git a/mediapipe/tasks/c/components/containers/BUILD b/mediapipe/tasks/c/components/containers/BUILD index 4bb580873..76ac50cc4 100644 --- a/mediapipe/tasks/c/components/containers/BUILD +++ b/mediapipe/tasks/c/components/containers/BUILD @@ -43,6 +43,33 @@ cc_test( ], ) +cc_library( + name = "landmark", + hdrs = ["landmark.h"], +) + +cc_library( + name = "landmark_converter", + srcs = ["landmark_converter.cc"], + hdrs = ["landmark_converter.h"], + deps = [ + ":landmark", + "//mediapipe/tasks/cc/components/containers:landmark", + ], +) + +cc_test( + name = "landmark_converter_test", + srcs = ["landmark_converter_test.cc"], + deps = [ + ":landmark", + ":landmark_converter", + "//mediapipe/framework/port:gtest", + "//mediapipe/tasks/cc/components/containers:landmark", + "@com_google_googletest//:gtest_main", + ], +) + cc_library( name = "classification_result", hdrs = ["classification_result.h"], @@ -121,3 +148,28 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) + +cc_library( + name = "gesture_recognizer_result_converter", + srcs = ["gesture_recognizer_result_converter.cc"], + hdrs = ["gesture_recognizer_result_converter.h"], + deps = [ + ":category_converter", + ":landmark_converter", + "//mediapipe/tasks/c/vision/gesture_recognizer:gesture_recognizer_result", + "//mediapipe/tasks/cc/vision/gesture_recognizer:gesture_recognizer_result", + ], +) + +cc_test( + name = "gesture_recognizer_result_converter_test", + srcs = ["gesture_recognizer_result_converter_test.cc"], + linkstatic = 1, + deps = [ + ":gesture_recognizer_result_converter", + "//mediapipe/framework/port:gtest", + "//mediapipe/tasks/c/vision/gesture_recognizer:gesture_recognizer_result", + "//mediapipe/tasks/cc/vision/gesture_recognizer:gesture_recognizer_result", + "@com_google_googletest//:gtest_main", + ], +) \ No newline at end of file diff --git a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc new file mode 100644 index 000000000..3b2da8eab --- /dev/null +++ b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc @@ -0,0 +1,181 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.h" + +#include +#include + +#include "mediapipe/tasks/c/components/containers/category_converter.h" +#include "mediapipe/tasks/c/components/containers/landmark_converter.h" +#include "mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_result.h" +#include "mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_result.h" + +namespace mediapipe::tasks::c::components::containers { + +using CppCategory = ::mediapipe::tasks::components::containers::Category; +using CppLandmark = ::mediapipe::tasks::components::containers::Landmark; +using CppNormalizedLandmark = + ::mediapipe::tasks::components::containers::NormalizedLandmark; + +void CppConvertToGestureRecognizerResult( + const mediapipe::tasks::vision::gesture_recognizer::GestureRecognizerResult& + in, + GestureRecognizerResult* out) { + out->gestures_count = in.gestures.size(); + out->gestures = new Category*[out->gestures_count]; + out->gestures_categories_counts = new uint32_t[out->gestures_count]; + + for (uint32_t i = 0; i < out->gestures_count; ++i) { + uint32_t categories_count = in.gestures[i].classification_size(); + out->gestures_categories_counts[i] = categories_count; + out->gestures[i] = new Category[categories_count]; + + for (uint32_t j = 0; j < categories_count; ++j) { + const auto& classification = in.gestures[i].classification(j); + + CppCategory cpp_category; + // Set fields from the Classification protobuf + if (classification.has_index()) { + cpp_category.index = classification.index(); + } + if (classification.has_score()) { + cpp_category.score = classification.score(); + } + if (classification.has_label()) { + cpp_category.category_name = classification.label(); + } + if (classification.has_display_name()) { + cpp_category.display_name = classification.display_name(); + } + + CppConvertToCategory(cpp_category, &out->gestures[i][j]); + } + } + + out->handedness_count = in.handedness.size(); + out->handedness = new Category*[out->handedness_count]; + out->handedness_categories_counts = new uint32_t[out->handedness_count]; + + for (uint32_t i = 0; i < out->handedness_count; ++i) { + uint32_t categories_count = in.handedness[i].classification_size(); + out->handedness_categories_counts[i] = categories_count; + out->handedness[i] = new Category[categories_count]; + + for (uint32_t j = 0; j < categories_count; ++j) { + const auto& classification = in.handedness[i].classification(j); + + CppCategory cpp_category; + // Set fields from the Classification protobuf + if (classification.has_index()) { + cpp_category.index = classification.index(); + } + if (classification.has_score()) { + cpp_category.score = classification.score(); + } + if (classification.has_label()) { + cpp_category.category_name = classification.label(); + } + if (classification.has_display_name()) { + cpp_category.display_name = classification.display_name(); + } + + CppConvertToCategory(cpp_category, &out->handedness[i][j]); + } + } + + out->hand_landmarks_count = in.hand_landmarks.size(); + out->hand_landmarks = new NormalizedLandmarks[out->hand_landmarks_count]; + for (uint32_t i = 0; i < out->hand_landmarks_count; ++i) { + std::vector cpp_normalized_landmarks; + for (uint32_t j = 0; j < in.hand_landmarks[i].landmark_size(); ++j) { + const auto& landmark = in.hand_landmarks[i].landmark(j); + CppNormalizedLandmark cpp_landmark; + cpp_landmark.x = landmark.x(); + cpp_landmark.y = landmark.y(); + cpp_landmark.z = landmark.z(); + if (landmark.has_presence()) { + cpp_landmark.presence = landmark.presence(); + } + if (landmark.has_visibility()) { + cpp_landmark.visibility = landmark.visibility(); + } + cpp_normalized_landmarks.push_back(cpp_landmark); + } + CppConvertToNormalizedLandmarks(cpp_normalized_landmarks, + &out->hand_landmarks[i]); + } + + out->hand_world_landmarks_count = in.hand_world_landmarks.size(); + out->hand_world_landmarks = new Landmarks[out->hand_world_landmarks_count]; + for (uint32_t i = 0; i < out->hand_world_landmarks_count; ++i) { + std::vector cpp_landmarks; + for (uint32_t j = 0; j < in.hand_world_landmarks[i].landmark_size(); ++j) { + const auto& landmark = in.hand_world_landmarks[i].landmark(j); + CppLandmark cpp_landmark; + cpp_landmark.x = landmark.x(); + cpp_landmark.y = landmark.y(); + cpp_landmark.z = landmark.z(); + if (landmark.has_presence()) { + cpp_landmark.presence = landmark.presence(); + } + if (landmark.has_visibility()) { + cpp_landmark.visibility = landmark.visibility(); + } + cpp_landmarks.push_back(cpp_landmark); + } + CppConvertToLandmarks(cpp_landmarks, &out->hand_world_landmarks[i]); + } +} + +void CppCloseGestureRecognizerResult(GestureRecognizerResult* result) { + for (uint32_t i = 0; i < result->gestures_count; ++i) { + for (uint32_t j = 0; j < result->gestures_categories_counts[i]; ++j) { + CppCloseCategory(&result->gestures[i][j]); + } + delete[] result->gestures[i]; + } + delete[] result->gestures; + + for (uint32_t i = 0; i < result->handedness_count; ++i) { + for (uint32_t j = 0; j < result->handedness_categories_counts[i]; ++j) { + CppCloseCategory(&result->handedness[i][j]); + } + delete[] result->handedness[i]; + } + delete[] result->handedness; + + for (uint32_t i = 0; i < result->hand_landmarks_count; ++i) { + CppCloseNormalizedLandmarks(&result->hand_landmarks[i]); + } + delete[] result->hand_landmarks; + + for (uint32_t i = 0; i < result->hand_world_landmarks_count; ++i) { + CppCloseLandmarks(&result->hand_world_landmarks[i]); + } + delete[] result->hand_world_landmarks; + + result->gestures = nullptr; + result->handedness = nullptr; + result->hand_landmarks = nullptr; + result->hand_world_landmarks = nullptr; + + result->gestures_count = 0; + result->handedness_count = 0; + result->hand_landmarks_count = 0; + result->hand_world_landmarks_count = 0; +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.h b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.h new file mode 100644 index 000000000..d5105acf6 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.h @@ -0,0 +1,33 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_GESTURE_RECOGNIZER_RESULT_CONVERTER_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_GESTURE_RECOGNIZER_RESULT_CONVERTER_H_ + +#include "mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_result.h" +#include "mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_result.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToGestureRecognizerResult( + const mediapipe::tasks::vision::gesture_recognizer::GestureRecognizerResult& + in, + GestureRecognizerResult* out); + +void CppCloseGestureRecognizerResult(GestureRecognizerResult* result); + +} // namespace mediapipe::tasks::c::components::containers + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_GESTURE_RECOGNIZER_RESULT_CONVERTER_H_ diff --git a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc new file mode 100644 index 000000000..551498e12 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc @@ -0,0 +1,24 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.h" + +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_result.h" +#include "mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_result.h" + +namespace mediapipe::tasks::c::components::containers { + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/landmark.h b/mediapipe/tasks/c/components/containers/landmark.h new file mode 100644 index 000000000..de6dd9928 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/landmark.h @@ -0,0 +1,90 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANDMARK_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANDMARK_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Landmark represents a point in 3D space with x, y, z coordinates. The +// landmark coordinates are in meters. z represents the landmark depth, and the +// smaller the value the closer the world landmark is to the camera. +struct Landmark { + float x; + float y; + float z; + + // For optional visibility. + bool has_visibility; + + // Landmark visibility. Should stay unset if not supported. + // Float score of whether landmark is visible or occluded by other objects. + // Landmark considered as invisible also if it is not present on the screen + // (out of scene bounds). Depending on the model, visibility value is either + // a sigmoid or an argument of sigmoid. + float visibility; + + // For optional presence. + bool has_presence; + + // Landmark presence. Should stay unset if not supported. + // Float score of whether landmark is present on the scene (located within + // scene bounds). Depending on the model, presence value is either a result + // of sigmoid or an argument of sigmoid function to get landmark presence + // probability. + float presence; + + // Landmark name. Should stay unset if not supported. + // Defaults to nullptr. + char* name; +}; + +// A normalized version of above Landmark struct. All coordinates should be +// within [0, 1]. +struct NormalizedLandmark { + float x; + float y; + float z; + + bool has_visibility; + float visibility; + + bool has_presence; + float presence; + + char* name; +}; + +// A list of Landmarks. +struct Landmarks { + struct Landmark* landmarks; + uint32_t landmarks_count; +}; + +// A list of NormalizedLandmarks. +struct NormalizedLandmarks { + struct NormalizedLandmark* landmarks; + uint32_t landmarks_count; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANDMARK_H_ diff --git a/mediapipe/tasks/c/components/containers/landmark_converter.cc b/mediapipe/tasks/c/components/containers/landmark_converter.cc new file mode 100644 index 000000000..5f4ab5ef2 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/landmark_converter.cc @@ -0,0 +1,132 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/landmark_converter.h" + +#include + +#include "mediapipe/tasks/c/components/containers/landmark.h" + +typedef Landmark LandmarkC; +typedef NormalizedLandmark NormalizedLandmarkC; +typedef Landmarks LandmarksC; +typedef NormalizedLandmarks NormalizedLandmarksC; + +#include "mediapipe/tasks/cc/components/containers/landmark.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToLandmark( + const mediapipe::tasks::components::containers::Landmark& in, + LandmarkC* out) { + out->x = in.x; + out->y = in.y; + out->z = in.z; + + if (in.visibility.has_value()) { + out->has_visibility = true; + out->visibility = in.visibility.value(); + } else { + out->has_visibility = false; + } + + if (in.presence.has_value()) { + out->has_presence = true; + out->presence = in.presence.value(); + } else { + out->has_presence = false; + } + + out->name = in.name.has_value() ? strdup(in.name->c_str()) : nullptr; +} + +void CppConvertToNormalizedLandmark( + const mediapipe::tasks::components::containers::NormalizedLandmark& in, + NormalizedLandmarkC* out) { + out->x = in.x; + out->y = in.y; + out->z = in.z; + + if (in.visibility.has_value()) { + out->has_visibility = true; + out->visibility = in.visibility.value(); + } else { + out->has_visibility = false; + } + + if (in.presence.has_value()) { + out->has_presence = true; + out->presence = in.presence.value(); + } else { + out->has_presence = false; + } + + out->name = in.name.has_value() ? strdup(in.name->c_str()) : nullptr; +} + +void CppConvertToLandmarks( + const std::vector& in, + LandmarksC* out) { + out->landmarks_count = in.size(); + out->landmarks = new LandmarkC[out->landmarks_count]; + for (uint32_t i = 0; i < out->landmarks_count; ++i) { + CppConvertToLandmark(in[i], &out->landmarks[i]); + } +} + +void CppConvertToNormalizedLandmarks( + const std::vector< + mediapipe::tasks::components::containers::NormalizedLandmark>& in, + NormalizedLandmarksC* out) { + out->landmarks_count = in.size(); + out->landmarks = new NormalizedLandmarkC[out->landmarks_count]; + for (uint32_t i = 0; i < out->landmarks_count; ++i) { + CppConvertToNormalizedLandmark(in[i], &out->landmarks[i]); + } +} + +void CppCloseLandmark(LandmarkC* in) { + if (in && in->name) { + free(in->name); + in->name = nullptr; + } +} + +void CppCloseLandmarks(LandmarksC* in) { + for (uint32_t i = 0; i < in->landmarks_count; ++i) { + CppCloseLandmark(&in->landmarks[i]); + } + delete[] in->landmarks; + in->landmarks = nullptr; + in->landmarks_count = 0; +} + +void CppCloseNormalizedLandmark(NormalizedLandmarkC* in) { + if (in && in->name) { + free(in->name); + in->name = nullptr; + } +} + +void CppCloseNormalizedLandmarks(NormalizedLandmarksC* in) { + for (uint32_t i = 0; i < in->landmarks_count; ++i) { + CppCloseNormalizedLandmark(&in->landmarks[i]); + } + delete[] in->landmarks; + in->landmarks = nullptr; + in->landmarks_count = 0; +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/landmark_converter.h b/mediapipe/tasks/c/components/containers/landmark_converter.h new file mode 100644 index 000000000..f59158112 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/landmark_converter.h @@ -0,0 +1,51 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANDMARK_CONVERTER_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANDMARK_CONVERTER_H_ + +#include "mediapipe/tasks/c/components/containers/landmark.h" +#include "mediapipe/tasks/cc/components/containers/landmark.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToLandmark( + const mediapipe::tasks::components::containers::Landmark& in, + Landmark* out); + +void CppConvertToNormalizedLandmark( + const mediapipe::tasks::components::containers::NormalizedLandmark& in, + NormalizedLandmark* out); + +void CppConvertToLandmarks( + const std::vector& in, + Landmarks* out); + +void CppConvertToNormalizedLandmarks( + const std::vector< + mediapipe::tasks::components::containers::NormalizedLandmark>& in, + NormalizedLandmarks* out); + +void CppCloseLandmark(struct Landmark* in); + +void CppCloseLandmarks(struct Landmarks* in); + +void CppCloseNormalizedLandmark(struct NormalizedLandmark* in); + +void CppCloseNormalizedLandmarks(struct NormalizedLandmarks* in); + +} // namespace mediapipe::tasks::c::components::containers + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANDMARK_CONVERTER_H_ diff --git a/mediapipe/tasks/c/components/containers/landmark_converter_test.cc b/mediapipe/tasks/c/components/containers/landmark_converter_test.cc new file mode 100644 index 000000000..cf2b5a0e9 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/landmark_converter_test.cc @@ -0,0 +1,28 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/landmark_converter.h" + +#include +#include +#include + +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/components/containers/landmark.h" +#include "mediapipe/tasks/cc/components/containers/landmark.h" + +namespace mediapipe::tasks::c::components::containers { + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/BUILD b/mediapipe/tasks/c/vision/gesture_recognizer/BUILD new file mode 100644 index 000000000..8b905b338 --- /dev/null +++ b/mediapipe/tasks/c/vision/gesture_recognizer/BUILD @@ -0,0 +1,76 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +cc_library( + name = "gesture_recognizer_result", + hdrs = ["gesture_recognizer_result.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/tasks/c/components/containers:category", + "//mediapipe/tasks/c/components/containers:landmark", + "//mediapipe/tasks/c/vision/core:common", + ], +) + +cc_library( + name = "gesture_recognizer_lib", + srcs = ["gesture_recognizer.cc"], + hdrs = ["gesture_recognizer.h"], + visibility = ["//visibility:public"], + deps = [ + ":gesture_recognizer_result", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/tasks/c/components/containers:gesture_recognizer_result_converter", + "//mediapipe/tasks/c/components/processors:classifier_options", + "//mediapipe/tasks/c/components/processors:classifier_options_converter", + "//mediapipe/tasks/c/core:base_options", + "//mediapipe/tasks/c/core:base_options_converter", + "//mediapipe/tasks/cc/vision/core:running_mode", + "//mediapipe/tasks/cc/vision/gesture_recognizer", + "//mediapipe/tasks/cc/vision/utils:image_utils", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], + alwayslink = 1, +) + +cc_test( + name = "gesture_recognizer_test", + srcs = ["gesture_recognizer_test.cc"], + data = [ + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + linkstatic = 1, + deps = [ + ":gesture_recognizer_lib", + "//mediapipe/framework/deps:file_path", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/port:gtest", + "//mediapipe/tasks/c/components/containers:landmark", + "//mediapipe/tasks/cc/vision/utils:image_utils", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.cc b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.cc new file mode 100644 index 000000000..f8c42dcaf --- /dev/null +++ b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.cc @@ -0,0 +1,294 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h" + +#include +#include +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.h" +#include "mediapipe/tasks/c/components/processors/classifier_options_converter.h" +#include "mediapipe/tasks/c/core/base_options_converter.h" +#include "mediapipe/tasks/cc/vision/core/running_mode.h" +#include "mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace mediapipe::tasks::c::vision::gesture_recognizer { + +namespace { + +using ::mediapipe::tasks::c::components::containers:: + CppCloseGestureRecognizerResult; +using ::mediapipe::tasks::c::components::containers:: + CppConvertToGestureRecognizerResult; +using ::mediapipe::tasks::c::components::processors:: + CppConvertToClassifierOptions; +using ::mediapipe::tasks::c::core::CppConvertToBaseOptions; +using ::mediapipe::tasks::vision::CreateImageFromBuffer; +using ::mediapipe::tasks::vision::core::RunningMode; +using ::mediapipe::tasks::vision::gesture_recognizer::GestureRecognizer; +typedef ::mediapipe::tasks::vision::gesture_recognizer::GestureRecognizerResult + CppGestureRecognizerResult; + +int CppProcessError(absl::Status status, char** error_msg) { + if (error_msg) { + *error_msg = strdup(status.ToString().c_str()); + } + return status.raw_code(); +} + +} // namespace + +void CppConvertToGestureRecognizerOptions( + const GestureRecognizerOptions& in, + mediapipe::tasks::vision::gesture_recognizer::GestureRecognizerOptions* + out) { + out->num_hands = in.num_hands; + out->min_hand_detection_confidence = in.min_hand_detection_confidence; + out->min_hand_presence_confidence = in.min_hand_presence_confidence; + out->min_tracking_confidence = in.min_tracking_confidence; + CppConvertToClassifierOptions(in.canned_gestures_classifier_options, + &out->canned_gestures_classifier_options); + CppConvertToClassifierOptions(in.custom_gestures_classifier_options, + &out->custom_gestures_classifier_options); +} + +GestureRecognizer* CppGestureRecognizerCreate( + const GestureRecognizerOptions& options, char** error_msg) { + auto cpp_options = + std::make_unique<::mediapipe::tasks::vision::gesture_recognizer:: + GestureRecognizerOptions>(); + + CppConvertToBaseOptions(options.base_options, &cpp_options->base_options); + CppConvertToGestureRecognizerOptions(options, cpp_options.get()); + cpp_options->running_mode = static_cast(options.running_mode); + + // Enable callback for processing live stream data when the running mode is + // set to RunningMode::LIVE_STREAM. + if (cpp_options->running_mode == RunningMode::LIVE_STREAM) { + if (options.result_callback == nullptr) { + const absl::Status status = absl::InvalidArgumentError( + "Provided null pointer to callback function."); + ABSL_LOG(ERROR) << "Failed to create GestureRecognizer: " << status; + CppProcessError(status, error_msg); + return nullptr; + } + + GestureRecognizerOptions::result_callback_fn result_callback = + options.result_callback; + cpp_options->result_callback = + [result_callback](absl::StatusOr cpp_result, + const Image& image, int64_t timestamp) { + char* error_msg = nullptr; + + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Recognition failed: " << cpp_result.status(); + CppProcessError(cpp_result.status(), &error_msg); + result_callback(nullptr, MpImage(), timestamp, error_msg); + free(error_msg); + return; + } + + // Result is valid for the lifetime of the callback function. + GestureRecognizerResult result; + CppConvertToGestureRecognizerResult(*cpp_result, &result); + + const auto& image_frame = image.GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = { + .format = static_cast<::ImageFormat>(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + result_callback(&result, mp_image, timestamp, + /* error_msg= */ nullptr); + + CppCloseGestureRecognizerResult(&result); + }; + } + + auto recognizer = GestureRecognizer::Create(std::move(cpp_options)); + if (!recognizer.ok()) { + ABSL_LOG(ERROR) << "Failed to create GestureRecognizer: " + << recognizer.status(); + CppProcessError(recognizer.status(), error_msg); + return nullptr; + } + return recognizer->release(); +} + +int CppGestureRecognizerRecognize(void* recognizer, const MpImage* image, + GestureRecognizerResult* result, + char** error_msg) { + if (image->type == MpImage::GPU_BUFFER) { + const absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet."); + + ABSL_LOG(ERROR) << "Recognition failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image->image_frame.format), + image->image_frame.image_buffer, image->image_frame.width, + image->image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_recognizer = static_cast(recognizer); + auto cpp_result = cpp_recognizer->Recognize(*img); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Recognition failed: " << cpp_result.status(); + return CppProcessError(cpp_result.status(), error_msg); + } + CppConvertToGestureRecognizerResult(*cpp_result, result); + return 0; +} + +int CppGestureRecognizerRecognizeForVideo(void* recognizer, + const MpImage* image, + int64_t timestamp_ms, + GestureRecognizerResult* result, + char** error_msg) { + if (image->type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Recognition failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image->image_frame.format), + image->image_frame.image_buffer, image->image_frame.width, + image->image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_recognizer = static_cast(recognizer); + auto cpp_result = cpp_recognizer->RecognizeForVideo(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Recognition failed: " << cpp_result.status(); + return CppProcessError(cpp_result.status(), error_msg); + } + CppConvertToGestureRecognizerResult(*cpp_result, result); + return 0; +} + +int CppGestureRecognizerRecognizeAsync(void* recognizer, const MpImage* image, + int64_t timestamp_ms, char** error_msg) { + if (image->type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Recognition failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image->image_frame.format), + image->image_frame.image_buffer, image->image_frame.width, + image->image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_recognizer = static_cast(recognizer); + auto cpp_result = cpp_recognizer->RecognizeAsync(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Data preparation for the image classification failed: " + << cpp_result; + return CppProcessError(cpp_result, error_msg); + } + return 0; +} + +void CppGestureRecognizerCloseResult(GestureRecognizerResult* result) { + CppCloseGestureRecognizerResult(result); +} + +int CppGestureRecognizerClose(void* recognizer, char** error_msg) { + auto cpp_recognizer = static_cast(recognizer); + auto result = cpp_recognizer->Close(); + if (!result.ok()) { + ABSL_LOG(ERROR) << "Failed to close GestureRecognizer: " << result; + return CppProcessError(result, error_msg); + } + delete cpp_recognizer; + return 0; +} + +} // namespace mediapipe::tasks::c::vision::gesture_recognizer + +extern "C" { + +void* gesture_recognizer_create(struct GestureRecognizerOptions* options, + char** error_msg) { + return mediapipe::tasks::c::vision::gesture_recognizer:: + CppGestureRecognizerCreate(*options, error_msg); +} + +int gesture_recognizer_recognize_image(void* recognizer, const MpImage* image, + GestureRecognizerResult* result, + char** error_msg) { + return mediapipe::tasks::c::vision::gesture_recognizer:: + CppGestureRecognizerRecognize(recognizer, image, result, error_msg); +} + +int gesture_recognizer_recognize_for_video(void* recognizer, + const MpImage* image, + int64_t timestamp_ms, + GestureRecognizerResult* result, + char** error_msg) { + return mediapipe::tasks::c::vision::gesture_recognizer:: + CppGestureRecognizerRecognizeForVideo(recognizer, image, timestamp_ms, + result, error_msg); +} + +int gesture_recognizer_recognize_async(void* recognizer, const MpImage* image, + int64_t timestamp_ms, char** error_msg) { + return mediapipe::tasks::c::vision::gesture_recognizer:: + CppGestureRecognizerRecognizeAsync(recognizer, image, timestamp_ms, + error_msg); +} + +void gesture_recognizer_close_result(GestureRecognizerResult* result) { + mediapipe::tasks::c::vision::gesture_recognizer:: + CppGestureRecognizerCloseResult(result); +} + +int gesture_recognizer_close(void* recognizer, char** error_ms) { + return mediapipe::tasks::c::vision::gesture_recognizer:: + CppGestureRecognizerClose(recognizer, error_ms); +} + +} // extern "C" diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h new file mode 100644 index 000000000..1c2b65112 --- /dev/null +++ b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h @@ -0,0 +1,156 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_VISION_GESTURE_RECOGNIZER_GESTURE_RECOGNIZER_H_ +#define MEDIAPIPE_TASKS_C_VISION_GESTURE_RECOGNIZER_GESTURE_RECOGNIZER_H_ + +#include "mediapipe/tasks/c/components/processors/classifier_options.h" +#include "mediapipe/tasks/c/core/base_options.h" +#include "mediapipe/tasks/c/vision/core/common.h" +#include "mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_result.h" + +#ifndef MP_EXPORT +#define MP_EXPORT __attribute__((visibility("default"))) +#endif // MP_EXPORT + +#ifdef __cplusplus +extern "C" { +#endif + +// The options for configuring a MediaPipe gesture recognizer task. +struct GestureRecognizerOptions { + // Base options for configuring MediaPipe Tasks, such as specifying the model + // file with metadata, accelerator options, op resolver, etc. + struct BaseOptions base_options; + + // The running mode of the task. Default to the image mode. + // GestureRecognizer has three running modes: + // 1) The image mode for recognizing hand gestures on single image inputs. + // 2) The video mode for recognizing hand gestures on the decoded frames of a + // video. + // 3) The live stream mode for recognizing hand gestures on the live stream of + // input data, such as from camera. In this mode, the "result_callback" + // below must be specified to receive the detection results asynchronously. + RunningMode running_mode; + + // The maximum number of hands can be detected by the GestureRecognizer. + int num_hands = 1; + + // The minimum confidence score for the hand detection to be considered + // successful. + float min_hand_detection_confidence = 0.5; + + // The minimum confidence score of hand presence score in the hand landmark + // detection. + float min_hand_presence_confidence = 0.5; + + // The minimum confidence score for the hand tracking to be considered + // successful. + float min_tracking_confidence = 0.5; + + // TODO Note this option is subject to change. + // Options for configuring the canned gestures classifier, such as score + // threshold, allow list and deny list of gestures. The categories for canned + // gesture classifier are: ["None", "Closed_Fist", "Open_Palm", + // "Pointing_Up", "Thumb_Down", "Thumb_Up", "Victory", "ILoveYou"] + struct ClassifierOptions canned_gestures_classifier_options; + + // TODO Note this option is subject to change. + // Options for configuring the custom gestures classifier, such as score + // threshold, allow list and deny list of gestures. + struct ClassifierOptions custom_gestures_classifier_options; + + // The user-defined result callback for processing live stream data. + // The result callback should only be specified when the running mode is set + // to RunningMode::LIVE_STREAM. Arguments of the callback function include: + // the pointer to recognition result, the image that result was obtained + // on, the timestamp relevant to recognition results and pointer to error + // message in case of any failure. The validity of the passed arguments is + // true for the lifetime of the callback function. + // + // A caller is responsible for closing gesture recognizer result. + typedef void (*result_callback_fn)(GestureRecognizerResult* result, + const MpImage image, int64_t timestamp_ms, + char* error_msg); + result_callback_fn result_callback; +}; + +// Creates an GestureRecognizer from provided `options`. +// Returns a pointer to the gesture recognizer on success. +// If an error occurs, returns `nullptr` and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT void* gesture_recognizer_create( + struct GestureRecognizerOptions* options, char** error_msg); + +// Performs gesture recognition on the input `image`. Returns `0` on success. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int gesture_recognizer_recognize_image( + void* recognizer, const MpImage* image, GestureRecognizerResult* result, + char** error_msg); + +// Performs gesture recognition on the provided video frame. +// Only use this method when the GestureRecognizer is created with the video +// running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide the video frame's timestamp (in milliseconds). The input timestamps +// must be monotonically increasing. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int gesture_recognizer_recognize_for_video( + void* recognizer, const MpImage* image, int64_t timestamp_ms, + GestureRecognizerResult* result, char** error_msg); + +// Sends live image data to gesture recognition, and the results will be +// available via the `result_callback` provided in the GestureRecognizerOptions. +// Only use this method when the GestureRecognizer is created with the live +// stream running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide a timestamp (in milliseconds) to indicate when the input image is +// sent to the gesture recognizer. The input timestamps must be monotonically +// increasing. +// The `result_callback` provides: +// - The recognition results as an GestureRecognizerResult object. +// - The const reference to the corresponding input image that the gesture +// recognizer runs on. Note that the const reference to the image will no +// longer be valid when the callback returns. To access the image data +// outside of the callback, callers need to make a copy of the image. +// - The input timestamp in milliseconds. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int gesture_recognizer_recognize_async(void* recognizer, + const MpImage* image, + int64_t timestamp_ms, + char** error_msg); + +// Frees the memory allocated inside a GestureRecognizerResult result. +// Does not free the result pointer itself. +MP_EXPORT void gesture_recognizer_close_result(GestureRecognizerResult* result); + +// Frees gesture recognizer. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int gesture_recognizer_close(void* recognizer, char** error_msg); + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_VISION_GESTURE_RECOGNIZER_GESTURE_RECOGNIZER_H_ diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_result.h b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_result.h new file mode 100644 index 000000000..e546fa509 --- /dev/null +++ b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_result.h @@ -0,0 +1,70 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_VISION_GESTURE_RECOGNIZER_RESULT_GESTURE_RECOGNIZER_RESULT_H_ +#define MEDIAPIPE_TASKS_C_VISION_GESTURE_RECOGNIZER_RESULT_GESTURE_RECOGNIZER_RESULT_H_ + +#include "mediapipe/tasks/c/components/containers/category.h" +#include "mediapipe/tasks/c/components/containers/landmark.h" +#include "mediapipe/tasks/c/vision/core/common.h" + +#ifndef MP_EXPORT +#define MP_EXPORT __attribute__((visibility("default"))) +#endif // MP_EXPORT + +#ifdef __cplusplus +extern "C" { +#endif + +// The gesture recognition result from GestureRecognizer, where each vector +// element represents a single hand detected in the image. +struct GestureRecognizerResult { + // Recognized hand gestures with sorted order such that the winning label is + // the first item in the list. + struct Category** gestures; + + // The number of elements in the gestures array. + uint32_t gestures_count; + + // The number of elements in the gestures categories array. + uint32_t* gestures_categories_counts; + + // Classification of handedness. + struct Category** handedness; + + // The number of elements in the handedness array. + uint32_t handedness_count; + + // The number of elements in the handedness categories array. + uint32_t* handedness_categories_counts; + + // Detected hand landmarks in normalized image coordinates. + struct NormalizedLandmarks* hand_landmarks; + + // The number of elements in the hand_landmarks array. + uint32_t hand_landmarks_count; + + // Detected hand landmarks in world coordinates. + struct Landmarks* hand_world_landmarks; + + // The number of elements in the hand_world_landmarks array. + uint32_t hand_world_landmarks_count; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_VISION_GESTURE_RECOGNIZER_RESULT_GESTURE_RECOGNIZER_RESULT_H_ diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc new file mode 100644 index 000000000..723ff9838 --- /dev/null +++ b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc @@ -0,0 +1,123 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h" + +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/strings/string_view.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/components/containers/landmark.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace { + +using ::mediapipe::file::JoinPath; +using ::mediapipe::tasks::vision::DecodeImageFromFile; +using testing::HasSubstr; + +constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; +constexpr char kModelName[] = "gesture_recognizer.task"; +constexpr float kPrecision = 1e-4; +constexpr float kLandmarkPrecision = 1e-3; +constexpr int kIterations = 100; + +std::string GetFullPath(absl::string_view file_name) { + return JoinPath("./", kTestDataDirectory, file_name); +} + +TEST(GestureRecognizerTest, ImageModeTest) { + const auto image = DecodeImageFromFile(GetFullPath("fist.jpg")); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + GestureRecognizerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* num_hands= */ 1, + /* min_hand_detection_confidence= */ 0.5, + /* min_hand_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}}; + + void* recognizer = + gesture_recognizer_create(&options, /* error_msg */ nullptr); + EXPECT_NE(recognizer, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + GestureRecognizerResult result; + gesture_recognizer_recognize_image(recognizer, &mp_image, &result, + /* error_msg */ nullptr); + + // Expects to have the same number of hands detected. + EXPECT_EQ(result.gestures_count, 1); + EXPECT_EQ(result.handedness_count, 1); + // Actual gesture with top score matches expected gesture. + EXPECT_EQ(std::string{result.gestures[0][0].category_name}, "Closed_Fist"); + EXPECT_NEAR(result.gestures[0][0].score, 0.9000f, kPrecision); + + // Actual handedness matches expected handedness. + EXPECT_EQ(std::string{result.handedness[0][0].category_name}, "Right"); + EXPECT_NEAR(result.handedness[0][0].score, 0.9893f, kPrecision); + + // Actual landmarks match expected landmarks. + EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].x, 0.477f, + kLandmarkPrecision); + EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].y, 0.661f, + kLandmarkPrecision); + EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].z, 0.0f, + kLandmarkPrecision); + EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].x, -0.009f, + kLandmarkPrecision); + EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].y, 0.082f, + kLandmarkPrecision); + EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].z, 0.006f, + kLandmarkPrecision); + + gesture_recognizer_close_result(&result); + gesture_recognizer_close(recognizer, /* error_msg */ nullptr); +} + +// TODO other tests + +} // namespace diff --git a/mediapipe/tasks/testdata/vision/BUILD b/mediapipe/tasks/testdata/vision/BUILD index 3f83118b0..e35e04a97 100644 --- a/mediapipe/tasks/testdata/vision/BUILD +++ b/mediapipe/tasks/testdata/vision/BUILD @@ -185,6 +185,7 @@ filegroup( "face_landmarker.task", "face_landmarker_v2.task", "face_stylizer_color_ink.task", + "gesture_recognizer.task", "hair_segmentation.tflite", "hand_landmark_full.tflite", "hand_landmark_lite.tflite", From 8d57a9e2e815dd11a7434cb764382cc7ce4831a5 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Mon, 27 Nov 2023 11:12:24 -0800 Subject: [PATCH 136/157] Add missing export declarations to DrawingUtils Fixes https://github.com/google/mediapipe/issues/4980 PiperOrigin-RevId: 585705106 --- mediapipe/tasks/web/vision/core/drawing_utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mediapipe/tasks/web/vision/core/drawing_utils.ts b/mediapipe/tasks/web/vision/core/drawing_utils.ts index 520f9e2b3..f3c3d5d75 100644 --- a/mediapipe/tasks/web/vision/core/drawing_utils.ts +++ b/mediapipe/tasks/web/vision/core/drawing_utils.ts @@ -391,6 +391,7 @@ export class DrawingUtils { drawCategoryMask( mask: MPMask, categoryToColorMap: RGBAColor[], background?: RGBAColor|ImageSource): void; + /** @export */ drawCategoryMask( mask: MPMask, categoryToColorMap: CategoryToColorMap, background: RGBAColor|ImageSource = [0, 0, 0, 255]): void { @@ -480,6 +481,7 @@ export class DrawingUtils { * frame, you can reduce the cost of re-uploading these images by passing a * `HTMLCanvasElement` instead. * + * @export * @param mask A confidence mask that was returned from a segmentation task. * @param defaultTexture An image or a four-channel color that will be used * when confidence values are low. From 1ff7e95295aed0b6a4c4c77d92432d48ff4ba041 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 27 Nov 2023 11:59:35 -0800 Subject: [PATCH 137/157] No public description PiperOrigin-RevId: 585719403 --- mediapipe/model_maker/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/model_maker/requirements.txt b/mediapipe/model_maker/requirements.txt index ff43fa3f0..ffb547b82 100644 --- a/mediapipe/model_maker/requirements.txt +++ b/mediapipe/model_maker/requirements.txt @@ -7,4 +7,4 @@ tensorflow-addons tensorflow-datasets tensorflow-hub tensorflow-text -tf-models-official>=2.13.1 +tf-models-official>=2.13.2 From 95601ff98b6ba5a5eb9bd4a84761dc25a9d5aab4 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 27 Nov 2023 15:48:37 -0800 Subject: [PATCH 138/157] Remove internal logs. PiperOrigin-RevId: 585782033 --- .../mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java index b673b00c9..813dba93c 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/imagesegmenter/ImageSegmenter.java @@ -15,7 +15,6 @@ package com.google.mediapipe.tasks.vision.imagesegmenter; import android.content.Context; -import android.util.Log; import com.google.auto.value.AutoValue; import com.google.mediapipe.proto.CalculatorOptionsProto.CalculatorOptions; import com.google.mediapipe.proto.CalculatorProto.CalculatorGraphConfig; From a898215c52a6c406cee993c97cb705ab6d66bc96 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 28 Nov 2023 14:33:15 -0800 Subject: [PATCH 139/157] Holistic Landmarker C++ Graph PiperOrigin-RevId: 586105983 --- .../tasks/cc/vision/hand_landmarker/BUILD | 31 + .../hand_roi_refinement_graph.cc | 154 +++++ .../tasks/cc/vision/holistic_landmarker/BUILD | 152 +++++ .../holistic_face_tracking.cc | 260 ++++++++ .../holistic_face_tracking.h | 89 +++ .../holistic_face_tracking_test.cc | 227 +++++++ .../holistic_hand_tracking.cc | 272 ++++++++ .../holistic_hand_tracking.h | 94 +++ .../holistic_hand_tracking_test.cc | 303 +++++++++ .../holistic_landmarker_graph.cc | 521 +++++++++++++++ .../holistic_landmarker_graph_test.cc | 595 ++++++++++++++++++ .../holistic_pose_tracking.cc | 307 +++++++++ .../holistic_pose_tracking.h | 110 ++++ .../holistic_pose_tracking_test.cc | 243 +++++++ .../cc/vision/holistic_landmarker/proto/BUILD | 44 ++ .../holistic_landmarker_graph_options.proto | 57 ++ .../proto/holistic_result.proto | 34 + 17 files changed, 3493 insertions(+) create mode 100644 mediapipe/tasks/cc/vision/hand_landmarker/hand_roi_refinement_graph.cc create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/BUILD create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking.cc create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking.h create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking_test.cc create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking.cc create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking.h create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking_test.cc create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/holistic_landmarker_graph.cc create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/holistic_landmarker_graph_test.cc create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking.cc create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking.h create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking_test.cc create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/proto/BUILD create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_landmarker_graph_options.proto create mode 100644 mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_result.proto diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/BUILD b/mediapipe/tasks/cc/vision/hand_landmarker/BUILD index 5b75ef8fc..6db49c668 100644 --- a/mediapipe/tasks/cc/vision/hand_landmarker/BUILD +++ b/mediapipe/tasks/cc/vision/hand_landmarker/BUILD @@ -155,6 +155,37 @@ cc_library( # TODO: open source hand joints graph +cc_library( + name = "hand_roi_refinement_graph", + srcs = ["hand_roi_refinement_graph.cc"], + deps = [ + "//mediapipe/calculators/tensor:image_to_tensor_calculator_cc_proto", + "//mediapipe/calculators/tensor:inference_calculator", + "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator", + "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/api2/stream:detections_to_rects", + "//mediapipe/framework/api2/stream:landmarks_projection", + "//mediapipe/framework/api2/stream:landmarks_to_detection", + "//mediapipe/framework/api2/stream:rect_transformation", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/formats:tensor", + "//mediapipe/framework/port:status", + "//mediapipe/framework/port:statusor", + "//mediapipe/tasks/cc/components/processors:image_preprocessing_graph", + "//mediapipe/tasks/cc/components/processors/proto:image_preprocessing_graph_options_cc_proto", + "//mediapipe/tasks/cc/core:model_task_graph", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_roi_refinement_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/utils:image_tensor_specs", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], + alwayslink = 1, +) + cc_library( name = "hand_landmarker_result", srcs = ["hand_landmarker_result.cc"], diff --git a/mediapipe/tasks/cc/vision/hand_landmarker/hand_roi_refinement_graph.cc b/mediapipe/tasks/cc/vision/hand_landmarker/hand_roi_refinement_graph.cc new file mode 100644 index 000000000..e7e9b94d0 --- /dev/null +++ b/mediapipe/tasks/cc/vision/hand_landmarker/hand_roi_refinement_graph.cc @@ -0,0 +1,154 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensors_to_landmarks_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/stream/detections_to_rects.h" +#include "mediapipe/framework/api2/stream/landmarks_projection.h" +#include "mediapipe/framework/api2/stream/landmarks_to_detection.h" +#include "mediapipe/framework/api2/stream/rect_transformation.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/tasks/cc/components/processors/image_preprocessing_graph.h" +#include "mediapipe/tasks/cc/components/processors/proto/image_preprocessing_graph_options.pb.h" +#include "mediapipe/tasks/cc/core/model_task_graph.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_roi_refinement_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/utils/image_tensor_specs.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace hand_landmarker { + +using ::mediapipe::api2::builder::ConvertAlignmentPointsDetectionToRect; +using ::mediapipe::api2::builder::ConvertLandmarksToDetection; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::ProjectLandmarks; +using ::mediapipe::api2::builder::ScaleAndShiftAndMakeSquareLong; +using ::mediapipe::api2::builder::Stream; + +// Refine the input hand RoI with hand_roi_refinement model. +// +// Inputs: +// IMAGE - Image +// The image to preprocess. +// NORM_RECT - NormalizedRect +// Coarse RoI of hand. +// Outputs: +// NORM_RECT - NormalizedRect +// Refined RoI of hand. +class HandRoiRefinementGraph : public core::ModelTaskGraph { + public: + absl::StatusOr GetConfig( + mediapipe::SubgraphContext* context) override { + Graph graph; + Stream image_in = graph.In("IMAGE").Cast(); + Stream roi_in = + graph.In("NORM_RECT").Cast(); + + auto& graph_options = + *context->MutableOptions(); + + MP_ASSIGN_OR_RETURN( + const auto* model_resources, + GetOrCreateModelResources( + context)); + + auto& preprocessing = graph.AddNode( + "mediapipe.tasks.components.processors.ImagePreprocessingGraph"); + bool use_gpu = + components::processors::DetermineImagePreprocessingGpuBackend( + graph_options.base_options().acceleration()); + auto& image_to_tensor_options = + *preprocessing + .GetOptions() + .mutable_image_to_tensor_options(); + image_to_tensor_options.set_keep_aspect_ratio(true); + image_to_tensor_options.set_border_mode( + mediapipe::ImageToTensorCalculatorOptions::BORDER_REPLICATE); + MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph( + *model_resources, use_gpu, graph_options.base_options().gpu_origin(), + &preprocessing.GetOptions())); + image_in >> preprocessing.In("IMAGE"); + roi_in >> preprocessing.In("NORM_RECT"); + auto tensors_in = preprocessing.Out("TENSORS"); + auto matrix = preprocessing.Out("MATRIX").Cast>(); + auto image_size = + preprocessing.Out("IMAGE_SIZE").Cast>(); + + auto& inference = AddInference( + *model_resources, graph_options.base_options().acceleration(), graph); + tensors_in >> inference.In("TENSORS"); + auto tensors_out = inference.Out("TENSORS").Cast>(); + + MP_ASSIGN_OR_RETURN(auto image_tensor_specs, + BuildInputImageTensorSpecs(*model_resources)); + + // Convert tensors to landmarks. Recrop model outputs two points, + // center point and guide point. + auto& to_landmarks = graph.AddNode("TensorsToLandmarksCalculator"); + auto& to_landmarks_opts = + to_landmarks + .GetOptions(); + to_landmarks_opts.set_num_landmarks(/*num_landmarks=*/2); + to_landmarks_opts.set_input_image_width(image_tensor_specs.image_width); + to_landmarks_opts.set_input_image_height(image_tensor_specs.image_height); + to_landmarks_opts.set_normalize_z(/*z_norm_factor=*/1.0f); + tensors_out.ConnectTo(to_landmarks.In("TENSORS")); + auto recrop_landmarks = to_landmarks.Out("NORM_LANDMARKS") + .Cast(); + + // Project landmarks. + auto projected_recrop_landmarks = + ProjectLandmarks(recrop_landmarks, matrix, graph); + + // Convert re-crop landmarks to detection. + auto recrop_detection = + ConvertLandmarksToDetection(projected_recrop_landmarks, graph); + + // Convert re-crop detection to rect. + auto recrop_rect = ConvertAlignmentPointsDetectionToRect( + recrop_detection, image_size, /*start_keypoint_index=*/0, + /*end_keypoint_index=*/1, /*target_angle=*/-90, graph); + + auto refined_roi = + ScaleAndShiftAndMakeSquareLong(recrop_rect, image_size, + /*scale_x_factor=*/1.0, + /*scale_y_factor=*/1.0, /*shift_x=*/0, + /*shift_y=*/-0.1, graph); + refined_roi >> graph.Out("NORM_RECT").Cast(); + return graph.GetConfig(); + } +}; + +REGISTER_MEDIAPIPE_GRAPH( + ::mediapipe::tasks::vision::hand_landmarker::HandRoiRefinementGraph); + +} // namespace hand_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/BUILD b/mediapipe/tasks/cc/vision/holistic_landmarker/BUILD new file mode 100644 index 000000000..446cf1e09 --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/BUILD @@ -0,0 +1,152 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package( + default_visibility = ["//mediapipe/tasks:internal"], + licenses = ["notice"], # Apache 2.0 +) + +cc_library( + name = "holistic_face_tracking", + srcs = ["holistic_face_tracking.cc"], + hdrs = ["holistic_face_tracking.h"], + deps = [ + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/api2/stream:detections_to_rects", + "//mediapipe/framework/api2/stream:image_size", + "//mediapipe/framework/api2/stream:landmarks_to_detection", + "//mediapipe/framework/api2/stream:loopback", + "//mediapipe/framework/api2/stream:rect_transformation", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:detection_cc_proto", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:status", + "//mediapipe/modules/holistic_landmark/calculators:roi_tracking_calculator", + "//mediapipe/modules/holistic_landmark/calculators:roi_tracking_calculator_cc_proto", + "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph", + "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/face_landmarker:face_blendshapes_graph", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_graph", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarks_detector_graph", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_blendshapes_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_cc_proto", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings:str_format", + ], +) + +cc_library( + name = "holistic_hand_tracking", + srcs = ["holistic_hand_tracking.cc"], + hdrs = ["holistic_hand_tracking.h"], + deps = [ + "//mediapipe/calculators/util:align_hand_to_pose_in_world_calculator", + "//mediapipe/calculators/util:align_hand_to_pose_in_world_calculator_cc_proto", + "//mediapipe/calculators/util:landmark_visibility_calculator", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/api2/stream:image_size", + "//mediapipe/framework/api2/stream:landmarks_to_detection", + "//mediapipe/framework/api2/stream:loopback", + "//mediapipe/framework/api2/stream:rect_transformation", + "//mediapipe/framework/api2/stream:split", + "//mediapipe/framework/api2/stream:threshold", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:status", + "//mediapipe/modules/holistic_landmark/calculators:hand_detections_from_pose_to_rects_calculator", + "//mediapipe/modules/holistic_landmark/calculators:roi_tracking_calculator", + "//mediapipe/modules/holistic_landmark/calculators:roi_tracking_calculator_cc_proto", + "//mediapipe/tasks/cc/components/utils:gate", + "//mediapipe/tasks/cc/vision/hand_landmarker:hand_landmarker_graph", + "//mediapipe/tasks/cc/vision/hand_landmarker:hand_landmarks_detector_graph", + "//mediapipe/tasks/cc/vision/hand_landmarker:hand_roi_refinement_graph", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_roi_refinement_graph_options_cc_proto", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_library( + name = "holistic_pose_tracking", + srcs = ["holistic_pose_tracking.cc"], + hdrs = ["holistic_pose_tracking.h"], + deps = [ + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/api2/stream:detections_to_rects", + "//mediapipe/framework/api2/stream:image_size", + "//mediapipe/framework/api2/stream:landmarks_to_detection", + "//mediapipe/framework/api2/stream:loopback", + "//mediapipe/framework/api2/stream:merge", + "//mediapipe/framework/api2/stream:presence", + "//mediapipe/framework/api2/stream:rect_transformation", + "//mediapipe/framework/api2/stream:segmentation_smoothing", + "//mediapipe/framework/api2/stream:smoothing", + "//mediapipe/framework/api2/stream:split", + "//mediapipe/framework/formats:detection_cc_proto", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "//mediapipe/tasks/cc/components/utils:gate", + "//mediapipe/tasks/cc/vision/pose_detector:pose_detector_graph", + "//mediapipe/tasks/cc/vision/pose_detector/proto:pose_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/pose_landmarker:pose_landmarks_detector_graph", + "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarks_detector_graph_options_cc_proto", + "@com_google_absl//absl/status:statusor", + ], +) + +cc_library( + name = "holistic_landmarker_graph", + srcs = ["holistic_landmarker_graph.cc"], + deps = [ + ":holistic_face_tracking", + ":holistic_hand_tracking", + ":holistic_pose_tracking", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/api2/stream:split", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + "//mediapipe/tasks/cc/core:model_asset_bundle_resources", + "//mediapipe/tasks/cc/core:model_resources_cache", + "//mediapipe/tasks/cc/core:model_task_graph", + "//mediapipe/tasks/cc/core:utils", + "//mediapipe/tasks/cc/metadata/utils:zip_utils", + "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_blendshapes_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_roi_refinement_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/holistic_landmarker/proto:holistic_landmarker_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/pose_detector/proto:pose_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/pose_landmarker:pose_topology", + "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarks_detector_graph_options_cc_proto", + "//mediapipe/util:graph_builder_utils", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + ], + alwayslink = 1, +) diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking.cc b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking.cc new file mode 100644 index 000000000..1116cda21 --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking.cc @@ -0,0 +1,260 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_format.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/stream/detections_to_rects.h" +#include "mediapipe/framework/api2/stream/image_size.h" +#include "mediapipe/framework/api2/stream/landmarks_to_detection.h" +#include "mediapipe/framework/api2/stream/loopback.h" +#include "mediapipe/framework/api2/stream/rect_transformation.h" +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/detection.pb.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/modules/holistic_landmark/calculators/roi_tracking_calculator.pb.h" +#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace holistic_landmarker { + +namespace { + +using ::mediapipe::NormalizedRect; +using ::mediapipe::api2::builder::ConvertDetectionsToRectUsingKeypoints; +using ::mediapipe::api2::builder::ConvertDetectionToRect; +using ::mediapipe::api2::builder::ConvertLandmarksToDetection; +using ::mediapipe::api2::builder::GetImageSize; +using ::mediapipe::api2::builder::GetLoopbackData; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Scale; +using ::mediapipe::api2::builder::ScaleAndMakeSquare; +using ::mediapipe::api2::builder::Stream; + +struct FaceLandmarksResult { + std::optional> landmarks; + std::optional> classifications; +}; + +absl::Status ValidateGraphOptions( + const face_detector::proto::FaceDetectorGraphOptions& + face_detector_graph_options, + const face_landmarker::proto::FaceLandmarksDetectorGraphOptions& + face_landmarks_detector_graph_options, + const HolisticFaceTrackingRequest& request) { + if (face_detector_graph_options.num_faces() != 1) { + return absl::InvalidArgumentError(absl::StrFormat( + "Only support num_faces to be 1, but got num_faces = %d.", + face_detector_graph_options.num_faces())); + } + if (request.classifications && !face_landmarks_detector_graph_options + .has_face_blendshapes_graph_options()) { + return absl::InvalidArgumentError( + "Blendshapes detection is requested, but " + "face_blendshapes_graph_options is not configured."); + } + return absl::OkStatus(); +} + +Stream GetFaceRoiFromPoseFaceLandmarks( + Stream pose_face_landmarks, + Stream> image_size, Graph& graph) { + Stream detection = + ConvertLandmarksToDetection(pose_face_landmarks, graph); + + // Refer the pose face landmarks indices here: + // https://developers.google.com/mediapipe/solutions/vision/pose_landmarker#pose_landmarker_model + Stream rect = ConvertDetectionToRect( + detection, image_size, /*start_keypoint_index=*/5, + /*end_keypoint_index=*/2, /*target_angle=*/0, graph); + + // Scale the face RoI from a tight rect enclosing the pose face landmarks, to + // a larger square so that the whole face is within the RoI. + return ScaleAndMakeSquare(rect, image_size, + /*scale_x_factor=*/3.0, + /*scale_y_factor=*/3.0, graph); +} + +Stream GetFaceRoiFromFaceLandmarks( + Stream face_landmarks, + Stream> image_size, Graph& graph) { + Stream detection = + ConvertLandmarksToDetection(face_landmarks, graph); + + Stream rect = ConvertDetectionToRect( + detection, image_size, /*start_keypoint_index=*/33, + /*end_keypoint_index=*/263, /*target_angle=*/0, graph); + + return Scale(rect, image_size, + /*scale_x_factor=*/1.5, + /*scale_y_factor=*/1.5, graph); +} + +Stream> GetFaceDetections( + Stream image, Stream roi, + const face_detector::proto::FaceDetectorGraphOptions& + face_detector_graph_options, + Graph& graph) { + auto& face_detector_graph = + graph.AddNode("mediapipe.tasks.vision.face_detector.FaceDetectorGraph"); + face_detector_graph + .GetOptions() = + face_detector_graph_options; + image >> face_detector_graph.In("IMAGE"); + roi >> face_detector_graph.In("NORM_RECT"); + return face_detector_graph.Out("DETECTIONS").Cast>(); +} + +Stream GetFaceRoiFromFaceDetections( + Stream> face_detections, + Stream> image_size, Graph& graph) { + // Convert detection to rect. + Stream rect = ConvertDetectionsToRectUsingKeypoints( + face_detections, image_size, /*start_keypoint_index=*/0, + /*end_keypoint_index=*/1, /*target_angle=*/0, graph); + + return ScaleAndMakeSquare(rect, image_size, + /*scale_x_factor=*/2.0, + /*scale_y_factor=*/2.0, graph); +} + +Stream TrackFaceRoi( + Stream prev_landmarks, Stream roi, + Stream> image_size, Graph& graph) { + // Gets face ROI from previous frame face landmarks. + Stream prev_roi = + GetFaceRoiFromFaceLandmarks(prev_landmarks, image_size, graph); + + auto& tracking_node = graph.AddNode("RoiTrackingCalculator"); + auto& tracking_node_opts = + tracking_node.GetOptions(); + auto* rect_requirements = tracking_node_opts.mutable_rect_requirements(); + rect_requirements->set_rotation_degrees(15.0); + rect_requirements->set_translation(0.1); + rect_requirements->set_scale(0.3); + auto* landmarks_requirements = + tracking_node_opts.mutable_landmarks_requirements(); + landmarks_requirements->set_recrop_rect_margin(-0.2); + prev_landmarks.ConnectTo(tracking_node.In("PREV_LANDMARKS")); + prev_roi.ConnectTo(tracking_node.In("PREV_LANDMARKS_RECT")); + roi.ConnectTo(tracking_node.In("RECROP_RECT")); + image_size.ConnectTo(tracking_node.In("IMAGE_SIZE")); + return tracking_node.Out("TRACKING_RECT").Cast(); +} + +FaceLandmarksResult GetFaceLandmarksDetection( + Stream image, Stream roi, + Stream> image_size, + const face_landmarker::proto::FaceLandmarksDetectorGraphOptions& + face_landmarks_detector_graph_options, + const HolisticFaceTrackingRequest& request, Graph& graph) { + FaceLandmarksResult result; + auto& face_landmarks_detector_graph = graph.AddNode( + "mediapipe.tasks.vision.face_landmarker." + "SingleFaceLandmarksDetectorGraph"); + face_landmarks_detector_graph + .GetOptions() = + face_landmarks_detector_graph_options; + image >> face_landmarks_detector_graph.In("IMAGE"); + roi >> face_landmarks_detector_graph.In("NORM_RECT"); + auto landmarks = face_landmarks_detector_graph.Out("NORM_LANDMARKS") + .Cast(); + result.landmarks = landmarks; + if (request.classifications) { + auto& blendshapes_graph = graph.AddNode( + "mediapipe.tasks.vision.face_landmarker.FaceBlendshapesGraph"); + blendshapes_graph + .GetOptions() = + face_landmarks_detector_graph_options.face_blendshapes_graph_options(); + landmarks >> blendshapes_graph.In("LANDMARKS"); + image_size >> blendshapes_graph.In("IMAGE_SIZE"); + result.classifications = + blendshapes_graph.Out("BLENDSHAPES").Cast(); + } + return result; +} + +} // namespace + +absl::StatusOr TrackHolisticFace( + Stream image, Stream pose_face_landmarks, + const face_detector::proto::FaceDetectorGraphOptions& + face_detector_graph_options, + const face_landmarker::proto::FaceLandmarksDetectorGraphOptions& + face_landmarks_detector_graph_options, + const HolisticFaceTrackingRequest& request, Graph& graph) { + MP_RETURN_IF_ERROR(ValidateGraphOptions(face_detector_graph_options, + face_landmarks_detector_graph_options, + request)); + + // Extracts image size from the input images. + Stream> image_size = GetImageSize(image, graph); + + // Gets face ROI from pose face landmarks. + Stream roi_from_pose = + GetFaceRoiFromPoseFaceLandmarks(pose_face_landmarks, image_size, graph); + + // Detects faces within ROI of pose face. + Stream> face_detections = GetFaceDetections( + image, roi_from_pose, face_detector_graph_options, graph); + + // Gets face ROI from face detector. + Stream roi_from_detection = + GetFaceRoiFromFaceDetections(face_detections, image_size, graph); + + // Loop for previous frame landmarks. + auto [prev_landmarks, set_prev_landmarks_fn] = + GetLoopbackData(/*tick=*/image_size, graph); + + // Tracks face ROI. + auto tracking_roi = + TrackFaceRoi(prev_landmarks, roi_from_detection, image_size, graph); + + // Predicts face landmarks. + auto landmarks_detection_result = GetFaceLandmarksDetection( + image, tracking_roi, image_size, face_landmarks_detector_graph_options, + request, graph); + + // Sets previous landmarks for ROI tracking. + set_prev_landmarks_fn(landmarks_detection_result.landmarks.value()); + + return {{.landmarks = landmarks_detection_result.landmarks, + .classifications = landmarks_detection_result.classifications, + .debug_output = { + .roi_from_pose = roi_from_pose, + .roi_from_detection = roi_from_detection, + .tracking_roi = tracking_roi, + }}}; +} + +} // namespace holistic_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking.h b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking.h new file mode 100644 index 000000000..835767ebc --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking.h @@ -0,0 +1,89 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_HOLISTIC_LANDMARKER_HOLISTIC_FACE_TRACKING_H_ +#define MEDIAPIPE_TASKS_CC_VISION_HOLISTIC_LANDMARKER_HOLISTIC_FACE_TRACKING_H_ + +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace holistic_landmarker { + +struct HolisticFaceTrackingRequest { + bool classifications = false; +}; + +struct HolisticFaceTrackingOutput { + std::optional> + landmarks; + std::optional> + classifications; + + struct DebugOutput { + api2::builder::Stream roi_from_pose; + api2::builder::Stream roi_from_detection; + api2::builder::Stream tracking_roi; + }; + + DebugOutput debug_output; +}; + +// Updates @graph to track a single face in @image based on pose landmarks. +// +// To track single face this subgraph uses pose face landmarks to obtain +// approximate face location, refines it with face detector model and then runs +// face landmarks model. It can also reuse face ROI from the previous frame if +// face hasn't moved too much. +// +// @image - Image to track a single face in. +// @pose_face_landmarks - Pose face landmarks to derive initial face location +// from. +// @face_detector_graph_options - face detector graph options used to detect the +// face within the RoI constructed from the pose face landmarks. +// @face_landmarks_detector_graph_options - face landmarks detector graph +// options used to detect face landmarks within the RoI given be the face +// detector graph. +// @request - object to request specific face tracking outputs. +// NOTE: Outputs that were not requested won't be returned and corresponding +// parts of the graph won't be genertaed. +// @graph - graph to update. +absl::StatusOr TrackHolisticFace( + api2::builder::Stream image, + api2::builder::Stream + pose_face_landmarks, + const face_detector::proto::FaceDetectorGraphOptions& + face_detector_graph_options, + const face_landmarker::proto::FaceLandmarksDetectorGraphOptions& + face_landmarks_detector_graph_options, + const HolisticFaceTrackingRequest& request, + mediapipe::api2::builder::Graph& graph); + +} // namespace holistic_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe + +#endif // MEDIAPIPE_TASKS_CC_VISION_HOLISTIC_LANDMARKER_HOLISTIC_FACE_TRACKING_H_ diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking_test.cc b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking_test.cc new file mode 100644 index 000000000..314c330b3 --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking_test.cc @@ -0,0 +1,227 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking.h" + +#include +#include +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" +#include "mediapipe/calculators/util/rect_to_render_data_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/stream/image_size.h" +#include "mediapipe/framework/api2/stream/split.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/framework/tool/test_util.h" +#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" +#include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h" +#include "mediapipe/tasks/cc/core/proto/base_options.pb.h" +#include "mediapipe/tasks/cc/core/proto/external_file.pb.h" +#include "mediapipe/tasks/cc/core/task_runner.h" +#include "mediapipe/tasks/cc/core/utils.h" +#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_connections.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_result.pb.h" +#include "mediapipe/tasks/cc/vision/utils/data_renderer.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" +#include "mediapipe/util/color.pb.h" +#include "mediapipe/util/render_data.pb.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace holistic_landmarker { + +namespace { + +using ::mediapipe::Image; +using ::mediapipe::api2::builder::GetImageSize; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::SplitToRanges; +using ::mediapipe::api2::builder::Stream; +using ::mediapipe::tasks::core::ModelAssetBundleResources; +using ::mediapipe::tasks::core::TaskRunner; +using ::mediapipe::tasks::core::proto::ExternalFile; +using ::testing::proto::Approximately; +using ::testing::proto::Partially; + +constexpr float kAbsMargin = 0.015; +constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; +constexpr char kTestImageFile[] = "male_full_height_hands.jpg"; +constexpr char kHolisticResultFile[] = + "male_full_height_hands_result_cpu.pbtxt"; +constexpr char kImageInStream[] = "image_in"; +constexpr char kPoseLandmarksInStream[] = "pose_landmarks_in"; +constexpr char kFaceLandmarksOutStream[] = "face_landmarks_out"; +constexpr char kRenderedImageOutStream[] = "rendered_image_out"; +constexpr char kFaceDetectorTFLiteName[] = "face_detector.tflite"; +constexpr char kFaceLandmarksDetectorTFLiteName[] = + "face_landmarks_detector.tflite"; + +std::string GetFilePath(absl::string_view filename) { + return file::JoinPath("./", kTestDataDirectory, filename); +} + +mediapipe::LandmarksToRenderDataCalculatorOptions GetFaceRendererOptions() { + mediapipe::LandmarksToRenderDataCalculatorOptions render_options; + for (const auto& connection : + face_landmarker::FaceLandmarksConnections::kFaceLandmarksConnectors) { + render_options.add_landmark_connections(connection[0]); + render_options.add_landmark_connections(connection[1]); + } + render_options.mutable_landmark_color()->set_r(255); + render_options.mutable_landmark_color()->set_g(255); + render_options.mutable_landmark_color()->set_b(255); + render_options.mutable_connection_color()->set_r(255); + render_options.mutable_connection_color()->set_g(255); + render_options.mutable_connection_color()->set_b(255); + render_options.set_thickness(0.5); + render_options.set_visualize_landmark_depth(false); + return render_options; +} + +absl::StatusOr> +CreateModelAssetBundleResources(const std::string& model_asset_filename) { + auto external_model_bundle = std::make_unique(); + external_model_bundle->set_file_name(model_asset_filename); + return ModelAssetBundleResources::Create("", + std::move(external_model_bundle)); +} + +// Helper function to create a TaskRunner. +absl::StatusOr> CreateTaskRunner() { + Graph graph; + Stream image = graph.In("IMAGE").Cast().SetName(kImageInStream); + Stream pose_landmarks = + graph.In("POSE_LANDMARKS") + .Cast() + .SetName(kPoseLandmarksInStream); + Stream face_landmarks_from_pose = + SplitToRanges(pose_landmarks, {{0, 11}}, graph)[0]; + // Create face landmarker model bundle. + MP_ASSIGN_OR_RETURN( + auto model_bundle, + CreateModelAssetBundleResources(GetFilePath("face_landmarker_v2.task"))); + face_detector::proto::FaceDetectorGraphOptions detector_options; + face_landmarker::proto::FaceLandmarksDetectorGraphOptions + landmarks_detector_options; + + // Set face detection model. + MP_ASSIGN_OR_RETURN(auto face_detector_model_file, + model_bundle->GetFile(kFaceDetectorTFLiteName)); + core::proto::FilePointerMeta face_detection_file_pointer; + face_detection_file_pointer.set_pointer( + reinterpret_cast(face_detector_model_file.data())); + face_detection_file_pointer.set_length(face_detector_model_file.size()); + detector_options.mutable_base_options() + ->mutable_model_asset() + ->mutable_file_pointer_meta() + ->Swap(&face_detection_file_pointer); + detector_options.set_num_faces(1); + + // Set face landmarks model. + MP_ASSIGN_OR_RETURN(auto face_landmarks_model_file, + model_bundle->GetFile(kFaceLandmarksDetectorTFLiteName)); + core::proto::FilePointerMeta face_landmarks_detector_file_pointer; + face_landmarks_detector_file_pointer.set_pointer( + reinterpret_cast(face_landmarks_model_file.data())); + face_landmarks_detector_file_pointer.set_length( + face_landmarks_model_file.size()); + landmarks_detector_options.mutable_base_options() + ->mutable_model_asset() + ->mutable_file_pointer_meta() + ->Swap(&face_landmarks_detector_file_pointer); + + // Track holistic face. + HolisticFaceTrackingRequest request; + MP_ASSIGN_OR_RETURN( + HolisticFaceTrackingOutput result, + TrackHolisticFace(image, face_landmarks_from_pose, detector_options, + landmarks_detector_options, request, graph)); + auto face_landmarks = + result.landmarks.value().SetName(kFaceLandmarksOutStream); + + auto image_size = GetImageSize(image, graph); + auto render_scale = utils::GetRenderScale( + image_size, result.debug_output.roi_from_pose, 0.0001, graph); + + auto face_landmarks_render_data = utils::RenderLandmarks( + face_landmarks, render_scale, GetFaceRendererOptions(), graph); + std::vector> render_list = { + face_landmarks_render_data}; + + auto rendered_image = + utils::Render( + image, absl::Span>(render_list), graph) + .SetName(kRenderedImageOutStream); + face_landmarks >> graph.Out("FACE_LANDMARKS"); + rendered_image >> graph.Out("RENDERED_IMAGE"); + + auto config = graph.GetConfig(); + core::FixGraphBackEdges(config); + return TaskRunner::Create( + config, std::make_unique()); +} + +class HolisticFaceTrackingTest : public ::testing::Test {}; + +TEST_F(HolisticFaceTrackingTest, SmokeTest) { + MP_ASSERT_OK_AND_ASSIGN(Image image, + DecodeImageFromFile(GetFilePath(kTestImageFile))); + + proto::HolisticResult holistic_result; + MP_ASSERT_OK(GetTextProto(GetFilePath(kHolisticResultFile), &holistic_result, + ::file::Defaults())); + MP_ASSERT_OK_AND_ASSIGN(auto task_runner, CreateTaskRunner()); + MP_ASSERT_OK_AND_ASSIGN( + auto output_packets, + task_runner->Process( + {{kImageInStream, MakePacket(image)}, + {kPoseLandmarksInStream, MakePacket( + holistic_result.pose_landmarks())}})); + ASSERT_TRUE(output_packets.find(kFaceLandmarksOutStream) != + output_packets.end()); + auto face_landmarks = output_packets.find(kFaceLandmarksOutStream) + ->second.Get(); + EXPECT_THAT( + face_landmarks, + Approximately(Partially(EqualsProto(holistic_result.face_landmarks())), + /*margin=*/kAbsMargin)); + auto rendered_image = output_packets.at(kRenderedImageOutStream).Get(); + MP_EXPECT_OK(SavePngTestOutput(*rendered_image.GetImageFrameSharedPtr(), + "holistic_face_landmarks")); +} + +} // namespace +} // namespace holistic_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking.cc b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking.cc new file mode 100644 index 000000000..2c57aa059 --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking.cc @@ -0,0 +1,272 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking.h" + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.h" +#include "mediapipe/calculators/util/align_hand_to_pose_in_world_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/stream/image_size.h" +#include "mediapipe/framework/api2/stream/landmarks_to_detection.h" +#include "mediapipe/framework/api2/stream/loopback.h" +#include "mediapipe/framework/api2/stream/rect_transformation.h" +#include "mediapipe/framework/api2/stream/split.h" +#include "mediapipe/framework/api2/stream/threshold.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/modules/holistic_landmark/calculators/roi_tracking_calculator.pb.h" +#include "mediapipe/tasks/cc/components/utils/gate.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_roi_refinement_graph_options.pb.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace holistic_landmarker { + +namespace { + +using ::mediapipe::NormalizedRect; +using ::mediapipe::api2::AlignHandToPoseInWorldCalculator; +using ::mediapipe::api2::builder::ConvertLandmarksToDetection; +using ::mediapipe::api2::builder::GetImageSize; +using ::mediapipe::api2::builder::GetLoopbackData; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::IsOverThreshold; +using ::mediapipe::api2::builder::ScaleAndShiftAndMakeSquareLong; +using ::mediapipe::api2::builder::SplitAndCombine; +using ::mediapipe::api2::builder::Stream; +using ::mediapipe::tasks::components::utils::AllowIf; + +struct HandLandmarksResult { + std::optional> landmarks; + std::optional> world_landmarks; +}; + +Stream AlignHandToPoseInWorldCalculator( + Stream hand_world_landmarks, + Stream pose_world_landmarks, int pose_wrist_idx, + Graph& graph) { + auto& node = graph.AddNode("AlignHandToPoseInWorldCalculator"); + auto& opts = node.GetOptions(); + opts.set_hand_wrist_idx(0); + opts.set_pose_wrist_idx(pose_wrist_idx); + hand_world_landmarks.ConnectTo( + node[AlignHandToPoseInWorldCalculator::kInHandLandmarks]); + pose_world_landmarks.ConnectTo( + node[AlignHandToPoseInWorldCalculator::kInPoseLandmarks]); + return node[AlignHandToPoseInWorldCalculator::kOutHandLandmarks]; +} + +Stream GetPosePalmVisibility( + Stream pose_palm_landmarks, Graph& graph) { + // Get wrist landmark. + auto pose_wrist = SplitAndCombine(pose_palm_landmarks, {0}, graph); + + // Get visibility score. + auto& score_node = graph.AddNode("LandmarkVisibilityCalculator"); + pose_wrist.ConnectTo(score_node.In("NORM_LANDMARKS")); + Stream score = score_node.Out("VISIBILITY").Cast(); + + // Convert score into flag. + return IsOverThreshold(score, /*threshold=*/0.1, graph); +} + +Stream GetHandRoiFromPosePalmLandmarks( + Stream pose_palm_landmarks, + Stream> image_size, Graph& graph) { + // Convert pose palm landmarks to detection. + auto detection = ConvertLandmarksToDetection(pose_palm_landmarks, graph); + + // Convert detection to rect. + auto& rect_node = graph.AddNode("HandDetectionsFromPoseToRectsCalculator"); + detection.ConnectTo(rect_node.In("DETECTION")); + image_size.ConnectTo(rect_node.In("IMAGE_SIZE")); + Stream rect = + rect_node.Out("NORM_RECT").Cast(); + + return ScaleAndShiftAndMakeSquareLong(rect, image_size, + /*scale_x_factor=*/2.7, + /*scale_y_factor=*/2.7, /*shift_x=*/0, + /*shift_y=*/-0.1, graph); +} + +absl::StatusOr> RefineHandRoi( + Stream image, Stream roi, + const hand_landmarker::proto::HandRoiRefinementGraphOptions& + hand_roi_refinenement_graph_options, + Graph& graph) { + auto& hand_roi_refinement = graph.AddNode( + "mediapipe.tasks.vision.hand_landmarker.HandRoiRefinementGraph"); + hand_roi_refinement + .GetOptions() = + hand_roi_refinenement_graph_options; + image >> hand_roi_refinement.In("IMAGE"); + roi >> hand_roi_refinement.In("NORM_RECT"); + return hand_roi_refinement.Out("NORM_RECT").Cast(); +} + +Stream TrackHandRoi( + Stream prev_landmarks, Stream roi, + Stream> image_size, Graph& graph) { + // Convert hand landmarks to tight rect. + auto& prev_rect_node = graph.AddNode("HandLandmarksToRectCalculator"); + prev_landmarks.ConnectTo(prev_rect_node.In("NORM_LANDMARKS")); + image_size.ConnectTo(prev_rect_node.In("IMAGE_SIZE")); + Stream prev_rect = + prev_rect_node.Out("NORM_RECT").Cast(); + + // Convert tight hand rect to hand roi. + Stream prev_roi = + ScaleAndShiftAndMakeSquareLong(prev_rect, image_size, + /*scale_x_factor=*/2.0, + /*scale_y_factor=*/2.0, /*shift_x=*/0, + /*shift_y=*/-0.1, graph); + + auto& tracking_node = graph.AddNode("RoiTrackingCalculator"); + auto& tracking_node_opts = + tracking_node.GetOptions(); + auto* rect_requirements = tracking_node_opts.mutable_rect_requirements(); + rect_requirements->set_rotation_degrees(40.0); + rect_requirements->set_translation(0.2); + rect_requirements->set_scale(0.4); + auto* landmarks_requirements = + tracking_node_opts.mutable_landmarks_requirements(); + landmarks_requirements->set_recrop_rect_margin(-0.1); + prev_landmarks.ConnectTo(tracking_node.In("PREV_LANDMARKS")); + prev_roi.ConnectTo(tracking_node.In("PREV_LANDMARKS_RECT")); + roi.ConnectTo(tracking_node.In("RECROP_RECT")); + image_size.ConnectTo(tracking_node.In("IMAGE_SIZE")); + return tracking_node.Out("TRACKING_RECT").Cast(); +} + +HandLandmarksResult GetHandLandmarksDetection( + Stream image, Stream roi, + const hand_landmarker::proto::HandLandmarksDetectorGraphOptions& + hand_landmarks_detector_graph_options, + const HolisticHandTrackingRequest& request, Graph& graph) { + HandLandmarksResult result; + auto& hand_landmarks_detector_graph = graph.AddNode( + "mediapipe.tasks.vision.hand_landmarker." + "SingleHandLandmarksDetectorGraph"); + hand_landmarks_detector_graph + .GetOptions() = + hand_landmarks_detector_graph_options; + + image >> hand_landmarks_detector_graph.In("IMAGE"); + roi >> hand_landmarks_detector_graph.In("HAND_RECT"); + + if (request.landmarks) { + result.landmarks = hand_landmarks_detector_graph.Out("LANDMARKS") + .Cast(); + } + if (request.world_landmarks) { + result.world_landmarks = + hand_landmarks_detector_graph.Out("WORLD_LANDMARKS") + .Cast(); + } + return result; +} + +} // namespace + +absl::StatusOr TrackHolisticHand( + Stream image, Stream pose_landmarks, + Stream pose_world_landmarks, + const hand_landmarker::proto::HandLandmarksDetectorGraphOptions& + hand_landmarks_detector_graph_options, + const hand_landmarker::proto::HandRoiRefinementGraphOptions& + hand_roi_refinement_graph_options, + const PoseIndices& pose_indices, const HolisticHandTrackingRequest& request, + Graph& graph) { + // Extracts pose palm landmarks. + Stream pose_palm_landmarks = SplitAndCombine( + pose_landmarks, + {pose_indices.wrist_idx, pose_indices.pinky_idx, pose_indices.index_idx}, + graph); + + // Get pose palm visibility. + Stream is_pose_palm_visible = + GetPosePalmVisibility(pose_palm_landmarks, graph); + + // Drop pose palm landmarks if pose palm is invisible. + pose_palm_landmarks = + AllowIf(pose_palm_landmarks, is_pose_palm_visible, graph); + + // Extracts image size from the input images. + Stream> image_size = GetImageSize(image, graph); + + // Get hand ROI from pose palm landmarks. + Stream roi_from_pose = + GetHandRoiFromPosePalmLandmarks(pose_palm_landmarks, image_size, graph); + + // Refine hand ROI with re-crop model. + MP_ASSIGN_OR_RETURN(Stream roi_from_recrop, + RefineHandRoi(image, roi_from_pose, + hand_roi_refinement_graph_options, graph)); + + // Loop for previous frame landmarks. + auto [prev_landmarks, set_prev_landmarks_fn] = + GetLoopbackData(/*tick=*/image_size, graph); + + // Track hand ROI. + auto tracking_roi = + TrackHandRoi(prev_landmarks, roi_from_recrop, image_size, graph); + + // Predict hand landmarks. + auto landmarks_detection_result = GetHandLandmarksDetection( + image, tracking_roi, hand_landmarks_detector_graph_options, request, + graph); + + // Set previous landmarks for ROI tracking. + set_prev_landmarks_fn(landmarks_detection_result.landmarks.value()); + + // Output landmarks. + std::optional> hand_landmarks; + if (request.landmarks) { + hand_landmarks = landmarks_detection_result.landmarks; + } + + // Output world landmarks. + std::optional> hand_world_landmarks; + if (request.world_landmarks) { + hand_world_landmarks = landmarks_detection_result.world_landmarks; + + // Align hand world landmarks with pose world landmarks. + hand_world_landmarks = AlignHandToPoseInWorldCalculator( + hand_world_landmarks.value(), pose_world_landmarks, + pose_indices.wrist_idx, graph); + } + + return {{.landmarks = hand_landmarks, + .world_landmarks = hand_world_landmarks, + .debug_output = { + .roi_from_pose = roi_from_pose, + .roi_from_recrop = roi_from_recrop, + .tracking_roi = tracking_roi, + }}}; +} + +} // namespace holistic_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking.h b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking.h new file mode 100644 index 000000000..463f4979b --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking.h @@ -0,0 +1,94 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_HOLISTIC_LANDMARKER_HOLISTIC_HAND_TRACKING_H_ +#define MEDIAPIPE_TASKS_CC_VISION_HOLISTIC_LANDMARKER_HOLISTIC_HAND_TRACKING_H_ + +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_roi_refinement_graph_options.pb.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace holistic_landmarker { + +struct PoseIndices { + int wrist_idx; + int pinky_idx; + int index_idx; +}; + +struct HolisticHandTrackingRequest { + bool landmarks = false; + bool world_landmarks = false; +}; + +struct HolisticHandTrackingOutput { + std::optional> + landmarks; + std::optional> world_landmarks; + + struct DebugOutput { + api2::builder::Stream roi_from_pose; + api2::builder::Stream roi_from_recrop; + api2::builder::Stream tracking_roi; + }; + + DebugOutput debug_output; +}; + +// Updates @graph to track a single hand in @image based on pose landmarks. +// +// To track single hand this subgraph uses pose palm landmarks to obtain +// approximate hand location, refines it with re-crop model and then runs hand +// landmarks model. It can also reuse hand ROI from the previous frame if hand +// hasn't moved too much. +// +// @image - ImageFrame/GpuBuffer to track a single hand in. +// @pose_landmarks - Pose landmarks to derive initial hand location from. +// @pose_world_landmarks - Pose world landmarks to align hand world landmarks +// wrist with. +// @ hand_landmarks_detector_graph_options - Options of the +// HandLandmarksDetectorGraph used to detect the hand landmarks. +// @ hand_roi_refinement_graph_options - Options of HandRoiRefinementGraph used +// to refine the hand RoIs got from Pose landmarks. +// @request - object to request specific hand tracking outputs. +// NOTE: Outputs that were not requested won't be returned and corresponding +// parts of the graph won't be genertaed. +// @graph - graph to update. +absl::StatusOr TrackHolisticHand( + api2::builder::Stream image, + api2::builder::Stream pose_landmarks, + api2::builder::Stream pose_world_landmarks, + const hand_landmarker::proto::HandLandmarksDetectorGraphOptions& + hand_landmarks_detector_graph_options, + const hand_landmarker::proto::HandRoiRefinementGraphOptions& + hand_roi_refinement_graph_options, + const PoseIndices& pose_indices, const HolisticHandTrackingRequest& request, + mediapipe::api2::builder::Graph& graph); + +} // namespace holistic_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe + +#endif // MEDIAPIPE_TASKS_CC_VISION_HOLISTIC_LANDMARKER_HOLISTIC_HAND_TRACKING_H_ diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking_test.cc b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking_test.cc new file mode 100644 index 000000000..4ae4a37ed --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking_test.cc @@ -0,0 +1,303 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking.h" + +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/status/statusor.h" +#include "absl/strings/substitute.h" +#include "absl/types/span.h" +#include "file/base/helpers.h" +#include "file/base/options.h" +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/stream/image_size.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/framework/tool/test_util.h" +#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" +#include "mediapipe/tasks/cc/core/proto/base_options.pb.h" +#include "mediapipe/tasks/cc/core/task_runner.h" +#include "mediapipe/tasks/cc/core/utils.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_connections.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_roi_refinement_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_result.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/pose_topology.h" +#include "mediapipe/tasks/cc/vision/utils/data_renderer.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" +#include "mediapipe/util/color.pb.h" +#include "mediapipe/util/render_data.pb.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace holistic_landmarker { + +namespace { + +using ::file::Defaults; +using ::file::GetTextProto; +using ::mediapipe::Image; +using ::mediapipe::api2::builder::GetImageSize; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Stream; +using ::mediapipe::tasks::core::TaskRunner; +using ::testing::proto::Approximately; +using ::testing::proto::Partially; + +constexpr float kAbsMargin = 0.018; +constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; +constexpr char kHolisticHandTrackingLeft[] = + "holistic_hand_tracking_left_hand_graph.pbtxt"; +constexpr char kTestImageFile[] = "male_full_height_hands.jpg"; +constexpr char kHolisticResultFile[] = + "male_full_height_hands_result_cpu.pbtxt"; +constexpr char kImageInStream[] = "image_in"; +constexpr char kPoseLandmarksInStream[] = "pose_landmarks_in"; +constexpr char kPoseWorldLandmarksInStream[] = "pose_world_landmarks_in"; +constexpr char kLeftHandLandmarksOutStream[] = "left_hand_landmarks_out"; +constexpr char kLeftHandWorldLandmarksOutStream[] = + "left_hand_world_landmarks_out"; +constexpr char kRightHandLandmarksOutStream[] = "right_hand_landmarks_out"; +constexpr char kRenderedImageOutStream[] = "rendered_image_out"; +constexpr char kHandLandmarksModelFile[] = "hand_landmark_full.tflite"; +constexpr char kHandRoiRefinementModelFile[] = + "handrecrop_2020_07_21_v0.f16.tflite"; + +std::string GetFilePath(const std::string& filename) { + return file::JoinPath("./", kTestDataDirectory, filename); +} + +mediapipe::LandmarksToRenderDataCalculatorOptions GetHandRendererOptions() { + mediapipe::LandmarksToRenderDataCalculatorOptions renderer_options; + for (const auto& connection : hand_landmarker::kHandConnections) { + renderer_options.add_landmark_connections(connection[0]); + renderer_options.add_landmark_connections(connection[1]); + } + renderer_options.mutable_landmark_color()->set_r(255); + renderer_options.mutable_landmark_color()->set_g(255); + renderer_options.mutable_landmark_color()->set_b(255); + renderer_options.mutable_connection_color()->set_r(255); + renderer_options.mutable_connection_color()->set_g(255); + renderer_options.mutable_connection_color()->set_b(255); + renderer_options.set_thickness(0.5); + renderer_options.set_visualize_landmark_depth(false); + return renderer_options; +} + +void ConfigHandTrackingModelsOptions( + hand_landmarker::proto::HandLandmarksDetectorGraphOptions& + hand_landmarks_detector_graph_options, + hand_landmarker::proto::HandRoiRefinementGraphOptions& + hand_roi_refinement_options) { + hand_landmarks_detector_graph_options.mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath(kHandLandmarksModelFile)); + + hand_roi_refinement_options.mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath(kHandRoiRefinementModelFile)); +} + +// Helper function to create a TaskRunner. +absl::StatusOr> CreateTaskRunner() { + Graph graph; + Stream image = graph.In("IMAGE").Cast().SetName(kImageInStream); + Stream pose_landmarks = + graph.In("POSE_LANDMARKS") + .Cast() + .SetName(kPoseLandmarksInStream); + Stream pose_world_landmarks = + graph.In("POSE_WORLD_LANDMARKS") + .Cast() + .SetName(kPoseWorldLandmarksInStream); + hand_landmarker::proto::HandLandmarksDetectorGraphOptions + hand_landmarks_detector_options; + hand_landmarker::proto::HandRoiRefinementGraphOptions + hand_roi_refinement_options; + ConfigHandTrackingModelsOptions(hand_landmarks_detector_options, + hand_roi_refinement_options); + HolisticHandTrackingRequest request; + request.landmarks = true; + MP_ASSIGN_OR_RETURN( + HolisticHandTrackingOutput left_hand_result, + TrackHolisticHand( + image, pose_landmarks, pose_world_landmarks, + hand_landmarks_detector_options, hand_roi_refinement_options, + PoseIndices{ + /*wrist_idx=*/static_cast( + pose_landmarker::PoseLandmarkName::kLeftWrist), + /*pinky_idx=*/ + static_cast(pose_landmarker::PoseLandmarkName::kLeftPinky1), + /*index_idx=*/ + static_cast(pose_landmarker::PoseLandmarkName::kLeftIndex1)}, + request, graph)); + MP_ASSIGN_OR_RETURN( + HolisticHandTrackingOutput right_hand_result, + TrackHolisticHand( + image, pose_landmarks, pose_world_landmarks, + hand_landmarks_detector_options, hand_roi_refinement_options, + PoseIndices{ + /*wrist_idx=*/static_cast( + pose_landmarker::PoseLandmarkName::kRightWrist), + /*pinky_idx=*/ + static_cast(pose_landmarker::PoseLandmarkName::kRightPinky1), + /*index_idx=*/ + static_cast( + pose_landmarker::PoseLandmarkName::kRightIndex1)}, + request, graph)); + + auto image_size = GetImageSize(image, graph); + auto left_hand_landmarks_render_data = utils::RenderLandmarks( + *left_hand_result.landmarks, + utils::GetRenderScale(image_size, + left_hand_result.debug_output.roi_from_pose, 0.0001, + graph), + GetHandRendererOptions(), graph); + auto right_hand_landmarks_render_data = utils::RenderLandmarks( + *right_hand_result.landmarks, + utils::GetRenderScale(image_size, + right_hand_result.debug_output.roi_from_pose, + 0.0001, graph), + GetHandRendererOptions(), graph); + std::vector> render_list = { + left_hand_landmarks_render_data, right_hand_landmarks_render_data}; + auto rendered_image = + utils::Render( + image, absl::Span>(render_list), graph) + .SetName(kRenderedImageOutStream); + left_hand_result.landmarks->SetName(kLeftHandLandmarksOutStream) >> + graph.Out("LEFT_HAND_LANDMARKS"); + right_hand_result.landmarks->SetName(kRightHandLandmarksOutStream) >> + graph.Out("RIGHT_HAND_LANDMARKS"); + rendered_image >> graph.Out("RENDERED_IMAGE"); + + auto config = graph.GetConfig(); + core::FixGraphBackEdges(config); + + return TaskRunner::Create( + config, std::make_unique()); +} + +class HolisticHandTrackingTest : public ::testing::Test {}; + +TEST_F(HolisticHandTrackingTest, VerifyGraph) { + Graph graph; + Stream image = graph.In("IMAGE").Cast().SetName(kImageInStream); + Stream pose_landmarks = + graph.In("POSE_LANDMARKS") + .Cast() + .SetName(kPoseLandmarksInStream); + Stream pose_world_landmarks = + graph.In("POSE_WORLD_LANDMARKS") + .Cast() + .SetName(kPoseWorldLandmarksInStream); + hand_landmarker::proto::HandLandmarksDetectorGraphOptions + hand_landmarks_detector_options; + hand_landmarker::proto::HandRoiRefinementGraphOptions + hand_roi_refinement_options; + ConfigHandTrackingModelsOptions(hand_landmarks_detector_options, + hand_roi_refinement_options); + HolisticHandTrackingRequest request; + request.landmarks = true; + request.world_landmarks = true; + MP_ASSERT_OK_AND_ASSIGN( + HolisticHandTrackingOutput left_hand_result, + TrackHolisticHand( + image, pose_landmarks, pose_world_landmarks, + hand_landmarks_detector_options, hand_roi_refinement_options, + PoseIndices{ + /*wrist_idx=*/static_cast( + pose_landmarker::PoseLandmarkName::kLeftWrist), + /*pinky_idx=*/ + static_cast(pose_landmarker::PoseLandmarkName::kLeftPinky1), + /*index_idx=*/ + static_cast(pose_landmarker::PoseLandmarkName::kLeftIndex1)}, + request, graph)); + left_hand_result.landmarks->SetName(kLeftHandLandmarksOutStream) >> + graph.Out("LEFT_HAND_LANDMARKS"); + left_hand_result.world_landmarks->SetName(kLeftHandWorldLandmarksOutStream) >> + graph.Out("LEFT_HAND_WORLD_LANDMARKS"); + + // Read the expected graph config. + std::string expected_graph_contents; + MP_ASSERT_OK(file::GetContents( + file::JoinPath("./", kTestDataDirectory, kHolisticHandTrackingLeft), + &expected_graph_contents)); + + // Need to replace the expected graph config with the test srcdir, because + // each run has different test dir on TAP. + expected_graph_contents = absl::Substitute( + expected_graph_contents, FLAGS_test_srcdir, FLAGS_test_srcdir); + CalculatorGraphConfig expected_graph = + ParseTextProtoOrDie(expected_graph_contents); + + EXPECT_THAT(graph.GetConfig(), testing::proto::IgnoringRepeatedFieldOrdering( + testing::EqualsProto(expected_graph))); +} + +TEST_F(HolisticHandTrackingTest, SmokeTest) { + MP_ASSERT_OK_AND_ASSIGN(Image image, + DecodeImageFromFile(GetFilePath(kTestImageFile))); + + proto::HolisticResult holistic_result; + MP_ASSERT_OK(GetTextProto(GetFilePath(kHolisticResultFile), &holistic_result, + Defaults())); + MP_ASSERT_OK_AND_ASSIGN(auto task_runner, CreateTaskRunner()); + MP_ASSERT_OK_AND_ASSIGN( + auto output_packets, + task_runner->Process( + {{kImageInStream, MakePacket(image)}, + {kPoseLandmarksInStream, MakePacket( + holistic_result.pose_landmarks())}, + {kPoseWorldLandmarksInStream, + MakePacket( + holistic_result.pose_world_landmarks())}})); + auto left_hand_landmarks = output_packets.at(kLeftHandLandmarksOutStream) + .Get(); + auto right_hand_landmarks = output_packets.at(kRightHandLandmarksOutStream) + .Get(); + EXPECT_THAT(left_hand_landmarks, + Approximately( + Partially(EqualsProto(holistic_result.left_hand_landmarks())), + /*margin=*/kAbsMargin)); + EXPECT_THAT( + right_hand_landmarks, + Approximately( + Partially(EqualsProto(holistic_result.right_hand_landmarks())), + /*margin=*/kAbsMargin)); + auto rendered_image = output_packets.at(kRenderedImageOutStream).Get(); + MP_EXPECT_OK(SavePngTestOutput(*rendered_image.GetImageFrameSharedPtr(), + "holistic_hand_landmarks")); +} + +} // namespace +} // namespace holistic_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_landmarker_graph.cc b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_landmarker_graph.cc new file mode 100644 index 000000000..2de358a6c --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_landmarker_graph.cc @@ -0,0 +1,521 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/stream/split.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h" +#include "mediapipe/tasks/cc/core/model_resources_cache.h" +#include "mediapipe/tasks/cc/core/model_task_graph.h" +#include "mediapipe/tasks/cc/core/utils.h" +#include "mediapipe/tasks/cc/metadata/utils/zip_utils.h" +#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_roi_refinement_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/holistic_landmarker/holistic_face_tracking.h" +#include "mediapipe/tasks/cc/vision/holistic_landmarker/holistic_hand_tracking.h" +#include "mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking.h" +#include "mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_landmarker_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/pose_detector/proto/pose_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/pose_topology.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.pb.h" +#include "mediapipe/util/graph_builder_utils.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace holistic_landmarker { +namespace { + +using ::mediapipe::api2::Output; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Stream; +using ::mediapipe::tasks::metadata::SetExternalFile; + +constexpr absl::string_view kHandLandmarksDetectorModelName = + "hand_landmarks_detector.tflite"; +constexpr absl::string_view kHandRoiRefinementModelName = + "hand_roi_refinement.tflite"; +constexpr absl::string_view kFaceDetectorModelName = "face_detector.tflite"; +constexpr absl::string_view kFaceLandmarksDetectorModelName = + "face_landmarks_detector.tflite"; +constexpr absl::string_view kFaceBlendshapesModelName = + "face_blendshapes.tflite"; +constexpr absl::string_view kPoseDetectorModelName = "pose_detector.tflite"; +constexpr absl::string_view kPoseLandmarksDetectorModelName = + "pose_landmarks_detector.tflite"; + +absl::Status SetGraphPoseOutputs( + const HolisticPoseTrackingRequest& pose_request, + const CalculatorGraphConfig::Node& node, + HolisticPoseTrackingOutput& pose_output, Graph& graph) { + // Main outputs. + if (pose_request.landmarks) { + RET_CHECK(pose_output.landmarks.has_value()) + << "POSE_LANDMARKS output is not supported."; + pose_output.landmarks->ConnectTo(graph.Out("POSE_LANDMARKS")); + } + if (pose_request.world_landmarks) { + RET_CHECK(pose_output.world_landmarks.has_value()) + << "POSE_WORLD_LANDMARKS output is not supported."; + pose_output.world_landmarks->ConnectTo(graph.Out("POSE_WORLD_LANDMARKS")); + } + if (pose_request.segmentation_mask) { + RET_CHECK(pose_output.segmentation_mask.has_value()) + << "POSE_SEGMENTATION_MASK output is not supported."; + pose_output.segmentation_mask->ConnectTo( + graph.Out("POSE_SEGMENTATION_MASK")); + } + + // Debug outputs. + if (HasOutput(node, "POSE_AUXILIARY_LANDMARKS")) { + pose_output.debug_output.auxiliary_landmarks.ConnectTo( + graph.Out("POSE_AUXILIARY_LANDMARKS")); + } + if (HasOutput(node, "POSE_LANDMARKS_ROI")) { + pose_output.debug_output.roi_from_landmarks.ConnectTo( + graph.Out("POSE_LANDMARKS_ROI")); + } + + return absl::OkStatus(); +} + +// Sets the base options in the sub tasks. +template +absl::Status SetSubTaskBaseOptions( + const core::ModelAssetBundleResources* resources, + proto::HolisticLandmarkerGraphOptions* options, T* sub_task_options, + absl::string_view model_name, bool is_copy) { + if (!sub_task_options->base_options().has_model_asset()) { + MP_ASSIGN_OR_RETURN(const auto model_file_content, + resources->GetFile(std::string(model_name))); + SetExternalFile( + model_file_content, + sub_task_options->mutable_base_options()->mutable_model_asset(), + is_copy); + } + sub_task_options->mutable_base_options()->mutable_acceleration()->CopyFrom( + options->base_options().acceleration()); + sub_task_options->mutable_base_options()->set_use_stream_mode( + options->base_options().use_stream_mode()); + sub_task_options->mutable_base_options()->set_gpu_origin( + options->base_options().gpu_origin()); + return absl::OkStatus(); +} + +void SetGraphHandOutputs(bool is_left, const CalculatorGraphConfig::Node& node, + HolisticHandTrackingOutput& hand_output, + Graph& graph) { + const std::string hand_side = is_left ? "LEFT" : "RIGHT"; + + if (hand_output.landmarks) { + hand_output.landmarks->ConnectTo(graph.Out(hand_side + "_HAND_LANDMARKS")); + } + if (hand_output.world_landmarks) { + hand_output.world_landmarks->ConnectTo( + graph.Out(hand_side + "_HAND_WORLD_LANDMARKS")); + } + + // Debug outputs. + if (HasOutput(node, hand_side + "_HAND_ROI_FROM_POSE")) { + hand_output.debug_output.roi_from_pose.ConnectTo( + graph.Out(hand_side + "_HAND_ROI_FROM_POSE")); + } + if (HasOutput(node, hand_side + "_HAND_ROI_FROM_RECROP")) { + hand_output.debug_output.roi_from_recrop.ConnectTo( + graph.Out(hand_side + "_HAND_ROI_FROM_RECROP")); + } + if (HasOutput(node, hand_side + "_HAND_TRACKING_ROI")) { + hand_output.debug_output.tracking_roi.ConnectTo( + graph.Out(hand_side + "_HAND_TRACKING_ROI")); + } +} + +void SetGraphFaceOutputs(const CalculatorGraphConfig::Node& node, + HolisticFaceTrackingOutput& face_output, + Graph& graph) { + if (face_output.landmarks) { + face_output.landmarks->ConnectTo(graph.Out("FACE_LANDMARKS")); + } + if (face_output.classifications) { + face_output.classifications->ConnectTo(graph.Out("FACE_BLENDSHAPES")); + } + + // Face detection debug outputs + if (HasOutput(node, "FACE_ROI_FROM_POSE")) { + face_output.debug_output.roi_from_pose.ConnectTo( + graph.Out("FACE_ROI_FROM_POSE")); + } + if (HasOutput(node, "FACE_ROI_FROM_DETECTION")) { + face_output.debug_output.roi_from_detection.ConnectTo( + graph.Out("FACE_ROI_FROM_DETECTION")); + } + if (HasOutput(node, "FACE_TRACKING_ROI")) { + face_output.debug_output.tracking_roi.ConnectTo( + graph.Out("FACE_TRACKING_ROI")); + } +} + +} // namespace + +// Tracks pose and detects hands and face. +// +// NOTE: for GPU works only with image having GpuOrigin::TOP_LEFT +// +// Inputs: +// IMAGE - Image +// Image to perform detection on. +// +// Outputs: +// POSE_LANDMARKS - NormalizedLandmarkList +// 33 landmarks (see pose_landmarker/pose_topology.h) +// 0 - nose +// 1 - left eye (inner) +// 2 - left eye +// 3 - left eye (outer) +// 4 - right eye (inner) +// 5 - right eye +// 6 - right eye (outer) +// 7 - left ear +// 8 - right ear +// 9 - mouth (left) +// 10 - mouth (right) +// 11 - left shoulder +// 12 - right shoulder +// 13 - left elbow +// 14 - right elbow +// 15 - left wrist +// 16 - right wrist +// 17 - left pinky +// 18 - right pinky +// 19 - left index +// 20 - right index +// 21 - left thumb +// 22 - right thumb +// 23 - left hip +// 24 - right hip +// 25 - left knee +// 26 - right knee +// 27 - left ankle +// 28 - right ankle +// 29 - left heel +// 30 - right heel +// 31 - left foot index +// 32 - right foot index +// POSE_WORLD_LANDMARKS - LandmarkList +// World landmarks are real world 3D coordinates with origin in hips center +// and coordinates in meters. To understand the difference: POSE_LANDMARKS +// stream provides coordinates (in pixels) of 3D object projected on a 2D +// surface of the image (check on how perspective projection works), while +// POSE_WORLD_LANDMARKS stream provides coordinates (in meters) of the 3D +// object itself. POSE_WORLD_LANDMARKS has the same landmarks topology, +// visibility and presence as POSE_LANDMARKS. +// POSE_SEGMENTATION_MASK - Image +// Separates person from background. Mask is stored as gray float32 image +// with [0.0, 1.0] range for pixels (1 for person and 0 for background) on +// CPU and, on GPU - RGBA texture with R channel indicating person vs. +// background probability. +// LEFT_HAND_LANDMARKS - NormalizedLandmarkList +// 21 left hand landmarks. +// RIGHT_HAND_LANDMARKS - NormalizedLandmarkList +// 21 right hand landmarks. +// FACE_LANDMARKS - NormalizedLandmarkList +// 468 face landmarks. +// FACE_BLENDSHAPES - ClassificationList +// Supplementary blendshape coefficients that are predicted directly from +// the input image. +// LEFT_HAND_WORLD_LANDMARKS - LandmarkList +// 21 left hand world 3D landmarks. +// Hand landmarks are aligned with pose landmarks: translated so that wrist +// from # hand matches wrist from pose in pose coordinates system. +// RIGHT_HAND_WORLD_LANDMARKS - LandmarkList +// 21 right hand world 3D landmarks. +// Hand landmarks are aligned with pose landmarks: translated so that wrist +// from # hand matches wrist from pose in pose coordinates system. +// IMAGE - Image +// The input image that the hiolistic landmarker runs on and has the pixel +// data stored on the target storage (CPU vs GPU). +// +// Debug outputs: +// POSE_AUXILIARY_LANDMARKS - NormalizedLandmarkList +// TODO: Return ROI rather than auxiliary landmarks +// Auxiliary landmarks for deriving the ROI in the subsequent image. +// 0 - hidden center point +// 1 - hidden scale point +// POSE_LANDMARKS_ROI - NormalizedRect +// Region of interest calculated based on landmarks. +// LEFT_HAND_ROI_FROM_POSE - NormalizedLandmarkList +// LEFT_HAND_ROI_FROM_RECROP - NormalizedLandmarkList +// LEFT_HAND_TRACKING_ROI - NormalizedLandmarkList +// RIGHT_HAND_ROI_FROM_POSE - NormalizedLandmarkList +// RIGHT_HAND_ROI_FROM_RECROP - NormalizedLandmarkList +// RIGHT_HAND_TRACKING_ROI - NormalizedLandmarkList +// FACE_ROI_FROM_POSE - NormalizedLandmarkList +// FACE_ROI_FROM_DETECTION - NormalizedLandmarkList +// FACE_TRACKING_ROI - NormalizedLandmarkList +// +// NOTE: failure is reported if some output has been requested, but specified +// model doesn't support it. +// +// NOTE: there will not be an output packet in an output stream for a +// particular timestamp if nothing is detected. However, the MediaPipe +// framework will internally inform the downstream calculators of the +// absence of this packet so that they don't wait for it unnecessarily. +// +// Example: +// node { +// calculator: +// "mediapipe.tasks.vision.holistic_landmarker.HolisticLandmarkerGraph" +// input_stream: "IMAGE:input_frames_image" +// output_stream: "POSE_LANDMARKS:pose_landmarks" +// output_stream: "POSE_WORLD_LANDMARKS:pose_world_landmarks" +// output_stream: "FACE_LANDMARKS:face_landmarks" +// output_stream: "FACE_BLENDSHAPES:extra_blendshapes" +// output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks" +// output_stream: "LEFT_HAND_WORLD_LANDMARKS:left_hand_world_landmarks" +// output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks" +// output_stream: "RIGHT_HAND_WORLD_LANDMARKS:right_hand_world_landmarks" +// node_options { +// [type.googleapis.com/mediapipe.tasks.vision.holistic_landmarker.proto.HolisticLandmarkerGraphOptions] +// { +// base_options { +// model_asset { +// file_name: +// "mediapipe/tasks/testdata/vision/holistic_landmarker.task" +// } +// } +// face_detector_graph_options: { +// num_faces: 1 +// } +// pose_detector_graph_options: { +// num_poses: 1 +// } +// } +// } +// } +class HolisticLandmarkerGraph : public core::ModelTaskGraph { + public: + absl::StatusOr GetConfig( + SubgraphContext* sc) override { + Graph graph; + const auto& holistic_node = sc->OriginalNode(); + proto::HolisticLandmarkerGraphOptions* holistic_options = + sc->MutableOptions(); + const core::ModelAssetBundleResources* model_asset_bundle_resources; + if (holistic_options->base_options().has_model_asset()) { + MP_ASSIGN_OR_RETURN(model_asset_bundle_resources, + CreateModelAssetBundleResources< + proto::HolisticLandmarkerGraphOptions>(sc)); + } + // Copies the file content instead of passing the pointer of file in + // memory if the subgraph model resource service is not available. + bool create_copy = + !sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService) + .IsAvailable(); + + Stream image = graph.In("IMAGE").Cast(); + + // Check whether Hand requested + const bool is_left_hand_requested = + HasOutput(holistic_node, "LEFT_HAND_LANDMARKS"); + const bool is_right_hand_requested = + HasOutput(holistic_node, "RIGHT_HAND_LANDMARKS"); + const bool is_left_hand_world_requested = + HasOutput(holistic_node, "LEFT_HAND_WORLD_LANDMARKS"); + const bool is_right_hand_world_requested = + HasOutput(holistic_node, "RIGHT_HAND_WORLD_LANDMARKS"); + const bool hands_requested = + is_left_hand_requested || is_right_hand_requested || + is_left_hand_world_requested || is_right_hand_world_requested; + if (hands_requested) { + MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( + model_asset_bundle_resources, holistic_options, + holistic_options->mutable_hand_landmarks_detector_graph_options(), + kHandLandmarksDetectorModelName, create_copy)); + MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( + model_asset_bundle_resources, holistic_options, + holistic_options->mutable_hand_roi_refinement_graph_options(), + kHandRoiRefinementModelName, create_copy)); + } + + // Check whether Face requested + const bool is_face_requested = HasOutput(holistic_node, "FACE_LANDMARKS"); + const bool is_face_blendshapes_requested = + HasOutput(holistic_node, "FACE_BLENDSHAPES"); + const bool face_requested = + is_face_requested || is_face_blendshapes_requested; + if (face_requested) { + MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( + model_asset_bundle_resources, holistic_options, + holistic_options->mutable_face_detector_graph_options(), + kFaceDetectorModelName, create_copy)); + // Forcely set num_faces to 1, because holistic landmarker only supports a + // single subject for now. + holistic_options->mutable_face_detector_graph_options()->set_num_faces(1); + MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( + model_asset_bundle_resources, holistic_options, + holistic_options->mutable_face_landmarks_detector_graph_options(), + kFaceLandmarksDetectorModelName, create_copy)); + if (is_face_blendshapes_requested) { + MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( + model_asset_bundle_resources, holistic_options, + holistic_options->mutable_face_landmarks_detector_graph_options() + ->mutable_face_blendshapes_graph_options(), + kFaceBlendshapesModelName, create_copy)); + } + } + + MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( + model_asset_bundle_resources, holistic_options, + holistic_options->mutable_pose_detector_graph_options(), + kPoseDetectorModelName, create_copy)); + // Forcely set num_poses to 1, because holistic landmarker sonly supports a + // single subject for now. + holistic_options->mutable_pose_detector_graph_options()->set_num_poses(1); + MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( + model_asset_bundle_resources, holistic_options, + holistic_options->mutable_pose_landmarks_detector_graph_options(), + kPoseLandmarksDetectorModelName, create_copy)); + + HolisticPoseTrackingRequest pose_request = { + .landmarks = HasOutput(holistic_node, "POSE_LANDMARKS") || + hands_requested || face_requested, + .world_landmarks = + HasOutput(holistic_node, "POSE_WORLD_LANDMARKS") || hands_requested, + .segmentation_mask = + HasOutput(holistic_node, "POSE_SEGMENTATION_MASK")}; + + // Detect and track pose. + MP_ASSIGN_OR_RETURN( + HolisticPoseTrackingOutput pose_output, + TrackHolisticPose( + image, holistic_options->pose_detector_graph_options(), + holistic_options->pose_landmarks_detector_graph_options(), + pose_request, graph)); + MP_RETURN_IF_ERROR( + SetGraphPoseOutputs(pose_request, holistic_node, pose_output, graph)); + + // Detect and track hand. + if (hands_requested) { + if (is_left_hand_requested || is_left_hand_world_requested) { + RET_CHECK(pose_output.landmarks.has_value()); + RET_CHECK(pose_output.world_landmarks.has_value()); + + PoseIndices pose_indices = { + .wrist_idx = + static_cast(pose_landmarker::PoseLandmarkName::kLeftWrist), + .pinky_idx = static_cast( + pose_landmarker::PoseLandmarkName::kLeftPinky1), + .index_idx = static_cast( + pose_landmarker::PoseLandmarkName::kLeftIndex1), + }; + HolisticHandTrackingRequest hand_request = { + .landmarks = is_left_hand_requested, + .world_landmarks = is_left_hand_world_requested, + }; + MP_ASSIGN_OR_RETURN( + HolisticHandTrackingOutput hand_output, + TrackHolisticHand( + image, *pose_output.landmarks, *pose_output.world_landmarks, + holistic_options->hand_landmarks_detector_graph_options(), + holistic_options->hand_roi_refinement_graph_options(), + pose_indices, hand_request, graph + + )); + SetGraphHandOutputs(/*is_left=*/true, holistic_node, hand_output, + graph); + } + + if (is_right_hand_requested || is_right_hand_world_requested) { + RET_CHECK(pose_output.landmarks.has_value()); + RET_CHECK(pose_output.world_landmarks.has_value()); + + PoseIndices pose_indices = { + .wrist_idx = static_cast( + pose_landmarker::PoseLandmarkName::kRightWrist), + .pinky_idx = static_cast( + pose_landmarker::PoseLandmarkName::kRightPinky1), + .index_idx = static_cast( + pose_landmarker::PoseLandmarkName::kRightIndex1), + }; + HolisticHandTrackingRequest hand_request = { + .landmarks = is_right_hand_requested, + .world_landmarks = is_right_hand_world_requested, + }; + MP_ASSIGN_OR_RETURN( + HolisticHandTrackingOutput hand_output, + TrackHolisticHand( + image, *pose_output.landmarks, *pose_output.world_landmarks, + holistic_options->hand_landmarks_detector_graph_options(), + holistic_options->hand_roi_refinement_graph_options(), + pose_indices, hand_request, graph + + )); + SetGraphHandOutputs(/*is_left=*/false, holistic_node, hand_output, + graph); + } + } + + // Detect and track face. + if (face_requested) { + RET_CHECK(pose_output.landmarks.has_value()); + + Stream face_landmarks_from_pose = + api2::builder::SplitToRanges(*pose_output.landmarks, {{0, 11}}, + graph)[0]; + + HolisticFaceTrackingRequest face_request = { + .classifications = is_face_blendshapes_requested, + }; + MP_ASSIGN_OR_RETURN( + HolisticFaceTrackingOutput face_output, + TrackHolisticFace( + image, face_landmarks_from_pose, + holistic_options->face_detector_graph_options(), + holistic_options->face_landmarks_detector_graph_options(), + face_request, graph)); + SetGraphFaceOutputs(holistic_node, face_output, graph); + } + + auto& pass_through = graph.AddNode("PassThroughCalculator"); + image >> pass_through.In(""); + pass_through.Out("") >> graph.Out("IMAGE"); + + auto config = graph.GetConfig(); + core::FixGraphBackEdges(config); + return config; + } +}; + +REGISTER_MEDIAPIPE_GRAPH( + ::mediapipe::tasks::vision::holistic_landmarker::HolisticLandmarkerGraph); + +} // namespace holistic_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_landmarker_graph_test.cc b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_landmarker_graph_test.cc new file mode 100644 index 000000000..c549a022b --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_landmarker_graph_test.cc @@ -0,0 +1,595 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" +#include "file/base/helpers.h" +#include "file/base/options.h" +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/stream/image_size.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/tool/test_util.h" +#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" +#include "mediapipe/tasks/cc/core/proto/base_options.pb.h" +#include "mediapipe/tasks/cc/core/proto/external_file.pb.h" +#include "mediapipe/tasks/cc/core/task_runner.h" +#include "mediapipe/tasks/cc/core/utils.h" +#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_connections.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_connections.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_roi_refinement_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_landmarker_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_result.pb.h" +#include "mediapipe/tasks/cc/vision/pose_detector/proto/pose_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarks_connections.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/utils/data_renderer.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" +#include "mediapipe/util/color.pb.h" +#include "mediapipe/util/render_data.pb.h" +#include "testing/base/public/gmock.h" +#include "testing/base/public/googletest.h" +#include "testing/base/public/gunit.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace holistic_landmarker { +namespace { + +using ::mediapipe::api2::builder::GetImageSize; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Stream; +using ::mediapipe::tasks::core::TaskRunner; +using ::testing::TestParamInfo; +using ::testing::TestWithParam; +using ::testing::Values; +using ::testing::proto::Approximately; +using ::testing::proto::Partially; + +constexpr float kAbsMargin = 0.025; +constexpr absl::string_view kTestDataDirectory = + "/mediapipe/tasks/testdata/vision/"; +constexpr char kHolisticResultFile[] = + "male_full_height_hands_result_cpu.pbtxt"; +constexpr absl::string_view kTestImageFile = "male_full_height_hands.jpg"; +constexpr absl::string_view kImageInStream = "image_in"; +constexpr absl::string_view kLeftHandLandmarksStream = "left_hand_landmarks"; +constexpr absl::string_view kRightHandLandmarksStream = "right_hand_landmarks"; +constexpr absl::string_view kFaceLandmarksStream = "face_landmarks"; +constexpr absl::string_view kFaceBlendshapesStream = "face_blendshapes"; +constexpr absl::string_view kPoseLandmarksStream = "pose_landmarks"; +constexpr absl::string_view kRenderedImageOutStream = "rendered_image_out"; +constexpr absl::string_view kPoseSegmentationMaskStream = + "pose_segmentation_mask"; +constexpr absl::string_view kHolisticLandmarkerModelBundleFile = + "holistic_landmarker.task"; +constexpr absl::string_view kHandLandmarksModelFile = + "hand_landmark_full.tflite"; +constexpr absl::string_view kHandRoiRefinementModelFile = + "handrecrop_2020_07_21_v0.f16.tflite"; +constexpr absl::string_view kPoseDetectionModelFile = "pose_detection.tflite"; +constexpr absl::string_view kPoseLandmarksModelFile = + "pose_landmark_lite.tflite"; +constexpr absl::string_view kFaceDetectionModelFile = + "face_detection_short_range.tflite"; +constexpr absl::string_view kFaceLandmarksModelFile = + "facemesh2_lite_iris_faceflag_2023_02_14.tflite"; +constexpr absl::string_view kFaceBlendshapesModelFile = + "face_blendshapes.tflite"; + +enum RenderPart { + HAND = 0, + POSE = 1, + FACE = 2, +}; + +mediapipe::Color GetColor(RenderPart render_part) { + mediapipe::Color color; + switch (render_part) { + case HAND: + color.set_b(255); + color.set_g(255); + color.set_r(255); + break; + case POSE: + color.set_b(0); + color.set_g(255); + color.set_r(0); + break; + case FACE: + color.set_b(0); + color.set_g(0); + color.set_r(255); + break; + } + return color; +} + +std::string GetFilePath(absl::string_view filename) { + return file::JoinPath("./", kTestDataDirectory, filename); +} + +template +mediapipe::LandmarksToRenderDataCalculatorOptions GetRendererOptions( + const std::array, N>& connections, + mediapipe::Color color) { + mediapipe::LandmarksToRenderDataCalculatorOptions renderer_options; + for (const auto& connection : connections) { + renderer_options.add_landmark_connections(connection[0]); + renderer_options.add_landmark_connections(connection[1]); + } + *renderer_options.mutable_landmark_color() = color; + *renderer_options.mutable_connection_color() = color; + renderer_options.set_thickness(0.5); + renderer_options.set_visualize_landmark_depth(false); + return renderer_options; +} + +void ConfigureHandProtoOptions(proto::HolisticLandmarkerGraphOptions& options) { + options.mutable_hand_landmarks_detector_graph_options() + ->mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath(kHandLandmarksModelFile)); + + options.mutable_hand_roi_refinement_graph_options() + ->mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath(kHandRoiRefinementModelFile)); +} + +void ConfigureFaceProtoOptions(proto::HolisticLandmarkerGraphOptions& options) { + // Set face detection model. + face_detector::proto::FaceDetectorGraphOptions& face_detector_graph_options = + *options.mutable_face_detector_graph_options(); + face_detector_graph_options.mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath(kFaceDetectionModelFile)); + face_detector_graph_options.set_num_faces(1); + + // Set face landmarks model. + face_landmarker::proto::FaceLandmarksDetectorGraphOptions& + face_landmarks_graph_options = + *options.mutable_face_landmarks_detector_graph_options(); + face_landmarks_graph_options.mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath(kFaceLandmarksModelFile)); + face_landmarks_graph_options.mutable_face_blendshapes_graph_options() + ->mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath(kFaceBlendshapesModelFile)); +} + +void ConfigurePoseProtoOptions(proto::HolisticLandmarkerGraphOptions& options) { + pose_detector::proto::PoseDetectorGraphOptions& pose_detector_graph_options = + *options.mutable_pose_detector_graph_options(); + pose_detector_graph_options.mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath(kPoseDetectionModelFile)); + pose_detector_graph_options.set_num_poses(1); + options.mutable_pose_landmarks_detector_graph_options() + ->mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath(kPoseLandmarksModelFile)); +} + +struct HolisticRequest { + bool is_left_hand_requested = false; + bool is_right_hand_requested = false; + bool is_face_requested = false; + bool is_face_blendshapes_requested = false; +}; + +// Helper function to create a TaskRunner. +absl::StatusOr> CreateTaskRunner( + bool use_model_bundle, HolisticRequest holistic_request) { + Graph graph; + + Stream image = graph.In("IMAEG").Cast().SetName(kImageInStream); + + auto& holistic_graph = graph.AddNode( + "mediapipe.tasks.vision.holistic_landmarker.HolisticLandmarkerGraph"); + proto::HolisticLandmarkerGraphOptions& options = + holistic_graph.GetOptions(); + if (use_model_bundle) { + options.mutable_base_options()->mutable_model_asset()->set_file_name( + GetFilePath(kHolisticLandmarkerModelBundleFile)); + } else { + ConfigureHandProtoOptions(options); + ConfigurePoseProtoOptions(options); + ConfigureFaceProtoOptions(options); + } + + std::vector> render_list; + image >> holistic_graph.In("IMAGE"); + Stream> image_size = GetImageSize(image, graph); + + if (holistic_request.is_left_hand_requested) { + Stream left_hand_landmarks = + holistic_graph.Out("LEFT_HAND_LANDMARKS") + .Cast() + .SetName(kLeftHandLandmarksStream); + Stream left_hand_tracking_roi = + holistic_graph.Out("LEFT_HAND_TRACKING_ROI").Cast(); + auto left_hand_landmarks_render_data = utils::RenderLandmarks( + left_hand_landmarks, + utils::GetRenderScale(image_size, left_hand_tracking_roi, 0.0001, + graph), + GetRendererOptions(hand_landmarker::kHandConnections, + GetColor(RenderPart::HAND)), + graph); + render_list.push_back(left_hand_landmarks_render_data); + left_hand_landmarks >> graph.Out("LEFT_HAND_LANDMARKS"); + } + if (holistic_request.is_right_hand_requested) { + Stream right_hand_landmarks = + holistic_graph.Out("RIGHT_HAND_LANDMARKS") + .Cast() + .SetName(kRightHandLandmarksStream); + Stream right_hand_tracking_roi = + holistic_graph.Out("RIGHT_HAND_TRACKING_ROI").Cast(); + auto right_hand_landmarks_render_data = utils::RenderLandmarks( + right_hand_landmarks, + utils::GetRenderScale(image_size, right_hand_tracking_roi, 0.0001, + graph), + GetRendererOptions(hand_landmarker::kHandConnections, + GetColor(RenderPart::HAND)), + graph); + render_list.push_back(right_hand_landmarks_render_data); + right_hand_landmarks >> graph.Out("RIGHT_HAND_LANDMARKS"); + } + if (holistic_request.is_face_requested) { + Stream face_landmarks = + holistic_graph.Out("FACE_LANDMARKS") + .Cast() + .SetName(kFaceLandmarksStream); + Stream face_tracking_roi = + holistic_graph.Out("FACE_TRACKING_ROI").Cast(); + auto face_landmarks_render_data = utils::RenderLandmarks( + face_landmarks, + utils::GetRenderScale(image_size, face_tracking_roi, 0.0001, graph), + GetRendererOptions( + face_landmarker::FaceLandmarksConnections::kFaceLandmarksConnectors, + GetColor(RenderPart::FACE)), + graph); + render_list.push_back(face_landmarks_render_data); + face_landmarks >> graph.Out("FACE_LANDMARKS"); + } + if (holistic_request.is_face_blendshapes_requested) { + Stream face_blendshapes = + holistic_graph.Out("FACE_BLENDSHAPES") + .Cast() + .SetName(kFaceBlendshapesStream); + face_blendshapes >> graph.Out("FACE_BLENDSHAPES"); + } + Stream pose_landmarks = + holistic_graph.Out("POSE_LANDMARKS") + .Cast() + .SetName(kPoseLandmarksStream); + Stream pose_tracking_roi = + holistic_graph.Out("POSE_LANDMARKS_ROI").Cast(); + Stream pose_segmentation_mask = + holistic_graph.Out("POSE_SEGMENTATION_MASK") + .Cast() + .SetName(kPoseSegmentationMaskStream); + + auto pose_landmarks_render_data = utils::RenderLandmarks( + pose_landmarks, + utils::GetRenderScale(image_size, pose_tracking_roi, 0.0001, graph), + GetRendererOptions(pose_landmarker::kPoseLandmarksConnections, + GetColor(RenderPart::POSE)), + graph); + render_list.push_back(pose_landmarks_render_data); + auto rendered_image = + utils::Render( + image, absl::Span>(render_list), graph) + .SetName(kRenderedImageOutStream); + + pose_landmarks >> graph.Out("POSE_LANDMARKS"); + pose_segmentation_mask >> graph.Out("POSE_SEGMENTATION_MASK"); + rendered_image >> graph.Out("RENDERED_IMAGE"); + + auto config = graph.GetConfig(); + core::FixGraphBackEdges(config); + + return TaskRunner::Create( + config, std::make_unique()); +} + +template +absl::StatusOr FetchResult(const core::PacketMap& output_packets, + absl::string_view stream_name) { + auto it = output_packets.find(std::string(stream_name)); + RET_CHECK(it != output_packets.end()); + return it->second.Get(); +} + +// Remove fields not to be checked in the result, since the model +// generating expected result is different from the testing model. +void RemoveUncheckedResult(proto::HolisticResult& holistic_result) { + for (auto& landmark : + *holistic_result.mutable_pose_landmarks()->mutable_landmark()) { + landmark.clear_z(); + landmark.clear_visibility(); + landmark.clear_presence(); + } + for (auto& landmark : + *holistic_result.mutable_face_landmarks()->mutable_landmark()) { + landmark.clear_z(); + landmark.clear_visibility(); + landmark.clear_presence(); + } + for (auto& landmark : + *holistic_result.mutable_left_hand_landmarks()->mutable_landmark()) { + landmark.clear_z(); + landmark.clear_visibility(); + landmark.clear_presence(); + } + for (auto& landmark : + *holistic_result.mutable_right_hand_landmarks()->mutable_landmark()) { + landmark.clear_z(); + landmark.clear_visibility(); + landmark.clear_presence(); + } +} + +std::string RequestToString(HolisticRequest request) { + return absl::StrFormat( + "%s_%s_%s_%s", + request.is_left_hand_requested ? "left_hand" : "no_left_hand", + request.is_right_hand_requested ? "right_hand" : "no_right_hand", + request.is_face_requested ? "face" : "no_face", + request.is_face_blendshapes_requested ? "face_blendshapes" + : "no_face_blendshapes"); +} + +struct TestParams { + // The name of this test, for convenience when displaying test results. + std::string test_name; + // The filename of test image. + std::string test_image_name; + // Whether to use holistic model bundle to test. + bool use_model_bundle; + // Requests of holistic parts. + HolisticRequest holistic_request; +}; + +class SmokeTest : public testing::TestWithParam {}; + +TEST_P(SmokeTest, Succeeds) { + MP_ASSERT_OK_AND_ASSIGN( + Image image, + DecodeImageFromFile(GetFilePath(GetParam().test_image_name))); + + proto::HolisticResult holistic_result; + MP_ASSERT_OK(GetTextProto(GetFilePath(kHolisticResultFile), &holistic_result, + ::file::Defaults())); + RemoveUncheckedResult(holistic_result); + + MP_ASSERT_OK_AND_ASSIGN(auto task_runner, + CreateTaskRunner(GetParam().use_model_bundle, + GetParam().holistic_request)); + MP_ASSERT_OK_AND_ASSIGN(auto output_packets, + task_runner->Process({{std::string(kImageInStream), + MakePacket(image)}})); + + // Check face landmarks + if (GetParam().holistic_request.is_face_requested) { + MP_ASSERT_OK_AND_ASSIGN(auto face_landmarks, + FetchResult( + output_packets, kFaceLandmarksStream)); + EXPECT_THAT( + face_landmarks, + Approximately(Partially(EqualsProto(holistic_result.face_landmarks())), + /*margin=*/kAbsMargin)); + } else { + ASSERT_FALSE(output_packets.contains(std::string(kFaceLandmarksStream))); + } + + if (GetParam().holistic_request.is_face_blendshapes_requested) { + MP_ASSERT_OK_AND_ASSIGN(auto face_blendshapes, + FetchResult( + output_packets, kFaceBlendshapesStream)); + EXPECT_THAT(face_blendshapes, + Approximately( + Partially(EqualsProto(holistic_result.face_blendshapes())), + /*margin=*/kAbsMargin)); + } else { + ASSERT_FALSE(output_packets.contains(std::string(kFaceBlendshapesStream))); + } + + // Check Pose landmarks + MP_ASSERT_OK_AND_ASSIGN(auto pose_landmarks, + FetchResult( + output_packets, kPoseLandmarksStream)); + EXPECT_THAT( + pose_landmarks, + Approximately(Partially(EqualsProto(holistic_result.pose_landmarks())), + /*margin=*/kAbsMargin)); + + // Check Hand landmarks + if (GetParam().holistic_request.is_left_hand_requested) { + MP_ASSERT_OK_AND_ASSIGN(auto left_hand_landmarks, + FetchResult( + output_packets, kLeftHandLandmarksStream)); + EXPECT_THAT( + left_hand_landmarks, + Approximately( + Partially(EqualsProto(holistic_result.left_hand_landmarks())), + /*margin=*/kAbsMargin)); + } else { + ASSERT_FALSE( + output_packets.contains(std::string(kLeftHandLandmarksStream))); + } + + if (GetParam().holistic_request.is_right_hand_requested) { + MP_ASSERT_OK_AND_ASSIGN(auto right_hand_landmarks, + FetchResult( + output_packets, kRightHandLandmarksStream)); + EXPECT_THAT( + right_hand_landmarks, + Approximately( + Partially(EqualsProto(holistic_result.right_hand_landmarks())), + /*margin=*/kAbsMargin)); + } else { + ASSERT_FALSE( + output_packets.contains(std::string(kRightHandLandmarksStream))); + } + + auto rendered_image = + output_packets.at(std::string(kRenderedImageOutStream)).Get(); + MP_EXPECT_OK(SavePngTestOutput( + *rendered_image.GetImageFrameSharedPtr(), + absl::StrCat("holistic_landmark_", + RequestToString(GetParam().holistic_request)))); + + auto pose_segmentation_mask = + output_packets.at(std::string(kPoseSegmentationMaskStream)).Get(); + + cv::Mat matting_mask = mediapipe::formats::MatView( + pose_segmentation_mask.GetImageFrameSharedPtr().get()); + cv::Mat visualized_mask; + matting_mask.convertTo(visualized_mask, CV_8UC1, 255); + ImageFrame visualized_image(mediapipe::ImageFormat::GRAY8, + visualized_mask.cols, visualized_mask.rows, + visualized_mask.step, visualized_mask.data, + [visualized_mask](uint8_t[]) {}); + + MP_EXPECT_OK( + SavePngTestOutput(visualized_image, "holistic_pose_segmentation_mask")); +} + +INSTANTIATE_TEST_SUITE_P( + HolisticLandmarkerGraphTest, SmokeTest, + Values(TestParams{ + /* test_name= */ "UseModelBundle", + /* test_image_name= */ std::string(kTestImageFile), + /* use_model_bundle= */ true, + /* holistic_request= */ + { + /*is_left_hand_requested= */ true, + /*is_right_hand_requested= */ true, + /*is_face_requested= */ true, + /*is_face_blendshapes_requested= */ true, + }, + }, + TestParams{ + /* test_name= */ "UseSeparateModelFiles", + /* test_image_name= */ std::string(kTestImageFile), + /* use_model_bundle= */ false, + /* holistic_request= */ + { + /*is_left_hand_requested= */ true, + /*is_right_hand_requested= */ true, + /*is_face_requested= */ true, + /*is_face_blendshapes_requested= */ true, + }, + }, + TestParams{ + /* test_name= */ "ModelBundleNoLeftHand", + /* test_image_name= */ std::string(kTestImageFile), + /* use_model_bundle= */ true, + /* holistic_request= */ + { + /*is_left_hand_requested= */ false, + /*is_right_hand_requested= */ true, + /*is_face_requested= */ true, + /*is_face_blendshapes_requested= */ true, + }, + }, + TestParams{ + /* test_name= */ "ModelBundleNoRightHand", + /* test_image_name= */ std::string(kTestImageFile), + /* use_model_bundle= */ true, + /* holistic_request= */ + { + /*is_left_hand_requested= */ true, + /*is_right_hand_requested= */ false, + /*is_face_requested= */ true, + /*is_face_blendshapes_requested= */ true, + }, + }, + TestParams{ + /* test_name= */ "ModelBundleNoHand", + /* test_image_name= */ std::string(kTestImageFile), + /* use_model_bundle= */ true, + /* holistic_request= */ + { + /*is_left_hand_requested= */ false, + /*is_right_hand_requested= */ false, + /*is_face_requested= */ true, + /*is_face_blendshapes_requested= */ true, + }, + }, + TestParams{ + /* test_name= */ "ModelBundleNoFace", + /* test_image_name= */ std::string(kTestImageFile), + /* use_model_bundle= */ true, + /* holistic_request= */ + { + /*is_left_hand_requested= */ true, + /*is_right_hand_requested= */ true, + /*is_face_requested= */ false, + /*is_face_blendshapes_requested= */ false, + }, + }, + TestParams{ + /* test_name= */ "ModelBundleNoFaceBlendshapes", + /* test_image_name= */ std::string(kTestImageFile), + /* use_model_bundle= */ true, + /* holistic_request= */ + { + /*is_left_hand_requested= */ true, + /*is_right_hand_requested= */ true, + /*is_face_requested= */ true, + /*is_face_blendshapes_requested= */ false, + }, + }), + [](const TestParamInfo& info) { + return info.param.test_name; + }); + +} // namespace +} // namespace holistic_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking.cc b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking.cc new file mode 100644 index 000000000..860035ad0 --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking.cc @@ -0,0 +1,307 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking.h" + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/stream/detections_to_rects.h" +#include "mediapipe/framework/api2/stream/image_size.h" +#include "mediapipe/framework/api2/stream/landmarks_to_detection.h" +#include "mediapipe/framework/api2/stream/loopback.h" +#include "mediapipe/framework/api2/stream/merge.h" +#include "mediapipe/framework/api2/stream/presence.h" +#include "mediapipe/framework/api2/stream/rect_transformation.h" +#include "mediapipe/framework/api2/stream/segmentation_smoothing.h" +#include "mediapipe/framework/api2/stream/smoothing.h" +#include "mediapipe/framework/api2/stream/split.h" +#include "mediapipe/framework/formats/detection.pb.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/tasks/cc/components/utils/gate.h" +#include "mediapipe/tasks/cc/vision/pose_detector/proto/pose_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.pb.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace holistic_landmarker { + +namespace { + +using ::mediapipe::NormalizedRect; +using ::mediapipe::api2::builder::ConvertAlignmentPointsDetectionsToRect; +using ::mediapipe::api2::builder::ConvertAlignmentPointsDetectionToRect; +using ::mediapipe::api2::builder::ConvertLandmarksToDetection; +using ::mediapipe::api2::builder::GenericNode; +using ::mediapipe::api2::builder::GetImageSize; +using ::mediapipe::api2::builder::GetLoopbackData; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::IsPresent; +using ::mediapipe::api2::builder::Merge; +using ::mediapipe::api2::builder::ScaleAndMakeSquare; +using ::mediapipe::api2::builder::SmoothLandmarks; +using ::mediapipe::api2::builder::SmoothLandmarksVisibility; +using ::mediapipe::api2::builder::SmoothSegmentationMask; +using ::mediapipe::api2::builder::SplitToRanges; +using ::mediapipe::api2::builder::Stream; +using ::mediapipe::tasks::components::utils::DisallowIf; +using Size = std::pair; + +constexpr int kAuxLandmarksStartKeypointIndex = 0; +constexpr int kAuxLandmarksEndKeypointIndex = 1; +constexpr float kAuxLandmarksTargetAngle = 90; +constexpr float kRoiFromDetectionScaleFactor = 1.25f; +constexpr float kRoiFromLandmarksScaleFactor = 1.25f; + +Stream CalculateRoiFromDetections( + Stream> detections, Stream image_size, + Graph& graph) { + auto roi = ConvertAlignmentPointsDetectionsToRect(detections, image_size, + /*start_keypoint_index=*/0, + /*end_keypoint_index=*/1, + /*target_angle=*/90, graph); + return ScaleAndMakeSquare( + roi, image_size, /*scale_x_factor=*/kRoiFromDetectionScaleFactor, + /*scale_y_factor=*/kRoiFromDetectionScaleFactor, graph); +} + +Stream CalculateScaleRoiFromAuxiliaryLandmarks( + Stream landmarks, Stream image_size, + Graph& graph) { + // TODO: consider calculating ROI directly from landmarks. + auto detection = ConvertLandmarksToDetection(landmarks, graph); + return ConvertAlignmentPointsDetectionToRect( + detection, image_size, kAuxLandmarksStartKeypointIndex, + kAuxLandmarksEndKeypointIndex, kAuxLandmarksTargetAngle, graph); +} + +Stream CalculateRoiFromAuxiliaryLandmarks( + Stream landmarks, Stream image_size, + Graph& graph) { + // TODO: consider calculating ROI directly from landmarks. + auto detection = ConvertLandmarksToDetection(landmarks, graph); + auto roi = ConvertAlignmentPointsDetectionToRect( + detection, image_size, kAuxLandmarksStartKeypointIndex, + kAuxLandmarksEndKeypointIndex, kAuxLandmarksTargetAngle, graph); + return ScaleAndMakeSquare( + roi, image_size, /*scale_x_factor=*/kRoiFromLandmarksScaleFactor, + /*scale_y_factor=*/kRoiFromLandmarksScaleFactor, graph); +} + +struct PoseLandmarksResult { + std::optional> landmarks; + std::optional> world_landmarks; + std::optional> auxiliary_landmarks; + std::optional> segmentation_mask; +}; + +PoseLandmarksResult RunLandmarksDetection( + Stream image, Stream roi, + const pose_landmarker::proto::PoseLandmarksDetectorGraphOptions& + pose_landmarks_detector_graph_options, + const HolisticPoseTrackingRequest& request, Graph& graph) { + GenericNode& landmarks_graph = graph.AddNode( + "mediapipe.tasks.vision.pose_landmarker." + "SinglePoseLandmarksDetectorGraph"); + landmarks_graph + .GetOptions() = + pose_landmarks_detector_graph_options; + image >> landmarks_graph.In("IMAGE"); + roi >> landmarks_graph.In("NORM_RECT"); + + PoseLandmarksResult result; + if (request.landmarks) { + result.landmarks = + landmarks_graph.Out("LANDMARKS").Cast(); + result.auxiliary_landmarks = landmarks_graph.Out("AUXILIARY_LANDMARKS") + .Cast(); + } + if (request.world_landmarks) { + result.world_landmarks = + landmarks_graph.Out("WORLD_LANDMARKS").Cast(); + } + if (request.segmentation_mask) { + result.segmentation_mask = + landmarks_graph.Out("SEGMENTATION_MASK").Cast(); + } + return result; +} + +} // namespace + +absl::StatusOr +TrackHolisticPoseUsingCustomPoseDetection( + Stream image, PoseDetectionFn pose_detection_fn, + const pose_landmarker::proto::PoseLandmarksDetectorGraphOptions& + pose_landmarks_detector_graph_options, + const HolisticPoseTrackingRequest& request, Graph& graph) { + // Calculate ROI from scratch (pose detection) or reuse one from the + // previous run if available. + auto [previous_roi, set_previous_roi_fn] = + GetLoopbackData(/*tick=*/image, graph); + auto is_previous_roi_available = IsPresent(previous_roi, graph); + auto image_for_detection = + DisallowIf(image, is_previous_roi_available, graph); + MP_ASSIGN_OR_RETURN(auto pose_detections, + pose_detection_fn(image_for_detection, graph)); + auto roi_from_detections = CalculateRoiFromDetections( + pose_detections, GetImageSize(image_for_detection, graph), graph); + // Take first non-empty. + auto roi = Merge(roi_from_detections, previous_roi, graph); + + // Calculate landmarks and other outputs (if requested) in the specified ROI. + auto landmarks_detection_result = RunLandmarksDetection( + image, roi, pose_landmarks_detector_graph_options, + { + // Landmarks are required for tracking, hence force-requesting them. + .landmarks = true, + .world_landmarks = request.world_landmarks, + .segmentation_mask = request.segmentation_mask, + }, + graph); + RET_CHECK(landmarks_detection_result.landmarks.has_value() && + landmarks_detection_result.auxiliary_landmarks.has_value()) + << "Failed to calculate landmarks required for tracking."; + + // Split landmarks to pose landmarks and auxiliary landmarks. + auto pose_landmarks_raw = *landmarks_detection_result.landmarks; + auto auxiliary_landmarks = *landmarks_detection_result.auxiliary_landmarks; + + auto image_size = GetImageSize(image, graph); + + // TODO: b/305750053 - Apply adaptive crop by adding AdaptiveCropCalculator. + + // Calculate ROI from smoothed auxiliary landmarks. + auto scale_roi = CalculateScaleRoiFromAuxiliaryLandmarks(auxiliary_landmarks, + image_size, graph); + auto auxiliary_landmarks_smoothed = SmoothLandmarks( + auxiliary_landmarks, image_size, scale_roi, + {// Min cutoff 0.01 results into ~0.002 alpha in landmark EMA filter when + // landmark is static. + .min_cutoff = 0.01, + // Beta 10.0 in combintation with min_cutoff 0.01 results into ~0.68 + // alpha in landmark EMA filter when landmark is moving fast. + .beta = 10.0, + // Derivative cutoff 1.0 results into ~0.17 alpha in landmark velocity + // EMA filter. + .derivate_cutoff = 1.0}, + graph); + auto roi_from_auxiliary_landmarks = CalculateRoiFromAuxiliaryLandmarks( + auxiliary_landmarks_smoothed, image_size, graph); + + // Make ROI from auxiliary landmarks to be used as "previous" ROI for a + // subsequent run. + set_previous_roi_fn(roi_from_auxiliary_landmarks); + + // Populate and smooth pose landmarks if corresponding output has been + // requested. + std::optional> pose_landmarks; + if (request.landmarks) { + pose_landmarks = SmoothLandmarksVisibility( + pose_landmarks_raw, /*low_pass_filter_alpha=*/0.1f, graph); + pose_landmarks = SmoothLandmarks( + *pose_landmarks, image_size, scale_roi, + {// Min cutoff 0.05 results into ~0.01 alpha in landmark EMA filter when + // landmark is static. + .min_cutoff = 0.05f, + // Beta 80.0 in combination with min_cutoff 0.05 results into ~0.94 + // alpha in landmark EMA filter when landmark is moving fast. + .beta = 80.0f, + // Derivative cutoff 1.0 results into ~0.17 alpha in landmark velocity + // EMA filter. + .derivate_cutoff = 1.0f}, + graph); + } + + // Populate and smooth world landmarks if available. + std::optional> world_landmarks; + if (landmarks_detection_result.world_landmarks) { + world_landmarks = SplitToRanges(*landmarks_detection_result.world_landmarks, + /*ranges*/ {{0, 33}}, graph)[0]; + world_landmarks = SmoothLandmarksVisibility( + *world_landmarks, /*low_pass_filter_alpha=*/0.1f, graph); + world_landmarks = SmoothLandmarks( + *world_landmarks, + /*scale_roi=*/std::nullopt, + {// Min cutoff 0.1 results into ~ 0.02 alpha in landmark EMA filter when + // landmark is static. + .min_cutoff = 0.1f, + // Beta 40.0 in combination with min_cutoff 0.1 results into ~0.8 + // alpha in landmark EMA filter when landmark is moving fast. + .beta = 40.0f, + // Derivative cutoff 1.0 results into ~0.17 alpha in landmark velocity + // EMA filter. + .derivate_cutoff = 1.0f}, + graph); + } + + // Populate and smooth segmentation mask if available. + std::optional> segmentation_mask; + if (landmarks_detection_result.segmentation_mask) { + auto mask = *landmarks_detection_result.segmentation_mask; + auto [prev_mask_as_img, set_prev_mask_as_img_fn] = + GetLoopbackData( + /*tick=*/*landmarks_detection_result.segmentation_mask, graph); + auto mask_smoothed = + SmoothSegmentationMask(mask, prev_mask_as_img, + /*combine_with_previous_ratio=*/0.7f, graph); + set_prev_mask_as_img_fn(mask_smoothed); + segmentation_mask = mask_smoothed; + } + + return {{/*landmarks=*/pose_landmarks, + /*world_landmarks=*/world_landmarks, + /*segmentation_mask=*/segmentation_mask, + /*debug_output=*/ + {/*auxiliary_landmarks=*/auxiliary_landmarks_smoothed, + /*roi_from_landmarks=*/roi_from_auxiliary_landmarks, + /*detections*/ pose_detections}}}; +} + +absl::StatusOr TrackHolisticPose( + Stream image, + const pose_detector::proto::PoseDetectorGraphOptions& + pose_detector_graph_options, + const pose_landmarker::proto::PoseLandmarksDetectorGraphOptions& + pose_landmarks_detector_graph_options, + const HolisticPoseTrackingRequest& request, Graph& graph) { + PoseDetectionFn pose_detection_fn = [&pose_detector_graph_options]( + Stream image, Graph& graph) + -> absl::StatusOr>> { + GenericNode& pose_detector = + graph.AddNode("mediapipe.tasks.vision.pose_detector.PoseDetectorGraph"); + pose_detector.GetOptions() = + pose_detector_graph_options; + image >> pose_detector.In("IMAGE"); + return pose_detector.Out("DETECTIONS") + .Cast>(); + }; + return TrackHolisticPoseUsingCustomPoseDetection( + image, pose_detection_fn, pose_landmarks_detector_graph_options, request, + graph); +} + +} // namespace holistic_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking.h b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking.h new file mode 100644 index 000000000..f51ccc283 --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking.h @@ -0,0 +1,110 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_CC_VISION_HOLISTIC_LANDMARKER_HOLISTIC_POSE_TRACKING_H_ +#define MEDIAPIPE_TASKS_CC_VISION_HOLISTIC_LANDMARKER_HOLISTIC_POSE_TRACKING_H_ + +#include +#include +#include + +#include "absl/status/statusor.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/formats/detection.pb.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/tasks/cc/vision/pose_detector/proto/pose_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.pb.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace holistic_landmarker { + +// Type of pose detection function that can be used to customize pose tracking, +// by supplying the function into a corresponding `TrackPose` function overload. +// +// Function should update provided graph with node/nodes that accept image +// stream and produce stream of detections. +using PoseDetectionFn = std::function< + absl::StatusOr>>( + api2::builder::Stream, api2::builder::Graph&)>; + +struct HolisticPoseTrackingRequest { + bool landmarks = false; + bool world_landmarks = false; + bool segmentation_mask = false; +}; + +struct HolisticPoseTrackingOutput { + std::optional> + landmarks; + std::optional> world_landmarks; + std::optional> segmentation_mask; + + struct DebugOutput { + api2::builder::Stream + auxiliary_landmarks; + api2::builder::Stream roi_from_landmarks; + api2::builder::Stream> detections; + }; + + DebugOutput debug_output; +}; + +// Updates @graph to track pose in @image. +// +// @image - ImageFrame/GpuBuffer to track pose in. +// @pose_detection_fn - pose detection function that takes @image as input and +// produces stream of pose detections. +// @pose_landmarks_detector_graph_options - options of the +// PoseLandmarksDetectorGraph used to detect the pose landmarks. +// @request - object to request specific pose tracking outputs. +// NOTE: Outputs that were not requested won't be returned and corresponding +// parts of the graph won't be genertaed all. +// @graph - graph to update. +absl::StatusOr +TrackHolisticPoseUsingCustomPoseDetection( + api2::builder::Stream image, PoseDetectionFn pose_detection_fn, + const pose_landmarker::proto::PoseLandmarksDetectorGraphOptions& + pose_landmarks_detector_graph_options, + const HolisticPoseTrackingRequest& request, api2::builder::Graph& graph); + +// Updates @graph to track pose in @image. +// +// @image - ImageFrame/GpuBuffer to track pose in. +// @pose_detector_graph_options - options of the PoseDetectorGraph used to +// detect the pose. +// @pose_landmarks_detector_graph_options - options of the +// PoseLandmarksDetectorGraph used to detect the pose landmarks. +// @request - object to request specific pose tracking outputs. +// NOTE: Outputs that were not requested won't be returned and corresponding +// parts of the graph won't be genertaed all. +// @graph - graph to update. +absl::StatusOr TrackHolisticPose( + api2::builder::Stream image, + const pose_detector::proto::PoseDetectorGraphOptions& + pose_detector_graph_options, + const pose_landmarker::proto::PoseLandmarksDetectorGraphOptions& + pose_landmarks_detector_graph_options, + const HolisticPoseTrackingRequest& request, api2::builder::Graph& graph); + +} // namespace holistic_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe + +#endif // MEDIAPIPE_TASKS_CC_VISION_HOLISTIC_LANDMARKER_HOLISTIC_POSE_TRACKING_H_ diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking_test.cc b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking_test.cc new file mode 100644 index 000000000..0bf7259e8 --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking_test.cc @@ -0,0 +1,243 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/cc/vision/holistic_landmarker/holistic_pose_tracking.h" + +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" +#include "absl/types/span.h" +#include "file/base/helpers.h" +#include "file/base/options.h" +#include "mediapipe/calculators/util/landmarks_to_render_data_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/stream/image_size.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/framework/tool/test_util.h" +#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" +#include "mediapipe/tasks/cc/core/proto/base_options.pb.h" +#include "mediapipe/tasks/cc/core/task_runner.h" +#include "mediapipe/tasks/cc/core/utils.h" +#include "mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_result.pb.h" +#include "mediapipe/tasks/cc/vision/pose_detector/proto/pose_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarks_connections.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/utils/data_renderer.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" +#include "mediapipe/util/color.pb.h" +#include "mediapipe/util/render_data.pb.h" +#include "testing/base/public/googletest.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace holistic_landmarker { + +namespace { + +using ::file::Defaults; +using ::file::GetTextProto; +using ::mediapipe::Image; +using ::mediapipe::api2::builder::GetImageSize; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Stream; +using ::mediapipe::tasks::core::TaskRunner; +using ::testing::proto::Approximately; +using ::testing::proto::Partially; + +constexpr float kAbsMargin = 0.025; +constexpr absl::string_view kTestDataDirectory = + "/mediapipe/tasks/testdata/vision/"; +constexpr absl::string_view kTestImageFile = "male_full_height_hands.jpg"; +constexpr absl::string_view kImageInStream = "image_in"; +constexpr absl::string_view kPoseLandmarksOutStream = "pose_landmarks_out"; +constexpr absl::string_view kPoseWorldLandmarksOutStream = + "pose_world_landmarks_out"; +constexpr absl::string_view kRenderedImageOutStream = "rendered_image_out"; +constexpr absl::string_view kHolisticResultFile = + "male_full_height_hands_result_cpu.pbtxt"; +constexpr absl::string_view kHolisticPoseTrackingGraph = + "holistic_pose_tracking_graph.pbtxt"; + +std::string GetFilePath(absl::string_view filename) { + return file::JoinPath("./", kTestDataDirectory, filename); +} + +mediapipe::LandmarksToRenderDataCalculatorOptions GetPoseRendererOptions() { + mediapipe::LandmarksToRenderDataCalculatorOptions renderer_options; + for (const auto& connection : pose_landmarker::kPoseLandmarksConnections) { + renderer_options.add_landmark_connections(connection[0]); + renderer_options.add_landmark_connections(connection[1]); + } + renderer_options.mutable_landmark_color()->set_r(255); + renderer_options.mutable_landmark_color()->set_g(255); + renderer_options.mutable_landmark_color()->set_b(255); + renderer_options.mutable_connection_color()->set_r(255); + renderer_options.mutable_connection_color()->set_g(255); + renderer_options.mutable_connection_color()->set_b(255); + renderer_options.set_thickness(0.5); + renderer_options.set_visualize_landmark_depth(false); + return renderer_options; +} + +// Helper function to create a TaskRunner. +absl::StatusOr> CreateTaskRunner() { + Graph graph; + Stream image = graph.In("IMAGE").Cast().SetName(kImageInStream); + pose_detector::proto::PoseDetectorGraphOptions pose_detector_graph_options; + pose_detector_graph_options.mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath("pose_detection.tflite")); + pose_detector_graph_options.set_num_poses(1); + pose_landmarker::proto::PoseLandmarksDetectorGraphOptions + pose_landmarks_detector_graph_options; + pose_landmarks_detector_graph_options.mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath("pose_landmark_lite.tflite")); + + HolisticPoseTrackingRequest request; + request.landmarks = true; + request.world_landmarks = true; + MP_ASSIGN_OR_RETURN( + HolisticPoseTrackingOutput result, + TrackHolisticPose(image, pose_detector_graph_options, + pose_landmarks_detector_graph_options, request, graph)); + + auto image_size = GetImageSize(image, graph); + auto render_data = utils::RenderLandmarks( + *result.landmarks, + utils::GetRenderScale(image_size, result.debug_output.roi_from_landmarks, + 0.0001, graph), + GetPoseRendererOptions(), graph); + std::vector> render_list = {render_data}; + auto rendered_image = + utils::Render( + image, absl::Span>(render_list), graph) + .SetName(kRenderedImageOutStream); + + rendered_image >> graph.Out("RENDERED_IMAGE"); + result.landmarks->SetName(kPoseLandmarksOutStream) >> + graph.Out("POSE_LANDMARKS"); + result.world_landmarks->SetName(kPoseWorldLandmarksOutStream) >> + graph.Out("POSE_WORLD_LANDMARKS"); + + auto config = graph.GetConfig(); + core::FixGraphBackEdges(config); + + return TaskRunner::Create( + config, std::make_unique()); +} + +// Remove fields not to be checked in the result, since the model +// generating expected result is different from the testing model. +void RemoveUncheckedResult(proto::HolisticResult& holistic_result) { + for (auto& landmark : + *holistic_result.mutable_pose_landmarks()->mutable_landmark()) { + landmark.clear_z(); + landmark.clear_visibility(); + landmark.clear_presence(); + } +} + +class HolisticPoseTrackingTest : public testing::Test {}; + +TEST_F(HolisticPoseTrackingTest, VerifyGraph) { + Graph graph; + Stream image = graph.In("IMAGE").Cast().SetName(kImageInStream); + pose_detector::proto::PoseDetectorGraphOptions pose_detector_graph_options; + pose_detector_graph_options.mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath("pose_detection.tflite")); + pose_detector_graph_options.set_num_poses(1); + pose_landmarker::proto::PoseLandmarksDetectorGraphOptions + pose_landmarks_detector_graph_options; + pose_landmarks_detector_graph_options.mutable_base_options() + ->mutable_model_asset() + ->set_file_name(GetFilePath("pose_landmark_lite.tflite")); + HolisticPoseTrackingRequest request; + request.landmarks = true; + request.world_landmarks = true; + MP_ASSERT_OK_AND_ASSIGN( + HolisticPoseTrackingOutput result, + TrackHolisticPose(image, pose_detector_graph_options, + pose_landmarks_detector_graph_options, request, graph)); + result.landmarks->SetName(kPoseLandmarksOutStream) >> + graph.Out("POSE_LANDMARKS"); + result.world_landmarks->SetName(kPoseWorldLandmarksOutStream) >> + graph.Out("POSE_WORLD_LANDMARKS"); + + auto config = graph.GetConfig(); + core::FixGraphBackEdges(config); + + // Read the expected graph config. + std::string expected_graph_contents; + MP_ASSERT_OK(file::GetContents( + file::JoinPath("./", kTestDataDirectory, kHolisticPoseTrackingGraph), + &expected_graph_contents)); + + // Need to replace the expected graph config with the test srcdir, because + // each run has different test dir on TAP. + expected_graph_contents = absl::Substitute( + expected_graph_contents, FLAGS_test_srcdir, FLAGS_test_srcdir); + CalculatorGraphConfig expected_graph = + ParseTextProtoOrDie(expected_graph_contents); + + EXPECT_THAT(config, testing::proto::IgnoringRepeatedFieldOrdering( + testing::EqualsProto(expected_graph))); +} + +TEST_F(HolisticPoseTrackingTest, SmokeTest) { + MP_ASSERT_OK_AND_ASSIGN(Image image, + DecodeImageFromFile(GetFilePath(kTestImageFile))); + + proto::HolisticResult holistic_result; + MP_ASSERT_OK(GetTextProto(GetFilePath(kHolisticResultFile), &holistic_result, + Defaults())); + RemoveUncheckedResult(holistic_result); + MP_ASSERT_OK_AND_ASSIGN(auto task_runner, CreateTaskRunner()); + MP_ASSERT_OK_AND_ASSIGN(auto output_packets, + task_runner->Process({{std::string(kImageInStream), + MakePacket(image)}})); + auto pose_landmarks = output_packets.at(std::string(kPoseLandmarksOutStream)) + .Get(); + EXPECT_THAT( + pose_landmarks, + Approximately(Partially(EqualsProto(holistic_result.pose_landmarks())), + /*margin=*/kAbsMargin)); + auto rendered_image = + output_packets.at(std::string(kRenderedImageOutStream)).Get(); + MP_EXPECT_OK(SavePngTestOutput(*rendered_image.GetImageFrameSharedPtr(), + "pose_landmarks")); +} + +} // namespace +} // namespace holistic_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/proto/BUILD b/mediapipe/tasks/cc/vision/holistic_landmarker/proto/BUILD new file mode 100644 index 000000000..147f3cc86 --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/proto/BUILD @@ -0,0 +1,44 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") + +package(default_visibility = [ + "//mediapipe/tasks:internal", +]) + +licenses(["notice"]) + +mediapipe_proto_library( + name = "holistic_result_proto", + srcs = ["holistic_result.proto"], + deps = [ + "//mediapipe/framework/formats:classification_proto", + "//mediapipe/framework/formats:landmark_proto", + ], +) + +mediapipe_proto_library( + name = "holistic_landmarker_graph_options_proto", + srcs = ["holistic_landmarker_graph_options.proto"], + deps = [ + "//mediapipe/tasks/cc/core/proto:base_options_proto", + "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_proto", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_proto", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_proto", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_roi_refinement_graph_options_proto", + "//mediapipe/tasks/cc/vision/pose_detector/proto:pose_detector_graph_options_proto", + "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarks_detector_graph_options_proto", + ], +) diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_landmarker_graph_options.proto b/mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_landmarker_graph_options.proto new file mode 100644 index 000000000..86aba8887 --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_landmarker_graph_options.proto @@ -0,0 +1,57 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +syntax = "proto3"; + +package mediapipe.tasks.vision.holistic_landmarker.proto; + +import "mediapipe/tasks/cc/core/proto/base_options.proto"; +import "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.proto"; +import "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.proto"; +import "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options.proto"; +import "mediapipe/tasks/cc/vision/hand_landmarker/proto/hand_roi_refinement_graph_options.proto"; +import "mediapipe/tasks/cc/vision/pose_detector/proto/pose_detector_graph_options.proto"; +import "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.proto"; + +option java_package = "com.google.mediapipe.tasks.vision.holisticlandmarker.proto"; +option java_outer_classname = "HolisticLandmarkerGraphOptionsProto"; + +message HolisticLandmarkerGraphOptions { + // Base options for configuring MediaPipe Tasks, such as specifying the model + // asset bundle file with metadata, accelerator options, etc. + core.proto.BaseOptions base_options = 1; + + // Options for hand landmarks graph. + hand_landmarker.proto.HandLandmarksDetectorGraphOptions + hand_landmarks_detector_graph_options = 2; + + // Options for hand roi refinement graph. + hand_landmarker.proto.HandRoiRefinementGraphOptions + hand_roi_refinement_graph_options = 3; + + // Options for face detector graph. + face_detector.proto.FaceDetectorGraphOptions face_detector_graph_options = 4; + + // Options for face landmarks detector graph. + face_landmarker.proto.FaceLandmarksDetectorGraphOptions + face_landmarks_detector_graph_options = 5; + + // Options for pose detector graph. + pose_detector.proto.PoseDetectorGraphOptions pose_detector_graph_options = 6; + + // Options for pose landmarks detector graph. + pose_landmarker.proto.PoseLandmarksDetectorGraphOptions + pose_landmarks_detector_graph_options = 7; +} diff --git a/mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_result.proto b/mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_result.proto new file mode 100644 index 000000000..356da45d9 --- /dev/null +++ b/mediapipe/tasks/cc/vision/holistic_landmarker/proto/holistic_result.proto @@ -0,0 +1,34 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +syntax = "proto3"; + +package mediapipe.tasks.vision.holistic_landmarker.proto; + +import "mediapipe/framework/formats/classification.proto"; +import "mediapipe/framework/formats/landmark.proto"; + +option java_package = "com.google.mediapipe.tasks.vision.holisticlandmarker"; +option java_outer_classname = "HolisticResultProto"; + +message HolisticResult { + mediapipe.NormalizedLandmarkList pose_landmarks = 1; + mediapipe.LandmarkList pose_world_landmarks = 7; + mediapipe.NormalizedLandmarkList left_hand_landmarks = 2; + mediapipe.NormalizedLandmarkList right_hand_landmarks = 3; + mediapipe.NormalizedLandmarkList face_landmarks = 4; + mediapipe.ClassificationList face_blendshapes = 6; + mediapipe.NormalizedLandmarkList auxiliary_landmarks = 5; +} From 62bafd39bb5ff53ffecbfff2de85b00d875e86d7 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 28 Nov 2023 14:57:41 -0800 Subject: [PATCH 140/157] HolisticLandmarker Java API PiperOrigin-RevId: 586113048 --- .../mediapipe/tasks/mediapipe_tasks_aar.bzl | 7 +- .../com/google/mediapipe/tasks/vision/BUILD | 12 + .../holisticlandmarker/AndroidManifest.xml | 8 + .../HolisticLandmarker.java | 668 ++++++++++++++++++ .../holisticlandmarker/AndroidManifest.xml | 24 + .../tasks/vision/holisticlandmarker/BUILD | 19 + .../HolisticLandmarkerTest.java | 512 ++++++++++++++ mediapipe/tasks/testdata/vision/BUILD | 1 + 8 files changed, 1248 insertions(+), 3 deletions(-) create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/AndroidManifest.xml create mode 100644 mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarker.java create mode 100644 mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/AndroidManifest.xml create mode 100644 mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/BUILD create mode 100644 mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarkerTest.java diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl b/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl index 916323372..e63695e31 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl @@ -47,13 +47,14 @@ _VISION_TASKS_JAVA_PROTO_LITE_TARGETS = [ "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_embedder_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/holistic_landmarker/proto:holistic_landmarker_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/image_classifier/proto:image_classifier_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/image_embedder/proto:image_embedder_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/image_segmenter/proto:image_segmenter_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/image_segmenter/proto:segmenter_options_java_proto_lite", - "//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_java_proto_lite", - "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_java_proto_lite", - "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/object_detector/proto:object_detector_options_java_proto_lite", "//mediapipe/tasks/cc/vision/pose_detector/proto:pose_detector_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarker_graph_options_java_proto_lite", diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD index fc56bfa27..2d5ef7a9c 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/BUILD @@ -67,6 +67,7 @@ cc_binary( "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_graph", "//mediapipe/tasks/cc/vision/face_stylizer:face_stylizer_graph", "//mediapipe/tasks/cc/vision/gesture_recognizer:gesture_recognizer_graph", + "//mediapipe/tasks/cc/vision/holistic_landmarker:holistic_landmarker_graph", "//mediapipe/tasks/cc/vision/image_classifier:image_classifier_graph", "//mediapipe/tasks/cc/vision/image_embedder:image_embedder_graph", "//mediapipe/tasks/cc/vision/image_segmenter:image_segmenter_graph", @@ -429,6 +430,7 @@ filegroup( android_library( name = "holisticlandmarker", srcs = [ + "holisticlandmarker/HolisticLandmarker.java", "holisticlandmarker/HolisticLandmarkerResult.java", ], javacopts = [ @@ -439,10 +441,20 @@ android_library( ":core", "//mediapipe/framework/formats:classification_java_proto_lite", "//mediapipe/framework/formats:landmark_java_proto_lite", + "//mediapipe/java/com/google/mediapipe/framework:android_framework", "//mediapipe/java/com/google/mediapipe/framework/image", + "//mediapipe/tasks/cc/core/proto:base_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/holistic_landmarker/proto:holistic_landmarker_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/pose_detector/proto:pose_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarks_detector_graph_options_java_proto_lite", "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:category", "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:landmark", "//mediapipe/tasks/java/com/google/mediapipe/tasks/components/containers:normalized_landmark", + "//mediapipe/tasks/java/com/google/mediapipe/tasks/core", + "//third_party:any_java_proto", "//third_party:autovalue", "@maven//:com_google_guava_guava", ], diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/AndroidManifest.xml b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/AndroidManifest.xml new file mode 100644 index 000000000..a90c388f4 --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/AndroidManifest.xml @@ -0,0 +1,8 @@ + + + + + + diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarker.java b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarker.java new file mode 100644 index 000000000..e80da4fca --- /dev/null +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarker.java @@ -0,0 +1,668 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.vision.holisticlandmarker; + +import android.content.Context; +import android.os.ParcelFileDescriptor; +import com.google.auto.value.AutoValue; +import com.google.mediapipe.formats.proto.LandmarkProto.LandmarkList; +import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList; +import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList; +import com.google.mediapipe.framework.AndroidPacketGetter; +import com.google.mediapipe.framework.MediaPipeException; +import com.google.mediapipe.framework.Packet; +import com.google.mediapipe.framework.PacketGetter; +import com.google.mediapipe.framework.image.BitmapImageBuilder; +import com.google.mediapipe.framework.image.ByteBufferImageBuilder; +import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.tasks.core.BaseOptions; +import com.google.mediapipe.tasks.core.ErrorListener; +import com.google.mediapipe.tasks.core.OutputHandler; +import com.google.mediapipe.tasks.core.OutputHandler.ResultListener; +import com.google.mediapipe.tasks.core.TaskInfo; +import com.google.mediapipe.tasks.core.TaskOptions; +import com.google.mediapipe.tasks.core.TaskRunner; +import com.google.mediapipe.tasks.core.proto.BaseOptionsProto; +import com.google.mediapipe.tasks.vision.core.BaseVisionTaskApi; +import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions; +import com.google.mediapipe.tasks.vision.core.RunningMode; +import com.google.mediapipe.tasks.vision.facedetector.proto.FaceDetectorGraphOptionsProto.FaceDetectorGraphOptions; +import com.google.mediapipe.tasks.vision.facelandmarker.proto.FaceLandmarksDetectorGraphOptionsProto.FaceLandmarksDetectorGraphOptions; +import com.google.mediapipe.tasks.vision.handlandmarker.proto.HandLandmarksDetectorGraphOptionsProto.HandLandmarksDetectorGraphOptions; +import com.google.mediapipe.tasks.vision.holisticlandmarker.proto.HolisticLandmarkerGraphOptionsProto.HolisticLandmarkerGraphOptions; +import com.google.mediapipe.tasks.vision.posedetector.proto.PoseDetectorGraphOptionsProto.PoseDetectorGraphOptions; +import com.google.mediapipe.tasks.vision.poselandmarker.proto.PoseLandmarksDetectorGraphOptionsProto.PoseLandmarksDetectorGraphOptions; +import com.google.protobuf.Any; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +/** + * Performs holistic landmarks detection on images. + * + *

This API expects a pre-trained holistic landmarks model asset bundle. + * + *

    + *
  • Input image {@link MPImage} + *
      + *
    • The image that holistic landmarks detection runs on. + *
    + *
  • Output {@link HolisticLandmarkerResult} + *
      + *
    • A HolisticLandmarkerResult containing holistic landmarks. + *
    + *
+ */ +public final class HolisticLandmarker extends BaseVisionTaskApi { + private static final String TAG = HolisticLandmarker.class.getSimpleName(); + + private static final String IMAGE_IN_STREAM_NAME = "image_in"; + private static final String POSE_LANDMARKS_STREAM = "pose_landmarks"; + private static final String POSE_WORLD_LANDMARKS_STREAM = "pose_world_landmarks"; + private static final String POSE_SEGMENTATION_MASK_STREAM = "pose_segmentation_mask"; + private static final String FACE_LANDMARKS_STREAM = "face_landmarks"; + private static final String FACE_BLENDSHAPES_STREAM = "extra_blendshapes"; + private static final String LEFT_HAND_LANDMARKS_STREAM = "left_hand_landmarks"; + private static final String LEFT_HAND_WORLD_LANDMARKS_STREAM = "left_hand_world_landmarks"; + private static final String RIGHT_HAND_LANDMARKS_STREAM = "right_hand_landmarks"; + private static final String RIGHT_HAND_WORLD_LANDMARKS_STREAM = "right_hand_world_landmarks"; + private static final String IMAGE_OUT_STREAM_NAME = "image_out"; + + private static final int FACE_LANDMARKS_OUT_STREAM_INDEX = 0; + private static final int POSE_LANDMARKS_OUT_STREAM_INDEX = 1; + private static final int POSE_WORLD_LANDMARKS_OUT_STREAM_INDEX = 2; + private static final int LEFT_HAND_LANDMARKS_OUT_STREAM_INDEX = 3; + private static final int LEFT_HAND_WORLD_LANDMARKS_OUT_STREAM_INDEX = 4; + private static final int RIGHT_HAND_LANDMARKS_OUT_STREAM_INDEX = 5; + private static final int RIGHT_HAND_WORLD_LANDMARKS_OUT_STREAM_INDEX = 6; + private static final int IMAGE_OUT_STREAM_INDEX = 7; + + private static final float DEFAULT_PRESENCE_THRESHOLD = 0.5f; + private static final float DEFAULT_SUPPRESION_THRESHOLD = 0.3f; + private static final boolean DEFAULT_OUTPUT_FACE_BLENDSHAPES = false; + private static final boolean DEFAULT_OUTPUT_SEGMENTATION_MASKS = false; + + private static final String TASK_GRAPH_NAME = + "mediapipe.tasks.vision.holistic_landmarker.HolisticLandmarkerGraph"; + + @SuppressWarnings("ConstantCaseForConstants") + private static final List INPUT_STREAMS = + Collections.unmodifiableList(Arrays.asList("IMAGE:" + IMAGE_IN_STREAM_NAME)); + + static { + System.loadLibrary("mediapipe_tasks_vision_jni"); + } + + /** + * Creates a {@link HolisticLandmarker} instance from a model asset bundle path and the default + * {@link HolisticLandmarkerOptions}. + * + * @param context an Android {@link Context}. + * @param modelAssetPath path to the holistic landmarks model with metadata in the assets. + * @throws MediaPipeException if there is an error during {@link HolisticLandmarker} creation. + */ + public static HolisticLandmarker createFromFile(Context context, String modelAssetPath) { + BaseOptions baseOptions = BaseOptions.builder().setModelAssetPath(modelAssetPath).build(); + return createFromOptions( + context, HolisticLandmarkerOptions.builder().setBaseOptions(baseOptions).build()); + } + + /** + * Creates a {@link HolisticLandmarker} instance from a model asset bundle file and the default + * {@link HolisticLandmarkerOptions}. + * + * @param context an Android {@link Context}. + * @param modelAssetFile the holistic landmarks model {@link File} instance. + * @throws IOException if an I/O error occurs when opening the tflite model file. + * @throws MediaPipeException if there is an error during {@link HolisticLandmarker} creation. + */ + public static HolisticLandmarker createFromFile(Context context, File modelAssetFile) + throws IOException { + try (ParcelFileDescriptor descriptor = + ParcelFileDescriptor.open(modelAssetFile, ParcelFileDescriptor.MODE_READ_ONLY)) { + BaseOptions baseOptions = + BaseOptions.builder().setModelAssetFileDescriptor(descriptor.getFd()).build(); + return createFromOptions( + context, HolisticLandmarkerOptions.builder().setBaseOptions(baseOptions).build()); + } + } + + /** + * Creates a {@link HolisticLandmarker} instance from a model asset bundle buffer and the default + * {@link HolisticLandmarkerOptions}. + * + * @param context an Android {@link Context}. + * @param modelAssetBuffer a direct {@link ByteBuffer} or a {@link MappedByteBuffer} of the + * detection model. + * @throws MediaPipeException if there is an error during {@link HolisticLandmarker} creation. + */ + public static HolisticLandmarker createFromBuffer( + Context context, final ByteBuffer modelAssetBuffer) { + BaseOptions baseOptions = BaseOptions.builder().setModelAssetBuffer(modelAssetBuffer).build(); + return createFromOptions( + context, HolisticLandmarkerOptions.builder().setBaseOptions(baseOptions).build()); + } + + /** + * Creates a {@link HolisticLandmarker} instance from a {@link HolisticLandmarkerOptions}. + * + * @param context an Android {@link Context}. + * @param landmarkerOptions a {@link HolisticLandmarkerOptions} instance. + * @throws MediaPipeException if there is an error during {@link HolisticLandmarker} creation. + */ + public static HolisticLandmarker createFromOptions( + Context context, HolisticLandmarkerOptions landmarkerOptions) { + List outputStreams = new ArrayList<>(); + outputStreams.add("FACE_LANDMARKS:" + FACE_LANDMARKS_STREAM); + outputStreams.add("POSE_LANDMARKS:" + POSE_LANDMARKS_STREAM); + outputStreams.add("POSE_WORLD_LANDMARKS:" + POSE_WORLD_LANDMARKS_STREAM); + outputStreams.add("LEFT_HAND_LANDMARKS:" + LEFT_HAND_LANDMARKS_STREAM); + outputStreams.add("LEFT_HAND_WORLD_LANDMARKS:" + LEFT_HAND_WORLD_LANDMARKS_STREAM); + outputStreams.add("RIGHT_HAND_LANDMARKS:" + RIGHT_HAND_LANDMARKS_STREAM); + outputStreams.add("RIGHT_HAND_WORLD_LANDMARKS:" + RIGHT_HAND_WORLD_LANDMARKS_STREAM); + outputStreams.add("IMAGE:" + IMAGE_OUT_STREAM_NAME); + + int[] faceBlendshapesOutStreamIndex = new int[] {-1}; + if (landmarkerOptions.outputFaceBlendshapes()) { + outputStreams.add("FACE_BLENDSHAPES:" + FACE_BLENDSHAPES_STREAM); + faceBlendshapesOutStreamIndex[0] = outputStreams.size() - 1; + } + + int[] poseSegmentationMasksOutStreamIndex = new int[] {-1}; + if (landmarkerOptions.outputPoseSegmentationMasks()) { + outputStreams.add("POSE_SEGMENTATION_MASK:" + POSE_SEGMENTATION_MASK_STREAM); + poseSegmentationMasksOutStreamIndex[0] = outputStreams.size() - 1; + } + + OutputHandler handler = new OutputHandler<>(); + handler.setOutputPacketConverter( + new OutputHandler.OutputPacketConverter() { + @Override + public HolisticLandmarkerResult convertToTaskResult(List packets) { + // If there are no detected landmarks, just returns empty lists. + if (packets.get(FACE_LANDMARKS_OUT_STREAM_INDEX).isEmpty()) { + return HolisticLandmarkerResult.createEmpty( + BaseVisionTaskApi.generateResultTimestampMs( + landmarkerOptions.runningMode(), + packets.get(FACE_LANDMARKS_OUT_STREAM_INDEX))); + } + + NormalizedLandmarkList faceLandmarkProtos = + PacketGetter.getProto( + packets.get(FACE_LANDMARKS_OUT_STREAM_INDEX), NormalizedLandmarkList.parser()); + Optional faceBlendshapeProtos = + landmarkerOptions.outputFaceBlendshapes() + ? Optional.of( + PacketGetter.getProto( + packets.get(faceBlendshapesOutStreamIndex[0]), + ClassificationList.parser())) + : Optional.empty(); + NormalizedLandmarkList poseLandmarkProtos = + PacketGetter.getProto( + packets.get(POSE_LANDMARKS_OUT_STREAM_INDEX), NormalizedLandmarkList.parser()); + LandmarkList poseWorldLandmarkProtos = + PacketGetter.getProto( + packets.get(POSE_WORLD_LANDMARKS_OUT_STREAM_INDEX), LandmarkList.parser()); + Optional segmentationMask = + landmarkerOptions.outputPoseSegmentationMasks() + ? Optional.of( + getSegmentationMask(packets, poseSegmentationMasksOutStreamIndex[0])) + : Optional.empty(); + NormalizedLandmarkList leftHandLandmarkProtos = + PacketGetter.getProto( + packets.get(LEFT_HAND_LANDMARKS_OUT_STREAM_INDEX), + NormalizedLandmarkList.parser()); + LandmarkList leftHandWorldLandmarkProtos = + PacketGetter.getProto( + packets.get(LEFT_HAND_WORLD_LANDMARKS_OUT_STREAM_INDEX), LandmarkList.parser()); + NormalizedLandmarkList rightHandLandmarkProtos = + PacketGetter.getProto( + packets.get(RIGHT_HAND_LANDMARKS_OUT_STREAM_INDEX), + NormalizedLandmarkList.parser()); + LandmarkList rightHandWorldLandmarkProtos = + PacketGetter.getProto( + packets.get(RIGHT_HAND_WORLD_LANDMARKS_OUT_STREAM_INDEX), + LandmarkList.parser()); + + return HolisticLandmarkerResult.create( + faceLandmarkProtos, + faceBlendshapeProtos, + poseLandmarkProtos, + poseWorldLandmarkProtos, + segmentationMask, + leftHandLandmarkProtos, + leftHandWorldLandmarkProtos, + rightHandLandmarkProtos, + rightHandWorldLandmarkProtos, + BaseVisionTaskApi.generateResultTimestampMs( + landmarkerOptions.runningMode(), packets.get(FACE_LANDMARKS_OUT_STREAM_INDEX))); + } + + @Override + public MPImage convertToTaskInput(List packets) { + return new BitmapImageBuilder( + AndroidPacketGetter.getBitmapFromRgb(packets.get(IMAGE_OUT_STREAM_INDEX))) + .build(); + } + }); + landmarkerOptions.resultListener().ifPresent(handler::setResultListener); + landmarkerOptions.errorListener().ifPresent(handler::setErrorListener); + TaskRunner runner = + TaskRunner.create( + context, + TaskInfo.builder() + .setTaskName(HolisticLandmarker.class.getSimpleName()) + .setTaskRunningModeName(landmarkerOptions.runningMode().name()) + .setTaskGraphName(TASK_GRAPH_NAME) + .setInputStreams(INPUT_STREAMS) + .setOutputStreams(outputStreams) + .setTaskOptions(landmarkerOptions) + .setEnableFlowLimiting(landmarkerOptions.runningMode() == RunningMode.LIVE_STREAM) + .build(), + handler); + return new HolisticLandmarker(runner, landmarkerOptions.runningMode()); + } + + /** + * Constructor to initialize an {@link HolisticLandmarker} from a {@link TaskRunner} and a {@link + * RunningMode}. + * + * @param taskRunner a {@link TaskRunner}. + * @param runningMode a mediapipe vision task {@link RunningMode}. + */ + private HolisticLandmarker(TaskRunner taskRunner, RunningMode runningMode) { + super(taskRunner, runningMode, IMAGE_IN_STREAM_NAME, /* normRectStreamName= */ ""); + } + + /** + * Performs holistic landmarks detection on the provided single image with default image + * processing options, i.e. without any rotation applied. Only use this method when the {@link + * HolisticLandmarker} is created with {@link RunningMode.IMAGE}. + * + *

{@link HolisticLandmarker} supports the following color space types: + * + *

    + *
  • {@link Bitmap.Config.ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @throws MediaPipeException if there is an internal error. + */ + public HolisticLandmarkerResult detect(MPImage image) { + return detect(image, ImageProcessingOptions.builder().build()); + } + + /** + * Performs holistic landmarks detection on the provided single image. Only use this method when + * the {@link HolisticLandmarker} is created with {@link RunningMode.IMAGE}. + * + *

{@link HolisticLandmarker} supports the following color space types: + * + *

    + *
  • {@link Bitmap.Config.ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the + * input image before running inference. Note that region-of-interest is not supported + * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in + * this method throwing an IllegalArgumentException. + * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a + * region-of-interest. + * @throws MediaPipeException if there is an internal error. + */ + public HolisticLandmarkerResult detect( + MPImage image, ImageProcessingOptions imageProcessingOptions) { + validateImageProcessingOptions(imageProcessingOptions); + return (HolisticLandmarkerResult) processImageData(image, imageProcessingOptions); + } + + /** + * Performs holistic landmarks detection on the provided video frame with default image processing + * options, i.e. without any rotation applied. Only use this method when the {@link + * HolisticLandmarker} is created with {@link RunningMode.VIDEO}. + * + *

It's required to provide the video frame"s timestamp (in milliseconds). The input timestamps + * must be monotonically increasing. + * + *

{@link HolisticLandmarker} supports the following color space types: + * + *

    + *
  • {@link Bitmap.Config.ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param timestampMs the input timestamp (in milliseconds). + * @throws MediaPipeException if there is an internal error. + */ + public HolisticLandmarkerResult detectForVideo(MPImage image, long timestampMs) { + return detectForVideo(image, ImageProcessingOptions.builder().build(), timestampMs); + } + + /** + * Performs holistic landmarks detection on the provided video frame. Only use this method when + * the {@link HolisticLandmarker} is created with {@link RunningMode.VIDEO}. + * + *

It's required to provide the video frame"s timestamp (in milliseconds). The input timestamps + * must be monotonically increasing. + * + *

{@link HolisticLandmarker} supports the following color space types: + * + *

    + *
  • {@link Bitmap.Config.ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the + * input image before running inference. Note that region-of-interest is not supported + * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in + * this method throwing an IllegalArgumentException. + * @param timestampMs the input timestamp (in milliseconds). + * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a + * region-of-interest. + * @throws MediaPipeException if there is an internal error. + */ + public HolisticLandmarkerResult detectForVideo( + MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { + validateImageProcessingOptions(imageProcessingOptions); + return (HolisticLandmarkerResult) processVideoData(image, imageProcessingOptions, timestampMs); + } + + /** + * Sends live image data to perform holistic landmarks detection with default image processing + * options, i.e. without any rotation applied, and the results will be available via the {@link + * ResultListener} provided in the {@link HolisticLandmarkerOptions}. Only use this method when + * the {@link HolisticLandmarker } is created with {@link RunningMode.LIVE_STREAM}. + * + *

It's required to provide a timestamp (in milliseconds) to indicate when the input image is + * sent to the holistic landmarker. The input timestamps must be monotonically increasing. + * + *

{@link HolisticLandmarker} supports the following color space types: + * + *

    + *
  • {@link Bitmap.Config.ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param timestampMs the input timestamp (in milliseconds). + * @throws MediaPipeException if there is an internal error. + */ + public void detectAsync(MPImage image, long timestampMs) { + detectAsync(image, ImageProcessingOptions.builder().build(), timestampMs); + } + + /** + * Sends live image data to perform holistic landmarks detection, and the results will be + * available via the {@link ResultListener} provided in the {@link HolisticLandmarkerOptions}. + * Only use this method when the {@link HolisticLandmarker} is created with {@link + * RunningMode.LIVE_STREAM}. + * + *

It's required to provide a timestamp (in milliseconds) to indicate when the input image is + * sent to the holistic landmarker. The input timestamps must be monotonically increasing. + * + *

{@link HolisticLandmarker} supports the following color space types: + * + *

    + *
  • {@link Bitmap.Config.ARGB_8888} + *
+ * + * @param image a MediaPipe {@link MPImage} object for processing. + * @param imageProcessingOptions the {@link ImageProcessingOptions} specifying how to process the + * input image before running inference. Note that region-of-interest is not supported + * by this task: specifying {@link ImageProcessingOptions#regionOfInterest()} will result in + * this method throwing an IllegalArgumentException. + * @param timestampMs the input timestamp (in milliseconds). + * @throws IllegalArgumentException if the {@link ImageProcessingOptions} specify a + * region-of-interest. + * @throws MediaPipeException if there is an internal error. + */ + public void detectAsync( + MPImage image, ImageProcessingOptions imageProcessingOptions, long timestampMs) { + validateImageProcessingOptions(imageProcessingOptions); + sendLiveStreamData(image, imageProcessingOptions, timestampMs); + } + + /** Options for setting up an {@link HolisticLandmarker}. */ + @AutoValue + public abstract static class HolisticLandmarkerOptions extends TaskOptions { + + /** Builder for {@link HolisticLandmarkerOptions}. */ + @AutoValue.Builder + public abstract static class Builder { + /** Sets the base options for the holistic landmarker task. */ + public abstract Builder setBaseOptions(BaseOptions value); + + /** + * Sets the running mode for the holistic landmarker task. Defaults to the image mode. + * Holistic landmarker has three modes: + * + *
    + *
  • IMAGE: The mode for detecting holistic landmarks on single image inputs. + *
  • VIDEO: The mode for detecting holistic landmarks on the decoded frames of a video. + *
  • LIVE_STREAM: The mode for for detecting holistic landmarks on a live stream of input + * data, such as from camera. In this mode, {@code setResultListener} must be called to + * set up a listener to receive the detection results asynchronously. + *
+ */ + public abstract Builder setRunningMode(RunningMode value); + + /** + * Sets minimum confidence score for the face detection to be considered successful. Defaults + * to 0.5. + */ + public abstract Builder setMinFaceDetectionConfidence(Float value); + + /** + * The minimum threshold for the face suppression score in the face detection. Defaults to + * 0.3. + */ + public abstract Builder setMinFaceSuppressionThreshold(Float value); + + /** + * Sets minimum confidence score for the face landmark detection to be considered successful. + * Defaults to 0.5. + */ + public abstract Builder setMinFaceLandmarksConfidence(Float value); + + /** + * The minimum confidence score for the pose detection to be considered successful. Defaults + * to 0.5. + */ + public abstract Builder setMinPoseDetectionConfidence(Float value); + + /** + * The minimum threshold for the pose suppression score in the pose detection. Defaults to + * 0.3. + */ + public abstract Builder setMinPoseSuppressionThreshold(Float value); + + /** + * The minimum confidence score for the pose landmarks detection to be considered successful. + * Defaults to 0.5. + */ + public abstract Builder setMinPoseLandmarksConfidence(Float value); + + /** + * The minimum confidence score for the hand landmark detection to be considered successful. + * Defaults to 0.5. + */ + public abstract Builder setMinHandLandmarksConfidence(Float value); + + /** Whether to output segmentation masks. Defaults to false. */ + public abstract Builder setOutputPoseSegmentationMasks(Boolean value); + + /** Whether to output face blendshapes. Defaults to false. */ + public abstract Builder setOutputFaceBlendshapes(Boolean value); + + /** + * Sets the result listener to receive the detection results asynchronously when the holistic + * landmarker is in the live stream mode. + */ + public abstract Builder setResultListener( + ResultListener value); + + /** Sets an optional error listener. */ + public abstract Builder setErrorListener(ErrorListener value); + + abstract HolisticLandmarkerOptions autoBuild(); + + /** + * Validates and builds the {@link HolisticLandmarkerOptions} instance. + * + * @throws IllegalArgumentException if the result listener and the running mode are not + * properly configured. The result listener should only be set when the holistic + * landmarker is in the live stream mode. + */ + public final HolisticLandmarkerOptions build() { + HolisticLandmarkerOptions options = autoBuild(); + if (options.runningMode() == RunningMode.LIVE_STREAM) { + if (!options.resultListener().isPresent()) { + throw new IllegalArgumentException( + "The holistic landmarker is in the live stream mode, a user-defined result listener" + + " must be provided in HolisticLandmarkerOptions."); + } + } else if (options.resultListener().isPresent()) { + throw new IllegalArgumentException( + "The holistic landmarker is in the image or the video mode, a user-defined result" + + " listener shouldn't be provided in HolisticLandmarkerOptions."); + } + return options; + } + } + + abstract BaseOptions baseOptions(); + + abstract RunningMode runningMode(); + + abstract Optional minFaceDetectionConfidence(); + + abstract Optional minFaceSuppressionThreshold(); + + abstract Optional minFaceLandmarksConfidence(); + + abstract Optional minPoseDetectionConfidence(); + + abstract Optional minPoseSuppressionThreshold(); + + abstract Optional minPoseLandmarksConfidence(); + + abstract Optional minHandLandmarksConfidence(); + + abstract Boolean outputFaceBlendshapes(); + + abstract Boolean outputPoseSegmentationMasks(); + + abstract Optional> resultListener(); + + abstract Optional errorListener(); + + public static Builder builder() { + return new AutoValue_HolisticLandmarker_HolisticLandmarkerOptions.Builder() + .setRunningMode(RunningMode.IMAGE) + .setMinFaceDetectionConfidence(DEFAULT_PRESENCE_THRESHOLD) + .setMinFaceSuppressionThreshold(DEFAULT_SUPPRESION_THRESHOLD) + .setMinFaceLandmarksConfidence(DEFAULT_PRESENCE_THRESHOLD) + .setMinPoseDetectionConfidence(DEFAULT_PRESENCE_THRESHOLD) + .setMinPoseSuppressionThreshold(DEFAULT_SUPPRESION_THRESHOLD) + .setMinPoseLandmarksConfidence(DEFAULT_PRESENCE_THRESHOLD) + .setMinHandLandmarksConfidence(DEFAULT_PRESENCE_THRESHOLD) + .setOutputFaceBlendshapes(DEFAULT_OUTPUT_FACE_BLENDSHAPES) + .setOutputPoseSegmentationMasks(DEFAULT_OUTPUT_SEGMENTATION_MASKS); + } + + /** Converts a {@link HolisticLandmarkerOptions} to a {@link Any} protobuf message. */ + @Override + public Any convertToAnyProto() { + HolisticLandmarkerGraphOptions.Builder holisticLandmarkerGraphOptions = + HolisticLandmarkerGraphOptions.newBuilder() + .setBaseOptions( + BaseOptionsProto.BaseOptions.newBuilder() + .setUseStreamMode(runningMode() != RunningMode.IMAGE) + .mergeFrom(convertBaseOptionsToProto(baseOptions())) + .build()); + + HandLandmarksDetectorGraphOptions.Builder handLandmarksDetectorGraphOptions = + HandLandmarksDetectorGraphOptions.newBuilder(); + FaceDetectorGraphOptions.Builder faceDetectorGraphOptions = + FaceDetectorGraphOptions.newBuilder(); + FaceLandmarksDetectorGraphOptions.Builder faceLandmarksDetectorGraphOptions = + FaceLandmarksDetectorGraphOptions.newBuilder(); + PoseDetectorGraphOptions.Builder poseDetectorGraphOptions = + PoseDetectorGraphOptions.newBuilder(); + PoseLandmarksDetectorGraphOptions.Builder poseLandmarkerGraphOptions = + PoseLandmarksDetectorGraphOptions.newBuilder(); + + // Configure hand detector options. + minHandLandmarksConfidence() + .ifPresent(handLandmarksDetectorGraphOptions::setMinDetectionConfidence); + + // Configure pose detector options. + minPoseDetectionConfidence().ifPresent(poseDetectorGraphOptions::setMinDetectionConfidence); + minPoseSuppressionThreshold().ifPresent(poseDetectorGraphOptions::setMinSuppressionThreshold); + minPoseLandmarksConfidence().ifPresent(poseLandmarkerGraphOptions::setMinDetectionConfidence); + + // Configure face detector options. + minFaceDetectionConfidence().ifPresent(faceDetectorGraphOptions::setMinDetectionConfidence); + minFaceSuppressionThreshold().ifPresent(faceDetectorGraphOptions::setMinSuppressionThreshold); + minFaceLandmarksConfidence() + .ifPresent(faceLandmarksDetectorGraphOptions::setMinDetectionConfidence); + + holisticLandmarkerGraphOptions + .setHandLandmarksDetectorGraphOptions(handLandmarksDetectorGraphOptions.build()) + .setFaceDetectorGraphOptions(faceDetectorGraphOptions.build()) + .setFaceLandmarksDetectorGraphOptions(faceLandmarksDetectorGraphOptions.build()) + .setPoseDetectorGraphOptions(poseDetectorGraphOptions.build()) + .setPoseLandmarksDetectorGraphOptions(poseLandmarkerGraphOptions.build()); + + return Any.newBuilder() + .setTypeUrl( + "type.googleapis.com/mediapipe.tasks.vision.holistic_landmarker.proto.HolisticLandmarkerGraphOptions") + .setValue(holisticLandmarkerGraphOptions.build().toByteString()) + .build(); + } + } + + /** + * Validates that the provided {@link ImageProcessingOptions} doesn"t contain a + * region-of-interest. + */ + private static void validateImageProcessingOptions( + ImageProcessingOptions imageProcessingOptions) { + if (imageProcessingOptions.regionOfInterest().isPresent()) { + throw new IllegalArgumentException("HolisticLandmarker doesn't support region-of-interest."); + } + } + + private static MPImage getSegmentationMask(List packets, int packetIndex) { + int width = PacketGetter.getImageWidth(packets.get(packetIndex)); + int height = PacketGetter.getImageHeight(packets.get(packetIndex)); + ByteBuffer buffer = ByteBuffer.allocateDirect(width * height * 4); + + if (!PacketGetter.getImageData(packets.get(packetIndex), buffer)) { + throw new MediaPipeException( + MediaPipeException.StatusCode.INTERNAL.ordinal(), + "There was an error getting the sefmentation mask."); + } + + ByteBufferImageBuilder builder = + new ByteBufferImageBuilder(buffer, width, height, MPImage.IMAGE_FORMAT_VEC32F1); + return builder.build(); + } +} diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/AndroidManifest.xml b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/AndroidManifest.xml new file mode 100644 index 000000000..22b19b702 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/AndroidManifest.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/BUILD b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/BUILD new file mode 100644 index 000000000..287602c85 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/BUILD @@ -0,0 +1,19 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +# TODO: Enable this in OSS diff --git a/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarkerTest.java b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarkerTest.java new file mode 100644 index 000000000..f8c87c798 --- /dev/null +++ b/mediapipe/tasks/javatests/com/google/mediapipe/tasks/vision/holisticlandmarker/HolisticLandmarkerTest.java @@ -0,0 +1,512 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.mediapipe.tasks.vision.holisticlandmarker; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import android.content.res.AssetManager; +import android.graphics.BitmapFactory; +import android.graphics.RectF; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.truth.Correspondence; +import com.google.mediapipe.formats.proto.LandmarkProto.LandmarkList; +import com.google.mediapipe.formats.proto.ClassificationProto.ClassificationList; +import com.google.mediapipe.framework.MediaPipeException; +import com.google.mediapipe.framework.image.BitmapImageBuilder; +import com.google.mediapipe.framework.image.ByteBufferImageBuilder; +import com.google.mediapipe.framework.image.MPImage; +import com.google.mediapipe.tasks.components.containers.Category; +import com.google.mediapipe.tasks.components.containers.NormalizedLandmark; +import com.google.mediapipe.tasks.core.BaseOptions; +import com.google.mediapipe.tasks.vision.core.ImageProcessingOptions; +import com.google.mediapipe.tasks.vision.core.RunningMode; +import com.google.mediapipe.tasks.vision.holisticlandmarker.HolisticLandmarker.HolisticLandmarkerOptions; +import com.google.mediapipe.tasks.vision.holisticlandmarker.HolisticResultProto.HolisticResult; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Optional; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Suite; +import org.junit.runners.Suite.SuiteClasses; + +/** Test for {@link HolisticLandmarker}. */ +@RunWith(Suite.class) +@SuiteClasses({HolisticLandmarkerTest.General.class, HolisticLandmarkerTest.RunningModeTest.class}) +public class HolisticLandmarkerTest { + private static final String HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE = "holistic_landmarker.task"; + private static final String POSE_IMAGE = "male_full_height_hands.jpg"; + private static final String CAT_IMAGE = "cat.jpg"; + private static final String HOLISTIC_RESULT = "male_full_height_hands_result_cpu.pb"; + private static final String TAG = "Holistic Landmarker Test"; + private static final float FACE_LANDMARKS_ERROR_TOLERANCE = 0.03f; + private static final float FACE_BLENDSHAPES_ERROR_TOLERANCE = 0.13f; + private static final MPImage PLACEHOLDER_MASK = + new ByteBufferImageBuilder( + ByteBuffer.allocate(0), /* widht= */ 0, /* height= */ 0, MPImage.IMAGE_FORMAT_VEC32F1) + .build(); + private static final int IMAGE_WIDTH = 638; + private static final int IMAGE_HEIGHT = 1000; + + private static final Correspondence VALIDATE_LANDMARRKS = + Correspondence.from( + (Correspondence.BinaryPredicate) + (actual, expected) -> { + return Correspondence.tolerance(FACE_LANDMARKS_ERROR_TOLERANCE) + .compare(actual.x(), expected.x()) + && Correspondence.tolerance(FACE_LANDMARKS_ERROR_TOLERANCE) + .compare(actual.y(), expected.y()); + }, + "landmarks approximately equal to"); + + private static final Correspondence VALIDATE_BLENDSHAPES = + Correspondence.from( + (Correspondence.BinaryPredicate) + (actual, expected) -> + Correspondence.tolerance(FACE_BLENDSHAPES_ERROR_TOLERANCE) + .compare(actual.score(), expected.score()) + && actual.index() == expected.index() + && actual.categoryName().equals(expected.categoryName()), + "face blendshapes approximately equal to"); + + @RunWith(AndroidJUnit4.class) + public static final class General extends HolisticLandmarkerTest { + + @Test + public void detect_successWithValidModels() throws Exception { + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .build(); + HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options); + HolisticLandmarkerResult actualResult = + holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE)); + HolisticLandmarkerResult expectedResult = + getExpectedHolisticLandmarkerResult( + HOLISTIC_RESULT, /* hasFaceBlendshapes= */ false, /* hasSegmentationMask= */ false); + assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult); + } + + @Test + public void detect_successWithBlendshapes() throws Exception { + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .setOutputFaceBlendshapes(true) + .build(); + HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options); + HolisticLandmarkerResult actualResult = + holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE)); + HolisticLandmarkerResult expectedResult = + getExpectedHolisticLandmarkerResult( + HOLISTIC_RESULT, /* hasFaceBlendshapes= */ true, /* hasSegmentationMask= */ false); + assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult); + } + + @Test + public void detect_successWithSegmentationMasks() throws Exception { + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .setOutputPoseSegmentationMasks(true) + .build(); + HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options); + HolisticLandmarkerResult actualResult = + holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE)); + HolisticLandmarkerResult expectedResult = + getExpectedHolisticLandmarkerResult( + HOLISTIC_RESULT, /* hasFaceBlendshapes= */ false, /* hasSegmentationMask= */ true); + assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult); + } + + @Test + public void detect_successWithEmptyResult() throws Exception { + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .build(); + HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options); + HolisticLandmarkerResult actualResult = + holisticLandmarker.detect(getImageFromAsset(CAT_IMAGE)); + assertThat(actualResult.faceLandmarks()).isEmpty(); + } + + @Test + public void detect_failsWithRegionOfInterest() throws Exception { + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .build(); + HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options); + ImageProcessingOptions imageProcessingOptions = + ImageProcessingOptions.builder().setRegionOfInterest(new RectF(0, 0, 1, 1)).build(); + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE), imageProcessingOptions)); + assertThat(exception) + .hasMessageThat() + .contains("HolisticLandmarker doesn't support region-of-interest"); + } + } + + @RunWith(AndroidJUnit4.class) + public static final class RunningModeTest extends HolisticLandmarkerTest { + private void assertCreationFailsWithResultListenerInNonLiveStreamMode(RunningMode runningMode) + throws Exception { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .setRunningMode(runningMode) + .setResultListener((HolisticLandmarkerResult, inputImage) -> {}) + .build()); + assertThat(exception) + .hasMessageThat() + .contains("a user-defined result listener shouldn't be provided"); + } + + @Test + public void create_failsWithIllegalResultListenerInVideoMode() throws Exception { + assertCreationFailsWithResultListenerInNonLiveStreamMode(RunningMode.VIDEO); + } + + @Test + public void create_failsWithIllegalResultListenerInImageMode() throws Exception { + assertCreationFailsWithResultListenerInNonLiveStreamMode(RunningMode.IMAGE); + } + + @Test + public void create_failsWithMissingResultListenerInLiveSteamMode() throws Exception { + IllegalArgumentException exception = + assertThrows( + IllegalArgumentException.class, + () -> + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .setRunningMode(RunningMode.LIVE_STREAM) + .build()); + assertThat(exception) + .hasMessageThat() + .contains("a user-defined result listener must be provided"); + } + + @Test + public void detect_failsWithCallingWrongApiInImageMode() throws Exception { + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .setRunningMode(RunningMode.IMAGE) + .build(); + + HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options); + MediaPipeException exception = + assertThrows( + MediaPipeException.class, + () -> + holisticLandmarker.detectForVideo( + getImageFromAsset(POSE_IMAGE), /* timestampsMs= */ 0)); + assertThat(exception).hasMessageThat().contains("not initialized with the video mode"); + exception = + assertThrows( + MediaPipeException.class, + () -> + holisticLandmarker.detectAsync( + getImageFromAsset(POSE_IMAGE), /* timestampsMs= */ 0)); + assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode"); + } + + @Test + public void detect_failsWithCallingWrongApiInVideoMode() throws Exception { + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .setRunningMode(RunningMode.VIDEO) + .build(); + + HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options); + MediaPipeException exception = + assertThrows( + MediaPipeException.class, + () -> holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE))); + assertThat(exception).hasMessageThat().contains("not initialized with the image mode"); + exception = + assertThrows( + MediaPipeException.class, + () -> + holisticLandmarker.detectAsync( + getImageFromAsset(POSE_IMAGE), /* timestampsMs= */ 0)); + assertThat(exception).hasMessageThat().contains("not initialized with the live stream mode"); + } + + @Test + public void detect_failsWithCallingWrongApiInLiveSteamMode() throws Exception { + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .setRunningMode(RunningMode.LIVE_STREAM) + .setResultListener((HolisticLandmarkerResult, inputImage) -> {}) + .build(); + + HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options); + MediaPipeException exception = + assertThrows( + MediaPipeException.class, + () -> holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE))); + assertThat(exception).hasMessageThat().contains("not initialized with the image mode"); + exception = + assertThrows( + MediaPipeException.class, + () -> + holisticLandmarker.detectForVideo( + getImageFromAsset(POSE_IMAGE), /* timestampsMs= */ 0)); + assertThat(exception).hasMessageThat().contains("not initialized with the video mode"); + } + + @Test + public void detect_successWithImageMode() throws Exception { + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .setRunningMode(RunningMode.IMAGE) + .build(); + + HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options); + HolisticLandmarkerResult actualResult = + holisticLandmarker.detect(getImageFromAsset(POSE_IMAGE)); + HolisticLandmarkerResult expectedResult = + getExpectedHolisticLandmarkerResult( + HOLISTIC_RESULT, /* hasFaceBlendshapes= */ false, /* hasSegmentationMask= */ false); + assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult); + } + + @Test + public void detect_successWithVideoMode() throws Exception { + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .setRunningMode(RunningMode.VIDEO) + .build(); + HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options); + HolisticLandmarkerResult expectedResult = + getExpectedHolisticLandmarkerResult( + HOLISTIC_RESULT, /* hasFaceBlendshapes= */ false, /* hasSegmentationMask= */ false); + for (int i = 0; i < 3; i++) { + HolisticLandmarkerResult actualResult = + holisticLandmarker.detectForVideo(getImageFromAsset(POSE_IMAGE), /* timestampsMs= */ i); + assertActualResultApproximatelyEqualsToExpectedResult(actualResult, expectedResult); + } + } + + @Test + public void detect_failsWithOutOfOrderInputTimestamps() throws Exception { + MPImage image = getImageFromAsset(POSE_IMAGE); + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .setRunningMode(RunningMode.LIVE_STREAM) + .setResultListener((actualResult, inputImage) -> {}) + .build(); + try (HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options)) { + holisticLandmarker.detectAsync(image, /* timestampsMs= */ 1); + MediaPipeException exception = + assertThrows( + MediaPipeException.class, + () -> holisticLandmarker.detectAsync(image, /* timestampsMs= */ 0)); + assertThat(exception) + .hasMessageThat() + .contains("having a smaller timestamp than the processed timestamp"); + } + } + + @Test + public void detect_successWithLiveSteamMode() throws Exception { + MPImage image = getImageFromAsset(POSE_IMAGE); + HolisticLandmarkerResult expectedResult = + getExpectedHolisticLandmarkerResult( + HOLISTIC_RESULT, /* hasFaceBlendshapes= */ false, /* hasSegmentationMask= */ false); + HolisticLandmarkerOptions options = + HolisticLandmarkerOptions.builder() + .setBaseOptions( + BaseOptions.builder() + .setModelAssetPath(HOLISTIC_LANDMARKER_BUNDLE_ASSET_FILE) + .build()) + .setRunningMode(RunningMode.LIVE_STREAM) + .setResultListener( + (actualResult, inputImage) -> { + assertActualResultApproximatelyEqualsToExpectedResult( + actualResult, expectedResult); + assertImageSizeIsExpected(inputImage); + }) + .build(); + try (HolisticLandmarker holisticLandmarker = + HolisticLandmarker.createFromOptions( + ApplicationProvider.getApplicationContext(), options)) { + for (int i = 0; i < 3; i++) { + holisticLandmarker.detectAsync(image, /* timestampsMs= */ i); + } + } + } + } + + private static MPImage getImageFromAsset(String filePath) throws Exception { + AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets(); + InputStream istr = assetManager.open(filePath); + return new BitmapImageBuilder(BitmapFactory.decodeStream(istr)).build(); + } + + private static HolisticLandmarkerResult getExpectedHolisticLandmarkerResult( + String resultPath, boolean hasFaceBlendshapes, boolean hasSegmentationMask) throws Exception { + AssetManager assetManager = ApplicationProvider.getApplicationContext().getAssets(); + + HolisticResult holisticResult = HolisticResult.parseFrom(assetManager.open(resultPath)); + + Optional blendshapes = + hasFaceBlendshapes + ? Optional.of(holisticResult.getFaceBlendshapes()) + : Optional.empty(); + Optional segmentationMask = + hasSegmentationMask ? Optional.of(PLACEHOLDER_MASK) : Optional.empty(); + + return HolisticLandmarkerResult.create( + holisticResult.getFaceLandmarks(), + blendshapes, + holisticResult.getPoseLandmarks(), + LandmarkList.getDefaultInstance(), + segmentationMask, + holisticResult.getLeftHandLandmarks(), + LandmarkList.getDefaultInstance(), + holisticResult.getRightHandLandmarks(), + LandmarkList.getDefaultInstance(), + /* timestampMs= */ 0); + } + + private static void assertActualResultApproximatelyEqualsToExpectedResult( + HolisticLandmarkerResult actualResult, HolisticLandmarkerResult expectedResult) { + // Expects to have the same number of holistics detected. + assertThat(actualResult.faceLandmarks()).hasSize(expectedResult.faceLandmarks().size()); + assertThat(actualResult.faceBlendshapes().isPresent()) + .isEqualTo(expectedResult.faceBlendshapes().isPresent()); + assertThat(actualResult.poseLandmarks()).hasSize(expectedResult.poseLandmarks().size()); + assertThat(actualResult.segmentationMask().isPresent()) + .isEqualTo(expectedResult.segmentationMask().isPresent()); + assertThat(actualResult.leftHandLandmarks()).hasSize(expectedResult.leftHandLandmarks().size()); + assertThat(actualResult.rightHandLandmarks()) + .hasSize(expectedResult.rightHandLandmarks().size()); + + // Actual face landmarks match expected face landmarks. + assertThat(actualResult.faceLandmarks()) + .comparingElementsUsing(VALIDATE_LANDMARRKS) + .containsExactlyElementsIn(expectedResult.faceLandmarks()); + + // Actual face blendshapes match expected face blendshapes. + if (actualResult.faceBlendshapes().isPresent()) { + assertThat(actualResult.faceBlendshapes().get()) + .comparingElementsUsing(VALIDATE_BLENDSHAPES) + .containsExactlyElementsIn(expectedResult.faceBlendshapes().get()); + } + + // Actual pose landmarks match expected pose landmarks. + assertThat(actualResult.poseLandmarks()) + .comparingElementsUsing(VALIDATE_LANDMARRKS) + .containsExactlyElementsIn(expectedResult.poseLandmarks()); + + if (actualResult.segmentationMask().isPresent()) { + assertImageSizeIsExpected(actualResult.segmentationMask().get()); + } + + // Actual left hand landmarks match expected left hand landmarks. + assertThat(actualResult.leftHandLandmarks()) + .comparingElementsUsing(VALIDATE_LANDMARRKS) + .containsExactlyElementsIn(expectedResult.leftHandLandmarks()); + + // Actual right hand landmarks match expected right hand landmarks. + assertThat(actualResult.rightHandLandmarks()) + .comparingElementsUsing(VALIDATE_LANDMARRKS) + .containsExactlyElementsIn(expectedResult.rightHandLandmarks()); + } + + private static void assertImageSizeIsExpected(MPImage inputImage) { + assertThat(inputImage).isNotNull(); + assertThat(inputImage.getWidth()).isEqualTo(IMAGE_WIDTH); + assertThat(inputImage.getHeight()).isEqualTo(IMAGE_HEIGHT); + } +} diff --git a/mediapipe/tasks/testdata/vision/BUILD b/mediapipe/tasks/testdata/vision/BUILD index 3f83118b0..422241081 100644 --- a/mediapipe/tasks/testdata/vision/BUILD +++ b/mediapipe/tasks/testdata/vision/BUILD @@ -224,6 +224,7 @@ filegroup( "hand_detector_result_one_hand.pbtxt", "hand_detector_result_one_hand_rotated.pbtxt", "hand_detector_result_two_hands.pbtxt", + "male_full_height_hands_result_cpu.pbtxt", "pointing_up_landmarks.pbtxt", "pointing_up_rotated_landmarks.pbtxt", "portrait_expected_detection.pbtxt", From 91589b10d3c684af00cf8e3d14e4683797ab55bd Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 28 Nov 2023 18:05:07 -0800 Subject: [PATCH 141/157] internal change. PiperOrigin-RevId: 586156439 --- mediapipe/python/solutions/drawing_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mediapipe/python/solutions/drawing_utils.py b/mediapipe/python/solutions/drawing_utils.py index a1acc0be2..78e931264 100644 --- a/mediapipe/python/solutions/drawing_utils.py +++ b/mediapipe/python/solutions/drawing_utils.py @@ -125,7 +125,8 @@ def draw_landmarks( color=RED_COLOR), connection_drawing_spec: Union[DrawingSpec, Mapping[Tuple[int, int], - DrawingSpec]] = DrawingSpec()): + DrawingSpec]] = DrawingSpec(), + is_drawing_landmarks: bool = True): """Draws the landmarks and the connections on the image. Args: @@ -142,6 +143,8 @@ def draw_landmarks( connections to the DrawingSpecs that specifies the connections' drawing settings such as color and line thickness. If this argument is explicitly set to None, no landmark connections will be drawn. + is_drawing_landmarks: Whether to draw landmarks. If set false, skip drawing + landmarks, only contours will be drawed. Raises: ValueError: If one of the followings: @@ -181,7 +184,7 @@ def draw_landmarks( drawing_spec.thickness) # Draws landmark points after finishing the connection lines, which is # aesthetically better. - if landmark_drawing_spec: + if is_drawing_landmarks and landmark_drawing_spec: for idx, landmark_px in idx_to_coordinates.items(): drawing_spec = landmark_drawing_spec[idx] if isinstance( landmark_drawing_spec, Mapping) else landmark_drawing_spec From 3c655e23345d93c78afb3f97229b70bdf25d44ed Mon Sep 17 00:00:00 2001 From: Kinar Date: Wed, 29 Nov 2023 03:08:09 -0800 Subject: [PATCH 142/157] Revised Gesture Recognizer API implementation and associated tests --- .../gesture_recognizer_result_converter.cc | 1 - ...esture_recognizer_result_converter_test.cc | 98 +++++++ .../containers/landmark_converter.cc | 26 +- .../containers/landmark_converter.h | 16 +- .../containers/landmark_converter_test.cc | 121 ++++++++ .../gesture_recognizer/gesture_recognizer.h | 4 +- .../gesture_recognizer_test.cc | 258 ++++++++++++++++-- 7 files changed, 467 insertions(+), 57 deletions(-) diff --git a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc index 3b2da8eab..6ac8b1370 100644 --- a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc +++ b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc @@ -16,7 +16,6 @@ limitations under the License. #include "mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.h" #include -#include #include "mediapipe/tasks/c/components/containers/category_converter.h" #include "mediapipe/tasks/c/components/containers/landmark_converter.h" diff --git a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc index 551498e12..6c1f2f798 100644 --- a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc +++ b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc @@ -21,4 +21,102 @@ limitations under the License. namespace mediapipe::tasks::c::components::containers { +TEST(GestureRecognizerResultConverterTest, ConvertsCustomResult) { + ::mediapipe::tasks::vision::gesture_recognizer::GestureRecognizerResult + cpp_result; + + // Initialize gestures + Classification classification_for_gestures; + classification_for_gestures.set_index(0); + classification_for_gestures.set_score(0.9f); + classification_for_gestures.set_label("gesture_label_1"); + classification_for_gestures.set_display_name("gesture_display_name_1"); + ClassificationList gestures_list; + *gestures_list.add_classification() = classification_for_gestures; + cpp_result.gestures.push_back(gestures_list); + + // Initialize handedness + Classification classification_for_handedness; + classification_for_handedness.set_index(1); + classification_for_handedness.set_score(0.8f); + classification_for_handedness.set_label("handeness_label_1"); + classification_for_handedness.set_display_name("handeness_display_name_1"); + ClassificationList handedness_list; + *handedness_list.add_classification() = classification_for_handedness; + cpp_result.handedness.push_back(handedness_list); + + // Initialize hand_landmarks + NormalizedLandmark normalized_landmark; + normalized_landmark.set_x(0.1f); + normalized_landmark.set_y(0.2f); + normalized_landmark.set_z(0.3f); + NormalizedLandmarkList normalized_landmark_list; + *normalized_landmark_list.add_landmark() = normalized_landmark; + cpp_result.hand_landmarks.push_back(normalized_landmark_list); + + // Initialize hand_world_landmarks + Landmark landmark; + landmark.set_x(1.0f); + landmark.set_y(1.1f); + landmark.set_z(1.2f); + + LandmarkList landmark_list; + *landmark_list.add_landmark() = landmark; + cpp_result.hand_world_landmarks.push_back(landmark_list); + + GestureRecognizerResult c_result; + CppConvertToGestureRecognizerResult(cpp_result, &c_result); + + // Verify conversion of gestures + EXPECT_NE(c_result.gestures, nullptr); + EXPECT_EQ(c_result.gestures_count, cpp_result.gestures.size()); + + for (uint32_t i = 0; i < c_result.gestures_count; ++i) { + EXPECT_EQ(c_result.gestures_categories_counts[i], + cpp_result.gestures[i].classification_size()); + for (uint32_t j = 0; j < c_result.gestures_categories_counts[i]; ++j) { + auto gesture = cpp_result.gestures[i].classification(j); + EXPECT_EQ(std::string(c_result.gestures[i][j].category_name), + gesture.label()); + EXPECT_FLOAT_EQ(c_result.gestures[i][j].score, gesture.score()); + } + } + + // Verify conversion of hand_landmarks + EXPECT_NE(c_result.hand_landmarks, nullptr); + EXPECT_EQ(c_result.hand_landmarks_count, cpp_result.hand_landmarks.size()); + + for (uint32_t i = 0; i < c_result.hand_landmarks_count; ++i) { + EXPECT_EQ(c_result.hand_landmarks[i].landmarks_count, + cpp_result.hand_landmarks[i].landmark_size()); + for (uint32_t j = 0; j < c_result.hand_landmarks[i].landmarks_count; ++j) { + const auto& landmark = cpp_result.hand_landmarks[i].landmark(j); + EXPECT_FLOAT_EQ(c_result.hand_landmarks[i].landmarks[j].x, landmark.x()); + EXPECT_FLOAT_EQ(c_result.hand_landmarks[i].landmarks[j].y, landmark.y()); + EXPECT_FLOAT_EQ(c_result.hand_landmarks[i].landmarks[j].z, landmark.z()); + } + } + + // Verify conversion of hand_world_landmarks + EXPECT_NE(c_result.hand_world_landmarks, nullptr); + EXPECT_EQ(c_result.hand_world_landmarks_count, + cpp_result.hand_world_landmarks.size()); + for (uint32_t i = 0; i < c_result.hand_world_landmarks_count; ++i) { + EXPECT_EQ(c_result.hand_world_landmarks[i].landmarks_count, + cpp_result.hand_world_landmarks[i].landmark_size()); + for (uint32_t j = 0; j < c_result.hand_world_landmarks[i].landmarks_count; + ++j) { + const auto& landmark = cpp_result.hand_world_landmarks[i].landmark(j); + EXPECT_FLOAT_EQ(c_result.hand_world_landmarks[i].landmarks[j].x, + landmark.x()); + EXPECT_FLOAT_EQ(c_result.hand_world_landmarks[i].landmarks[j].y, + landmark.y()); + EXPECT_FLOAT_EQ(c_result.hand_world_landmarks[i].landmarks[j].z, + landmark.z()); + } + } + + CppCloseGestureRecognizerResult(&c_result); +} + } // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/landmark_converter.cc b/mediapipe/tasks/c/components/containers/landmark_converter.cc index 5f4ab5ef2..f5643ffaa 100644 --- a/mediapipe/tasks/c/components/containers/landmark_converter.cc +++ b/mediapipe/tasks/c/components/containers/landmark_converter.cc @@ -18,19 +18,13 @@ limitations under the License. #include #include "mediapipe/tasks/c/components/containers/landmark.h" - -typedef Landmark LandmarkC; -typedef NormalizedLandmark NormalizedLandmarkC; -typedef Landmarks LandmarksC; -typedef NormalizedLandmarks NormalizedLandmarksC; - #include "mediapipe/tasks/cc/components/containers/landmark.h" namespace mediapipe::tasks::c::components::containers { void CppConvertToLandmark( const mediapipe::tasks::components::containers::Landmark& in, - LandmarkC* out) { + ::Landmark* out) { out->x = in.x; out->y = in.y; out->z = in.z; @@ -54,7 +48,7 @@ void CppConvertToLandmark( void CppConvertToNormalizedLandmark( const mediapipe::tasks::components::containers::NormalizedLandmark& in, - NormalizedLandmarkC* out) { + ::NormalizedLandmark* out) { out->x = in.x; out->y = in.y; out->z = in.z; @@ -78,9 +72,9 @@ void CppConvertToNormalizedLandmark( void CppConvertToLandmarks( const std::vector& in, - LandmarksC* out) { + ::Landmarks* out) { out->landmarks_count = in.size(); - out->landmarks = new LandmarkC[out->landmarks_count]; + out->landmarks = new ::Landmark[out->landmarks_count]; for (uint32_t i = 0; i < out->landmarks_count; ++i) { CppConvertToLandmark(in[i], &out->landmarks[i]); } @@ -89,22 +83,22 @@ void CppConvertToLandmarks( void CppConvertToNormalizedLandmarks( const std::vector< mediapipe::tasks::components::containers::NormalizedLandmark>& in, - NormalizedLandmarksC* out) { + ::NormalizedLandmarks* out) { out->landmarks_count = in.size(); - out->landmarks = new NormalizedLandmarkC[out->landmarks_count]; + out->landmarks = new ::NormalizedLandmark[out->landmarks_count]; for (uint32_t i = 0; i < out->landmarks_count; ++i) { CppConvertToNormalizedLandmark(in[i], &out->landmarks[i]); } } -void CppCloseLandmark(LandmarkC* in) { +void CppCloseLandmark(::Landmark* in) { if (in && in->name) { free(in->name); in->name = nullptr; } } -void CppCloseLandmarks(LandmarksC* in) { +void CppCloseLandmarks(::Landmarks* in) { for (uint32_t i = 0; i < in->landmarks_count; ++i) { CppCloseLandmark(&in->landmarks[i]); } @@ -113,14 +107,14 @@ void CppCloseLandmarks(LandmarksC* in) { in->landmarks_count = 0; } -void CppCloseNormalizedLandmark(NormalizedLandmarkC* in) { +void CppCloseNormalizedLandmark(::NormalizedLandmark* in) { if (in && in->name) { free(in->name); in->name = nullptr; } } -void CppCloseNormalizedLandmarks(NormalizedLandmarksC* in) { +void CppCloseNormalizedLandmarks(::NormalizedLandmarks* in) { for (uint32_t i = 0; i < in->landmarks_count; ++i) { CppCloseNormalizedLandmark(&in->landmarks[i]); } diff --git a/mediapipe/tasks/c/components/containers/landmark_converter.h b/mediapipe/tasks/c/components/containers/landmark_converter.h index f59158112..1b3626386 100644 --- a/mediapipe/tasks/c/components/containers/landmark_converter.h +++ b/mediapipe/tasks/c/components/containers/landmark_converter.h @@ -23,28 +23,28 @@ namespace mediapipe::tasks::c::components::containers { void CppConvertToLandmark( const mediapipe::tasks::components::containers::Landmark& in, - Landmark* out); + ::Landmark* out); void CppConvertToNormalizedLandmark( const mediapipe::tasks::components::containers::NormalizedLandmark& in, - NormalizedLandmark* out); + ::NormalizedLandmark* out); void CppConvertToLandmarks( const std::vector& in, - Landmarks* out); + ::Landmarks* out); void CppConvertToNormalizedLandmarks( const std::vector< mediapipe::tasks::components::containers::NormalizedLandmark>& in, - NormalizedLandmarks* out); + ::NormalizedLandmarks* out); -void CppCloseLandmark(struct Landmark* in); +void CppCloseLandmark(struct ::Landmark* in); -void CppCloseLandmarks(struct Landmarks* in); +void CppCloseLandmarks(struct ::Landmarks* in); -void CppCloseNormalizedLandmark(struct NormalizedLandmark* in); +void CppCloseNormalizedLandmark(struct ::NormalizedLandmark* in); -void CppCloseNormalizedLandmarks(struct NormalizedLandmarks* in); +void CppCloseNormalizedLandmarks(struct ::NormalizedLandmarks* in); } // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/landmark_converter_test.cc b/mediapipe/tasks/c/components/containers/landmark_converter_test.cc index cf2b5a0e9..d6cbe3e85 100644 --- a/mediapipe/tasks/c/components/containers/landmark_converter_test.cc +++ b/mediapipe/tasks/c/components/containers/landmark_converter_test.cc @@ -25,4 +25,125 @@ limitations under the License. namespace mediapipe::tasks::c::components::containers { +TEST(LandmarkConverterTest, ConvertsCustomLandmark) { + mediapipe::tasks::components::containers::Landmark cpp_landmark = {0.1f, 0.2f, + 0.3f}; + + ::Landmark c_landmark; + CppConvertToLandmark(cpp_landmark, &c_landmark); + EXPECT_FLOAT_EQ(c_landmark.x, cpp_landmark.x); + EXPECT_FLOAT_EQ(c_landmark.y, cpp_landmark.y); + EXPECT_FLOAT_EQ(c_landmark.z, cpp_landmark.z); + CppCloseLandmark(&c_landmark); +} + +TEST(LandmarksConverterTest, ConvertsCustomLandmarks) { + std::vector + cpp_landmarks = { + {0.1f, 0.2f, 0.3f}, // First Landmark + {0.4f, 0.5f, 0.6f} // Second Landmark + }; + + ::Landmarks c_landmarks; + CppConvertToLandmarks(cpp_landmarks, &c_landmarks); + + EXPECT_EQ(c_landmarks.landmarks_count, cpp_landmarks.size()); + for (size_t i = 0; i < c_landmarks.landmarks_count; ++i) { + EXPECT_FLOAT_EQ(c_landmarks.landmarks[i].x, cpp_landmarks[i].x); + EXPECT_FLOAT_EQ(c_landmarks.landmarks[i].y, cpp_landmarks[i].y); + EXPECT_FLOAT_EQ(c_landmarks.landmarks[i].z, cpp_landmarks[i].z); + } + + CppCloseLandmarks(&c_landmarks); +} + +TEST(NormalizedLandmarkConverterTest, ConvertsCustomNormalizedLandmark) { + mediapipe::tasks::components::containers::NormalizedLandmark + cpp_normalized_landmark = {0.7f, 0.8f, 0.9f}; + + ::NormalizedLandmark c_normalized_landmark; + CppConvertToNormalizedLandmark(cpp_normalized_landmark, + &c_normalized_landmark); + + EXPECT_FLOAT_EQ(c_normalized_landmark.x, cpp_normalized_landmark.x); + EXPECT_FLOAT_EQ(c_normalized_landmark.y, cpp_normalized_landmark.y); + EXPECT_FLOAT_EQ(c_normalized_landmark.z, cpp_normalized_landmark.z); + + CppCloseNormalizedLandmark(&c_normalized_landmark); +} + +TEST(NormalizedLandmarksConverterTest, ConvertsCustomNormalizedLandmarks) { + std::vector + cpp_normalized_landmarks = { + {0.1f, 0.2f, 0.3f}, // First NormalizedLandmark + {0.4f, 0.5f, 0.6f} // Second NormalizedLandmark + }; + + ::NormalizedLandmarks c_normalized_landmarks; + CppConvertToNormalizedLandmarks(cpp_normalized_landmarks, + &c_normalized_landmarks); + + EXPECT_EQ(c_normalized_landmarks.landmarks_count, + cpp_normalized_landmarks.size()); + for (size_t i = 0; i < c_normalized_landmarks.landmarks_count; ++i) { + EXPECT_FLOAT_EQ(c_normalized_landmarks.landmarks[i].x, + cpp_normalized_landmarks[i].x); + EXPECT_FLOAT_EQ(c_normalized_landmarks.landmarks[i].y, + cpp_normalized_landmarks[i].y); + EXPECT_FLOAT_EQ(c_normalized_landmarks.landmarks[i].z, + cpp_normalized_landmarks[i].z); + } + + CppCloseNormalizedLandmarks(&c_normalized_landmarks); +} + +TEST(LandmarkConverterTest, FreesMemory) { + mediapipe::tasks::components::containers::Landmark cpp_landmark = { + 0.1f, 0.2f, 0.3f, 0.0f, 0.0f, "foo"}; + + ::Landmark c_landmark; + CppConvertToLandmark(cpp_landmark, &c_landmark); + EXPECT_NE(c_landmark.name, nullptr); + + CppCloseLandmark(&c_landmark); + EXPECT_EQ(c_landmark.name, nullptr); +} + +TEST(NormalizedLandmarkConverterTest, FreesMemory) { + mediapipe::tasks::components::containers::NormalizedLandmark cpp_landmark = { + 0.1f, 0.2f, 0.3f, 0.0f, 0.0f, "foo"}; + + ::NormalizedLandmark c_landmark; + CppConvertToNormalizedLandmark(cpp_landmark, &c_landmark); + EXPECT_NE(c_landmark.name, nullptr); + + CppCloseNormalizedLandmark(&c_landmark); + EXPECT_EQ(c_landmark.name, nullptr); +} + +TEST(LandmarksConverterTest, FreesMemory) { + std::vector + cpp_landmarks = {{0.1f, 0.2f, 0.3f}, {0.4f, 0.5f, 0.6f}}; + + ::Landmarks c_landmarks; + CppConvertToLandmarks(cpp_landmarks, &c_landmarks); + EXPECT_NE(c_landmarks.landmarks, nullptr); + + CppCloseLandmarks(&c_landmarks); + EXPECT_EQ(c_landmarks.landmarks, nullptr); +} + +TEST(NormalizedLandmarksConverterTest, FreesMemory) { + std::vector + cpp_normalized_landmarks = {{0.1f, 0.2f, 0.3f}, {0.4f, 0.5f, 0.6f}}; + + ::NormalizedLandmarks c_normalized_landmarks; + CppConvertToNormalizedLandmarks(cpp_normalized_landmarks, + &c_normalized_landmarks); + EXPECT_NE(c_normalized_landmarks.landmarks, nullptr); + + CppCloseNormalizedLandmarks(&c_normalized_landmarks); + EXPECT_EQ(c_normalized_landmarks.landmarks, nullptr); +} + } // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h index 1c2b65112..39f4a1734 100644 --- a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h +++ b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h @@ -82,12 +82,12 @@ struct GestureRecognizerOptions { // // A caller is responsible for closing gesture recognizer result. typedef void (*result_callback_fn)(GestureRecognizerResult* result, - const MpImage image, int64_t timestamp_ms, + const MpImage& image, int64_t timestamp_ms, char* error_msg); result_callback_fn result_callback; }; -// Creates an GestureRecognizer from provided `options`. +// Creates an GestureRecognizer from the provided `options`. // Returns a pointer to the gesture recognizer on success. // If an error occurs, returns `nullptr` and sets the error parameter to an // an error message (if `error_msg` is not `nullptr`). You must free the memory diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc index 723ff9838..ce3f5df5a 100644 --- a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc +++ b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc @@ -36,16 +36,46 @@ using testing::HasSubstr; constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; constexpr char kModelName[] = "gesture_recognizer.task"; -constexpr float kPrecision = 1e-4; -constexpr float kLandmarkPrecision = 1e-3; +constexpr char kImageFile[] = "fist.jpg"; +constexpr float kScorePrecision = 1e-2; +constexpr float kLandmarkPrecision = 1e-1; constexpr int kIterations = 100; std::string GetFullPath(absl::string_view file_name) { return JoinPath("./", kTestDataDirectory, file_name); } +void MatchesGestureRecognizerResult(GestureRecognizerResult* result, + const float score_precision, + const float landmark_precision) { + // Expects to have the same number of hands detected. + EXPECT_EQ(result->gestures_count, 1); + EXPECT_EQ(result->handedness_count, 1); + // Actual gesture with top score matches expected gesture. + EXPECT_EQ(std::string{result->gestures[0][0].category_name}, "Closed_Fist"); + EXPECT_NEAR(result->gestures[0][0].score, 0.91f, score_precision); + + // Actual handedness matches expected handedness. + EXPECT_EQ(std::string{result->handedness[0][0].category_name}, "Right"); + EXPECT_NEAR(result->handedness[0][0].score, 0.9893f, score_precision); + + // Actual landmarks match expected landmarks. + EXPECT_NEAR(result->hand_landmarks[0].landmarks[0].x, 0.477f, + landmark_precision); + EXPECT_NEAR(result->hand_landmarks[0].landmarks[0].y, 0.661f, + landmark_precision); + EXPECT_NEAR(result->hand_landmarks[0].landmarks[0].z, 0.0f, + landmark_precision); + EXPECT_NEAR(result->hand_world_landmarks[0].landmarks[0].x, -0.009f, + landmark_precision); + EXPECT_NEAR(result->hand_world_landmarks[0].landmarks[0].y, 0.082f, + landmark_precision); + EXPECT_NEAR(result->hand_world_landmarks[0].landmarks[0].z, 0.006f, + landmark_precision); +} + TEST(GestureRecognizerTest, ImageModeTest) { - const auto image = DecodeImageFromFile(GetFullPath("fist.jpg")); + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); ASSERT_TRUE(image.ok()); const std::string model_path = GetFullPath(kModelName); @@ -88,36 +118,204 @@ TEST(GestureRecognizerTest, ImageModeTest) { GestureRecognizerResult result; gesture_recognizer_recognize_image(recognizer, &mp_image, &result, /* error_msg */ nullptr); - - // Expects to have the same number of hands detected. - EXPECT_EQ(result.gestures_count, 1); - EXPECT_EQ(result.handedness_count, 1); - // Actual gesture with top score matches expected gesture. - EXPECT_EQ(std::string{result.gestures[0][0].category_name}, "Closed_Fist"); - EXPECT_NEAR(result.gestures[0][0].score, 0.9000f, kPrecision); - - // Actual handedness matches expected handedness. - EXPECT_EQ(std::string{result.handedness[0][0].category_name}, "Right"); - EXPECT_NEAR(result.handedness[0][0].score, 0.9893f, kPrecision); - - // Actual landmarks match expected landmarks. - EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].x, 0.477f, - kLandmarkPrecision); - EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].y, 0.661f, - kLandmarkPrecision); - EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].z, 0.0f, - kLandmarkPrecision); - EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].x, -0.009f, - kLandmarkPrecision); - EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].y, 0.082f, - kLandmarkPrecision); - EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].z, 0.006f, - kLandmarkPrecision); - + MatchesGestureRecognizerResult(&result, kScorePrecision, kLandmarkPrecision); gesture_recognizer_close_result(&result); gesture_recognizer_close(recognizer, /* error_msg */ nullptr); } -// TODO other tests +TEST(GestureRecognizerTest, VideoModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + GestureRecognizerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::VIDEO, + /* num_hands= */ 1, + /* min_hand_detection_confidence= */ 0.5, + /* min_hand_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}}; + + void* recognizer = + gesture_recognizer_create(&options, /* error_msg */ nullptr); + EXPECT_NE(recognizer, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + GestureRecognizerResult result; + gesture_recognizer_recognize_for_video(recognizer, &mp_image, i, &result, + /* error_msg */ nullptr); + + MatchesGestureRecognizerResult(&result, kScorePrecision, + kLandmarkPrecision); + gesture_recognizer_close_result(&result); + } + gesture_recognizer_close(recognizer, /* error_msg */ nullptr); +} + +// A structure to support LiveStreamModeTest below. This structure holds a +// static method `Fn` for a callback function of C API. A `static` qualifier +// allows to take an address of the method to follow API style. Another static +// struct member is `last_timestamp` that is used to verify that current +// timestamp is greater than the previous one. +struct LiveStreamModeCallback { + static int64_t last_timestamp; + static void Fn(GestureRecognizerResult* recognizer_result, + const MpImage& image, int64_t timestamp, char* error_msg) { + ASSERT_NE(recognizer_result, nullptr); + ASSERT_EQ(error_msg, nullptr); + MatchesGestureRecognizerResult(recognizer_result, kScorePrecision, + kLandmarkPrecision); + EXPECT_GT(image.image_frame.width, 0); + EXPECT_GT(image.image_frame.height, 0); + EXPECT_GT(timestamp, last_timestamp); + last_timestamp++; + } +}; +int64_t LiveStreamModeCallback::last_timestamp = -1; + +TEST(GestureRecognizerTest, LiveStreamModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + + GestureRecognizerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::LIVE_STREAM, + /* num_hands= */ 1, + /* min_hand_detection_confidence= */ 0.5, + /* min_hand_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + /* result_callback= */ LiveStreamModeCallback::Fn, + }; + + void* recognizer = + gesture_recognizer_create(&options, /* error_msg */ nullptr); + EXPECT_NE(recognizer, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + EXPECT_GE(gesture_recognizer_recognize_async(recognizer, &mp_image, i, + /* error_msg */ nullptr), + 0); + } + gesture_recognizer_close(recognizer, /* error_msg */ nullptr); + + // Due to the flow limiter, the total of outputs might be smaller than the + // number of iterations. + EXPECT_LE(LiveStreamModeCallback::last_timestamp, kIterations); + EXPECT_GT(LiveStreamModeCallback::last_timestamp, 0); +} + +TEST(GestureRecognizerTest, InvalidArgumentHandling) { + // It is an error to set neither the asset buffer nor the path. + GestureRecognizerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ nullptr}, + /* running_mode= */ RunningMode::IMAGE, + /* num_hands= */ 1, + /* min_hand_detection_confidence= */ 0.5, + /* min_hand_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + {}, + {}}; + + char* error_msg; + void* recognizer = gesture_recognizer_create(&options, &error_msg); + EXPECT_EQ(recognizer, nullptr); + + EXPECT_THAT(error_msg, HasSubstr("ExternalFile must specify")); + + free(error_msg); +} + +TEST(GestureRecognizerTest, FailedRecognitionHandling) { + const std::string model_path = GetFullPath(kModelName); + GestureRecognizerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* num_hands= */ 1, + /* min_hand_detection_confidence= */ 0.5, + /* min_hand_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + }; + + void* recognizer = gesture_recognizer_create(&options, /* error_msg */ + nullptr); + EXPECT_NE(recognizer, nullptr); + + const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}}; + GestureRecognizerResult result; + char* error_msg; + gesture_recognizer_recognize_image(recognizer, &mp_image, &result, + &error_msg); + EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet")); + free(error_msg); + gesture_recognizer_close(recognizer, /* error_msg */ nullptr); +} } // namespace From d19d5a50be08c5569e5d8f8087313fbd239a14b6 Mon Sep 17 00:00:00 2001 From: Kinar Date: Wed, 29 Nov 2023 03:17:35 -0800 Subject: [PATCH 143/157] Added FreeMemory test for GestureRecognizerResult --- ...esture_recognizer_result_converter_test.cc | 42 +++++++++++++++---- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc index 6c1f2f798..75c8645ce 100644 --- a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc +++ b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc @@ -21,10 +21,9 @@ limitations under the License. namespace mediapipe::tasks::c::components::containers { -TEST(GestureRecognizerResultConverterTest, ConvertsCustomResult) { - ::mediapipe::tasks::vision::gesture_recognizer::GestureRecognizerResult - cpp_result; - +void InitGestureRecognizerResult( + ::mediapipe::tasks::vision::gesture_recognizer::GestureRecognizerResult* + cpp_result) { // Initialize gestures Classification classification_for_gestures; classification_for_gestures.set_index(0); @@ -33,7 +32,7 @@ TEST(GestureRecognizerResultConverterTest, ConvertsCustomResult) { classification_for_gestures.set_display_name("gesture_display_name_1"); ClassificationList gestures_list; *gestures_list.add_classification() = classification_for_gestures; - cpp_result.gestures.push_back(gestures_list); + cpp_result->gestures.push_back(gestures_list); // Initialize handedness Classification classification_for_handedness; @@ -43,7 +42,7 @@ TEST(GestureRecognizerResultConverterTest, ConvertsCustomResult) { classification_for_handedness.set_display_name("handeness_display_name_1"); ClassificationList handedness_list; *handedness_list.add_classification() = classification_for_handedness; - cpp_result.handedness.push_back(handedness_list); + cpp_result->handedness.push_back(handedness_list); // Initialize hand_landmarks NormalizedLandmark normalized_landmark; @@ -52,7 +51,7 @@ TEST(GestureRecognizerResultConverterTest, ConvertsCustomResult) { normalized_landmark.set_z(0.3f); NormalizedLandmarkList normalized_landmark_list; *normalized_landmark_list.add_landmark() = normalized_landmark; - cpp_result.hand_landmarks.push_back(normalized_landmark_list); + cpp_result->hand_landmarks.push_back(normalized_landmark_list); // Initialize hand_world_landmarks Landmark landmark; @@ -62,7 +61,13 @@ TEST(GestureRecognizerResultConverterTest, ConvertsCustomResult) { LandmarkList landmark_list; *landmark_list.add_landmark() = landmark; - cpp_result.hand_world_landmarks.push_back(landmark_list); + cpp_result->hand_world_landmarks.push_back(landmark_list); +} + +TEST(GestureRecognizerResultConverterTest, ConvertsCustomResult) { + ::mediapipe::tasks::vision::gesture_recognizer::GestureRecognizerResult + cpp_result; + InitGestureRecognizerResult(&cpp_result); GestureRecognizerResult c_result; CppConvertToGestureRecognizerResult(cpp_result, &c_result); @@ -119,4 +124,25 @@ TEST(GestureRecognizerResultConverterTest, ConvertsCustomResult) { CppCloseGestureRecognizerResult(&c_result); } +TEST(GestureRecognizerResultConverterTest, FreesMemory) { + ::mediapipe::tasks::vision::gesture_recognizer::GestureRecognizerResult + cpp_result; + InitGestureRecognizerResult(&cpp_result); + + GestureRecognizerResult c_result; + CppConvertToGestureRecognizerResult(cpp_result, &c_result); + + EXPECT_NE(c_result.gestures, nullptr); + EXPECT_NE(c_result.handedness, nullptr); + EXPECT_NE(c_result.hand_landmarks, nullptr); + EXPECT_NE(c_result.hand_world_landmarks, nullptr); + + CppCloseGestureRecognizerResult(&c_result); + + EXPECT_EQ(c_result.gestures, nullptr); + EXPECT_EQ(c_result.handedness, nullptr); + EXPECT_EQ(c_result.hand_landmarks, nullptr); + EXPECT_EQ(c_result.hand_world_landmarks, nullptr); +} + } // namespace mediapipe::tasks::c::components::containers From 3532503354e4ea7c09e9456c200754a10cd4f388 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Nov 2023 02:05:23 +0530 Subject: [PATCH 144/157] Added iOS interactive segmenter options --- .../ios/vision/interactive_segmenter/BUILD | 26 ++++++++++++ .../sources/MPPInteractiveSegmenterOptions.h | 41 +++++++++++++++++++ .../sources/MPPInteractiveSegmenterOptions.m | 38 +++++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/interactive_segmenter/BUILD create mode 100644 mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h create mode 100644 mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.m diff --git a/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD b/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD new file mode 100644 index 000000000..f3cd98833 --- /dev/null +++ b/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD @@ -0,0 +1,26 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPInteractiveSegmenterOptions", + srcs = ["sources/MPPInteractiveSegmenterOptions.m"], + hdrs = ["sources/MPPInteractiveSegmenterOptions.h"], + deps = [ + "//mediapipe/tasks/ios/core:MPPTaskOptions", + ], +) diff --git a/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h new file mode 100644 index 000000000..9ae45b13f --- /dev/null +++ b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h @@ -0,0 +1,41 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +@class MPPInteractiveSegmenter; + +/** Options for setting up a `InteractiveSegmenter`. */ +NS_SWIFT_NAME(InteractiveSegmenterOptions) +@interface MPPInteractiveSegmenterOptions : MPPTaskOptions + +/** + * The locale to use for display names specified through the TFLite Model Metadata, if any. Defaults + * to English. + */ +@property(nonatomic, copy) NSString *displayNamesLocale; + +/** Represents whether to output confidence masks. */ +@property(nonatomic) BOOL shouldOutputConfidenceMasks; + +/** Represents whether to output category mask. */ +@property(nonatomic) BOOL shouldOutputCategoryMask; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.m b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.m new file mode 100644 index 000000000..798ac11ce --- /dev/null +++ b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.m @@ -0,0 +1,38 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h" + +@implementation MPPInteractiveSegmenterOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _displayNamesLocale = @"en"; + _shouldOutputConfidenceMasks = YES; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPInteractiveSegmenterOptions *interactiveSegmenterOptions = [super copyWithZone:zone]; + + interactiveSegmenterOptions.shouldOutputConfidenceMasks = self.shouldOutputConfidenceMasks; + interactiveSegmenterOptions.shouldOutputCategoryMask = self.shouldOutputCategoryMask; + interactiveSegmenterOptions.displayNamesLocale = self.displayNamesLocale; + + return interactiveSegmenterOptions; +} + +@end From 4137dbcbf5e69e7b3254010b54996e445d39fb09 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Nov 2023 02:30:42 +0530 Subject: [PATCH 145/157] Added iOS region of interest --- .../tasks/ios/components/containers/BUILD | 7 ++ .../containers/sources/MPPRegionOfInterest.h | 67 +++++++++++++++++++ .../containers/sources/MPPRegionOfInterest.m | 35 ++++++++++ 3 files changed, 109 insertions(+) create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h create mode 100644 mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.m diff --git a/mediapipe/tasks/ios/components/containers/BUILD b/mediapipe/tasks/ios/components/containers/BUILD index 0477d288a..4effb74b2 100644 --- a/mediapipe/tasks/ios/components/containers/BUILD +++ b/mediapipe/tasks/ios/components/containers/BUILD @@ -66,3 +66,10 @@ objc_library( srcs = ["sources/MPPConnection.m"], hdrs = ["sources/MPPConnection.h"], ) + +objc_library( + name = "MPPRegionOfInterest", + srcs = ["sources/MPPRegionOfInterest.m"], + hdrs = ["sources/MPPRegionOfInterest.h"], + deps = [":MPPDetection"], +) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h b/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h new file mode 100644 index 000000000..a6b30269d --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h @@ -0,0 +1,67 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPDetection.h" + +NS_ASSUME_NONNULL_BEGIN + +/** The Region-Of-Interest (ROI) to interact with in an interactive segmentation inference. */ +/** An instance can contain erither a single normalized point pointing to the object that the user + * wants to segment or array of normalized key points that make up scribbles over the object that + * the user wants to segment.*/ +NS_SWIFT_NAME(RegionOfInterest) +@interface MPPRegionOfInterest : NSObject + +/** The normalized point pointing to the object that the user wants to segment. `nil if `scribbles` + * is not `nil` */ +@property(nonatomic, readonly, nullable) MPPNormalizedKeypoint *keypoint; + +/** The array of normalized key points that make up scribbles over the object that the user wants to + * segment. `nil if `keypoint` is not `nil` */ +@property(nonatomic, readonly, nullable) NSArray *scribbles; + +/** + * Initializes a new `RegionOfInterest` that represents a single normalized point pointing to the + * object that the user wants to segment. + * + * @param normalizedKeypoint The normalized key point pointing to the object that the user wants to + * segment. + * + * @return An instance of `RegionOfInterest` initialized with the given normalized key point + * pointing to the object that the user wants to segment. + */ +- (instancetype)initWithNormalizedKeyPoint:(MPPNormalizedKeypoint *)normalizedKeypoint + NS_DESIGNATED_INITIALIZER; + +/** + * Initializes a new `RegionOfInterest` that represents scribbles over the object that the user + * wants to segment. + * + * @param scribbles The array of normalized key points that make up scribbles over the object that + * the user wants to segment. + * + * @return An instance of `RegionOfInterest` initialized with the given normalized key points that + * make up scribbles over the object that the user wants to segment. + */ +- (instancetype)initWitScribbles:(NSArray *)scribbles + NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.m b/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.m new file mode 100644 index 000000000..0dfa0e5b4 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.m @@ -0,0 +1,35 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h" + +@implementation MPPRegionOfInterest + +- (instancetype)initWithNormalizedKeyPoint:(MPPNormalizedKeypoint *)normalizedKeypoint { + self = [super init]; + if (self) { + _keypoint = normalizedKeypoint; + } + return self; +} + +- (instancetype)initWitScribbles:(NSArray *)scribbles { + self = [super init]; + if (self) { + _scribbles = scribbles; + } + return self; +} + +@end From 4c02980b3f62d10ae409b629468012b7de8988c0 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Nov 2023 02:31:11 +0530 Subject: [PATCH 146/157] Added iOS region of interest helpers --- .../ios/components/containers/utils/BUILD | 13 +++++ .../sources/MPPRegionOfInterest+Helpers.h | 35 ++++++++++++ .../sources/MPPRegionOfInterest+Helpers.mm | 56 +++++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h create mode 100644 mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.mm diff --git a/mediapipe/tasks/ios/components/containers/utils/BUILD b/mediapipe/tasks/ios/components/containers/utils/BUILD index 5f0311f51..6da949858 100644 --- a/mediapipe/tasks/ios/components/containers/utils/BUILD +++ b/mediapipe/tasks/ios/components/containers/utils/BUILD @@ -16,6 +16,7 @@ package(default_visibility = ["//mediapipe/tasks:internal"]) licenses(["notice"]) + objc_library( name = "MPPCategoryHelpers", srcs = ["sources/MPPCategory+Helpers.mm"], @@ -84,3 +85,15 @@ objc_library( "//mediapipe/tasks/ios/components/containers:MPPLandmark", ], ) + +objc_library( + name = "MPPRegionOfInterestHelpers", + srcs = ["sources/MPPRegionOfInterest+Helpers.mm"], + hdrs = ["sources/MPPRegionOfInterest+Helpers.h"], + deps = [ + "//mediapipe/util:render_data_cc_proto", + "//mediapipe/tasks/ios/common:MPPCommon", + "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "//mediapipe/tasks/ios/components/containers:MPPRegionOfInterest", + ], +) diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h b/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h new file mode 100644 index 000000000..026d4f2e9 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h @@ -0,0 +1,35 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/util/render_data.pb.h" + +#import "mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPRegionOfInterest (Helpers) + +/** + * Creates a `RenderData` from the region of interest. + * + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return A `RenderData1 proto created from the region of interest. + */ +- (std::optional)getRenderDataWithError:(NSError **)error; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.mm new file mode 100644 index 000000000..c7f0bc9a0 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.mm @@ -0,0 +1,56 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/common/sources/MPPCommon.h" +#import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h" + +namespace { +using RenderData = ::mediapipe::RenderData; +using RenderAnnotation = ::mediapipe::RenderAnnotation; + +} // namespace + +@implementation MPPRegionOfInterest (Helpers) + +- (std::optional)getRenderDataWithError:(NSError**)error { + RenderData result; + if (self.keypoint) { + auto* annotation = result.add_render_annotations(); + annotation->mutable_color()->set_r(255); + auto* point = annotation->mutable_point(); + point->set_normalized(true); + point->set_x(self.keypoint.location.x); + point->set_y(self.keypoint.location.y); + return result; + } else if (self.scribbles) { + auto* annotation = result.add_render_annotations(); + annotation->mutable_color()->set_r(255); + for (MPPNormalizedKeypoint* keypoint in self.scribbles) { + auto* point = annotation->mutable_scribble()->add_point(); + point->set_normalized(true); + point->set_x(self.keypoint.location.x); + point->set_y(self.keypoint.location.y); + } + return result; + } + + [MPPCommonUtils createCustomError:error + withCode:MPPTasksErrorCodeInvalidArgumentError + description:@"RegionOfInterest does not include a valid user interaction."]; + + return std::nullopt; +} + +@end From f5ac0637a2112350f4a698635bcef113585ca3e9 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Nov 2023 02:31:48 +0530 Subject: [PATCH 147/157] Updated iOS vision/core to add methods for processing region of interest --- mediapipe/tasks/ios/vision/core/BUILD | 2 ++ .../core/sources/MPPVisionPacketCreator.h | 15 +++++++++++++ .../core/sources/MPPVisionPacketCreator.mm | 14 ++++++++++++ .../vision/core/sources/MPPVisionTaskRunner.h | 22 +++++++++++++++++++ 4 files changed, 53 insertions(+) diff --git a/mediapipe/tasks/ios/vision/core/BUILD b/mediapipe/tasks/ios/vision/core/BUILD index 711b4ff95..edb202316 100644 --- a/mediapipe/tasks/ios/vision/core/BUILD +++ b/mediapipe/tasks/ios/vision/core/BUILD @@ -55,6 +55,8 @@ objc_library( "//mediapipe/framework/formats:image", "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/tasks/ios/vision/core/utils:MPPImageUtils", + "//mediapipe/tasks/ios/components/containers:MPPRegionOfInterest", + "//mediapipe/tasks/ios/components/containers/utils:MPPRegionOfInterestHelpers", ], ) diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h index ed07c6d90..9d6ed34c3 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h @@ -14,6 +14,7 @@ #import +#import "mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" #include "mediapipe/framework/formats/rect.pb.h" @@ -73,4 +74,18 @@ + (mediapipe::Packet)createPacketWithNormalizedRect:(mediapipe::NormalizedRect &)normalizedRect timestampInMilliseconds:(NSInteger)timestampInMilliseconds; +/** + * Creates a MediapPipe Packet wrapping a `RenderData` constructed from an `MPPRegionOfInterest`. + * + * @param regionOfInterest The `MPPRegionOfInterest` to send to the MediaPipe graph. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return The MediaPipe packet containing the `RenderData` constructed from the given + * `MPPRegionOfInterest`. + */ ++ (std::optional)createRenderDataPacketWithRegionOfInterest: + (MPPRegionOfInterest *)regionOfInterest + error:(NSError **)error; + @end diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm index 92c33e09b..ae6efaa35 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.mm @@ -13,6 +13,7 @@ // limitations under the License. #import "mediapipe/tasks/ios/vision/core/sources/MPPVisionPacketCreator.h" +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPRegionOfInterest+Helpers.h" #import "mediapipe/tasks/ios/vision/core/utils/sources/MPPImage+Utils.h" #include "mediapipe/framework/formats/image.h" @@ -26,6 +27,7 @@ using ::mediapipe::ImageFrame; using ::mediapipe::MakePacket; using ::mediapipe::NormalizedRect; using ::mediapipe::Packet; +using ::mediapipe::RenderData; using ::mediapipe::Timestamp; } // namespace @@ -64,4 +66,16 @@ using ::mediapipe::Timestamp; .At(Timestamp(int64(timestampInMilliseconds * kMicrosecondsPerMillisecond))); } ++ (std::optional)createRenderDataPacketWithRegionOfInterest: + (MPPRegionOfInterest *)regionOfInterest + error:(NSError **)error { + std::optional renderData = [regionOfInterest getRenderDataWithError:error]; + + if (!renderData.has_value()) { + return std::nullopt; + } + + return MakePacket(std::move(renderData.value())); +} + @end diff --git a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h index aa0307d71..8c18260c1 100644 --- a/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h +++ b/mediapipe/tasks/ios/vision/core/sources/MPPVisionTaskRunner.h @@ -20,6 +20,8 @@ #import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" #import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#include "mediapipe/framework/packet.h" + NS_ASSUME_NONNULL_BEGIN /** @@ -190,6 +192,26 @@ NS_ASSUME_NONNULL_BEGIN timestampInMilliseconds:(NSInteger)timeStampInMilliseconds error:(NSError **)error; +/** + * This method creates an input packet map to the C++ task runner with the image and normalized rect + * calculated from the region of interest specified within the bounds of an image. Tasks which need + * to add more entries to the input packet map and build their own custom logic for processing + * images can use this method. + * + * @param image An `MPPImage` input to the task. + * @param regionOfInterest A `CGRect` specifying the region of interest within the given image data + * of type `MPPImage`, on which inference should be performed. + * @param error Pointer to the memory location where errors if any should be saved. If @c NULL, no + * error will be saved. + * + * @return A `BOOL` indicating if the creation of the input packet map with the image and the + * normalized rect calculated from the region of interest was successful. + */ +- (std::optional>) + inputPacketMapWithMPPImage:(MPPImage *)image + regionOfInterest:(CGRect)roi + error:(NSError **)error; + /** * This method returns a unique dispatch queue name by adding the given suffix and a `UUID` to the * pre-defined queue name prefix for vision tasks. The vision tasks can use this method to get From 90622475a29457acfc4b13097e4f211acd463b95 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 30 Nov 2023 02:32:08 +0530 Subject: [PATCH 148/157] Added iOS interactive segmenter header --- .../ios/vision/interactive_segmenter/BUILD | 12 ++ .../sources/MPPInteractiveSegmenter.h | 138 ++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenter.h diff --git a/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD b/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD index f3cd98833..9e435485d 100644 --- a/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD +++ b/mediapipe/tasks/ios/vision/interactive_segmenter/BUILD @@ -24,3 +24,15 @@ objc_library( "//mediapipe/tasks/ios/core:MPPTaskOptions", ], ) + +objc_library( + name = "MPPInteractiveSegmenter", + hdrs = ["sources/MPPInteractiveSegmenter.h"], + module_name = "MPPInteractiveSegmenter", + deps = [ + ":MPPInteractiveSegmenterOptions", + "//mediapipe/tasks/ios/vision/components/containers:MPPRegionOfInterest", + "//mediapipe/tasks/ios/vision/core:MPPImage", + "//mediapipe/tasks/ios/vision/image_segmenter:MPPImageSegmenterResult", + ], +) diff --git a/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenter.h b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenter.h new file mode 100644 index 000000000..214deaec0 --- /dev/null +++ b/mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenter.h @@ -0,0 +1,138 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/components/containers/sources/MPPRegionOfInterest.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPImage.h" +#import "mediapipe/tasks/ios/vision/image_segmenter/sources/MPPImageSegmenterResult.h" +#import "mediapipe/tasks/ios/vision/interactive_segmenter/sources/MPPInteractiveSegmenterOptions.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * @brief Class that performs interactive segmentation on images. + * + * Users can represent user interaction through `RegionOfInterest`, which gives a hint to + * `InteractiveSegmenter` to perform segmentation focusing on the given region of interest. + * + * The API expects a TFLite model with mandatory TFLite Model Metadata. + * + * Input tensor: + * (kTfLiteUInt8/kTfLiteFloat32) + * - image input of size `[batch x height x width x channels]`. + * - batch inference is not supported (`batch` is required to be 1). + * - RGB and greyscale inputs are supported (`channels` is required to be + * 1 or 3). + * - if type is kTfLiteFloat32, NormalizationOptions are required to be attached to the metadata + * for input normalization. Output tensors: (kTfLiteUInt8/kTfLiteFloat32) + * - list of segmented masks. + * - if `output_type` is CATEGORY_MASK, uint8 Image, Image vector of size 1. + * - if `output_type` is CONFIDENCE_MASK, float32 Image list of size `channels`. + * - batch is always 1. + * + * An example of such model can be found at: + * https://tfhub.dev/tensorflow/lite-model/deeplabv3/1/metadata/2 + */ +NS_SWIFT_NAME(InteractiveSegmenter) +@interface MPPInteractiveSegmenter : NSObject + +/** + * Get the category label list of the `InteractiveSegmenter` can recognize. For CATEGORY_MASK type, + * the index in the category mask corresponds to the category in the label list. For CONFIDENCE_MASK + * type, the output mask list at index corresponds to the category in the label list. If there is no + * labelmap provided in the model file, empty array is returned. + */ +@property(nonatomic, readonly) NSArray *labels; + +/** + * Creates a new instance of `InteractiveSegmenter` from an absolute path to a TensorFlow Lite model + * file stored locally on the device and the default `InteractiveSegmenterOptions`. + * + * @param modelPath An absolute path to a TensorFlow Lite model file stored locally on the device. + * + * @return A new instance of `InteractiveSegmenter` with the given model path. `nil` if there is an + * error in initializing the interactive segmenter. + */ +- (nullable instancetype)initWithModelPath:(NSString *)modelPath error:(NSError **)error; + +/** + * Creates a new instance of `InteractiveSegmenter` from the given `InteractiveSegmenterOptions`. + * + * @param options The options of type `InteractiveSegmenterOptions` to use for configuring the + * `InteractiveSegmenter`. + * + * @return A new instance of `InteractiveSegmenter` with the given options. `nil` if there is an + * error in initializing the interactive segmenter. + */ +- (nullable instancetype)initWithOptions:(MPPInteractiveSegmenterOptions *)options + error:(NSError **)error NS_DESIGNATED_INITIALIZER; + +/** + * Performs segmentation on the provided MPPImage using the specified user's region of interest. + * Rotation will be applied according to the `orientation` property of the provided `MPImage`. Only + * use this method when the `InteractiveSegmenter` is created with running mode, `image`. + * + * This method supports RGBA images. If your `MPImage` has a source type of `pixelBuffer` or + * `sampleBuffer`, the underlying pixel buffer must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPImage` has a source type of `.image` ensure that the color space is RGB with an Alpha + * channel. + * + * @param image The `MPImage` on which segmentation is to be performed. + * + * @return An `ImageSegmenterResult` that contains the segmented masks. + */ +- (nullable MPPImageSegmenterResult *)segmentImage:(MPPImage *)image + regionOfInterest:(MPPRegionOfInterest *)regionOfInterest + error:(NSError **)error + NS_SWIFT_NAME(segment(image:regionOfInterest:)); + +/** + * Performs segmentation on the provided MPPImage using the specified user's region of interest and + * invokes the given completion handler block with the response. The method returns synchronously + * once the completion handler returns. + * + * Rotation will be applied according to the `orientation` property of the provided `MPImage`. Only + * use this method when the `InteractiveSegmenter` is created with running mode, `image`. + * + * This method supports RGBA images. If your `MPImage` has a source type of `pixelBuffer` or + * `sampleBuffer`, the underlying pixel buffer must have one of the following pixel format types: + * 1. kCVPixelFormatType_32BGRA + * 2. kCVPixelFormatType_32RGBA + * + * If your `MPImage` has a source type of `image` ensure that the color space is RGB with an Alpha + * channel. + * + * @param image The `MPImage` on which segmentation is to be performed. + * @param completionHandler A block to be invoked with the results of performing segmentation on the + * image. The block takes two arguments, the optional `ImageSegmenterResult` that contains the + * segmented masks if the segmentation was successful and an optional error populated upon failure. + * The lifetime of the returned masks is only guaranteed for the duration of the block. + */ +- (void)segmentImage:(MPPImage *)image + regionOfInterest:(MPPRegionOfInterest *)regionOfInterest + withCompletionHandler:(void (^)(MPPImageSegmenterResult *_Nullable result, + NSError *_Nullable error))completionHandler + NS_SWIFT_NAME(segment(image:regionOfInterest:completion:)); + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END From 9a5aa1b360e9f42a365a8154bcf561c686d7fbf6 Mon Sep 17 00:00:00 2001 From: Kinar Date: Thu, 30 Nov 2023 09:13:10 -0800 Subject: [PATCH 149/157] Refactor GestureRecognizerResult conversion for default initialization --- .../gesture_recognizer_result_converter.cc | 42 ++++++++----------- .../gesture_recognizer/gesture_recognizer.cc | 36 ++++++++-------- .../gesture_recognizer/gesture_recognizer.h | 6 +-- .../gesture_recognizer_test.cc | 9 ++-- 4 files changed, 43 insertions(+), 50 deletions(-) diff --git a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc index 6ac8b1370..5aca374fc 100644 --- a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc +++ b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc @@ -47,18 +47,15 @@ void CppConvertToGestureRecognizerResult( CppCategory cpp_category; // Set fields from the Classification protobuf - if (classification.has_index()) { - cpp_category.index = classification.index(); - } - if (classification.has_score()) { - cpp_category.score = classification.score(); - } - if (classification.has_label()) { - cpp_category.category_name = classification.label(); - } - if (classification.has_display_name()) { - cpp_category.display_name = classification.display_name(); - } + cpp_category.index = + classification.has_index() ? classification.index() : 0; + cpp_category.score = + classification.has_score() ? classification.score() : 0.0f; + cpp_category.category_name = + classification.has_label() ? classification.label() : ""; + cpp_category.display_name = classification.has_display_name() + ? classification.display_name() + : ""; CppConvertToCategory(cpp_category, &out->gestures[i][j]); } @@ -78,18 +75,15 @@ void CppConvertToGestureRecognizerResult( CppCategory cpp_category; // Set fields from the Classification protobuf - if (classification.has_index()) { - cpp_category.index = classification.index(); - } - if (classification.has_score()) { - cpp_category.score = classification.score(); - } - if (classification.has_label()) { - cpp_category.category_name = classification.label(); - } - if (classification.has_display_name()) { - cpp_category.display_name = classification.display_name(); - } + cpp_category.index = + classification.has_index() ? classification.index() : 0; + cpp_category.score = + classification.has_score() ? classification.score() : 0.0f; + cpp_category.category_name = + classification.has_label() ? classification.label() : ""; + cpp_category.display_name = classification.has_display_name() + ? classification.display_name() + : ""; CppConvertToCategory(cpp_category, &out->handedness[i][j]); } diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.cc b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.cc index f8c42dcaf..692e3776b 100644 --- a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.cc +++ b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.cc @@ -138,10 +138,10 @@ GestureRecognizer* CppGestureRecognizerCreate( return recognizer->release(); } -int CppGestureRecognizerRecognize(void* recognizer, const MpImage* image, +int CppGestureRecognizerRecognize(void* recognizer, const MpImage& image, GestureRecognizerResult* result, char** error_msg) { - if (image->type == MpImage::GPU_BUFFER) { + if (image.type == MpImage::GPU_BUFFER) { const absl::Status status = absl::InvalidArgumentError("GPU Buffer not supported yet."); @@ -150,9 +150,9 @@ int CppGestureRecognizerRecognize(void* recognizer, const MpImage* image, } const auto img = CreateImageFromBuffer( - static_cast(image->image_frame.format), - image->image_frame.image_buffer, image->image_frame.width, - image->image_frame.height); + static_cast(image.image_frame.format), + image.image_frame.image_buffer, image.image_frame.width, + image.image_frame.height); if (!img.ok()) { ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); @@ -170,11 +170,11 @@ int CppGestureRecognizerRecognize(void* recognizer, const MpImage* image, } int CppGestureRecognizerRecognizeForVideo(void* recognizer, - const MpImage* image, + const MpImage& image, int64_t timestamp_ms, GestureRecognizerResult* result, char** error_msg) { - if (image->type == MpImage::GPU_BUFFER) { + if (image.type == MpImage::GPU_BUFFER) { absl::Status status = absl::InvalidArgumentError("GPU Buffer not supported yet"); @@ -183,9 +183,9 @@ int CppGestureRecognizerRecognizeForVideo(void* recognizer, } const auto img = CreateImageFromBuffer( - static_cast(image->image_frame.format), - image->image_frame.image_buffer, image->image_frame.width, - image->image_frame.height); + static_cast(image.image_frame.format), + image.image_frame.image_buffer, image.image_frame.width, + image.image_frame.height); if (!img.ok()) { ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); @@ -202,9 +202,9 @@ int CppGestureRecognizerRecognizeForVideo(void* recognizer, return 0; } -int CppGestureRecognizerRecognizeAsync(void* recognizer, const MpImage* image, +int CppGestureRecognizerRecognizeAsync(void* recognizer, const MpImage& image, int64_t timestamp_ms, char** error_msg) { - if (image->type == MpImage::GPU_BUFFER) { + if (image.type == MpImage::GPU_BUFFER) { absl::Status status = absl::InvalidArgumentError("GPU Buffer not supported yet"); @@ -213,9 +213,9 @@ int CppGestureRecognizerRecognizeAsync(void* recognizer, const MpImage* image, } const auto img = CreateImageFromBuffer( - static_cast(image->image_frame.format), - image->image_frame.image_buffer, image->image_frame.width, - image->image_frame.height); + static_cast(image.image_frame.format), + image.image_frame.image_buffer, image.image_frame.width, + image.image_frame.height); if (!img.ok()) { ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); @@ -257,7 +257,7 @@ void* gesture_recognizer_create(struct GestureRecognizerOptions* options, CppGestureRecognizerCreate(*options, error_msg); } -int gesture_recognizer_recognize_image(void* recognizer, const MpImage* image, +int gesture_recognizer_recognize_image(void* recognizer, const MpImage& image, GestureRecognizerResult* result, char** error_msg) { return mediapipe::tasks::c::vision::gesture_recognizer:: @@ -265,7 +265,7 @@ int gesture_recognizer_recognize_image(void* recognizer, const MpImage* image, } int gesture_recognizer_recognize_for_video(void* recognizer, - const MpImage* image, + const MpImage& image, int64_t timestamp_ms, GestureRecognizerResult* result, char** error_msg) { @@ -274,7 +274,7 @@ int gesture_recognizer_recognize_for_video(void* recognizer, result, error_msg); } -int gesture_recognizer_recognize_async(void* recognizer, const MpImage* image, +int gesture_recognizer_recognize_async(void* recognizer, const MpImage& image, int64_t timestamp_ms, char** error_msg) { return mediapipe::tasks::c::vision::gesture_recognizer:: CppGestureRecognizerRecognizeAsync(recognizer, image, timestamp_ms, diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h index 39f4a1734..4d59df62d 100644 --- a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h +++ b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h @@ -100,7 +100,7 @@ MP_EXPORT void* gesture_recognizer_create( // an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT int gesture_recognizer_recognize_image( - void* recognizer, const MpImage* image, GestureRecognizerResult* result, + void* recognizer, const MpImage& image, GestureRecognizerResult* result, char** error_msg); // Performs gesture recognition on the provided video frame. @@ -113,7 +113,7 @@ MP_EXPORT int gesture_recognizer_recognize_image( // an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT int gesture_recognizer_recognize_for_video( - void* recognizer, const MpImage* image, int64_t timestamp_ms, + void* recognizer, const MpImage& image, int64_t timestamp_ms, GestureRecognizerResult* result, char** error_msg); // Sends live image data to gesture recognition, and the results will be @@ -135,7 +135,7 @@ MP_EXPORT int gesture_recognizer_recognize_for_video( // an error message (if `error_msg` is not `nullptr`). You must free the memory // allocated for the error message. MP_EXPORT int gesture_recognizer_recognize_async(void* recognizer, - const MpImage* image, + const MpImage& image, int64_t timestamp_ms, char** error_msg); diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc index ce3f5df5a..bd95d7e52 100644 --- a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc +++ b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc @@ -116,7 +116,7 @@ TEST(GestureRecognizerTest, ImageModeTest) { .height = image_frame->Height()}}; GestureRecognizerResult result; - gesture_recognizer_recognize_image(recognizer, &mp_image, &result, + gesture_recognizer_recognize_image(recognizer, mp_image, &result, /* error_msg */ nullptr); MatchesGestureRecognizerResult(&result, kScorePrecision, kLandmarkPrecision); gesture_recognizer_close_result(&result); @@ -166,7 +166,7 @@ TEST(GestureRecognizerTest, VideoModeTest) { for (int i = 0; i < kIterations; ++i) { GestureRecognizerResult result; - gesture_recognizer_recognize_for_video(recognizer, &mp_image, i, &result, + gesture_recognizer_recognize_for_video(recognizer, mp_image, i, &result, /* error_msg */ nullptr); MatchesGestureRecognizerResult(&result, kScorePrecision, @@ -242,7 +242,7 @@ TEST(GestureRecognizerTest, LiveStreamModeTest) { .height = image_frame->Height()}}; for (int i = 0; i < kIterations; ++i) { - EXPECT_GE(gesture_recognizer_recognize_async(recognizer, &mp_image, i, + EXPECT_GE(gesture_recognizer_recognize_async(recognizer, mp_image, i, /* error_msg */ nullptr), 0); } @@ -311,8 +311,7 @@ TEST(GestureRecognizerTest, FailedRecognitionHandling) { const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}}; GestureRecognizerResult result; char* error_msg; - gesture_recognizer_recognize_image(recognizer, &mp_image, &result, - &error_msg); + gesture_recognizer_recognize_image(recognizer, mp_image, &result, &error_msg); EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet")); free(error_msg); gesture_recognizer_close(recognizer, /* error_msg */ nullptr); From e5c7ebec12e3ae63facf6310af46bd7f70a778de Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 30 Nov 2023 15:24:22 -0800 Subject: [PATCH 150/157] Add libtext and libvision build rules PiperOrigin-RevId: 586804071 --- mediapipe/tasks/c/text/BUILD | 57 ++++++++++++++++++++++++++++++++++ mediapipe/tasks/c/vision/BUILD | 57 ++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 mediapipe/tasks/c/text/BUILD create mode 100644 mediapipe/tasks/c/vision/BUILD diff --git a/mediapipe/tasks/c/text/BUILD b/mediapipe/tasks/c/text/BUILD new file mode 100644 index 000000000..61fdf5969 --- /dev/null +++ b/mediapipe/tasks/c/text/BUILD @@ -0,0 +1,57 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +TEXT_LIBRARIES = [ + "//mediapipe/tasks/c/text/language_detector:language_detector_lib", + "//mediapipe/tasks/c/text/text_classifier:text_classifier_lib", + "//mediapipe/tasks/c/text/text_embedder:text_embedder_lib", +] + +# bazel build -c opt --linkopt -s --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/text:libtext.so +cc_binary( + name = "libtext.so", + linkopts = [ + "-Wl,-soname=libtext.so", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = TEXT_LIBRARIES, +) + +# bazel build --config darwin_arm64 -c opt --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/text:libtext:.dylib +cc_binary( + name = "libtext.dylib", + linkopts = [ + "-Wl,-install_name,libtext.dylib", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = TEXT_LIBRARIES, +) diff --git a/mediapipe/tasks/c/vision/BUILD b/mediapipe/tasks/c/vision/BUILD new file mode 100644 index 000000000..1b3f6fa19 --- /dev/null +++ b/mediapipe/tasks/c/vision/BUILD @@ -0,0 +1,57 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +VISION_LIBRARIES = [ + "//mediapipe/tasks/c/vision/image_classifier:image_classifier_lib", + "//mediapipe/tasks/c/vision/image_embedder:image_embedder_lib", + "//mediapipe/tasks/c/vision/object_detector:object_detector_lib", +] + +# bazel build -c opt --linkopt -s --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision:libvision.so +cc_binary( + name = "libvision.so", + linkopts = [ + "-Wl,-soname=libvision.so", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = VISION_LIBRARIES, +) + +# bazel build --config darwin_arm64 -c opt --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision:libvision:.dylib +cc_binary( + name = "libvision.dylib", + linkopts = [ + "-Wl,-install_name,libvision.dylib", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = VISION_LIBRARIES, +) From 2b53891a7cf83ca151d7912bc78947b9023c4cf0 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 30 Nov 2023 15:29:19 -0800 Subject: [PATCH 151/157] Add lib targets for all C vision tasks PiperOrigin-RevId: 586805240 --- .../tasks/c/vision/image_classifier/BUILD | 34 +++++++++++++++++++ mediapipe/tasks/c/vision/image_embedder/BUILD | 34 +++++++++++++++++++ .../tasks/c/vision/object_detector/BUILD | 34 +++++++++++++++++++ 3 files changed, 102 insertions(+) diff --git a/mediapipe/tasks/c/vision/image_classifier/BUILD b/mediapipe/tasks/c/vision/image_classifier/BUILD index 08a0801d3..b1930fb0e 100644 --- a/mediapipe/tasks/c/vision/image_classifier/BUILD +++ b/mediapipe/tasks/c/vision/image_classifier/BUILD @@ -64,3 +64,37 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) + +# bazel build -c opt --linkopt -s --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision/image_classifier:libimage_classifier.so +cc_binary( + name = "libimage_classifier.so", + linkopts = [ + "-Wl,-soname=libimage_classifier.so", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [":image_classifier_lib"], +) + +# bazel build --config darwin_arm64 -c opt --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision/image_classifier:libimage_classifier.dylib +cc_binary( + name = "libimage_classifier.dylib", + linkopts = [ + "-Wl,-install_name,libimage_classifier.dylib", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [":image_classifier_lib"], +) diff --git a/mediapipe/tasks/c/vision/image_embedder/BUILD b/mediapipe/tasks/c/vision/image_embedder/BUILD index 4e06e03f9..5d96d90e8 100644 --- a/mediapipe/tasks/c/vision/image_embedder/BUILD +++ b/mediapipe/tasks/c/vision/image_embedder/BUILD @@ -65,3 +65,37 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) + +# bazel build -c opt --linkopt -s --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision/image_embedder:libimage_embedder.so +cc_binary( + name = "libimage_embedder.so", + linkopts = [ + "-Wl,-soname=libimage_embedder.so", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [":image_embedder_lib"], +) + +# bazel build --config darwin_arm64 -c opt --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision/image_embedder:libimage_embedder.dylib +cc_binary( + name = "libimage_embedder.dylib", + linkopts = [ + "-Wl,-install_name,libimage_embedder.dylib", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [":image_embedder_lib"], +) diff --git a/mediapipe/tasks/c/vision/object_detector/BUILD b/mediapipe/tasks/c/vision/object_detector/BUILD index 28bb6fa91..01c6d772d 100644 --- a/mediapipe/tasks/c/vision/object_detector/BUILD +++ b/mediapipe/tasks/c/vision/object_detector/BUILD @@ -63,3 +63,37 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) + +# bazel build -c opt --linkopt -s --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision/object_detector:libobject_detector.so +cc_binary( + name = "libobject_detector.so", + linkopts = [ + "-Wl,-soname=libobject_detector.so", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [":object_detector_lib"], +) + +# bazel build --config darwin_arm64 -c opt --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision/object_detector:libobject_detector.dylib +cc_binary( + name = "libobject_detector.dylib", + linkopts = [ + "-Wl,-install_name,libobject_detector.dylib", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [":object_detector_lib"], +) From 3433ba083a2a24c2b4c661099111b6fa6497fc17 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 30 Nov 2023 15:56:39 -0800 Subject: [PATCH 152/157] Move LanguageDetectorResult converter to LanguageDetector task PiperOrigin-RevId: 586812754 --- mediapipe/tasks/c/components/containers/BUILD | 23 ----------------- .../tasks/c/text/language_detector/BUILD | 25 ++++++++++++++++++- .../language_detector/language_detector.cc | 12 ++++----- .../language_detector_result_converter.cc} | 6 ++--- .../language_detector_result_converter.h} | 10 ++++---- ...anguage_detector_result_converter_test.cc} | 16 ++++++------ 6 files changed, 46 insertions(+), 46 deletions(-) rename mediapipe/tasks/c/{components/containers/language_detection_result_converter.cc => text/language_detector/language_detector_result_converter.cc} (90%) rename mediapipe/tasks/c/{components/containers/language_detection_result_converter.h => text/language_detector/language_detector_result_converter.h} (78%) rename mediapipe/tasks/c/{components/containers/language_detection_result_converter_test.cc => text/language_detector/language_detector_result_converter_test.cc} (77%) diff --git a/mediapipe/tasks/c/components/containers/BUILD b/mediapipe/tasks/c/components/containers/BUILD index a75e8b22f..3c4b557b3 100644 --- a/mediapipe/tasks/c/components/containers/BUILD +++ b/mediapipe/tasks/c/components/containers/BUILD @@ -212,26 +212,3 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) - -cc_library( - name = "language_detection_result_converter", - srcs = ["language_detection_result_converter.cc"], - hdrs = ["language_detection_result_converter.h"], - deps = [ - "//mediapipe/tasks/c/text/language_detector", - "//mediapipe/tasks/cc/text/language_detector", - ], -) - -cc_test( - name = "language_detection_result_converter_test", - srcs = ["language_detection_result_converter_test.cc"], - linkstatic = 1, - deps = [ - ":language_detection_result_converter", - "//mediapipe/framework/port:gtest", - "//mediapipe/tasks/c/text/language_detector", - "//mediapipe/tasks/cc/text/language_detector", - "@com_google_googletest//:gtest_main", - ], -) diff --git a/mediapipe/tasks/c/text/language_detector/BUILD b/mediapipe/tasks/c/text/language_detector/BUILD index 9a3ce21e7..f4a674a35 100644 --- a/mediapipe/tasks/c/text/language_detector/BUILD +++ b/mediapipe/tasks/c/text/language_detector/BUILD @@ -22,7 +22,7 @@ cc_library( hdrs = ["language_detector.h"], visibility = ["//visibility:public"], deps = [ - "//mediapipe/tasks/c/components/containers:language_detection_result_converter", + ":language_detector_result_converter", "//mediapipe/tasks/c/components/processors:classifier_options", "//mediapipe/tasks/c/components/processors:classifier_options_converter", "//mediapipe/tasks/c/core:base_options", @@ -77,6 +77,29 @@ cc_library( ], ) +cc_library( + name = "language_detector_result_converter", + srcs = ["language_detector_result_converter.cc"], + hdrs = ["language_detector_result_converter.h"], + deps = [ + ":language_detector", + "//mediapipe/tasks/cc/text/language_detector", + ], +) + +cc_test( + name = "language_detector_result_converter_test", + srcs = ["language_detector_result_converter_test.cc"], + linkstatic = 1, + deps = [ + ":language_detector", + ":language_detector_result_converter", + "//mediapipe/framework/port:gtest", + "//mediapipe/tasks/cc/text/language_detector", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "language_detector_test", srcs = ["language_detector_test.cc"], diff --git a/mediapipe/tasks/c/text/language_detector/language_detector.cc b/mediapipe/tasks/c/text/language_detector/language_detector.cc index c71433fdc..ec2aba521 100644 --- a/mediapipe/tasks/c/text/language_detector/language_detector.cc +++ b/mediapipe/tasks/c/text/language_detector/language_detector.cc @@ -20,9 +20,9 @@ limitations under the License. #include "absl/log/absl_log.h" #include "absl/status/status.h" -#include "mediapipe/tasks/c/components/containers/language_detection_result_converter.h" #include "mediapipe/tasks/c/components/processors/classifier_options_converter.h" #include "mediapipe/tasks/c/core/base_options_converter.h" +#include "mediapipe/tasks/c/text/language_detector/language_detector_result_converter.h" #include "mediapipe/tasks/cc/text/language_detector/language_detector.h" namespace mediapipe::tasks::c::text::language_detector { @@ -30,9 +30,9 @@ namespace mediapipe::tasks::c::text::language_detector { namespace { using ::mediapipe::tasks::c::components::containers:: - CppCloseLanguageDetectionResult; + CppCloseLanguageDetectorResult; using ::mediapipe::tasks::c::components::containers:: - CppConvertToLanguageDetectionResult; + CppConvertToLanguageDetectorResult; using ::mediapipe::tasks::c::components::processors:: CppConvertToClassifierOptions; using ::mediapipe::tasks::c::core::CppConvertToBaseOptions; @@ -72,16 +72,16 @@ int CppLanguageDetectorDetect(void* detector, const char* utf8_str, auto cpp_detector = static_cast(detector); auto cpp_result = cpp_detector->Detect(utf8_str); if (!cpp_result.ok()) { - ABSL_LOG(ERROR) << "Language Detection failed: " << cpp_result.status(); + ABSL_LOG(ERROR) << "Language Detector failed: " << cpp_result.status(); return CppProcessError(cpp_result.status(), error_msg); } - CppConvertToLanguageDetectionResult(*cpp_result, result); + CppConvertToLanguageDetectorResult(*cpp_result, result); return 0; } void CppLanguageDetectorCloseResult(LanguageDetectorResult* result) { - CppCloseLanguageDetectionResult(result); + CppCloseLanguageDetectorResult(result); } int CppLanguageDetectorClose(void* detector, char** error_msg) { diff --git a/mediapipe/tasks/c/components/containers/language_detection_result_converter.cc b/mediapipe/tasks/c/text/language_detector/language_detector_result_converter.cc similarity index 90% rename from mediapipe/tasks/c/components/containers/language_detection_result_converter.cc rename to mediapipe/tasks/c/text/language_detector/language_detector_result_converter.cc index 89b112e45..435c3d1dc 100644 --- a/mediapipe/tasks/c/components/containers/language_detection_result_converter.cc +++ b/mediapipe/tasks/c/text/language_detector/language_detector_result_converter.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "mediapipe/tasks/c/components/containers/language_detection_result_converter.h" +#include "mediapipe/tasks/c/text/language_detector/language_detector_result_converter.h" #include #include @@ -23,7 +23,7 @@ limitations under the License. namespace mediapipe::tasks::c::components::containers { -void CppConvertToLanguageDetectionResult( +void CppConvertToLanguageDetectorResult( const mediapipe::tasks::text::language_detector::LanguageDetectorResult& in, LanguageDetectorResult* out) { out->predictions_count = in.size(); @@ -42,7 +42,7 @@ void CppConvertToLanguageDetectionResult( } } -void CppCloseLanguageDetectionResult(LanguageDetectorResult* in) { +void CppCloseLanguageDetectorResult(LanguageDetectorResult* in) { for (uint32_t i = 0; i < in->predictions_count; ++i) { auto prediction_in = in->predictions[i]; diff --git a/mediapipe/tasks/c/components/containers/language_detection_result_converter.h b/mediapipe/tasks/c/text/language_detector/language_detector_result_converter.h similarity index 78% rename from mediapipe/tasks/c/components/containers/language_detection_result_converter.h rename to mediapipe/tasks/c/text/language_detector/language_detector_result_converter.h index c9cfd55bd..c8b0c04ed 100644 --- a/mediapipe/tasks/c/components/containers/language_detection_result_converter.h +++ b/mediapipe/tasks/c/text/language_detector/language_detector_result_converter.h @@ -13,20 +13,20 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANGUAGE_DETECTION_RESULT_CONVERTER_H_ -#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANGUAGE_DETECTION_RESULT_CONVERTER_H_ +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANGUAGE_DETECTOR_RESULT_CONVERTER_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANGUAGE_DETECTOR_RESULT_CONVERTER_H_ #include "mediapipe/tasks/c/text/language_detector/language_detector.h" #include "mediapipe/tasks/cc/text/language_detector/language_detector.h" namespace mediapipe::tasks::c::components::containers { -void CppConvertToLanguageDetectionResult( +void CppConvertToLanguageDetectorResult( const mediapipe::tasks::text::language_detector::LanguageDetectorResult& in, LanguageDetectorResult* out); -void CppCloseLanguageDetectionResult(LanguageDetectorResult* in); +void CppCloseLanguageDetectorResult(LanguageDetectorResult* in); } // namespace mediapipe::tasks::c::components::containers -#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANGUAGE_DETECTION_RESULT_CONVERTER_H_ +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_LANGUAGE_DETECTOR_RESULT_CONVERTER_H_ diff --git a/mediapipe/tasks/c/components/containers/language_detection_result_converter_test.cc b/mediapipe/tasks/c/text/language_detector/language_detector_result_converter_test.cc similarity index 77% rename from mediapipe/tasks/c/components/containers/language_detection_result_converter_test.cc rename to mediapipe/tasks/c/text/language_detector/language_detector_result_converter_test.cc index 633b77eae..4c93aa232 100644 --- a/mediapipe/tasks/c/components/containers/language_detection_result_converter_test.cc +++ b/mediapipe/tasks/c/text/language_detector/language_detector_result_converter_test.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "mediapipe/tasks/c/components/containers/language_detection_result_converter.h" +#include "mediapipe/tasks/c/text/language_detector/language_detector_result_converter.h" #include "mediapipe/framework/port/gtest.h" #include "mediapipe/tasks/c/text/language_detector/language_detector.h" @@ -21,8 +21,8 @@ limitations under the License. namespace mediapipe::tasks::c::components::containers { -TEST(LanguageDetectionResultConverterTest, - ConvertsLanguageDetectionResultCustomResult) { +TEST(LanguageDetectorResultConverterTest, + ConvertsLanguageDetectorResultCustomResult) { mediapipe::tasks::text::language_detector::LanguageDetectorResult cpp_detector_result = {{/* language_code= */ "fr", /* probability= */ 0.5}, @@ -30,24 +30,24 @@ TEST(LanguageDetectionResultConverterTest, /* probability= */ 0.5}}; LanguageDetectorResult c_detector_result; - CppConvertToLanguageDetectionResult(cpp_detector_result, &c_detector_result); + CppConvertToLanguageDetectorResult(cpp_detector_result, &c_detector_result); EXPECT_NE(c_detector_result.predictions, nullptr); EXPECT_EQ(c_detector_result.predictions_count, 2); EXPECT_NE(c_detector_result.predictions[0].language_code, "fr"); EXPECT_EQ(c_detector_result.predictions[0].probability, 0.5); - CppCloseLanguageDetectionResult(&c_detector_result); + CppCloseLanguageDetectorResult(&c_detector_result); } -TEST(LanguageDetectionResultConverterTest, FreesMemory) { +TEST(LanguageDetectorResultConverterTest, FreesMemory) { mediapipe::tasks::text::language_detector::LanguageDetectorResult cpp_detector_result = {{"fr", 0.5}}; LanguageDetectorResult c_detector_result; - CppConvertToLanguageDetectionResult(cpp_detector_result, &c_detector_result); + CppConvertToLanguageDetectorResult(cpp_detector_result, &c_detector_result); EXPECT_NE(c_detector_result.predictions, nullptr); - CppCloseLanguageDetectionResult(&c_detector_result); + CppCloseLanguageDetectorResult(&c_detector_result); EXPECT_EQ(c_detector_result.predictions, nullptr); } From 7013b237850849cc6c849636a50233d5e34ebde7 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Thu, 30 Nov 2023 16:01:16 -0800 Subject: [PATCH 153/157] No public description PiperOrigin-RevId: 586813896 --- .../tensors_to_image_calculator.cc | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.cc b/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.cc index 6b67e43aa..1a99cb88c 100644 --- a/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.cc +++ b/mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.cc @@ -33,6 +33,7 @@ #include "mediapipe/framework/port/opencv_imgproc_inc.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/gpu/gpu_origin.pb.h" +#include "mediapipe/gpu/gpu_service.h" #include "mediapipe/tasks/cc/vision/face_stylizer/calculators/tensors_to_image_calculator.pb.h" #if !MEDIAPIPE_DISABLE_GPU @@ -145,7 +146,8 @@ absl::Status TensorsToImageCalculator::UpdateContract(CalculatorContract* cc) { #if MEDIAPIPE_METAL_ENABLED MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); #else - return GlCalculatorHelper::UpdateContract(cc); + return GlCalculatorHelper::UpdateContract(cc, + /*requesst_gpu_as_optional=*/true); #endif // MEDIAPIPE_METAL_ENABLED #endif // !MEDIAPIPE_DISABLE_GPU return absl::OkStatus(); @@ -153,16 +155,7 @@ absl::Status TensorsToImageCalculator::UpdateContract(CalculatorContract* cc) { absl::Status TensorsToImageCalculator::Open(CalculatorContext* cc) { options_ = cc->Options(); - if (CanUseGpu()) { -#if !MEDIAPIPE_DISABLE_GPU -#if MEDIAPIPE_METAL_ENABLED - gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; - RET_CHECK(gpu_helper_); -#else - MP_RETURN_IF_ERROR(gl_helper_.Open(cc)); -#endif // MEDIAPIPE_METAL_ENABLED -#endif // !MEDIAPIPE_DISABLE_GPU - } else { + if (!CanUseGpu()) { ABSL_CHECK(options_.has_input_tensor_float_range() ^ options_.has_input_tensor_uint_range()) << "Must specify either `input_tensor_float_range` or " @@ -179,7 +172,9 @@ absl::Status TensorsToImageCalculator::Process(CalculatorContext* cc) { #if MEDIAPIPE_METAL_ENABLED return MetalProcess(cc); #else - return GlProcess(cc); + if (cc->Service(kGpuService).IsAvailable()) { + return GlProcess(cc); + } #endif // MEDIAPIPE_METAL_ENABLED #endif // !MEDIAPIPE_DISABLE_GPU } @@ -188,14 +183,16 @@ absl::Status TensorsToImageCalculator::Process(CalculatorContext* cc) { absl::Status TensorsToImageCalculator::Close(CalculatorContext* cc) { #if !MEDIAPIPE_DISABLE_GPU && !MEDIAPIPE_METAL_ENABLED - gl_helper_.RunInGlContext([this] { + if (gl_initialized_) { + gl_helper_.RunInGlContext([this] { #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - gl_compute_program_.reset(); + gl_compute_program_.reset(); #else - if (program_) glDeleteProgram(program_); - program_ = 0; + if (program_) glDeleteProgram(program_); + program_ = 0; #endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 - }); + }); + } #endif // !MEDIAPIPE_DISABLE_GPU && !MEDIAPIPE_METAL_ENABLED return absl::OkStatus(); } @@ -315,6 +312,9 @@ absl::Status TensorsToImageCalculator::MetalProcess(CalculatorContext* cc) { } absl::Status TensorsToImageCalculator::MetalSetup(CalculatorContext* cc) { + gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; + RET_CHECK(gpu_helper_); + id device = gpu_helper_.mtlDevice; const std::string shader_source = R"( @@ -450,6 +450,10 @@ absl::Status TensorsToImageCalculator::GlSetup(CalculatorContext* cc) { } absl::Status TensorsToImageCalculator::GlProcess(CalculatorContext* cc) { + if (!gl_initialized_) { + MP_RETURN_IF_ERROR(gl_helper_.Open(cc)); + } + return gl_helper_.RunInGlContext([this, cc]() -> absl::Status { if (!gl_initialized_) { MP_RETURN_IF_ERROR(GlSetup(cc)); From a0eda45baf82d2f4823571b786b507ca6b57f9b8 Mon Sep 17 00:00:00 2001 From: Youchuan Hu Date: Thu, 30 Nov 2023 16:14:31 -0800 Subject: [PATCH 154/157] Add TensorsToSegmentationCalculator test utilities. PiperOrigin-RevId: 586817713 --- mediapipe/calculators/tensor/BUILD | 29 ++++- ...tensors_to_segmentation_calculator_test.cc | 56 +-------- ...s_to_segmentation_calculator_test_utils.cc | 111 ++++++++++++++++++ ...rs_to_segmentation_calculator_test_utils.h | 57 +++++++++ ...segmentation_calculator_test_utils_test.cc | 50 ++++++++ 5 files changed, 249 insertions(+), 54 deletions(-) create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.cc create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h create mode 100644 mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils_test.cc diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index 96a29089e..618624430 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -1555,12 +1555,37 @@ cc_library( ], ) +cc_library( + name = "tensors_to_segmentation_calculator_test_utils", + testonly = 1, + srcs = ["tensors_to_segmentation_calculator_test_utils.cc"], + hdrs = ["tensors_to_segmentation_calculator_test_utils.h"], + deps = [ + ":tensors_to_segmentation_calculator_cc_proto", + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/framework/port:parse_text_proto", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/strings", + ], +) + +cc_test( + name = "tensors_to_segmentation_calculator_test_utils_test", + srcs = ["tensors_to_segmentation_calculator_test_utils_test.cc"], + deps = [ + ":tensors_to_segmentation_calculator_cc_proto", + ":tensors_to_segmentation_calculator_test_utils", + "//mediapipe/framework/port:gtest_main", + ], +) + cc_test( name = "tensors_to_segmentation_calculator_test", srcs = ["tensors_to_segmentation_calculator_test.cc"], deps = [ ":tensors_to_segmentation_calculator", ":tensors_to_segmentation_calculator_cc_proto", + ":tensors_to_segmentation_calculator_test_utils", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", "//mediapipe/framework:packet", @@ -1571,10 +1596,6 @@ cc_test( "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/framework/formats:tensor", "//mediapipe/framework/port:gtest_main", - "//mediapipe/framework/port:parse_text_proto", - "@com_google_absl//absl/log", - "@com_google_absl//absl/log:absl_log", - "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc index e5c6b8ade..9ac63f31a 100644 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test.cc @@ -17,10 +17,8 @@ #include #include -#include "absl/log/absl_log.h" -#include "absl/log/log.h" -#include "absl/strings/substitute.h" #include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/formats/image.h" @@ -30,7 +28,6 @@ #include "mediapipe/framework/formats/tensor.h" #include "mediapipe/framework/packet.h" #include "mediapipe/framework/port/gtest.h" -#include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/status_matchers.h" #include "mediapipe/framework/timestamp.h" @@ -40,58 +37,17 @@ namespace { using ::testing::SizeIs; using ::testing::TestWithParam; using Options = mediapipe::TensorsToSegmentationCalculatorOptions; +namespace test_utils = ::mediapipe::tensors_to_segmentation_utils; -std::string ActivationTypeToString(Options::Activation activation) { - switch (activation) { - case Options::NONE: - return "NONE"; - case Options::SIGMOID: - return "SIGMOID"; - case Options::SOFTMAX: - return "SOFTMAX"; - default: - ABSL_LOG(FATAL) << "Unknown activation type: " << activation; - return "UNKNOWN"; - } -} - -struct FormattingTestCase { - std::string test_name; - std::vector inputs; - std::vector expected_outputs; - Options::Activation activation; - int rows = 1; - int cols = 1; - int rows_new = 1; - int cols_new = 1; - int channels = 1; - double max_abs_diff = 1e-7; -}; - -using TensorsToSegmentationCalculatorTest = TestWithParam; +using TensorsToSegmentationCalculatorTest = + TestWithParam; TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) { const auto& [test_name, inputs, expected_outputs, activation, rows, cols, rows_new, cols_new, channels, max_abs_diff] = GetParam(); auto graph_config = - mediapipe::ParseTextProtoOrDie(absl::Substitute( - R"pb( - input_stream: "tensors" - input_stream: "size" - node { - calculator: "TensorsToSegmentationCalculator" - input_stream: "TENSORS:tensors" - input_stream: "OUTPUT_SIZE:size" - output_stream: "MASK:image_as_mask" - options: { - [mediapipe.TensorsToSegmentationCalculatorOptions.ext] { - activation: $0 - } - } - } - )pb", - ActivationTypeToString(activation))); + test_utils::CreateGraphConfigForTest(/*test_gpu=*/false, activation); std::vector output_packets; tool::AddVectorSink("image_as_mask", &graph_config, &output_packets); @@ -151,7 +107,7 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) { INSTANTIATE_TEST_SUITE_P( TensorsToSegmentationCalculatorTests, TensorsToSegmentationCalculatorTest, - testing::ValuesIn({ + testing::ValuesIn({ {.test_name = "NoActivationAndNoOutputResize", .inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0}, diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.cc new file mode 100644 index 000000000..2fc9019c2 --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.cc @@ -0,0 +1,111 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h" + +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/strings/substitute.h" +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/port/parse_text_proto.h" + +namespace mediapipe { +namespace tensors_to_segmentation_utils { + +std::string ActivationTypeToString( + const TensorsToSegmentationCalculatorOptions::Activation& activation) { + switch (activation) { + case TensorsToSegmentationCalculatorOptions::NONE: + return "NONE"; + case TensorsToSegmentationCalculatorOptions::SIGMOID: + return "SIGMOID"; + case TensorsToSegmentationCalculatorOptions::SOFTMAX: + return "SOFTMAX"; + } + ABSL_LOG(FATAL) << "Unknown activation type: " << activation; + return "UNKNOWN"; +} + +std::vector ArrayFloatToUnsignedChar( + const std::vector& array) { + std::vector result; + result.reserve(array.size()); + for (int i = 0; i < array.size(); ++i) { + result.push_back(static_cast(array[i])); + } + return result; +} + +std::vector MakeRedAlphaMatrix(const std::vector& values) { + std::vector result; + result.reserve(values.size() * 4); + for (const float& value : values) { + result.push_back(value); + result.push_back(0); + result.push_back(0); + result.push_back(value); + } + return result; +} + +// For GPU tests, the input tensor needs to be moved to GPU, using +// TensorViewRequestor. After calculation, the output needs to be moved back +// to CPU, using ToImageCalculator. The output is an ImageFrame. +mediapipe::CalculatorGraphConfig CreateGraphConfigForTest( + bool test_gpu, + const TensorsToSegmentationCalculatorOptions::Activation& activation) { + std::string pre_process = R"pb( + node { + calculator: "mediapipe.aimatter.TensorViewRequestor" + input_stream: "TENSORS:tensors" + output_stream: "TENSORS:tensors_gpu" + options { + [mediapipe.aimatter.TensorViewRequestorOptions.ext] { gpu {} } + } + } + )pb"; + std::string post_process = R"pb( + node { + calculator: "FromImageCalculator" + input_stream: "IMAGE:image_as_mask_gpu" + output_stream: "IMAGE_CPU:image_as_mask" + } + )pb"; + return mediapipe::ParseTextProtoOrDie( + absl::Substitute( + R"pb( + input_stream: "tensors" + input_stream: "size" $0 + node { + calculator: "TensorsToSegmentationCalculator" + input_stream: "TENSORS:tensors$1" + input_stream: "OUTPUT_SIZE:size" + output_stream: "MASK:image_as_mask$2" + options: { + [mediapipe.TensorsToSegmentationCalculatorOptions.ext] { + activation: $3 + gpu_origin: TOP_LEFT + } + } + } $4 + )pb", + test_gpu ? pre_process : "", test_gpu ? "_gpu" : "", + test_gpu ? "_gpu" : "", ActivationTypeToString(activation), + test_gpu ? post_process : "")); +} +} // namespace tensors_to_segmentation_utils +} // namespace mediapipe diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h new file mode 100644 index 000000000..abeda546b --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h @@ -0,0 +1,57 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_ +#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_ + +#include +#include +#include +#include + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/framework/calculator.pb.h" + +namespace mediapipe { +namespace tensors_to_segmentation_utils { +std::string ActivationTypeToString( + const mediapipe::TensorsToSegmentationCalculatorOptions::Activation& + activation); + +std::vector ArrayFloatToUnsignedChar( + const std::vector& array); + +std::vector MakeRedAlphaMatrix(const std::vector& values); + +mediapipe::CalculatorGraphConfig CreateGraphConfigForTest( + bool test_gpu, + const mediapipe::TensorsToSegmentationCalculatorOptions::Activation& + activation); + +struct FormattingTestCase { + std::string test_name; + std::vector inputs; + std::vector expected_outputs; + mediapipe::TensorsToSegmentationCalculatorOptions::Activation activation; + int rows = 1; + int cols = 1; + int rows_new = 1; + int cols_new = 1; + int channels = 1; + double max_abs_diff = 1e-7; +}; +} // namespace tensors_to_segmentation_utils +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_ diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils_test.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils_test.cc new file mode 100644 index 000000000..3f048c62d --- /dev/null +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils_test.cc @@ -0,0 +1,50 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h" + +#include + +#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h" +#include "mediapipe/framework/port/gtest.h" + +namespace mediapipe::tensors_to_segmentation_utils { +namespace { + +using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions; + +TEST(TensorsToSegmentationCalculatorTestUtilsTest, + ActivationTypeToStringWorksCorrectly) { + EXPECT_EQ(ActivationTypeToString(Options::NONE), "NONE"); + EXPECT_EQ(ActivationTypeToString(Options::SIGMOID), "SIGMOID"); + EXPECT_EQ(ActivationTypeToString(Options::SOFTMAX), "SOFTMAX"); +} + +TEST(TensorsToSegmentationCalculatorTestUtilsTest, + ArrayFloatToUnsignedCharWorksCorrectly) { + std::vector input = {1.0, 2.0, 3.0}; + std::vector expected = {1, 2, 3}; + EXPECT_EQ(ArrayFloatToUnsignedChar(input), expected); +} + +TEST(TensorsToSegmentationCalculatorTestUtilsTest, + MakeRedAlphaMatrixWorksCorrectly) { + std::vector input = {1.0, 2.0, 3.0}; + std::vector expected = {1.0, 0.0, 0.0, 1.0, 2.0, 0.0, + 0.0, 2.0, 3.0, 0.0, 0.0, 3.0}; + EXPECT_EQ(MakeRedAlphaMatrix(input), expected); +} + +} // namespace +} // namespace mediapipe::tensors_to_segmentation_utils From f35ecb6c8b804c0c5843ce91bbee2f224df55ea9 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Fri, 1 Dec 2023 12:17:40 -0800 Subject: [PATCH 155/157] Add dependency on hand_roi_refinement_graph_options_proto PiperOrigin-RevId: 587082550 --- .../java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl | 1 + 1 file changed, 1 insertion(+) diff --git a/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl b/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl index e63695e31..f2e4d485f 100644 --- a/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl +++ b/mediapipe/tasks/java/com/google/mediapipe/tasks/mediapipe_tasks_aar.bzl @@ -85,6 +85,7 @@ _VISION_TASKS_IMAGE_GENERATOR_JAVA_PROTO_LITE_TARGETS = [ "//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_java_proto_lite", "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_java_proto_lite", + "//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_roi_refinement_graph_options_proto", ] _TEXT_TASKS_JAVA_PROTO_LITE_TARGETS = [ From 507d677d44e7f16be867ec0b3cb0e4252c49a43a Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Sat, 2 Dec 2023 09:40:19 -0800 Subject: [PATCH 156/157] Internal change PiperOrigin-RevId: 587325154 --- .../components/GlSurfaceViewRenderer.java | 63 ++++++------------- 1 file changed, 20 insertions(+), 43 deletions(-) diff --git a/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java index 2ecb9d1e6..e063e7ef0 100644 --- a/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java +++ b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java @@ -17,7 +17,6 @@ package com.google.mediapipe.components; import static java.lang.Math.max; import static java.lang.Math.min; -import android.graphics.Bitmap; import android.graphics.SurfaceTexture; import android.opengl.GLES11Ext; import android.opengl.GLES20; @@ -52,11 +51,9 @@ import javax.microedition.khronos.opengles.GL10; * {@link TextureFrame} (call {@link #setNextFrame(TextureFrame)}). */ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { - /** - * Listener for Bitmap capture requests. - */ - public interface BitmapCaptureListener { - void onBitmapCaptured(Bitmap result); + /** Listener for image capture requests. */ + public interface ImageCaptureListener { + void onImageCaptured(int width, int height, int[] data); } /** @@ -87,8 +84,8 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { private float[] textureTransformMatrix = new float[16]; private SurfaceTexture surfaceTexture = null; private final AtomicReference nextFrame = new AtomicReference<>(); - private final AtomicBoolean captureNextFrameBitmap = new AtomicBoolean(); - private BitmapCaptureListener bitmapCaptureListener; + private final AtomicBoolean captureNextFrameImage = new AtomicBoolean(); + private ImageCaptureListener imageCaptureListener; // Specifies whether a black CLAMP_TO_BORDER effect should be used. private boolean shouldClampToBorder = false; private Scale scale = Scale.FILL; @@ -96,21 +93,19 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { private float zoomFactor = 1.0f; private Pair zoomLocation = new Pair<>(0.5f, 0.5f); - /** - * Sets the {@link BitmapCaptureListener}. - */ - public void setBitmapCaptureListener(BitmapCaptureListener bitmapCaptureListener) { - this.bitmapCaptureListener = bitmapCaptureListener; + /** Sets the {@link ImageCaptureListener}. */ + public void setImageCaptureListener(ImageCaptureListener imageCaptureListener) { + this.imageCaptureListener = imageCaptureListener; } /** - * Request to capture Bitmap of the next frame. + * Request to capture Image of the next frame. * - * The result will be provided to the {@link BitmapCaptureListener} if one is set. Please note + *

The result will be provided to the {@link ImageCaptureListener} if one is set. Please note * this is an expensive operation and the result may not be available for a while. */ - public void captureNextFrameBitmap() { - captureNextFrameBitmap.set(true); + public void captureNextFrameImage() { + captureNextFrameImage.set(true); } @Override @@ -206,9 +201,9 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); ShaderUtil.checkGlError("glDrawArrays"); - // Capture Bitmap if requested. - BitmapCaptureListener bitmapCaptureListener = this.bitmapCaptureListener; - if (captureNextFrameBitmap.getAndSet(false) && bitmapCaptureListener != null) { + // Capture image if requested. + ImageCaptureListener imageCaptureListener = this.imageCaptureListener; + if (captureNextFrameImage.getAndSet(false) && imageCaptureListener != null) { // Find the name of the bound texture. int[] texName = new int[1]; if (surfaceTexture != null) { @@ -223,8 +218,8 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { GLES31.glGetTexLevelParameteriv(textureTarget, 0, GLES31.GL_TEXTURE_HEIGHT, texDims, 1); int texWidth = texDims[0]; int texHeight = texDims[1]; - int bitmapSize = texWidth * texHeight; - ByteBuffer byteBuffer = ByteBuffer.allocateDirect(bitmapSize * 4); + int imageSize = texWidth * texHeight; + ByteBuffer byteBuffer = ByteBuffer.allocateDirect(imageSize * 4); byteBuffer.order(ByteOrder.nativeOrder()); // Read pixels from texture. @@ -239,27 +234,9 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer { GLES20.glDeleteFramebuffers(1, fbo, 0); ShaderUtil.checkGlError("capture frame"); - int[] pixelBuffer = new int[bitmapSize]; - byteBuffer.asIntBuffer().get(pixelBuffer); - for (int i = 0; i < bitmapSize; i++) { - // Swap R and B channels. - pixelBuffer[i] = - (pixelBuffer[i] & 0xff00ff00) - | ((pixelBuffer[i] & 0x000000ff) << 16) - | ((pixelBuffer[i] & 0x00ff0000) >> 16); - } - - // Send bitmap. - Bitmap bitmap = Bitmap.createBitmap(texWidth, texHeight, Bitmap.Config.ARGB_8888); - bitmap.setPixels( - pixelBuffer, - /* offset= */ bitmapSize - texWidth, - /* stride= */ -texWidth, - /* x= */ 0, - /* y= */ 0, - texWidth, - texHeight); - bitmapCaptureListener.onBitmapCaptured(bitmap); + int[] data = new int[imageSize]; + byteBuffer.asIntBuffer().get(data); + imageCaptureListener.onImageCaptured(texWidth, texHeight, data); } GLES20.glBindTexture(textureTarget, 0); From 3d8b715dd64dcd1ccb888986a36765b00ad6252b Mon Sep 17 00:00:00 2001 From: Dmitri Gribenko Date: Sun, 3 Dec 2023 17:50:51 -0800 Subject: [PATCH 157/157] No public description PiperOrigin-RevId: 587559637 --- .../calculators/tensorflow/tensor_to_vector_int_calculator.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator.cc b/mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator.cc index 7adb26daa..b4fcf6a01 100644 --- a/mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator.cc @@ -15,9 +15,9 @@ // Calculator converts from one-dimensional Tensor of DT_FLOAT to vector // OR from (batched) two-dimensional Tensor of DT_FLOAT to vector. +#include #include -#include "absl/base/integral_types.h" #include "mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/status.h"