diff --git a/README.md b/README.md index 012ea3a27..a82c88ab1 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,20 @@ nav_order: 1 ![MediaPipe](https://mediapipe.dev/images/mediapipe_small.png) +---- + +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +*This notice and web page will be removed on June 1, 2023.* + +---- + +









+









+









+ -------------------------------------------------------------------------------- ## Live ML anywhere @@ -21,15 +35,6 @@ ML solutions for live and streaming media. ---- -**Attention:** *Thanks for your interest in MediaPipe! We are moving to -[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) -as the primary developer documentation -site for MediaPipe starting April 3, 2023.* - -*This notice and web page will be removed on April 3, 2023.* - ----- - ## ML solutions in MediaPipe Face Detection | Face Mesh | Iris | Hands | Pose | Holistic diff --git a/docs/framework_concepts/building_graphs_cpp.md b/docs/framework_concepts/building_graphs_cpp.md index 250cd89e2..26cdfe1e4 100644 --- a/docs/framework_concepts/building_graphs_cpp.md +++ b/docs/framework_concepts/building_graphs_cpp.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/framework/framework_concepts/graphs_cpp title: Building Graphs in C++ parent: Graphs nav_order: 1 @@ -12,6 +13,12 @@ nav_order: 1 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + C++ graph builder is a powerful tool for: * Building complex graphs diff --git a/docs/framework_concepts/calculators.md b/docs/framework_concepts/calculators.md index 5c51a3ec5..3a3661dd4 100644 --- a/docs/framework_concepts/calculators.md +++ b/docs/framework_concepts/calculators.md @@ -13,6 +13,12 @@ nav_order: 1 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + Each calculator is a node of a graph. We describe how to create a new calculator, how to initialize a calculator, how to perform its calculations, input and output streams, timestamps, and options. Each node in the graph is diff --git a/docs/framework_concepts/framework_concepts.md b/docs/framework_concepts/framework_concepts.md index 5d953480e..004c75cff 100644 --- a/docs/framework_concepts/framework_concepts.md +++ b/docs/framework_concepts/framework_concepts.md @@ -14,6 +14,12 @@ has_toc: false {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ## The basics ### Packet diff --git a/docs/framework_concepts/gpu.md b/docs/framework_concepts/gpu.md index 3c411d55c..8900ab3b4 100644 --- a/docs/framework_concepts/gpu.md +++ b/docs/framework_concepts/gpu.md @@ -13,6 +13,12 @@ nav_order: 5 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ## Overview MediaPipe supports calculator nodes for GPU compute and rendering, and allows combining multiple GPU nodes, as well as mixing them with CPU based calculator nodes. There exist several GPU APIs on mobile platforms (eg, OpenGL ES, Metal and Vulkan). MediaPipe does not attempt to offer a single cross-API GPU abstraction. Individual nodes can be written using different APIs, allowing them to take advantage of platform specific features when needed. diff --git a/docs/framework_concepts/graphs.md b/docs/framework_concepts/graphs.md index 0d38c75fc..5f9c68e08 100644 --- a/docs/framework_concepts/graphs.md +++ b/docs/framework_concepts/graphs.md @@ -13,6 +13,12 @@ nav_order: 2 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ## Graph A `CalculatorGraphConfig` proto specifies the topology and functionality of a diff --git a/docs/framework_concepts/packets.md b/docs/framework_concepts/packets.md index 100bc6b01..1bfad376d 100644 --- a/docs/framework_concepts/packets.md +++ b/docs/framework_concepts/packets.md @@ -13,6 +13,12 @@ nav_order: 3 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + Calculators communicate by sending and receiving packets. Typically a single packet is sent along each input stream at each input timestamp. A packet can contain any kind of data, such as a single frame of video or a single integer diff --git a/docs/framework_concepts/realtime_streams.md b/docs/framework_concepts/realtime_streams.md index 43d147f55..60f586cb9 100644 --- a/docs/framework_concepts/realtime_streams.md +++ b/docs/framework_concepts/realtime_streams.md @@ -13,6 +13,12 @@ nav_order: 6 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ## Real-time timestamps MediaPipe calculator graphs are often used to process streams of video or audio diff --git a/docs/framework_concepts/synchronization.md b/docs/framework_concepts/synchronization.md index e12d077a7..8a0a907c5 100644 --- a/docs/framework_concepts/synchronization.md +++ b/docs/framework_concepts/synchronization.md @@ -13,6 +13,12 @@ nav_order: 4 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ## Scheduling mechanics Data processing in a MediaPipe graph occurs inside processing nodes defined as diff --git a/docs/getting_started/android.md b/docs/getting_started/android.md index cb99a6fef..83fbd1c93 100644 --- a/docs/getting_started/android.md +++ b/docs/getting_started/android.md @@ -15,6 +15,12 @@ nav_order: 1 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + Please follow instructions below to build Android example apps in the supported MediaPipe [solutions](../solutions/solutions.md). To learn more about these example apps, start from [Hello World! on Android](./hello_world_android.md). diff --git a/docs/getting_started/android_archive_library.md b/docs/getting_started/android_archive_library.md index 9b92ef498..7d98b32c5 100644 --- a/docs/getting_started/android_archive_library.md +++ b/docs/getting_started/android_archive_library.md @@ -14,6 +14,12 @@ nav_order: 3 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ***Experimental Only*** The MediaPipe Android Archive (AAR) library is a convenient way to use MediaPipe diff --git a/docs/getting_started/android_solutions.md b/docs/getting_started/android_solutions.md index 0c492c1bb..159d1358d 100644 --- a/docs/getting_started/android_solutions.md +++ b/docs/getting_started/android_solutions.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/ title: MediaPipe Android Solutions parent: MediaPipe on Android grand_parent: Getting Started @@ -13,14 +14,9 @@ nav_order: 2 {:toc} --- -**Attention:** *Thanks for your interest in MediaPipe! We are moving to +**Attention:** *Thanks for your interest in MediaPipe! We have moved to [https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) -as the primary developer documentation -site for MediaPipe starting April 3, 2023. This content will not be moved to -the new site, but will remain available in the source code repository on an -as-is basis.* - -*This notice and web page will be removed on April 3, 2023.* +as the primary developer documentation site for MediaPipe as of April 3, 2023.* ---- diff --git a/docs/getting_started/building_examples.md b/docs/getting_started/building_examples.md index 20c30bef2..a77f6ea66 100644 --- a/docs/getting_started/building_examples.md +++ b/docs/getting_started/building_examples.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/ title: Building MediaPipe Examples parent: Getting Started nav_exclude: true @@ -12,14 +13,9 @@ nav_exclude: true {:toc} --- -**Attention:** *Thanks for your interest in MediaPipe! We are moving to +**Attention:** *Thanks for your interest in MediaPipe! We have moved to [https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) -as the primary developer documentation -site for MediaPipe starting April 3, 2023. This content will not be moved to -the new site, but will remain available in the source code repository on an -as-is basis.* - -*This notice and web page will be removed on April 3, 2023.* +as the primary developer documentation site for MediaPipe as of April 3, 2023.* ---- diff --git a/docs/getting_started/cpp.md b/docs/getting_started/cpp.md index 3afde767f..d708866a7 100644 --- a/docs/getting_started/cpp.md +++ b/docs/getting_started/cpp.md @@ -15,6 +15,12 @@ nav_order: 5 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + Please follow instructions below to build C++ command-line example apps in the supported MediaPipe [solutions](../solutions/solutions.md). To learn more about these example apps, start from [Hello World! in C++](./hello_world_cpp.md). diff --git a/docs/getting_started/faq.md b/docs/getting_started/faq.md index b7c24e6ec..ca50ae530 100644 --- a/docs/getting_started/faq.md +++ b/docs/getting_started/faq.md @@ -13,6 +13,12 @@ nav_order: 9 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ### How to convert ImageFrames and GpuBuffers The Calculators [`ImageFrameToGpuBufferCalculator`] and diff --git a/docs/getting_started/getting_started.md b/docs/getting_started/getting_started.md index fea9cfa73..db605b4b4 100644 --- a/docs/getting_started/getting_started.md +++ b/docs/getting_started/getting_started.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/ title: Getting Started nav_order: 2 has_children: true @@ -12,13 +13,8 @@ has_children: true {:toc} --- -**Attention:** *Thanks for your interest in MediaPipe! We are moving to +**Attention:** *Thanks for your interest in MediaPipe! We have moved to [https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) -as the primary developer documentation -site for MediaPipe starting April 3, 2023. This content will not be moved to -the new site, but will remain available in the source code repository on an -as-is basis.* - -*This notice and web page will be removed on April 3, 2023.* +as the primary developer documentation site for MediaPipe as of April 3, 2023.* ---- diff --git a/docs/getting_started/gpu_support.md b/docs/getting_started/gpu_support.md index 4bd1efeb8..6c0e8be0f 100644 --- a/docs/getting_started/gpu_support.md +++ b/docs/getting_started/gpu_support.md @@ -13,6 +13,12 @@ nav_order: 7 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ## OpenGL ES Support MediaPipe supports OpenGL ES up to version 3.2 on Android/Linux and up to ES 3.0 diff --git a/docs/getting_started/hello_world_android.md b/docs/getting_started/hello_world_android.md index 012743048..1148ff5a9 100644 --- a/docs/getting_started/hello_world_android.md +++ b/docs/getting_started/hello_world_android.md @@ -14,6 +14,12 @@ nav_order: 1 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ## Introduction This codelab uses MediaPipe on an Android device. diff --git a/docs/getting_started/hello_world_cpp.md b/docs/getting_started/hello_world_cpp.md index 880248725..7c8f9be3e 100644 --- a/docs/getting_started/hello_world_cpp.md +++ b/docs/getting_started/hello_world_cpp.md @@ -14,6 +14,12 @@ nav_order: 1 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + 1. Ensure you have a working version of MediaPipe. See [installation instructions](./install.md). diff --git a/docs/getting_started/hello_world_ios.md b/docs/getting_started/hello_world_ios.md index 713dbc79a..4be097646 100644 --- a/docs/getting_started/hello_world_ios.md +++ b/docs/getting_started/hello_world_ios.md @@ -14,6 +14,12 @@ nav_order: 1 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ## Introduction This codelab uses MediaPipe on an iOS device. diff --git a/docs/getting_started/help.md b/docs/getting_started/help.md index 3ba052741..2cb6b9e68 100644 --- a/docs/getting_started/help.md +++ b/docs/getting_started/help.md @@ -13,6 +13,12 @@ nav_order: 8 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ## Technical questions For help with technical or algorithmic questions, visit diff --git a/docs/getting_started/install.md b/docs/getting_started/install.md index 05f291e5c..b30284779 100644 --- a/docs/getting_started/install.md +++ b/docs/getting_started/install.md @@ -13,6 +13,12 @@ nav_order: 6 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + Note: To interoperate with OpenCV, OpenCV 3.x to 4.1 are preferred. OpenCV 2.x currently works but interoperability support may be deprecated in the future. diff --git a/docs/getting_started/ios.md b/docs/getting_started/ios.md index c4b8fb99e..798017aa3 100644 --- a/docs/getting_started/ios.md +++ b/docs/getting_started/ios.md @@ -15,6 +15,12 @@ nav_order: 2 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + Please follow instructions below to build iOS example apps in the supported MediaPipe [solutions](../solutions/solutions.md). To learn more about these example apps, start from, start from diff --git a/docs/getting_started/javascript.md b/docs/getting_started/javascript.md index 79269827b..e68d40917 100644 --- a/docs/getting_started/javascript.md +++ b/docs/getting_started/javascript.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/ title: MediaPipe in JavaScript parent: Getting Started nav_order: 4 @@ -14,12 +15,7 @@ nav_order: 4 **Attention:** *Thanks for your interest in MediaPipe! We are moving to [https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) -as the primary developer documentation -site for MediaPipe starting April 3, 2023. This content will not be moved to -the new site, but will remain available in the source code repository on an -as-is basis.* - -*This notice and web page will be removed on April 3, 2023.* +as the primary developer documentation site for MediaPipe starting April 3, 2023.* ---- diff --git a/docs/getting_started/python.md b/docs/getting_started/python.md index 880d5c85d..43f452a50 100644 --- a/docs/getting_started/python.md +++ b/docs/getting_started/python.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/ title: MediaPipe in Python parent: Getting Started has_children: true @@ -14,6 +15,12 @@ nav_order: 3 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ## Ready-to-use Python Solutions MediaPipe offers ready-to-use yet customizable Python solutions as a prebuilt diff --git a/docs/getting_started/python_framework.md b/docs/getting_started/python_framework.md index db5bc0cd4..6d4b7d450 100644 --- a/docs/getting_started/python_framework.md +++ b/docs/getting_started/python_framework.md @@ -12,6 +12,11 @@ nav_order: 1 1. TOC {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- The MediaPipe Python framework grants direct access to the core components of the MediaPipe C++ framework such as Timestamp, Packet, and CalculatorGraph, diff --git a/docs/getting_started/troubleshooting.md b/docs/getting_started/troubleshooting.md index 0da25497d..e7dff332c 100644 --- a/docs/getting_started/troubleshooting.md +++ b/docs/getting_started/troubleshooting.md @@ -13,6 +13,12 @@ nav_order: 10 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + ## Missing Python binary path The error message: diff --git a/docs/index.md b/docs/index.md index 012ea3a27..a82c88ab1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,6 +6,20 @@ nav_order: 1 ![MediaPipe](https://mediapipe.dev/images/mediapipe_small.png) +---- + +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +*This notice and web page will be removed on June 1, 2023.* + +---- + +









+









+









+ -------------------------------------------------------------------------------- ## Live ML anywhere @@ -21,15 +35,6 @@ ML solutions for live and streaming media. ---- -**Attention:** *Thanks for your interest in MediaPipe! We are moving to -[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) -as the primary developer documentation -site for MediaPipe starting April 3, 2023.* - -*This notice and web page will be removed on April 3, 2023.* - ----- - ## ML solutions in MediaPipe Face Detection | Face Mesh | Iris | Hands | Pose | Holistic diff --git a/docs/index.rst b/docs/index.rst index 4563284bd..fc7a6f50f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,3 +1,3 @@ MediaPipe ===================================== -Please see https://docs.mediapipe.dev. +Please see https://developers.google.com/mediapipe/ diff --git a/docs/solutions/autoflip.md b/docs/solutions/autoflip.md index d0a763436..9c5ca8766 100644 --- a/docs/solutions/autoflip.md +++ b/docs/solutions/autoflip.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/guide#legacy title: AutoFlip (Saliency-aware Video Cropping) parent: Solutions nav_order: 14 @@ -20,12 +21,10 @@ nav_order: 14 **Attention:** *Thank you for your interest in MediaPipe Solutions. We have ended support for this MediaPipe Legacy Solution as of March 1, 2023. -For more information, see the new +For more information, see the [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/box_tracking.md b/docs/solutions/box_tracking.md index 4fecc5150..944059b80 100644 --- a/docs/solutions/box_tracking.md +++ b/docs/solutions/box_tracking.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/guide#legacy title: Box Tracking parent: Solutions nav_order: 10 @@ -20,12 +21,10 @@ nav_order: 10 **Attention:** *Thank you for your interest in MediaPipe Solutions. We have ended support for this MediaPipe Legacy Solution as of March 1, 2023. -For more information, see the new +For more information, see the [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/face_detection.md b/docs/solutions/face_detection.md index 789d9b3dd..3966fab9d 100644 --- a/docs/solutions/face_detection.md +++ b/docs/solutions/face_detection.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/vision/face_detector/ title: Face Detection parent: Solutions nav_order: 1 @@ -20,12 +21,10 @@ nav_order: 1 **Attention:** *Thank you for your interest in MediaPipe Solutions. As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe -Solution. For more information, see the new +Solution. For more information, see the [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/face_mesh.md b/docs/solutions/face_mesh.md index 84fbb22a5..406e405b5 100644 --- a/docs/solutions/face_mesh.md +++ b/docs/solutions/face_mesh.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/vision/face_landmarker/ title: Face Mesh parent: Solutions nav_order: 2 @@ -20,12 +21,10 @@ nav_order: 2 **Attention:** *Thank you for your interest in MediaPipe Solutions. As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe -Solution. For more information, see the new +Solution. For more information, see the [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/hair_segmentation.md b/docs/solutions/hair_segmentation.md index 481cd0058..a59bb93b8 100644 --- a/docs/solutions/hair_segmentation.md +++ b/docs/solutions/hair_segmentation.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/vision/image_segmenter/ title: Hair Segmentation parent: Solutions nav_order: 8 @@ -19,13 +20,11 @@ nav_order: 8 --- **Attention:** *Thank you for your interest in MediaPipe Solutions. -As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe -Solution. For more information, see the new -[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) +As of April 4, 2023, this solution was upgraded to a new MediaPipe +Solution. For more information, see the +[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/image_segmenter/) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ![hair_segmentation_android_gpu_gif](https://mediapipe.dev/images/mobile/hair_segmentation_android_gpu.gif) diff --git a/docs/solutions/hands.md b/docs/solutions/hands.md index a4cd90baa..280677f0f 100644 --- a/docs/solutions/hands.md +++ b/docs/solutions/hands.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/vision/hand_landmarker title: Hands parent: Solutions nav_order: 4 @@ -19,13 +20,11 @@ nav_order: 4 --- **Attention:** *Thank you for your interest in MediaPipe Solutions. -As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe -Solution. For more information, see the new -[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) +As of March 1, 2023, this solution was upgraded to a new MediaPipe +Solution. For more information, see the +[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/hand_landmarker) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/holistic.md b/docs/solutions/holistic.md index 876a88572..70e6b5aff 100644 --- a/docs/solutions/holistic.md +++ b/docs/solutions/holistic.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://github.com/google/mediapipe/blob/master/docs/solutions/holistic.md title: Holistic parent: Solutions nav_order: 6 @@ -20,12 +21,10 @@ nav_order: 6 **Attention:** *Thank you for your interest in MediaPipe Solutions. As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe -Solution. For more information, see the new +Solution. For more information, see the [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/instant_motion_tracking.md b/docs/solutions/instant_motion_tracking.md index 1e714bdc8..76e36d12e 100644 --- a/docs/solutions/instant_motion_tracking.md +++ b/docs/solutions/instant_motion_tracking.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/guide#legacy title: Instant Motion Tracking parent: Solutions nav_order: 11 @@ -20,12 +21,10 @@ nav_order: 11 **Attention:** *Thank you for your interest in MediaPipe Solutions. We have ended support for this MediaPipe Legacy Solution as of March 1, 2023. -For more information, see the new +For more information, see the [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/iris.md b/docs/solutions/iris.md index b8459a0e3..1f5486afd 100644 --- a/docs/solutions/iris.md +++ b/docs/solutions/iris.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/vision/face_landmarker/ title: Iris parent: Solutions nav_order: 3 @@ -20,12 +21,10 @@ nav_order: 3 **Attention:** *Thank you for your interest in MediaPipe Solutions. As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe -Solution. For more information, see the new +Solution. For more information, see the [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/knift.md b/docs/solutions/knift.md index ad5d39f22..50bc1df62 100644 --- a/docs/solutions/knift.md +++ b/docs/solutions/knift.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/guide#legacy title: KNIFT (Template-based Feature Matching) parent: Solutions nav_order: 13 @@ -20,12 +21,10 @@ nav_order: 13 **Attention:** *Thank you for your interest in MediaPipe Solutions. We have ended support for this MediaPipe Legacy Solution as of March 1, 2023. -For more information, see the new +For more information, see the [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/media_sequence.md b/docs/solutions/media_sequence.md index 5c479ea4c..769292a76 100644 --- a/docs/solutions/media_sequence.md +++ b/docs/solutions/media_sequence.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/guide#legacy title: Dataset Preparation with MediaSequence parent: Solutions nav_order: 15 @@ -24,8 +25,6 @@ For more information, see the new [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/models.md b/docs/solutions/models.md index c45aefa44..d0aae8e77 100644 --- a/docs/solutions/models.md +++ b/docs/solutions/models.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/guide#legacy title: Models and Model Cards parent: Solutions nav_order: 30 @@ -22,8 +23,6 @@ MediaPipe Legacy Solutions will continue to be provided on an as-is basis. We encourage you to check out the new MediaPipe Solutions at: [https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)* -*This notice and web page will be removed on April 3, 2023.* - ---- ### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection) diff --git a/docs/solutions/object_detection.md b/docs/solutions/object_detection.md index ef7db8671..22eaed563 100644 --- a/docs/solutions/object_detection.md +++ b/docs/solutions/object_detection.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/vision/object_detector/ title: Object Detection parent: Solutions nav_order: 9 @@ -19,13 +20,11 @@ nav_order: 9 --- **Attention:** *Thank you for your interest in MediaPipe Solutions. -As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe -Solution. For more information, see the new -[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) +As of March 1, 2023, this solution was upgraded to a new MediaPipe +Solution. For more information, see the +[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/object_detector/) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ![object_detection_android_gpu.gif](https://mediapipe.dev/images/mobile/object_detection_android_gpu.gif) diff --git a/docs/solutions/object_detection_saved_model.md b/docs/solutions/object_detection_saved_model.md index 6acac0a1b..262bacbab 100644 --- a/docs/solutions/object_detection_saved_model.md +++ b/docs/solutions/object_detection_saved_model.md @@ -1,4 +1,31 @@ -## TensorFlow/TFLite Object Detection Model +--- +layout: forward +target: https://developers.google.com/mediapipe/solutions/vision/object_detector +title: Object Detection +parent: Solutions +nav_order: 9 +--- + +# MediaPipe Object Detection +{: .no_toc } + +
+ + Table of contents + + {: .text-delta } +1. TOC +{:toc} +
+--- + +**Attention:** *Thank you for your interest in MediaPipe Solutions. +As of March 1, 2023, this solution was upgraded to a new MediaPipe +Solution. For more information, see the +[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/object_detector) +site.* + +---- ### TensorFlow model diff --git a/docs/solutions/objectron.md b/docs/solutions/objectron.md index 4ffb27bd0..4eee7c31d 100644 --- a/docs/solutions/objectron.md +++ b/docs/solutions/objectron.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/guide#legacy title: Objectron (3D Object Detection) parent: Solutions nav_order: 12 @@ -20,12 +21,10 @@ nav_order: 12 **Attention:** *Thank you for your interest in MediaPipe Solutions. We have ended support for this MediaPipe Legacy Solution as of March 1, 2023. -For more information, see the new +For more information, see the [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/pose.md b/docs/solutions/pose.md index ce0197ebd..fe9ffc0dc 100644 --- a/docs/solutions/pose.md +++ b/docs/solutions/pose.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/ title: Pose parent: Solutions has_children: true @@ -22,12 +23,10 @@ nav_order: 5 **Attention:** *Thank you for your interest in MediaPipe Solutions. As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe -Solution. For more information, see the new -[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) +Solution. For more information, see the +[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/pose_classification.md b/docs/solutions/pose_classification.md index 24f20f727..d2c4d5575 100644 --- a/docs/solutions/pose_classification.md +++ b/docs/solutions/pose_classification.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/ title: Pose Classification parent: Pose grand_parent: Solutions @@ -21,12 +22,10 @@ nav_order: 1 **Attention:** *Thank you for your interest in MediaPipe Solutions. As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe -Solution. For more information, see the new -[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) +Solution. For more information, see the +[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/selfie_segmentation.md b/docs/solutions/selfie_segmentation.md index 5febf29f0..54be628cd 100644 --- a/docs/solutions/selfie_segmentation.md +++ b/docs/solutions/selfie_segmentation.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/vision/image_segmenter/ title: Selfie Segmentation parent: Solutions nav_order: 7 @@ -19,13 +20,11 @@ nav_order: 7 --- **Attention:** *Thank you for your interest in MediaPipe Solutions. -As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe -Solution. For more information, see the new -[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) +As of April 4, 2023, this solution was upgraded to a new MediaPipe +Solution. For more information, see the +[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/image_segmenter/) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- ## Overview diff --git a/docs/solutions/solutions.md b/docs/solutions/solutions.md index b65390af7..7bc32d169 100644 --- a/docs/solutions/solutions.md +++ b/docs/solutions/solutions.md @@ -1,12 +1,12 @@ --- layout: default -title: Solutions +title: MediaPipe Legacy Solutions nav_order: 3 has_children: true has_toc: false --- -# Solutions +# MediaPipe Legacy Solutions {: .no_toc } 1. TOC @@ -29,6 +29,12 @@ Solutions at: ---- +









+









+









+ +---- + MediaPipe offers open source cross-platform, customizable ML solutions for live and streaming media. diff --git a/docs/solutions/youtube_8m.md b/docs/solutions/youtube_8m.md index 2e82b85d3..3a615a8a7 100644 --- a/docs/solutions/youtube_8m.md +++ b/docs/solutions/youtube_8m.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/solutions/guide#legacy title: YouTube-8M Feature Extraction and Model Inference parent: Solutions nav_order: 16 @@ -20,12 +21,10 @@ nav_order: 16 **Attention:** *Thank you for your interest in MediaPipe Solutions. We have ended support for this MediaPipe Legacy Solution as of March 1, 2023. -For more information, see the new +For more information, see the [MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy) site.* -*This notice and web page will be removed on April 3, 2023.* - ---- MediaPipe is a useful and general framework for media processing that can assist diff --git a/docs/tools/performance_benchmarking.md b/docs/tools/performance_benchmarking.md index f0d334f58..fedbb6e8a 100644 --- a/docs/tools/performance_benchmarking.md +++ b/docs/tools/performance_benchmarking.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/ title: Performance Benchmarking parent: Tools nav_order: 3 @@ -12,6 +13,12 @@ nav_order: 3 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +--- + *Coming soon.* Future mediapipe releases will include tools for visualizing and analysing the diff --git a/docs/tools/tools.md b/docs/tools/tools.md index 568ba76a7..8e4c55db3 100644 --- a/docs/tools/tools.md +++ b/docs/tools/tools.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/ title: Tools nav_order: 4 has_children: true @@ -11,3 +12,9 @@ has_children: true 1. TOC {:toc} --- + +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- diff --git a/docs/tools/tracing_and_profiling.md b/docs/tools/tracing_and_profiling.md index 861564c99..0ed6f57ab 100644 --- a/docs/tools/tracing_and_profiling.md +++ b/docs/tools/tracing_and_profiling.md @@ -1,5 +1,6 @@ --- -layout: default +layout: forward +target: https://developers.google.com/mediapipe/ title: Tracing and Profiling parent: Tools nav_order: 2 @@ -12,6 +13,12 @@ nav_order: 2 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +---- + The MediaPipe framework includes a built-in tracer and profiler. The tracer records various timing events related to packet processing, including the start and end time of each Calculator::Process call. The tracer writes trace log files diff --git a/docs/tools/visualizer.md b/docs/tools/visualizer.md index 45111a36e..eb24a7fc5 100644 --- a/docs/tools/visualizer.md +++ b/docs/tools/visualizer.md @@ -13,6 +13,12 @@ nav_order: 1 {:toc} --- +**Attention:** *Thanks for your interest in MediaPipe! We have moved to +[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe) +as the primary developer documentation site for MediaPipe as of April 3, 2023.* + +--- + To help users understand the structure of their calculator graphs and to understand the overall behavior of their machine learning inference pipelines, we have built the [MediaPipe Visualizer](https://viz.mediapipe.dev/) diff --git a/mediapipe/tasks/cc/vision/pose_landmarker/BUILD b/mediapipe/tasks/cc/vision/pose_landmarker/BUILD index 86b6c67b2..c5964c648 100644 --- a/mediapipe/tasks/cc/vision/pose_landmarker/BUILD +++ b/mediapipe/tasks/cc/vision/pose_landmarker/BUILD @@ -59,3 +59,44 @@ cc_library( "@com_google_absl//absl/status:statusor", ], ) + +cc_library( + name = "pose_landmarker_graph", + srcs = ["pose_landmarker_graph.cc"], + deps = [ + ":pose_landmarks_detector_graph", + "//mediapipe/calculators/core:clip_vector_size_calculator", + "//mediapipe/calculators/core:clip_vector_size_calculator_cc_proto", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:gate_calculator_cc_proto", + "//mediapipe/calculators/core:pass_through_calculator", + "//mediapipe/calculators/core:previous_loopback_calculator", + "//mediapipe/calculators/image:image_properties_calculator", + "//mediapipe/calculators/util:association_calculator_cc_proto", + "//mediapipe/calculators/util:association_norm_rect_calculator", + "//mediapipe/calculators/util:collection_has_min_size_calculator", + "//mediapipe/calculators/util:collection_has_min_size_calculator_cc_proto", + "//mediapipe/framework/api2:builder", + "//mediapipe/framework/api2:port", + "//mediapipe/framework/formats:detection_cc_proto", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", + "//mediapipe/framework/formats:tensor", + "//mediapipe/framework/port:status", + "//mediapipe/tasks/cc:common", + "//mediapipe/tasks/cc/components/utils:gate", + "//mediapipe/tasks/cc/core:model_asset_bundle_resources", + "//mediapipe/tasks/cc/core:model_resources_cache", + "//mediapipe/tasks/cc/core:model_task_graph", + "//mediapipe/tasks/cc/core:utils", + "//mediapipe/tasks/cc/metadata/utils:zip_utils", + "//mediapipe/tasks/cc/vision/pose_detector:pose_detector_graph", + "//mediapipe/tasks/cc/vision/pose_detector/proto:pose_detector_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarker_graph_options_cc_proto", + "//mediapipe/tasks/cc/vision/pose_landmarker/proto:pose_landmarks_detector_graph_options_cc_proto", + "//mediapipe/util:graph_builder_utils", + "@com_google_absl//absl/strings:str_format", + ], + alwayslink = 1, +) diff --git a/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_graph.cc b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_graph.cc new file mode 100644 index 000000000..ae3a7482e --- /dev/null +++ b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_graph.cc @@ -0,0 +1,384 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include +#include +#include + +#include "absl/strings/str_format.h" +#include "mediapipe/calculators/core/clip_vector_size_calculator.pb.h" +#include "mediapipe/calculators/core/gate_calculator.pb.h" +#include "mediapipe/calculators/util/association_calculator.pb.h" +#include "mediapipe/calculators/util/collection_has_min_size_calculator.pb.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/port.h" +#include "mediapipe/framework/formats/detection.pb.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/formats/tensor.h" +#include "mediapipe/framework/port/status_macros.h" +#include "mediapipe/tasks/cc/common.h" +#include "mediapipe/tasks/cc/components/utils/gate.h" +#include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h" +#include "mediapipe/tasks/cc/core/model_resources_cache.h" +#include "mediapipe/tasks/cc/core/model_task_graph.h" +#include "mediapipe/tasks/cc/core/utils.h" +#include "mediapipe/tasks/cc/metadata/utils/zip_utils.h" +#include "mediapipe/tasks/cc/vision/pose_detector/proto/pose_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarker_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.pb.h" +#include "mediapipe/util/graph_builder_utils.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace pose_landmarker { + +namespace { + +using ::mediapipe::NormalizedRect; +using ::mediapipe::api2::Input; +using ::mediapipe::api2::Output; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::SidePacket; +using ::mediapipe::api2::builder::Source; +using ::mediapipe::tasks::components::utils::DisallowIf; +using ::mediapipe::tasks::core::ModelAssetBundleResources; +using ::mediapipe::tasks::metadata::SetExternalFile; +using ::mediapipe::tasks::vision::pose_detector::proto:: + PoseDetectorGraphOptions; +using ::mediapipe::tasks::vision::pose_landmarker::proto:: + PoseLandmarkerGraphOptions; +using ::mediapipe::tasks::vision::pose_landmarker::proto:: + PoseLandmarksDetectorGraphOptions; + +constexpr char kImageTag[] = "IMAGE"; +constexpr char kNormRectTag[] = "NORM_RECT"; +constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS"; +constexpr char kWorldLandmarksTag[] = "WORLD_LANDMARKS"; +constexpr char kAuxiliaryLandmarksTag[] = "AUXILIARY_LANDMARKS"; +constexpr char kPoseRectsNextFrameTag[] = "POSE_RECTS_NEXT_FRAME"; +constexpr char kExpandedPoseRectsTag[] = "EXPANDED_POSE_RECTS"; +constexpr char kDetectionsTag[] = "DETECTIONS"; +constexpr char kLoopTag[] = "LOOP"; +constexpr char kPrevLoopTag[] = "PREV_LOOP"; +constexpr char kMainTag[] = "MAIN"; +constexpr char kIterableTag[] = "ITERABLE"; +constexpr char kSegmentationMaskTag[] = "SEGMENTATION_MASK"; + +constexpr char kPoseDetectorTFLiteName[] = "pose_detector.tflite"; +constexpr char kPoseLandmarksDetectorTFLiteName[] = + "pose_landmarks_detector.tflite"; + +struct PoseLandmarkerOutputs { + Source> landmark_lists; + Source> world_landmark_lists; + Source> auxiliary_landmark_lists; + Source> pose_rects_next_frame; + Source> pose_detections; + Source> segmentation_masks; + Source image; +}; + +// Sets the base options in the sub tasks. +absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources, + PoseLandmarkerGraphOptions* options, + bool is_copy) { + auto* pose_detector_graph_options = + options->mutable_pose_detector_graph_options(); + if (!pose_detector_graph_options->base_options().has_model_asset()) { + ASSIGN_OR_RETURN(const auto pose_detector_file, + resources.GetFile(kPoseDetectorTFLiteName)); + SetExternalFile(pose_detector_file, + pose_detector_graph_options->mutable_base_options() + ->mutable_model_asset(), + is_copy); + } + pose_detector_graph_options->mutable_base_options() + ->mutable_acceleration() + ->CopyFrom(options->base_options().acceleration()); + pose_detector_graph_options->mutable_base_options()->set_use_stream_mode( + options->base_options().use_stream_mode()); + auto* pose_landmarks_detector_graph_options = + options->mutable_pose_landmarks_detector_graph_options(); + if (!pose_landmarks_detector_graph_options->base_options() + .has_model_asset()) { + ASSIGN_OR_RETURN(const auto pose_landmarks_detector_file, + resources.GetFile(kPoseLandmarksDetectorTFLiteName)); + SetExternalFile( + pose_landmarks_detector_file, + pose_landmarks_detector_graph_options->mutable_base_options() + ->mutable_model_asset(), + is_copy); + } + pose_landmarks_detector_graph_options->mutable_base_options() + ->mutable_acceleration() + ->CopyFrom(options->base_options().acceleration()); + pose_landmarks_detector_graph_options->mutable_base_options() + ->set_use_stream_mode(options->base_options().use_stream_mode()); + + return absl::OkStatus(); +} + +} // namespace + +// A "mediapipe.tasks.vision.pose_landmarker.PoseLandmarkerGraph" performs pose +// landmarks detection. The PoseLandmarkerGraph consists of two subgraphs: +// PoseDetectorGraph, MultiplePoseLandmarksDetectorGraph +// +// MultiplePoseLandmarksDetectorGraph detects landmarks from bounding boxes +// produced by PoseDetectorGraph. PoseLandmarkerGraph tracks the landmarks over +// time, and skips the PoseDetectorGraph. If the tracking is lost or the +// detected poses are less than configured max number poses, PoseDetectorGraph +// would be triggered to detect poses. +// +// +// Inputs: +// IMAGE - Image +// Image to perform pose landmarks detection on. +// NORM_RECT - NormalizedRect @Optional +// Describes image rotation and region of image to perform landmarks +// detection on. If not provided, whole image is used for pose landmarks +// detection. +// +// +// Outputs: +// NORM_LANDMARKS: - std::vector +// Vector of detected pose landmarks. +// WORLD_LANDMARKS: std::vector +// Vector of detected world pose landmarks. +// AUXILIARY_LANDMARKS: - std::vector +// Vector of detected auxiliary landmarks. +// POSE_RECTS_NEXT_FRAME - std::vector +// Vector of the expanded rects enclosing the whole pose RoI for landmark +// detection on the next frame. +// POSE_RECTS - std::vector +// Detected pose bounding boxes in normalized coordinates from pose +// detection. +// SEGMENTATION_MASK - std::vector +// Segmentation masks. +// IMAGE - Image +// The input image that the pose landmarker runs on and has the pixel data +// stored on the target storage (CPU vs GPU). +// All returned coordinates are in the unrotated and uncropped input image +// coordinates system. +// +// Example: +// node { +// calculator: "mediapipe.tasks.vision.pose_landmarker.PoseLandmarkerGraph" +// input_stream: "IMAGE:image_in" +// input_stream: "NORM_RECT:norm_rect" +// output_stream: "NORM_LANDMARKS:pose_landmarks" +// output_stream: "LANDMARKS:world_landmarks" +// output_stream: "NORM_LANDMAKRS:auxiliary_landmarks" +// output_stream: "POSE_RECTS_NEXT_FRAME:pose_rects_next_frame" +// output_stream: "POSE_RECTS:pose_rects" +// output_stream: "SEGMENTATION_MASK:segmentation_masks" +// output_stream: "IMAGE:image_out" +// options { +// [mediapipe.tasks.vision.pose_landmarker.proto.PoseLandmarkerGraphOptions.ext] +// { +// base_options { +// model_asset { +// file_name: "pose_landmarker.task" +// } +// } +// pose_detector_graph_options { +// min_detection_confidence: 0.5 +// num_poses: 2 +// } +// pose_landmarks_detector_graph_options { +// min_detection_confidence: 0.5 +// } +// } +// } +// } +class PoseLandmarkerGraph : public core::ModelTaskGraph { + public: + absl::StatusOr GetConfig( + SubgraphContext* sc) override { + Graph graph; + if (sc->Options() + .base_options() + .has_model_asset()) { + ASSIGN_OR_RETURN( + const auto* model_asset_bundle_resources, + CreateModelAssetBundleResources(sc)); + // Copies the file content instead of passing the pointer of file in + // memory if the subgraph model resource service is not available. + MP_RETURN_IF_ERROR(SetSubTaskBaseOptions( + *model_asset_bundle_resources, + sc->MutableOptions(), + !sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService) + .IsAvailable())); + } + ASSIGN_OR_RETURN( + auto outs, + BuildPoseLandmarkerGraph( + *sc->MutableOptions(), + graph[Input(kImageTag)], + graph[Input::Optional(kNormRectTag)], graph)); + outs.landmark_lists >> + graph[Output>(kNormLandmarksTag)]; + outs.world_landmark_lists >> + graph[Output>(kWorldLandmarksTag)]; + outs.auxiliary_landmark_lists >> + graph[Output>( + kAuxiliaryLandmarksTag)]; + outs.pose_rects_next_frame >> + graph[Output>(kPoseRectsNextFrameTag)]; + outs.segmentation_masks >> + graph[Output>(kSegmentationMaskTag)]; + outs.pose_detections >> + graph[Output>(kDetectionsTag)]; + outs.image >> graph[Output(kImageTag)]; + + // TODO remove when support is fixed. + // As mediapipe GraphBuilder currently doesn't support configuring + // InputStreamInfo, modifying the CalculatorGraphConfig proto directly. + CalculatorGraphConfig config = graph.GetConfig(); + for (int i = 0; i < config.node_size(); ++i) { + if (config.node(i).calculator() == "PreviousLoopbackCalculator") { + auto* info = config.mutable_node(i)->add_input_stream_info(); + info->set_tag_index(kLoopTag); + info->set_back_edge(true); + break; + } + } + + return config; + } + + private: + // Adds a mediapipe pose landmarker graph into the provided builder::Graph + // instance. + // + // tasks_options: the mediapipe tasks module PoseLandmarkerGraphOptions. + // image_in: (mediapipe::Image) stream to run pose landmark detection on. + // graph: the mediapipe graph instance to be updated. + absl::StatusOr BuildPoseLandmarkerGraph( + PoseLandmarkerGraphOptions& tasks_options, Source image_in, + Source norm_rect_in, Graph& graph) { + const int max_num_poses = + tasks_options.pose_detector_graph_options().num_poses(); + + auto& pose_detector = + graph.AddNode("mediapipe.tasks.vision.pose_detector.PoseDetectorGraph"); + pose_detector.GetOptions().Swap( + tasks_options.mutable_pose_detector_graph_options()); + auto& clip_pose_rects = + graph.AddNode("ClipNormalizedRectVectorSizeCalculator"); + clip_pose_rects.GetOptions() + .set_max_vec_size(max_num_poses); + auto clipped_pose_rects = clip_pose_rects.Out(""); + + auto& pose_landmarks_detector_graph = graph.AddNode( + "mediapipe.tasks.vision.pose_landmarker." + "MultiplePoseLandmarksDetectorGraph"); + pose_landmarks_detector_graph + .GetOptions() + .Swap(tasks_options.mutable_pose_landmarks_detector_graph_options()); + image_in >> pose_landmarks_detector_graph.In(kImageTag); + clipped_pose_rects >> pose_landmarks_detector_graph.In(kNormRectTag); + + // TODO: Add landmarks smoothing calculators to + // PoseLandmarkerGraph + auto landmarks = pose_landmarks_detector_graph.Out("LANDMARKS") + .Cast>(); + auto world_landmarks = pose_landmarks_detector_graph.Out(kWorldLandmarksTag) + .Cast>(); + auto aux_landmarks = + pose_landmarks_detector_graph.Out(kAuxiliaryLandmarksTag) + .Cast>(); + auto pose_rects_for_next_frame = + pose_landmarks_detector_graph.Out(kPoseRectsNextFrameTag) + .Cast>(); + auto segmentation_masks = + pose_landmarks_detector_graph.Out(kSegmentationMaskTag) + .Cast>(); + + if (tasks_options.base_options().use_stream_mode()) { + auto& previous_loopback = graph.AddNode("PreviousLoopbackCalculator"); + image_in >> previous_loopback.In(kMainTag); + auto prev_pose_rects_from_landmarks = + previous_loopback[Output>(kPrevLoopTag)]; + + auto& min_size_node = + graph.AddNode("NormalizedRectVectorHasMinSizeCalculator"); + prev_pose_rects_from_landmarks >> min_size_node.In(kIterableTag); + min_size_node.GetOptions() + .set_min_size(max_num_poses); + auto has_enough_poses = min_size_node.Out("").Cast(); + + // While in stream mode, skip pose detector graph when we successfully + // track the poses from the last frame. + auto image_for_pose_detector = + DisallowIf(image_in, has_enough_poses, graph); + auto norm_rect_in_for_pose_detector = + DisallowIf(norm_rect_in, has_enough_poses, graph); + image_for_pose_detector >> pose_detector.In(kImageTag); + norm_rect_in_for_pose_detector >> pose_detector.In(kNormRectTag); + auto expanded_pose_rects_from_pose_detector = + pose_detector.Out(kExpandedPoseRectsTag); + auto& pose_association = graph.AddNode("AssociationNormRectCalculator"); + pose_association.GetOptions() + .set_min_similarity_threshold( + tasks_options.min_tracking_confidence()); + prev_pose_rects_from_landmarks >> + pose_association[Input>::Multiple("")][0]; + expanded_pose_rects_from_pose_detector >> + pose_association[Input>::Multiple("")][1]; + auto pose_rects = pose_association.Out(""); + pose_rects >> clip_pose_rects.In(""); + // Back edge. + pose_rects_for_next_frame >> previous_loopback.In(kLoopTag); + } else { + // While not in stream mode, the input images are not guaranteed to be in + // series, and we don't want to enable the tracking and rect associations + // between input images. Always use the pose detector graph. + image_in >> pose_detector.In(kImageTag); + norm_rect_in >> pose_detector.In(kNormRectTag); + auto pose_rects = pose_detector.Out(kExpandedPoseRectsTag); + pose_rects >> clip_pose_rects.In(""); + } + + // TODO: Replace PassThroughCalculator with a calculator that + // converts the pixel data to be stored on the target storage (CPU vs GPU). + auto& pass_through = graph.AddNode("PassThroughCalculator"); + image_in >> pass_through.In(""); + + return {{ + /* landmark_lists= */ landmarks, + /* world_landmarks= */ world_landmarks, + /* aux_landmarks= */ aux_landmarks, + /* pose_rects_next_frame= */ pose_rects_for_next_frame, + /* pose_detections */ + pose_detector.Out(kDetectionsTag).Cast>(), + /* segmentation_masks= */ segmentation_masks, + /* image= */ + pass_through[Output("")], + }}; + } +}; + +REGISTER_MEDIAPIPE_GRAPH( + ::mediapipe::tasks::vision::pose_landmarker::PoseLandmarkerGraph); + +} // namespace pose_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_graph_test.cc b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_graph_test.cc new file mode 100644 index 000000000..6a4cc93b6 --- /dev/null +++ b/mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_graph_test.cc @@ -0,0 +1,190 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include "absl/flags/flag.h" +#include "absl/status/statusor.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "mediapipe/framework/api2/builder.h" +#include "mediapipe/framework/api2/port.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" +#include "mediapipe/tasks/cc/core/proto/base_options.pb.h" +#include "mediapipe/tasks/cc/core/proto/external_file.pb.h" +#include "mediapipe/tasks/cc/core/task_runner.h" +#include "mediapipe/tasks/cc/vision/pose_detector/proto/pose_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarker_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.pb.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace mediapipe { +namespace tasks { +namespace vision { +namespace pose_landmarker { +namespace { + +using ::file::Defaults; +using ::file::GetTextProto; +using ::mediapipe::api2::Input; +using ::mediapipe::api2::Output; +using ::mediapipe::api2::builder::Graph; +using ::mediapipe::api2::builder::Source; +using ::mediapipe::file::JoinPath; +using ::mediapipe::tasks::core::TaskRunner; +using ::mediapipe::tasks::vision::DecodeImageFromFile; +using ::mediapipe::tasks::vision::pose_landmarker::proto:: + PoseLandmarkerGraphOptions; +using ::testing::EqualsProto; +using ::testing::Pointwise; +using ::testing::TestParamInfo; +using ::testing::TestWithParam; +using ::testing::Values; +using ::testing::proto::Approximately; +using ::testing::proto::Partially; + +constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; +constexpr char kPoseLandmarkerModelBundleName[] = "pose_landmarker.task"; +constexpr char kPoseImageName[] = "pose.jpg"; +constexpr char kExpectedPoseLandmarksName[] = + "expected_pose_landmarks.prototxt"; + +constexpr char kImageTag[] = "IMAGE"; +constexpr char kImageName[] = "image"; +constexpr char kNormRectTag[] = "NORM_RECT"; +constexpr char kNormRectName[] = "norm_rect"; +constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS"; +constexpr char kNormLandmarksName[] = "norm_landmarks"; + +constexpr float kLiteModelFractionDiff = 0.05; // percentage + +template +ProtoT GetExpectedProto(absl::string_view filename) { + ProtoT expected_proto; + MP_EXPECT_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, filename), + &expected_proto, Defaults())); + return expected_proto; +} + +// Struct holding the parameters for parameterized PoseLandmarkerGraphTest +// class. +struct PoseLandmarkerGraphTestParams { + // The name of this test, for convenience when displaying test results. + std::string test_name; + // The filename of the model to test. + std::string input_model_name; + // The filename of the test image. + std::string test_image_name; + // The expected output landmarks positions. + std::optional> expected_landmarks_list; + // The max value difference between expected_positions and detected positions. + float landmarks_diff_threshold; +}; + +// Helper function to create a PoseLandmarkerGraph TaskRunner. +absl::StatusOr> CreatePoseLandmarkerGraphTaskRunner( + absl::string_view model_name) { + Graph graph; + + auto& pose_landmarker = graph.AddNode( + "mediapipe.tasks.vision.pose_landmarker." + "PoseLandmarkerGraph"); + + auto* options = &pose_landmarker.GetOptions(); + options->mutable_base_options()->mutable_model_asset()->set_file_name( + JoinPath("./", kTestDataDirectory, model_name)); + options->mutable_pose_detector_graph_options()->set_num_poses(1); + options->mutable_base_options()->set_use_stream_mode(true); + + graph[Input(kImageTag)].SetName(kImageName) >> + pose_landmarker.In(kImageTag); + graph[Input(kNormRectTag)].SetName(kNormRectName) >> + pose_landmarker.In(kNormRectTag); + + pose_landmarker.Out(kNormLandmarksTag).SetName(kNormLandmarksName) >> + graph[Output>(kNormLandmarksTag)]; + + return TaskRunner::Create( + graph.GetConfig(), + absl::make_unique()); +} + +// Helper function to construct NormalizeRect proto. +NormalizedRect MakeNormRect(float x_center, float y_center, float width, + float height, float rotation) { + NormalizedRect pose_rect; + pose_rect.set_x_center(x_center); + pose_rect.set_y_center(y_center); + pose_rect.set_width(width); + pose_rect.set_height(height); + pose_rect.set_rotation(rotation); + return pose_rect; +} + +class PoseLandmarkerGraphTest + : public testing::TestWithParam {}; + +TEST_P(PoseLandmarkerGraphTest, Succeeds) { + MP_ASSERT_OK_AND_ASSIGN( + Image image, DecodeImageFromFile(JoinPath("./", kTestDataDirectory, + GetParam().test_image_name))); + MP_ASSERT_OK_AND_ASSIGN(auto task_runner, CreatePoseLandmarkerGraphTaskRunner( + GetParam().input_model_name)); + + auto output_packets = task_runner->Process( + {{kImageName, MakePacket(std::move(image))}, + {kNormRectName, + MakePacket(MakeNormRect(0.5, 0.5, 1.0, 1.0, 0))}}); + MP_ASSERT_OK(output_packets); + + if (GetParam().expected_landmarks_list) { + const std::vector& landmarks_lists = + (*output_packets)[kNormLandmarksName] + .Get>(); + EXPECT_THAT(landmarks_lists, + Pointwise(Approximately(Partially(EqualsProto()), + GetParam().landmarks_diff_threshold), + *GetParam().expected_landmarks_list)); + } +} + +INSTANTIATE_TEST_SUITE_P( + PoseLandmarkerGraphTests, PoseLandmarkerGraphTest, + Values(PoseLandmarkerGraphTestParams{ + /* test_name= */ "PoseLandmarkerLite", + /* input_model_name= */ kPoseLandmarkerModelBundleName, + /* test_image_name= */ kPoseImageName, + /* expected_landmarks_list= */ + {{GetExpectedProto( + kExpectedPoseLandmarksName)}}, + /* landmarks_diff_threshold= */ kLiteModelFractionDiff}), + [](const TestParamInfo& info) { + return info.param.test_name; + }); + +} // namespace +} // namespace pose_landmarker +} // namespace vision +} // namespace tasks +} // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/pose_landmarker/proto/BUILD b/mediapipe/tasks/cc/vision/pose_landmarker/proto/BUILD index 6a95762c0..a2ad7b0b1 100644 --- a/mediapipe/tasks/cc/vision/pose_landmarker/proto/BUILD +++ b/mediapipe/tasks/cc/vision/pose_landmarker/proto/BUILD @@ -29,3 +29,15 @@ mediapipe_proto_library( "//mediapipe/tasks/cc/core/proto:base_options_proto", ], ) + +mediapipe_proto_library( + name = "pose_landmarker_graph_options_proto", + srcs = ["pose_landmarker_graph_options.proto"], + deps = [ + ":pose_landmarks_detector_graph_options_proto", + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + "//mediapipe/tasks/cc/core/proto:base_options_proto", + "//mediapipe/tasks/cc/vision/pose_detector/proto:pose_detector_graph_options_proto", + ], +) diff --git a/mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarker_graph_options.proto b/mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarker_graph_options.proto new file mode 100644 index 000000000..bde314bad --- /dev/null +++ b/mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarker_graph_options.proto @@ -0,0 +1,48 @@ +/* Copyright 2023 The MediaPipe Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +syntax = "proto2"; + +package mediapipe.tasks.vision.pose_landmarker.proto; + +import "mediapipe/framework/calculator.proto"; +import "mediapipe/framework/calculator_options.proto"; +import "mediapipe/tasks/cc/core/proto/base_options.proto"; +import "mediapipe/tasks/cc/vision/pose_detector/proto/pose_detector_graph_options.proto"; +import "mediapipe/tasks/cc/vision/pose_landmarker/proto/pose_landmarks_detector_graph_options.proto"; + +option java_package = "com.google.mediapipe.tasks.vision.poselandmarker.proto"; +option java_outer_classname = "PoseLandmarkerGraphOptionsProto"; + +message PoseLandmarkerGraphOptions { + extend mediapipe.CalculatorOptions { + optional PoseLandmarkerGraphOptions ext = 516587230; + } + // Base options for configuring Task library, such as specifying the TfLite + // model file with metadata, accelerator options, etc. + optional core.proto.BaseOptions base_options = 1; + + // Options for pose detector graph. + optional pose_detector.proto.PoseDetectorGraphOptions + pose_detector_graph_options = 2; + + // Options for pose landmarks detector graph. + optional PoseLandmarksDetectorGraphOptions + pose_landmarks_detector_graph_options = 3; + + // Minimum confidence for pose landmarks tracking to be considered + // successfully. + optional float min_tracking_confidence = 4 [default = 0.5]; +} diff --git a/mediapipe/tasks/ios/common/utils/BUILD b/mediapipe/tasks/ios/common/utils/BUILD index a29c700da..da093a775 100644 --- a/mediapipe/tasks/ios/common/utils/BUILD +++ b/mediapipe/tasks/ios/common/utils/BUILD @@ -24,7 +24,6 @@ objc_library( "//mediapipe/tasks/cc:common", "//mediapipe/tasks/ios/common:MPPCommon", "@com_google_absl//absl/status", - "@com_google_absl//absl/strings", "@com_google_absl//absl/strings:cord", ], ) diff --git a/mediapipe/tasks/ios/components/containers/BUILD b/mediapipe/tasks/ios/components/containers/BUILD index 36c8ef2e0..3ed7669cb 100644 --- a/mediapipe/tasks/ios/components/containers/BUILD +++ b/mediapipe/tasks/ios/components/containers/BUILD @@ -44,3 +44,13 @@ objc_library( hdrs = ["sources/MPPEmbeddingResult.h"], deps = [":MPPEmbedding"], ) + +objc_library( + name = "MPPDetection", + srcs = ["sources/MPPDetection.m"], + hdrs = ["sources/MPPDetection.h"], + deps = [ + ":MPPCategory", + "//third_party/apple_frameworks:UIKit", + ], +) diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPDetection.h b/mediapipe/tasks/ios/components/containers/sources/MPPDetection.h new file mode 100644 index 000000000..e085007a6 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPDetection.h @@ -0,0 +1,103 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPCategory.h" + +NS_ASSUME_NONNULL_BEGIN + +/** + * Normalized keypoint represents a point in 2D space with x, y coordinates. x and y are normalized + * to [0.0, 1.0] by the image width and height respectively. + */ +NS_SWIFT_NAME(NormalizedKeypoint) +@interface MPPNormalizedKeypoint : NSObject + +/** The (x,y) coordinates location of the normalized keypoint. */ +@property(nonatomic, readonly) CGPoint location; + +/** The optional label of the normalized keypoint. */ +@property(nonatomic, readonly, nullable) NSString *label; + +/** The optional score of the normalized keypoint. If score is absent, it will be equal to 0.0. */ +@property(nonatomic, readonly) float score; + +/** + * Initializes a new `MPPNormalizedKeypoint` object with the given location, label and score. + * You must pass 0.0 for `score` if it is not present. + * + * @param location The (x,y) coordinates location of the normalized keypoint. + * @param label The optional label of the normalized keypoint. + * @param score The optional score of the normalized keypoint. You must pass 0.0 for score if it + * is not present. + * + * @return An instance of `MPPNormalizedKeypoint` initialized with the given given location, label + * and score. + */ +- (instancetype)initWithLocation:(CGPoint)location + label:(nullable NSString *)label + score:(float)score NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +/** Represents one detected object in the results of `MPPObjectDetector`. */ +NS_SWIFT_NAME(Detection) +@interface MPPDetection : NSObject + +/** An array of `MPPCategory` objects containing the predicted categories. */ +@property(nonatomic, readonly) NSArray *categories; + +/** The bounding box of the detected object. */ +@property(nonatomic, readonly) CGRect boundingBox; + +/** + * An optional array of `MPPNormalizedKeypoint` objects associated with the detection. Keypoints + * represent interesting points related to the detection. For example, the keypoints represent the + * eyes, ear and mouth from the from detection model. In template matching detection, e.g. KNIFT, + * they can instead represent the feature points for template matching. + */ +@property(nonatomic, readonly, nullable) NSArray *keypoints; + +/** + * Initializes a new `MPPDetection` object with the given array of categories, bounding box and + * optional array of keypoints; + * + * @param categories A list of `MPPCategory` objects that contain category name, display name, + * score, and the label index. + * @param boundingBox A `CGRect` that represents the bounding box. + * @param keypoints: An optional array of `MPPNormalizedKeypoint` objects associated with the + * detection. Keypoints represent interesting points related to the detection. For example, the + * keypoints represent the eyes, ear and mouth from the face detection model. In template matching + * detection, e.g. KNIFT, they can instead represent the feature points for template matching. + * + * @return An instance of `MPPDetection` initialized with the given array of categories, bounding + * box and `nil` keypoints. + */ +- (instancetype)initWithCategories:(NSArray *)categories + boundingBox:(CGRect)boundingBox + keypoints:(nullable NSArray *)keypoints + NS_DESIGNATED_INITIALIZER; + +- (instancetype)init NS_UNAVAILABLE; + ++ (instancetype)new NS_UNAVAILABLE; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/sources/MPPDetection.m b/mediapipe/tasks/ios/components/containers/sources/MPPDetection.m new file mode 100644 index 000000000..c245478db --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/sources/MPPDetection.m @@ -0,0 +1,68 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/sources/MPPDetection.h" + +@implementation MPPNormalizedKeypoint + +- (instancetype)initWithLocation:(CGPoint)location + label:(nullable NSString *)label + score:(float)score { + self = [super init]; + if (self) { + _location = location; + _label = label; + _score = score; + } + return self; +} + +// TODO: Implement hash + +- (BOOL)isEqual:(nullable id)object { + if (!object) { + return NO; + } + + if (self == object) { + return YES; + } + + if (![object isKindOfClass:[MPPNormalizedKeypoint class]]) { + return NO; + } + + MPPNormalizedKeypoint *otherKeypoint = (MPPNormalizedKeypoint *)object; + + return CGPointEqualToPoint(self.location, otherKeypoint.location) && + (self.label == otherKeypoint.label) && (self.score == otherKeypoint.score); +} + +@end + +@implementation MPPDetection + +- (instancetype)initWithCategories:(NSArray *)categories + boundingBox:(CGRect)boundingBox + keypoints:(nullable NSArray *)keypoints { + self = [super init]; + if (self) { + _categories = categories; + _boundingBox = boundingBox; + _keypoints = keypoints; + } + return self; +} + +@end diff --git a/mediapipe/tasks/ios/components/containers/utils/BUILD b/mediapipe/tasks/ios/components/containers/utils/BUILD index 64ca29b88..3f93c0f36 100644 --- a/mediapipe/tasks/ios/components/containers/utils/BUILD +++ b/mediapipe/tasks/ios/components/containers/utils/BUILD @@ -61,3 +61,15 @@ objc_library( "//mediapipe/tasks/ios/components/containers:MPPEmbeddingResult", ], ) + +objc_library( + name = "MPPDetectionHelpers", + srcs = ["sources/MPPDetection+Helpers.mm"], + hdrs = ["sources/MPPDetection+Helpers.h"], + deps = [ + "//mediapipe/framework/formats:detection_cc_proto", + "//mediapipe/framework/formats:location_data_cc_proto", + "//mediapipe/tasks/ios/common/utils:NSStringHelpers", + "//mediapipe/tasks/ios/components/containers:MPPDetection", + ], +) diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.h b/mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.h new file mode 100644 index 000000000..c06c04d3e --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.h @@ -0,0 +1,26 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/framework/formats/detection.pb.h" +#import "mediapipe/tasks/ios/components/containers/sources/MPPDetection.h" + +NS_ASSUME_NONNULL_BEGIN + +@interface MPPDetection (Helpers) + ++ (MPPDetection *)detectionWithProto:(const mediapipe::Detection &)detectionProto; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.mm b/mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.mm new file mode 100644 index 000000000..f485b9110 --- /dev/null +++ b/mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.mm @@ -0,0 +1,83 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/components/containers/utils/sources/MPPDetection+Helpers.h" +#import "mediapipe/framework/formats/location_data.pb.h" + +#import "mediapipe/tasks/ios/common/utils/sources/NSString+Helpers.h" + +static const NSInteger kDefaultCategoryIndex = -1; + +namespace { +using DetectionProto = ::mediapipe::Detection; +using BoundingBoxProto = ::mediapipe::LocationData::BoundingBox; +} // namespace + +@implementation MPPDetection (Helpers) + ++ (MPPDetection *)detectionWithProto:(const DetectionProto &)detectionProto { + NSMutableArray *categories = + [NSMutableArray arrayWithCapacity:(NSUInteger)detectionProto.score_size()]; + + for (int idx = 0; idx < detectionProto.score_size(); ++idx) { + NSInteger categoryIndex = + detectionProto.label_id_size() > idx ? detectionProto.label_id(idx) : kDefaultCategoryIndex; + NSString *categoryName = detectionProto.label_size() > idx + ? [NSString stringWithCppString:detectionProto.label(idx)] + : nil; + + NSString *displayName = detectionProto.display_name_size() > idx + ? [NSString stringWithCppString:detectionProto.display_name(idx)] + : nil; + + [categories addObject:[[MPPCategory alloc] initWithIndex:categoryIndex + score:detectionProto.score(idx) + categoryName:categoryName + displayName:displayName]]; + } + + CGRect boundingBox = CGRectZero; + + if (detectionProto.location_data().has_bounding_box()) { + const BoundingBoxProto &boundingBoxProto = detectionProto.location_data().bounding_box(); + boundingBox.origin.x = boundingBoxProto.xmin(); + boundingBox.origin.y = boundingBoxProto.ymin(); + boundingBox.size.width = boundingBoxProto.width(); + boundingBox.size.height = boundingBoxProto.height(); + } + + NSMutableArray *normalizedKeypoints; + + if (!detectionProto.location_data().relative_keypoints().empty()) { + normalizedKeypoints = [NSMutableArray + arrayWithCapacity:(NSUInteger)detectionProto.location_data().relative_keypoints_size()]; + for (const auto &keypoint : detectionProto.location_data().relative_keypoints()) { + NSString *label = keypoint.has_keypoint_label() + ? [NSString stringWithCppString:keypoint.keypoint_label()] + : nil; + CGPoint location = CGPointMake(keypoint.x(), keypoint.y()); + float score = keypoint.has_score() ? keypoint.score() : 0.0f; + + [normalizedKeypoints addObject:[[MPPNormalizedKeypoint alloc] initWithLocation:location + label:label + score:score]]; + } + } + + return [[MPPDetection alloc] initWithCategories:categories + boundingBox:boundingBox + keypoints:normalizedKeypoints]; +} + +@end diff --git a/mediapipe/tasks/ios/test/vision/utils/BUILD b/mediapipe/tasks/ios/test/vision/utils/BUILD index cf7626dbd..c5acf7a46 100644 --- a/mediapipe/tasks/ios/test/vision/utils/BUILD +++ b/mediapipe/tasks/ios/test/vision/utils/BUILD @@ -7,7 +7,5 @@ objc_library( srcs = ["sources/MPPImage+TestUtils.m"], hdrs = ["sources/MPPImage+TestUtils.h"], module_name = "MPPImageTestUtils", - deps = [ - "//mediapipe/tasks/ios/vision/core:MPPImage", - ], + deps = ["//mediapipe/tasks/ios/vision/core:MPPImage"], ) diff --git a/mediapipe/tasks/ios/vision/object_detector/BUILD b/mediapipe/tasks/ios/vision/object_detector/BUILD new file mode 100644 index 000000000..f1325b050 --- /dev/null +++ b/mediapipe/tasks/ios/vision/object_detector/BUILD @@ -0,0 +1,38 @@ +# Copyright 2023 The MediaPipe Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +objc_library( + name = "MPPObjectDetectionResult", + srcs = ["sources/MPPObjectDetectionResult.m"], + hdrs = ["sources/MPPObjectDetectionResult.h"], + deps = [ + "//mediapipe/tasks/ios/components/containers:MPPDetection", + "//mediapipe/tasks/ios/core:MPPTaskResult", + ], +) + +objc_library( + name = "MPPObjectDetectorOptions", + srcs = ["sources/MPPObjectDetectorOptions.m"], + hdrs = ["sources/MPPObjectDetectorOptions.h"], + deps = [ + ":MPPObjectDetectionResult", + "//mediapipe/tasks/ios/core:MPPTaskOptions", + "//mediapipe/tasks/ios/vision/core:MPPRunningMode", + ], +) diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectionResult.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectionResult.h new file mode 100644 index 000000000..590867bf8 --- /dev/null +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectionResult.h @@ -0,0 +1,49 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import +#import "mediapipe/tasks/ios/components/containers/sources/MPPDetection.h" +#import "mediapipe/tasks/ios/core/sources/MPPTaskResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Represents the detection results generated by `MPPObjectDetector`. */ +NS_SWIFT_NAME(ObjectDetectionResult) +@interface MPPObjectDetectionResult : MPPTaskResult + +/** + * The array of `MPPDetection` objects each of which has a bounding box that is expressed in the + * unrotated input frame of reference coordinates system, i.e. in `[0,image_width) x + * [0,image_height)`, which are the dimensions of the underlying image data. + */ +@property(nonatomic, readonly) NSArray *detections; + +/** + * Initializes a new `MPPObjectDetectionResult` with the given array of detections and timestamp (in + * milliseconds). + * + * @param detections An array of `MPPDetection` objects each of which has a bounding box that is + * expressed in the unrotated input frame of reference coordinates system, i.e. in `[0,image_width) + * x [0,image_height)`, which are the dimensions of the underlying image data. + * @param timestampMs The timestamp for this result. + * + * @return An instance of `MPPObjectDetectionResult` initialized with the given array of detections + * and timestamp (in milliseconds). + */ +- (instancetype)initWithDetections:(NSArray *)detections + timestampMs:(NSInteger)timestampMs; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectionResult.m b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectionResult.m new file mode 100644 index 000000000..ac24c19fa --- /dev/null +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectionResult.m @@ -0,0 +1,28 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectionResult.h" + +@implementation MPPObjectDetectionResult + +- (instancetype)initWithDetections:(NSArray *)detections + timestampMs:(NSInteger)timestampMs { + self = [super initWithTimestampMs:timestampMs]; + if (self) { + _detections = detections; + } + return self; +} + +@end diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h new file mode 100644 index 000000000..bf92e9a44 --- /dev/null +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h @@ -0,0 +1,71 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import + +#import "mediapipe/tasks/ios/core/sources/MPPTaskOptions.h" +#import "mediapipe/tasks/ios/vision/core/sources/MPPRunningMode.h" +#import "mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectionResult.h" + +NS_ASSUME_NONNULL_BEGIN + +/** Options for setting up a `MPPObjectDetector`. */ +NS_SWIFT_NAME(ObjectDetectorOptions) +@interface MPPObjectDetectorOptions : MPPTaskOptions + +@property(nonatomic) MPPRunningMode runningMode; + +/** + * The user-defined result callback for processing live stream data. The result callback should only + * be specified when the running mode is set to the live stream mode. + * TODO: Add parameter `MPPImage` in the callback. + */ +@property(nonatomic, copy) void (^completion) + (MPPObjectDetectionResult *result, NSInteger timestampMs, NSError *error); + +/** + * The locale to use for display names specified through the TFLite Model Metadata, if any. Defaults + * to English. + */ +@property(nonatomic, copy) NSString *displayNamesLocale; + +/** + * The maximum number of top-scored classification results to return. If < 0, all available results + * will be returned. If 0, an invalid argument error is returned. + */ +@property(nonatomic) NSInteger maxResults; + +/** + * Score threshold to override the one provided in the model metadata (if any). Results below this + * value are rejected. + */ +@property(nonatomic) float scoreThreshold; + +/** + * The allowlist of category names. If non-empty, detection results whose category name is not in + * this set will be filtered out. Duplicate or unknown category names are ignored. Mutually + * exclusive with categoryDenylist. + */ +@property(nonatomic, copy) NSArray *categoryAllowlist; + +/** + * The denylist of category names. If non-empty, detection results whose category name is in this + * set will be filtered out. Duplicate or unknown category names are ignored. Mutually exclusive + * with categoryAllowlist. + */ +@property(nonatomic, copy) NSArray *categoryDenylist; + +@end + +NS_ASSUME_NONNULL_END diff --git a/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.m b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.m new file mode 100644 index 000000000..73f8ce5b5 --- /dev/null +++ b/mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.m @@ -0,0 +1,41 @@ +// Copyright 2023 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#import "mediapipe/tasks/ios/vision/object_detector/sources/MPPObjectDetectorOptions.h" + +@implementation MPPObjectDetectorOptions + +- (instancetype)init { + self = [super init]; + if (self) { + _maxResults = -1; + _scoreThreshold = 0; + } + return self; +} + +- (id)copyWithZone:(NSZone *)zone { + MPPObjectDetectorOptions *objectDetectorOptions = [super copyWithZone:zone]; + + objectDetectorOptions.scoreThreshold = self.scoreThreshold; + objectDetectorOptions.maxResults = self.maxResults; + objectDetectorOptions.categoryDenylist = self.categoryDenylist; + objectDetectorOptions.categoryAllowlist = self.categoryAllowlist; + objectDetectorOptions.displayNamesLocale = self.displayNamesLocale; + objectDetectorOptions.completion = self.completion; + + return objectDetectorOptions; +} + +@end diff --git a/mediapipe/tasks/testdata/vision/BUILD b/mediapipe/tasks/testdata/vision/BUILD index ed0ee6ea5..0de0c255c 100644 --- a/mediapipe/tasks/testdata/vision/BUILD +++ b/mediapipe/tasks/testdata/vision/BUILD @@ -78,6 +78,7 @@ mediapipe_files(srcs = [ "pose.jpg", "pose_detection.tflite", "pose_landmark_lite.tflite", + "pose_landmarker.task", "right_hands.jpg", "right_hands_rotated.jpg", "segmentation_golden_rotation0.png", @@ -187,6 +188,7 @@ filegroup( "palm_detection_full.tflite", "pose_detection.tflite", "pose_landmark_lite.tflite", + "pose_landmarker.task", "selfie_segm_128_128_3.tflite", "selfie_segm_144_256_3.tflite", "selfie_segmentation.tflite", diff --git a/mediapipe/tasks/testdata/vision/pose_landmarker.task b/mediapipe/tasks/testdata/vision/pose_landmarker.task new file mode 100644 index 000000000..d57dd9e0d Binary files /dev/null and b/mediapipe/tasks/testdata/vision/pose_landmarker.task differ diff --git a/mediapipe/tasks/web/components/containers/BUILD b/mediapipe/tasks/web/components/containers/BUILD index 927146c04..477ca15c3 100644 --- a/mediapipe/tasks/web/components/containers/BUILD +++ b/mediapipe/tasks/web/components/containers/BUILD @@ -4,6 +4,11 @@ load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_declaration") package(default_visibility = ["//mediapipe/tasks:internal"]) +mediapipe_ts_declaration( + name = "bounding_box", + srcs = ["bounding_box.d.ts"], +) + mediapipe_ts_declaration( name = "category", srcs = ["category.d.ts"], @@ -15,6 +20,15 @@ mediapipe_ts_declaration( deps = [":category"], ) +mediapipe_ts_declaration( + name = "detection_result", + srcs = ["detection_result.d.ts"], + deps = [ + ":bounding_box", + ":category", + ], +) + mediapipe_ts_declaration( name = "keypoint", srcs = ["keypoint.d.ts"], diff --git a/mediapipe/tasks/web/components/containers/bounding_box.d.ts b/mediapipe/tasks/web/components/containers/bounding_box.d.ts new file mode 100644 index 000000000..69174f4c7 --- /dev/null +++ b/mediapipe/tasks/web/components/containers/bounding_box.d.ts @@ -0,0 +1,27 @@ +/** + * Copyright 2023 The MediaPipe Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** An integer bounding box, axis aligned. */ +export declare interface BoundingBox { + /** The X coordinate of the top-left corner, in pixels. */ + originX: number; + /** The Y coordinate of the top-left corner, in pixels. */ + originY: number; + /** The width of the bounding box, in pixels. */ + width: number; + /** The height of the bounding box, in pixels. */ + height: number; +} diff --git a/mediapipe/tasks/web/components/containers/detection_result.d.ts b/mediapipe/tasks/web/components/containers/detection_result.d.ts new file mode 100644 index 000000000..a338cc901 --- /dev/null +++ b/mediapipe/tasks/web/components/containers/detection_result.d.ts @@ -0,0 +1,33 @@ +/** + * Copyright 2023 The MediaPipe Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {BoundingBox} from '../../../../tasks/web/components/containers/bounding_box'; +import {Category} from '../../../../tasks/web/components/containers/category'; + +/** Represents one detection by a detection task. */ +export declare interface Detection { + /** A list of `Category` objects. */ + categories: Category[]; + + /** The bounding box of the detected objects. */ + boundingBox?: BoundingBox; +} + +/** Detection results of a model. */ +export interface DetectionResult { + /** A list of Detections. */ + detections: Detection[]; +} diff --git a/mediapipe/tasks/web/components/processors/BUILD b/mediapipe/tasks/web/components/processors/BUILD index b83f73eb2..a5f93a147 100644 --- a/mediapipe/tasks/web/components/processors/BUILD +++ b/mediapipe/tasks/web/components/processors/BUILD @@ -56,6 +56,27 @@ jasmine_node_test( deps = [":classifier_result_test_lib"], ) +mediapipe_ts_library( + name = "detection_result", + srcs = ["detection_result.ts"], + deps = [ + "//mediapipe/framework/formats:detection_jspb_proto", + "//mediapipe/tasks/web/components/containers:detection_result", + ], +) + +mediapipe_ts_library( + name = "detection_result_test_lib", + testonly = True, + srcs = ["detection_result.test.ts"], + deps = [ + ":detection_result", + "//mediapipe/framework/formats:detection_jspb_proto", + "//mediapipe/framework/formats:location_data_jspb_proto", + "//mediapipe/tasks/web/components/containers:detection_result", + ], +) + mediapipe_ts_library( name = "embedder_result", srcs = ["embedder_result.ts"], diff --git a/mediapipe/tasks/web/components/processors/detection_result.test.ts b/mediapipe/tasks/web/components/processors/detection_result.test.ts new file mode 100644 index 000000000..26f8bd8a5 --- /dev/null +++ b/mediapipe/tasks/web/components/processors/detection_result.test.ts @@ -0,0 +1,76 @@ +/** + * Copyright 2023 The MediaPipe Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import 'jasmine'; + +import {Detection as DetectionProto} from '../../../../framework/formats/detection_pb'; +import {LocationData} from '../../../../framework/formats/location_data_pb'; + +import {convertFromDetectionProto} from './detection_result'; + +// The OSS JS API does not support the builder pattern. +// tslint:disable:jspb-use-builder-pattern + +describe('convertFromDetectionProto()', () => { + it('transforms custom values', () => { + const detection = new DetectionProto(); + detection.addScore(0.1); + detection.addLabelId(1); + detection.addLabel('foo'); + detection.addDisplayName('bar'); + const locationData = new LocationData(); + const boundingBox = new LocationData.BoundingBox(); + boundingBox.setXmin(1); + boundingBox.setYmin(2); + boundingBox.setWidth(3); + boundingBox.setHeight(4); + locationData.setBoundingBox(boundingBox); + detection.setLocationData(locationData); + + const result = convertFromDetectionProto(detection); + + expect(result).toEqual({ + categories: [{ + score: 0.1, + index: 1, + categoryName: 'foo', + displayName: 'bar', + }], + boundingBox: {originX: 1, originY: 2, width: 3, height: 4} + }); + }); + + it('transforms default values', () => { + const detection = new DetectionProto(); + detection.addScore(0.2); + const locationData = new LocationData(); + const boundingBox = new LocationData.BoundingBox(); + locationData.setBoundingBox(boundingBox); + detection.setLocationData(locationData); + + const result = convertFromDetectionProto(detection); + + expect(result).toEqual({ + categories: [{ + score: 0.2, + index: -1, + categoryName: '', + displayName: '', + }], + boundingBox: {originX: 0, originY: 0, width: 0, height: 0} + }); + }); +}); diff --git a/mediapipe/tasks/web/components/processors/detection_result.ts b/mediapipe/tasks/web/components/processors/detection_result.ts new file mode 100644 index 000000000..01041c915 --- /dev/null +++ b/mediapipe/tasks/web/components/processors/detection_result.ts @@ -0,0 +1,50 @@ +/** + * Copyright 2023 The MediaPipe Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {Detection as DetectionProto} from '../../../../framework/formats/detection_pb'; +import {Detection} from '../../../../tasks/web/components/containers/detection_result'; + +const DEFAULT_CATEGORY_INDEX = -1; + +/** Converts a Detection proto into a Detection object. */ +export function convertFromDetectionProto(source: DetectionProto): Detection { + const scores = source.getScoreList(); + const indexes = source.getLabelIdList(); + const labels = source.getLabelList(); + const displayNames = source.getDisplayNameList(); + + const detection: Detection = {categories: []}; + for (let i = 0; i < scores.length; i++) { + detection.categories.push({ + score: scores[i], + index: indexes[i] ?? DEFAULT_CATEGORY_INDEX, + categoryName: labels[i] ?? '', + displayName: displayNames[i] ?? '', + }); + } + + const boundingBox = source.getLocationData()?.getBoundingBox(); + if (boundingBox) { + detection.boundingBox = { + originX: boundingBox.getXmin() ?? 0, + originY: boundingBox.getYmin() ?? 0, + width: boundingBox.getWidth() ?? 0, + height: boundingBox.getHeight() ?? 0 + }; + } + + return detection; +} diff --git a/mediapipe/tasks/web/vision/object_detector/BUILD b/mediapipe/tasks/web/vision/object_detector/BUILD index 76fa589c8..2c0fcbdf8 100644 --- a/mediapipe/tasks/web/vision/object_detector/BUILD +++ b/mediapipe/tasks/web/vision/object_detector/BUILD @@ -22,6 +22,7 @@ mediapipe_ts_library( "//mediapipe/tasks/cc/core/proto:base_options_jspb_proto", "//mediapipe/tasks/cc/vision/object_detector/proto:object_detector_options_jspb_proto", "//mediapipe/tasks/web/components/containers:category", + "//mediapipe/tasks/web/components/processors:detection_result", "//mediapipe/tasks/web/core", "//mediapipe/tasks/web/vision/core:image_processing_options", "//mediapipe/tasks/web/vision/core:vision_task_runner", @@ -37,7 +38,9 @@ mediapipe_ts_declaration( ], visibility = ["//visibility:public"], deps = [ + "//mediapipe/tasks/web/components/containers:bounding_box", "//mediapipe/tasks/web/components/containers:category", + "//mediapipe/tasks/web/components/containers:detection_result", "//mediapipe/tasks/web/core", "//mediapipe/tasks/web/core:classifier_options", "//mediapipe/tasks/web/vision/core:vision_task_options", diff --git a/mediapipe/tasks/web/vision/object_detector/object_detector.ts b/mediapipe/tasks/web/vision/object_detector/object_detector.ts index 42b62c9e2..5d406f1a0 100644 --- a/mediapipe/tasks/web/vision/object_detector/object_detector.ts +++ b/mediapipe/tasks/web/vision/object_detector/object_detector.ts @@ -19,6 +19,7 @@ import {CalculatorOptions} from '../../../../framework/calculator_options_pb'; import {Detection as DetectionProto} from '../../../../framework/formats/detection_pb'; import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/base_options_pb'; import {ObjectDetectorOptions as ObjectDetectorOptionsProto} from '../../../../tasks/cc/vision/object_detector/proto/object_detector_options_pb'; +import {convertFromDetectionProto} from '../../../../tasks/web/components/processors/detection_result'; import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset'; import {ImageProcessingOptions} from '../../../../tasks/web/vision/core/image_processing_options'; import {VisionGraphRunner, VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner'; @@ -26,15 +27,13 @@ import {ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner // Placeholder for internal dependency on trusted resource url import {ObjectDetectorOptions} from './object_detector_options'; -import {Detection} from './object_detector_result'; +import {ObjectDetectorResult} from './object_detector_result'; const IMAGE_STREAM = 'input_frame_gpu'; const NORM_RECT_STREAM = 'norm_rect'; const DETECTIONS_STREAM = 'detections'; const OBJECT_DETECTOR_GRAPH = 'mediapipe.tasks.vision.ObjectDetectorGraph'; -const DEFAULT_CATEGORY_INDEX = -1; - export * from './object_detector_options'; export * from './object_detector_result'; export {ImageSource}; // Used in the public API @@ -44,7 +43,7 @@ export {ImageSource}; // Used in the public API /** Performs object detection on images. */ export class ObjectDetector extends VisionTaskRunner { - private detections: Detection[] = []; + private result: ObjectDetectorResult = {detections: []}; private readonly options = new ObjectDetectorOptionsProto(); /** @@ -163,13 +162,13 @@ export class ObjectDetector extends VisionTaskRunner { * @param image An image to process. * @param imageProcessingOptions the `ImageProcessingOptions` specifying how * to process the input image before running inference. - * @return The list of detected objects + * @return A result containing a list of detected objects. */ detect(image: ImageSource, imageProcessingOptions?: ImageProcessingOptions): - Detection[] { - this.detections = []; + ObjectDetectorResult { + this.result = {detections: []}; this.processImageData(image, imageProcessingOptions); - return [...this.detections]; + return this.result; } /** @@ -181,46 +180,21 @@ export class ObjectDetector extends VisionTaskRunner { * @param timestamp The timestamp of the current frame, in ms. * @param imageProcessingOptions the `ImageProcessingOptions` specifying how * to process the input image before running inference. - * @return The list of detected objects + * @return A result containing a list of detected objects. */ detectForVideo( videoFrame: ImageSource, timestamp: number, - imageProcessingOptions?: ImageProcessingOptions): Detection[] { - this.detections = []; + imageProcessingOptions?: ImageProcessingOptions): ObjectDetectorResult { + this.result = {detections: []}; this.processVideoData(videoFrame, imageProcessingOptions, timestamp); - return [...this.detections]; + return this.result; } /** Converts raw data into a Detection, and adds it to our detection list. */ private addJsObjectDetections(data: Uint8Array[]): void { for (const binaryProto of data) { const detectionProto = DetectionProto.deserializeBinary(binaryProto); - const scores = detectionProto.getScoreList(); - const indexes = detectionProto.getLabelIdList(); - const labels = detectionProto.getLabelList(); - const displayNames = detectionProto.getDisplayNameList(); - - const detection: Detection = {categories: []}; - for (let i = 0; i < scores.length; i++) { - detection.categories.push({ - score: scores[i], - index: indexes[i] ?? DEFAULT_CATEGORY_INDEX, - categoryName: labels[i] ?? '', - displayName: displayNames[i] ?? '', - }); - } - - const boundingBox = detectionProto.getLocationData()?.getBoundingBox(); - if (boundingBox) { - detection.boundingBox = { - originX: boundingBox.getXmin() ?? 0, - originY: boundingBox.getYmin() ?? 0, - width: boundingBox.getWidth() ?? 0, - height: boundingBox.getHeight() ?? 0 - }; - } - - this.detections.push(detection); + this.result.detections.push(convertFromDetectionProto(detectionProto)); } } diff --git a/mediapipe/tasks/web/vision/object_detector/object_detector_result.d.ts b/mediapipe/tasks/web/vision/object_detector/object_detector_result.d.ts index c9c87a1bf..54bc6f907 100644 --- a/mediapipe/tasks/web/vision/object_detector/object_detector_result.d.ts +++ b/mediapipe/tasks/web/vision/object_detector/object_detector_result.d.ts @@ -14,27 +14,6 @@ * limitations under the License. */ -import {Category} from '../../../../tasks/web/components/containers/category'; - -export {Category}; - -/** An integer bounding box, axis aligned. */ -export declare interface BoundingBox { - /** The X coordinate of the top-left corner, in pixels. */ - originX: number; - /** The Y coordinate of the top-left corner, in pixels. */ - originY: number; - /** The width of the bounding box, in pixels. */ - width: number; - /** The height of the bounding box, in pixels. */ - height: number; -} - -/** Represents one object detected by the `ObjectDetector`. */ -export declare interface Detection { - /** A list of `Category` objects. */ - categories: Category[]; - - /** The bounding box of the detected objects. */ - boundingBox?: BoundingBox; -} +export {BoundingBox} from '../../../../tasks/web/components/containers/bounding_box'; +export {Category} from '../../../../tasks/web/components/containers/category'; +export {Detection, DetectionResult as ObjectDetectorResult} from '../../../../tasks/web/components/containers/detection_result'; diff --git a/mediapipe/tasks/web/vision/object_detector/object_detector_test.ts b/mediapipe/tasks/web/vision/object_detector/object_detector_test.ts index 9dd64c0b6..18e4a2062 100644 --- a/mediapipe/tasks/web/vision/object_detector/object_detector_test.ts +++ b/mediapipe/tasks/web/vision/object_detector/object_detector_test.ts @@ -179,56 +179,29 @@ describe('ObjectDetector', () => { }); it('transforms results', async () => { - const detectionProtos: Uint8Array[] = []; - - // Add a detection with all optional properties - let detection = new DetectionProto(); + const detection = new DetectionProto(); detection.addScore(0.1); - detection.addLabelId(1); - detection.addLabel('foo'); - detection.addDisplayName('bar'); - let locationData = new LocationData(); - let boundingBox = new LocationData.BoundingBox(); - boundingBox.setXmin(1); - boundingBox.setYmin(2); - boundingBox.setWidth(3); - boundingBox.setHeight(4); + const locationData = new LocationData(); + const boundingBox = new LocationData.BoundingBox(); locationData.setBoundingBox(boundingBox); detection.setLocationData(locationData); - detectionProtos.push(detection.serializeBinary()); - // Add a detection without optional properties - detection = new DetectionProto(); - detection.addScore(0.2); - locationData = new LocationData(); - boundingBox = new LocationData.BoundingBox(); - locationData.setBoundingBox(boundingBox); - detection.setLocationData(locationData); - detectionProtos.push(detection.serializeBinary()); + const binaryProto = detection.serializeBinary(); // Pass the test data to our listener objectDetector.fakeWasmModule._waitUntilIdle.and.callFake(() => { verifyListenersRegistered(objectDetector); - objectDetector.protoListener!(detectionProtos, 1337); + objectDetector.protoListener!([binaryProto], 1337); }); // Invoke the object detector - const detections = objectDetector.detect({} as HTMLImageElement); + const {detections} = objectDetector.detect({} as HTMLImageElement); expect(objectDetector.fakeWasmModule._waitUntilIdle).toHaveBeenCalled(); - expect(detections.length).toEqual(2); + expect(detections.length).toEqual(1); expect(detections[0]).toEqual({ categories: [{ score: 0.1, - index: 1, - categoryName: 'foo', - displayName: 'bar', - }], - boundingBox: {originX: 1, originY: 2, width: 3, height: 4} - }); - expect(detections[1]).toEqual({ - categories: [{ - score: 0.2, index: -1, categoryName: '', displayName: '', diff --git a/setup.py b/setup.py index 992430cf1..fe8f2dc68 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ from setuptools.command import install __version__ = 'dev' IS_WINDOWS = (platform.system() == 'Windows') +IS_MAC = (platform.system() == 'Darwin') MP_ROOT_PATH = os.path.dirname(os.path.abspath(__file__)) MP_DIR_INIT_PY = os.path.join(MP_ROOT_PATH, 'mediapipe/__init__.py') MP_THIRD_PARTY_BUILD = os.path.join(MP_ROOT_PATH, 'third_party/BUILD') @@ -343,11 +344,39 @@ class BuildExtension(build_ext.build_ext): def run(self): _check_bazel() - for ext in self.extensions: - self._build_binary(ext) + if IS_MAC: + for ext in self.extensions: + target_name = self.get_ext_fullpath(ext.name) + # Build x86 + self._build_binary( + ext, + ['--cpu=darwin_x86_64', '--ios_multi_cpus=i386,x86_64,armv7,arm64'], + ) + x86_name = self.get_ext_fullpath(ext.name) + # Build Arm64 + ext.name = ext.name + '.arm64' + self._build_binary( + ext, + ['--cpu=darwin_arm64', '--ios_multi_cpus=i386,x86_64,armv7,arm64'], + ) + arm64_name = self.get_ext_fullpath(ext.name) + # Merge architectures + lipo_command = [ + 'lipo', + '-create', + '-output', + target_name, + x86_name, + arm64_name, + ] + if subprocess.call(lipo_command) != 0: + sys.exit(-1) + else: + for ext in self.extensions: + self._build_binary(ext) build_ext.build_ext.run(self) - def _build_binary(self, ext): + def _build_binary(self, ext, extra_args=None): if not os.path.exists(self.build_temp): os.makedirs(self.build_temp) bazel_command = [ @@ -359,6 +388,8 @@ class BuildExtension(build_ext.build_ext): '--action_env=PYTHON_BIN_PATH=' + _normalize_path(sys.executable), str(ext.bazel_target + '.so'), ] + if extra_args: + bazel_command += extra_args if not self.link_opencv and not IS_WINDOWS: bazel_command.append('--define=OPENCV=source') if subprocess.call(bazel_command) != 0: diff --git a/third_party/external_files.bzl b/third_party/external_files.bzl index e438afd98..d32c973a0 100644 --- a/third_party/external_files.bzl +++ b/third_party/external_files.bzl @@ -67,13 +67,7 @@ def external_files(): http_file( name = "com_google_mediapipe_BUILD", sha256 = "d2b2a8346202691d7f831887c84e9642e974f64ed67851d9a58cf15c94b1f6b3", - urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD?generation=166187566369397616783235763936531678737479599640"], - ) - - http_file( - name = "com_google_mediapipe_BUILD_orig", - sha256 = "d86b98b82e00dd87cd46bd1429bf5eaa007b500c1a24d9316b73309f2e6c8df8", - urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD.orig?generation=1679955080207504"], + urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD?generation=1661875663693976167832357639365316787374795996401679955080207504"], ) http_file( @@ -262,6 +256,12 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/dummy_gesture_recognizer.task?generation=1665707319890725"], ) + http_file( + name = "com_google_mediapipe_dynamic_input_classifier_tflite", + sha256 = "fb34b05e1cd4081f3c2bb882092f617efb19266b3353d51b3790a172cae09784", + urls = ["https://storage.googleapis.com/mediapipe-assets/dynamic_input_classifier.tflite?generation=1680543275416843"], + ) + http_file( name = "com_google_mediapipe_efficientdet_lite0_v1_json", sha256 = "7a9e1fb625a6130a251e612637fc546cfc8cfabfadc7dbdade44c87f1d8996ca", @@ -306,8 +306,8 @@ def external_files(): http_file( name = "com_google_mediapipe_expected_pose_landmarks_prototxt", - sha256 = "75dfd2825fc23f51e3906f3a0a050caa8ae9f502cc358af1e7c9fda7ea89c9a5", - urls = ["https://storage.googleapis.com/mediapipe-assets/expected_pose_landmarks.prototxt?generation=1679955083449778"], + sha256 = "0bb27e9d9729c4171419abf7edd746b4234cb91198d663f3a4363248a49dad1a", + urls = ["https://storage.googleapis.com/mediapipe-assets/expected_pose_landmarks.prototxt?generation=1680543279295598"], ) http_file( @@ -934,12 +934,24 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/pose_expected_detection.pbtxt?generation=1678737492211540"], ) + http_file( + name = "com_google_mediapipe_pose_expected_expanded_rect_pbtxt", + sha256 = "babb2a2d50077f6fa9ee15e30d81abb6e98a920e35acad7542bb3d27b5ce7ffd", + urls = ["https://storage.googleapis.com/mediapipe-assets/pose_expected_expanded_rect.pbtxt?generation=1680543294008098"], + ) + http_file( name = "com_google_mediapipe_pose_jpg", sha256 = "c8a830ed683c0276d713dd5aeda28f415f10cd6291972084a40d0d8b934ed62b", urls = ["https://storage.googleapis.com/mediapipe-assets/pose.jpg?generation=1678737494661975"], ) + http_file( + name = "com_google_mediapipe_pose_landmarker_task", + sha256 = "ca4137626f0dc04f87893ccf2ad01949a3b1d4b55fa85ba957dde44a29dd956e", + urls = ["https://storage.googleapis.com/mediapipe-assets/pose_landmarker.task?generation=1680543298177615"], + ) + http_file( name = "com_google_mediapipe_pose_landmark_full_tflite", sha256 = "e9a5c5cb17f736fafd4c2ec1da3b3d331d6edbe8a0d32395855aeb2cdfd64b9f", @@ -1258,12 +1270,6 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/vocab_with_index.txt?generation=1661875977280658"], ) - http_file( - name = "com_google_mediapipe_w_avg_npy", - sha256 = "a044e35609986d18a972532f2980e939832b5b7d559659959d11ecc752a58bbe", - urls = ["https://storage.googleapis.com/mediapipe-assets/w_avg.npy?generation=1679955100435717"], - ) - http_file( name = "com_google_mediapipe_yamnet_audio_classifier_with_metadata_tflite", sha256 = "10c95ea3eb9a7bb4cb8bddf6feb023250381008177ac162ce169694d05c317de", @@ -1276,60 +1282,6 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/yamnet_embedding_metadata.tflite?generation=1668295071595506"], ) - http_file( - name = "com_google_mediapipe_decoder_fingerprint_pb", - sha256 = "0bf6239c4855d78edb60f3349b46cdb2c6f83def64f1b31589b6e298e5cbec3c", - urls = ["https://storage.googleapis.com/mediapipe-assets/decoder/fingerprint.pb?generation=1679955102906559"], - ) - - http_file( - name = "com_google_mediapipe_decoder_keras_metadata_pb", - sha256 = "1631ee698455aea52d4467fe6118800718a86ec49c29f4f3c904785b72f425ff", - urls = ["https://storage.googleapis.com/mediapipe-assets/decoder/keras_metadata.pb?generation=1679955105294959"], - ) - - http_file( - name = "com_google_mediapipe_decoder_saved_model_pb", - sha256 = "b424d30c63548e93390b2944b9bd9dc29773a56197bb462d3bd9e7a0bd1270ff", - urls = ["https://storage.googleapis.com/mediapipe-assets/decoder/saved_model.pb?generation=1679955107808916"], - ) - - http_file( - name = "com_google_mediapipe_discriminator_fingerprint_pb", - sha256 = "1fa6201d253c9218f7054138b9ce273266ce431e00cbce2d74d557f6b97223fd", - urls = ["https://storage.googleapis.com/mediapipe-assets/discriminator/fingerprint.pb?generation=1679955110094297"], - ) - - http_file( - name = "com_google_mediapipe_discriminator_keras_metadata_pb", - sha256 = "59a8601790d615dd37ec24e788743ce737e9999ce6ea6593fcf1ee43f674987f", - urls = ["https://storage.googleapis.com/mediapipe-assets/discriminator/keras_metadata.pb?generation=1679955112486389"], - ) - - http_file( - name = "com_google_mediapipe_discriminator_saved_model_pb", - sha256 = "280d6097a9b3d4c3756e028c597fe3d3c76eb14f76f24d49a22ed7b6df1e3878", - urls = ["https://storage.googleapis.com/mediapipe-assets/discriminator/saved_model.pb?generation=1679955114873053"], - ) - - http_file( - name = "com_google_mediapipe_encoder_fingerprint_pb", - sha256 = "06cb4319f8178edf447a7a2442e89303a14a48cc4fc5ae27354eac2ba11ae120", - urls = ["https://storage.googleapis.com/mediapipe-assets/encoder/fingerprint.pb?generation=1679955117213209"], - ) - - http_file( - name = "com_google_mediapipe_encoder_keras_metadata_pb", - sha256 = "8b1429ee95c130fad0c78077b2b544cd03c9e288658aae93e81df4959b84009e", - urls = ["https://storage.googleapis.com/mediapipe-assets/encoder/keras_metadata.pb?generation=1679955119546778"], - ) - - http_file( - name = "com_google_mediapipe_encoder_saved_model_pb", - sha256 = "ce48392c71485ecd9b142b46e54442581a299df5560102337038b76a62e02a09", - urls = ["https://storage.googleapis.com/mediapipe-assets/encoder/saved_model.pb?generation=1679955122069362"], - ) - http_file( name = "com_google_mediapipe_gesture_embedder_keras_metadata_pb", sha256 = "c76b856101e2284293a5e5963b7c445e407a0b3e56ec63eb78f64d883e51e3aa", @@ -1342,24 +1294,6 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/gesture_embedder/saved_model.pb?generation=1668550484904822"], ) - http_file( - name = "com_google_mediapipe_mapping_fingerprint_pb", - sha256 = "6320890f1b9a57e5f4e50e3b56d96fd39d815aa2de51dd1c9b635aa6107d982b", - urls = ["https://storage.googleapis.com/mediapipe-assets/mapping/fingerprint.pb?generation=1679955124430234"], - ) - - http_file( - name = "com_google_mediapipe_mapping_keras_metadata_pb", - sha256 = "22582a2ec1d4883b52f50e628c1a2d69a2610b38d72a48a0bd9939c26be304f6", - urls = ["https://storage.googleapis.com/mediapipe-assets/mapping/keras_metadata.pb?generation=1679955126858694"], - ) - - http_file( - name = "com_google_mediapipe_mapping_saved_model_pb", - sha256 = "6a79de45d00f49110304bf0a6746bc717c45f77824cad22690f700f2fbdc1470", - urls = ["https://storage.googleapis.com/mediapipe-assets/mapping/saved_model.pb?generation=1679955129259768"], - ) - http_file( name = "com_google_mediapipe_mobilebert_tiny_keras_metadata_pb", sha256 = "cef8131a414c602b9d4742ac57f4f90bc5d8a42baec36b65deece884e2d0cf0f", @@ -1408,42 +1342,6 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/object_detection_saved_model/saved_model.pb?generation=1661875999264354"], ) - http_file( - name = "com_google_mediapipe_decoder_variables_variables_data-00000-of-00001", - sha256 = "d720ddf354036f17fa210951f9ebfb009453b244913a493f494f1441cfc2eca3", - urls = ["https://storage.googleapis.com/mediapipe-assets/decoder/variables/variables.data-00000-of-00001?generation=1679955132326947"], - ) - - http_file( - name = "com_google_mediapipe_decoder_variables_variables_index", - sha256 = "245f69af6e53fb8b163059fe9936f57b68a7844e15d696393fcddf94c771dfcc", - urls = ["https://storage.googleapis.com/mediapipe-assets/decoder/variables/variables.index?generation=1679955134518344"], - ) - - http_file( - name = "com_google_mediapipe_discriminator_variables_variables_data-00000-of-00001", - sha256 = "50b00e1898a573588fb0d5d24d74346d99b7153b5d79441d0350c2c6ca89fb02", - urls = ["https://storage.googleapis.com/mediapipe-assets/discriminator/variables/variables.data-00000-of-00001?generation=1679955138489595"], - ) - - http_file( - name = "com_google_mediapipe_discriminator_variables_variables_index", - sha256 = "e5cb4be6442a5741504ce7da9487445637ad89b1f4b6a993bb9e762c7bd5621d", - urls = ["https://storage.googleapis.com/mediapipe-assets/discriminator/variables/variables.index?generation=1679955140891136"], - ) - - http_file( - name = "com_google_mediapipe_encoder_variables_variables_data-00000-of-00001", - sha256 = "09bcd1e2f1c6261bd1842af2da95651d54c9b4b9343eb9b8f0004a97f9bc84bf", - urls = ["https://storage.googleapis.com/mediapipe-assets/encoder/variables/variables.data-00000-of-00001?generation=1679955144875765"], - ) - - http_file( - name = "com_google_mediapipe_encoder_variables_variables_index", - sha256 = "964f5ac6ced7b19f76b7856d9dad47594a5b2fa89c52840f82996b809372aec9", - urls = ["https://storage.googleapis.com/mediapipe-assets/encoder/variables/variables.index?generation=1679955147123313"], - ) - http_file( name = "com_google_mediapipe_gesture_embedder_variables_variables_data-00000-of-00001", sha256 = "c156c9654c9ffb1091bb9f06c71080bd1e428586276d3f39c33fbab27fe0522d", @@ -1456,18 +1354,6 @@ def external_files(): urls = ["https://storage.googleapis.com/mediapipe-assets/gesture_embedder/variables/variables.index?generation=1668550490691823"], ) - http_file( - name = "com_google_mediapipe_mapping_variables_variables_data-00000-of-00001", - sha256 = "4187055e7f69fcc913ee2b11151a56149dda3017c75621d1e160596bde874c07", - urls = ["https://storage.googleapis.com/mediapipe-assets/mapping/variables/variables.data-00000-of-00001?generation=1679955149680908"], - ) - - http_file( - name = "com_google_mediapipe_mapping_variables_variables_index", - sha256 = "a04fcae7083715613f93ac89943f5fe1f5ba2e6efb9efd14eee7314f25502e4a", - urls = ["https://storage.googleapis.com/mediapipe-assets/mapping/variables/variables.index?generation=1679955152034297"], - ) - http_file( name = "com_google_mediapipe_mobilebert_tiny_assets_vocab_txt", sha256 = "07eced375cec144d27c900241f3e339478dec958f92fddbc551f295c992038a3",