diff --git a/MANIFEST.in b/MANIFEST.in
index 14afffebe..37900d0ea 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -7,14 +7,4 @@ include MANIFEST.in
include README.md
include requirements.txt
-recursive-include mediapipe/modules *.tflite *.txt *.binarypb
-exclude mediapipe/modules/face_detection/face_detection_full_range.tflite
-exclude mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite
-exclude mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite
-exclude mediapipe/modules/objectron/object_detection_3d_sneakers.tflite
-exclude mediapipe/modules/objectron/object_detection_3d_chair.tflite
-exclude mediapipe/modules/objectron/object_detection_3d_camera.tflite
-exclude mediapipe/modules/objectron/object_detection_3d_cup.tflite
-exclude mediapipe/modules/objectron/object_detection_ssd_mobilenetv2_oidv4_fp16.tflite
-exclude mediapipe/modules/pose_landmark/pose_landmark_lite.tflite
-exclude mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite
+recursive-include mediapipe/modules *.txt
diff --git a/README.md b/README.md
index 9c81095c5..e10952bcd 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ title: Home
nav_order: 1
---
-
+
--------------------------------------------------------------------------------
@@ -13,21 +13,21 @@ nav_order: 1
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
ML solutions for live and streaming media.
- | 
+ | 
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
- | 
+ | 
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
## ML solutions in MediaPipe
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :------:
-[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/holistic)
+[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/holistic)
Hair Segmentation | Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
-[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
+[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
diff --git a/WORKSPACE b/WORKSPACE
index 7a75537db..d3cc40fbe 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -2,6 +2,12 @@ workspace(name = "mediapipe")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+# Protobuf expects an //external:python_headers target
+bind(
+ name = "python_headers",
+ actual = "@local_config_python//:python_headers",
+)
+
http_archive(
name = "bazel_skylib",
type = "tar.gz",
@@ -142,12 +148,50 @@ http_archive(
],
)
+load("//third_party/flatbuffers:workspace.bzl", flatbuffers = "repo")
+flatbuffers()
+
http_archive(
name = "com_google_audio_tools",
strip_prefix = "multichannel-audio-tools-master",
urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
)
+# sentencepiece
+http_archive(
+ name = "com_google_sentencepiece",
+ strip_prefix = "sentencepiece-1.0.0",
+ sha256 = "c05901f30a1d0ed64cbcf40eba08e48894e1b0e985777217b7c9036cac631346",
+ urls = [
+ "https://github.com/google/sentencepiece/archive/1.0.0.zip",
+ ],
+ repo_mapping = {"@com_google_glog" : "@com_github_glog_glog"},
+)
+
+http_archive(
+ name = "org_tensorflow_text",
+ sha256 = "f64647276f7288d1b1fe4c89581d51404d0ce4ae97f2bcc4c19bd667549adca8",
+ strip_prefix = "text-2.2.0",
+ urls = [
+ "https://github.com/tensorflow/text/archive/v2.2.0.zip",
+ ],
+ patches = [
+ "//third_party:tensorflow_text_remove_tf_deps.diff",
+ "//third_party:tensorflow_text_a0f49e63.diff",
+ ],
+ patch_args = ["-p1"],
+ repo_mapping = {"@com_google_re2": "@com_googlesource_code_re2"},
+)
+
+http_archive(
+ name = "com_googlesource_code_re2",
+ sha256 = "e06b718c129f4019d6e7aa8b7631bee38d3d450dd980246bfaf493eb7db67868",
+ strip_prefix = "re2-fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8",
+ urls = [
+ "https://github.com/google/re2/archive/fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8.tar.gz",
+ ],
+)
+
# 2020-07-09
http_archive(
name = "pybind11_bazel",
@@ -167,6 +211,15 @@ http_archive(
build_file = "@pybind11_bazel//:pybind11.BUILD",
)
+http_archive(
+ name = "pybind11_protobuf",
+ sha256 = "baa1f53568283630a5055c85f0898b8810f7a6431bd01bbaedd32b4c1defbcb1",
+ strip_prefix = "pybind11_protobuf-3594106f2df3d725e65015ffb4c7886d6eeee683",
+ urls = [
+ "https://github.com/pybind/pybind11_protobuf/archive/3594106f2df3d725e65015ffb4c7886d6eeee683.tar.gz",
+ ],
+)
+
# Point to the commit that deprecates the usage of Eigen::MappedSparseMatrix.
http_archive(
name = "ceres_solver",
@@ -377,10 +430,29 @@ http_archive(
],
)
-# Tensorflow repo should always go after the other external dependencies.
-# 2022-02-15
-_TENSORFLOW_GIT_COMMIT = "a3419acc751dfc19caf4d34a1594e1f76810ec58"
-_TENSORFLOW_SHA256 = "b95b2a83632d4055742ae1a2dcc96b45da6c12a339462dbc76c8bca505308e3a"
+# Load Zlib before initializing TensorFlow to guarantee that the target
+# @zlib//:mini_zlib is available
+http_archive(
+ name = "zlib",
+ build_file = "//third_party:zlib.BUILD",
+ sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
+ strip_prefix = "zlib-1.2.11",
+ urls = [
+ "http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
+ "http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15
+ ],
+ patches = [
+ "@//third_party:zlib.diff",
+ ],
+ patch_args = [
+ "-p1",
+ ],
+)
+
+# TensorFlow repo should always go after the other external dependencies.
+# TF on 2022-08-10.
+_TENSORFLOW_GIT_COMMIT = "af1d5bc4fbb66d9e6cc1cf89503014a99233583b"
+_TENSORFLOW_SHA256 = "f85a5443264fc58a12d136ca6a30774b5bc25ceaf7d114d97f252351b3c3a2cb"
http_archive(
name = "org_tensorflow",
urls = [
@@ -417,3 +489,6 @@ libedgetpu_dependencies()
load("@coral_crosstool//:configure.bzl", "cc_crosstool")
cc_crosstool(name = "crosstool")
+
+load("//third_party:external_files.bzl", "external_files")
+external_files()
diff --git a/docs/_config.yml b/docs/_config.yml
index a48c21d6d..35d61eff6 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -20,7 +20,7 @@ aux_links:
- "//github.com/google/mediapipe"
# Footer content appears at the bottom of every page's main content
-footer_content: "© 2020 GOOGLE LLC | PRIVACY POLICY | TERMS OF SERVICE"
+footer_content: "© GOOGLE LLC | PRIVACY POLICY | TERMS OF SERVICE"
# Color scheme currently only supports "dark", "light"/nil (default), or a custom scheme that you define
color_scheme: mediapipe
diff --git a/docs/framework_concepts/calculators.md b/docs/framework_concepts/calculators.md
index 9548fa461..614abbbfa 100644
--- a/docs/framework_concepts/calculators.md
+++ b/docs/framework_concepts/calculators.md
@@ -133,7 +133,7 @@ write outputs. After Close returns, the calculator is destroyed.
Calculators with no inputs are referred to as sources. A source calculator
continues to have `Process()` called as long as it returns an `Ok` status. A
source calculator indicates that it is exhausted by returning a stop status
-(i.e. MediaPipe::tool::StatusStop).
+(i.e. [`mediaPipe::tool::StatusStop()`](https://github.com/google/mediapipe/tree/master/mediapipe/framework/tool/status_util.cc).).
## Identifying inputs and outputs
@@ -459,6 +459,6 @@ node {
The diagram below shows how the `PacketClonerCalculator` defines its output
packets (bottom) based on its series of input packets (top).
- |
+ |
:--------------------------------------------------------------------------: |
*Each time it receives a packet on its TICK input stream, the PacketClonerCalculator outputs the most recent packet from each of its input streams. The sequence of output packets (bottom) is determined by the sequence of input packets (top) and their timestamps. The timestamps are shown along the right side of the diagram.* |
diff --git a/docs/framework_concepts/gpu.md b/docs/framework_concepts/gpu.md
index 8f9df6067..b089dd6f8 100644
--- a/docs/framework_concepts/gpu.md
+++ b/docs/framework_concepts/gpu.md
@@ -149,7 +149,7 @@ When possible, these calculators use platform-specific functionality to share da
The below diagram shows the data flow in a mobile application that captures video from the camera, runs it through a MediaPipe graph, and renders the output on the screen in real time. The dashed line indicates which parts are inside the MediaPipe graph proper. This application runs a Canny edge-detection filter on the CPU using OpenCV, and overlays it on top of the original video using the GPU.
-
+
Video frames from the camera are fed into the graph as `GpuBuffer` packets. The
input stream is accessed by two calculators in parallel.
diff --git a/docs/framework_concepts/graphs.md b/docs/framework_concepts/graphs.md
index d7d972be5..f951b506d 100644
--- a/docs/framework_concepts/graphs.md
+++ b/docs/framework_concepts/graphs.md
@@ -159,7 +159,7 @@ Please use the `CalculatorGraphTest.Cycle` unit test in
below is the cyclic graph in the test. The `sum` output of the adder is the sum
of the integers generated by the integer source calculator.
-
+
This simple graph illustrates all the issues in supporting cyclic graphs.
diff --git a/docs/getting_started/android_archive_library.md b/docs/getting_started/android_archive_library.md
index a5752c6d5..5cef2b516 100644
--- a/docs/getting_started/android_archive_library.md
+++ b/docs/getting_started/android_archive_library.md
@@ -102,7 +102,7 @@ each project.
/path/to/your/app/libs/
```
- 
+ 
3. Make app/src/main/assets and copy assets (graph, model, and etc) into
app/src/main/assets.
@@ -120,7 +120,7 @@ each project.
cp mediapipe/modules/face_detection/face_detection_short_range.tflite /path/to/your/app/src/main/assets/
```
- 
+ 
4. Modify app/build.gradle to add MediaPipe dependencies and MediaPipe AAR.
diff --git a/docs/getting_started/android_solutions.md b/docs/getting_started/android_solutions.md
index 9df98043f..2333cd664 100644
--- a/docs/getting_started/android_solutions.md
+++ b/docs/getting_started/android_solutions.md
@@ -55,18 +55,18 @@ To build these apps:
2. Import mediapipe/examples/android/solutions directory into Android Studio.
- 
+ 
3. For Windows users, run `create_win_symlinks.bat` as administrator to create
res directory symlinks.
- 
+ 
4. Select "File" -> "Sync Project with Gradle Files" to sync project.
5. Run solution example app in Android Studio.
- 
+ 
6. (Optional) Run solutions on CPU.
diff --git a/docs/getting_started/hello_world_android.md b/docs/getting_started/hello_world_android.md
index 6674d4023..a80432173 100644
--- a/docs/getting_started/hello_world_android.md
+++ b/docs/getting_started/hello_world_android.md
@@ -27,7 +27,7 @@ graph on Android.
A simple camera app for real-time Sobel edge detection applied to a live video
stream on an Android device.
-
+
## Setup
@@ -69,7 +69,7 @@ node: {
A visualization of the graph is shown below:
-
+
This graph has a single input stream named `input_video` for all incoming frames
that will be provided by your device's camera.
@@ -260,7 +260,7 @@ adb install bazel-bin/$APPLICATION_PATH/helloworld.apk
Open the application on your device. It should display a screen with the text
`Hello World!`.
-
+
## Using the camera via `CameraX`
@@ -377,7 +377,7 @@ Add the following line in the `$APPLICATION_PATH/res/values/strings.xml` file:
When the user doesn't grant camera permission, the screen will now look like
this:
-
+
Now, we will add the [`SurfaceTexture`] and [`SurfaceView`] objects to
`MainActivity`:
@@ -753,7 +753,7 @@ And that's it! You should now be able to successfully build and run the
application on the device and see Sobel edge detection running on a live camera
feed! Congrats!
-
+
If you ran into any issues, please see the full code of the tutorial
[here](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/basic).
diff --git a/docs/getting_started/hello_world_cpp.md b/docs/getting_started/hello_world_cpp.md
index e3d34d9b4..b1bd54bef 100644
--- a/docs/getting_started/hello_world_cpp.md
+++ b/docs/getting_started/hello_world_cpp.md
@@ -85,7 +85,7 @@ nav_order: 1
This graph consists of 1 graph input stream (`in`) and 1 graph output stream
(`out`), and 2 [`PassThroughCalculator`]s connected serially.
- 
+ 
4. Before running the graph, an `OutputStreamPoller` object is connected to the
output stream in order to later retrieve the graph output, and a graph run
diff --git a/docs/getting_started/hello_world_ios.md b/docs/getting_started/hello_world_ios.md
index dd75d416a..b03b995f0 100644
--- a/docs/getting_started/hello_world_ios.md
+++ b/docs/getting_started/hello_world_ios.md
@@ -27,7 +27,7 @@ on iOS.
A simple camera app for real-time Sobel edge detection applied to a live video
stream on an iOS device.
-
+
## Setup
@@ -67,7 +67,7 @@ node: {
A visualization of the graph is shown below:
-
+
This graph has a single input stream named `input_video` for all incoming frames
that will be provided by your device's camera.
@@ -580,7 +580,7 @@ Update the interface definition of `ViewController` with `MPPGraphDelegate`:
And that is all! Build and run the app on your iOS device. You should see the
results of running the edge detection graph on a live video feed. Congrats!
-
+
Please note that the iOS examples now use a [common] template app. The code in
this tutorial is used in the [common] template app. The [helloworld] app has the
diff --git a/docs/getting_started/python.md b/docs/getting_started/python.md
index 83550be84..289988e55 100644
--- a/docs/getting_started/python.md
+++ b/docs/getting_started/python.md
@@ -113,9 +113,8 @@ Nvidia Jetson and Raspberry Pi, please read
Download the latest protoc win64 zip from
[the Protobuf GitHub repo](https://github.com/protocolbuffers/protobuf/releases),
- unzip the file, and copy the protoc.exe executable to a preferred
- location. Please ensure that location is added into the Path environment
- variable.
+ unzip the file, and copy the protoc.exe executable to a preferred location.
+ Please ensure that location is added into the Path environment variable.
3. Activate a Python virtual environment.
@@ -131,16 +130,14 @@ Nvidia Jetson and Raspberry Pi, please read
(mp_env)mediapipe$ pip3 install -r requirements.txt
```
-6. Generate and install MediaPipe package.
+6. Build and install MediaPipe package.
```bash
- (mp_env)mediapipe$ python3 setup.py gen_protos
(mp_env)mediapipe$ python3 setup.py install --link-opencv
```
or
```bash
- (mp_env)mediapipe$ python3 setup.py gen_protos
(mp_env)mediapipe$ python3 setup.py bdist_wheel
```
diff --git a/docs/images/accelerated.png b/docs/images/accelerated.png
deleted file mode 100644
index 8c9d241ca..000000000
Binary files a/docs/images/accelerated.png and /dev/null differ
diff --git a/docs/images/accelerated_small.png b/docs/images/accelerated_small.png
deleted file mode 100644
index 759542dc4..000000000
Binary files a/docs/images/accelerated_small.png and /dev/null differ
diff --git a/docs/images/add_ipa.png b/docs/images/add_ipa.png
deleted file mode 100644
index 6fb793487..000000000
Binary files a/docs/images/add_ipa.png and /dev/null differ
diff --git a/docs/images/app_ipa.png b/docs/images/app_ipa.png
deleted file mode 100644
index ebbe0ec87..000000000
Binary files a/docs/images/app_ipa.png and /dev/null differ
diff --git a/docs/images/app_ipa_added.png b/docs/images/app_ipa_added.png
deleted file mode 100644
index e6b1efd1b..000000000
Binary files a/docs/images/app_ipa_added.png and /dev/null differ
diff --git a/docs/images/attention_mesh_architecture.png b/docs/images/attention_mesh_architecture.png
deleted file mode 100644
index 3a38de5c9..000000000
Binary files a/docs/images/attention_mesh_architecture.png and /dev/null differ
diff --git a/docs/images/autoflip_edited_example.gif b/docs/images/autoflip_edited_example.gif
deleted file mode 100644
index c36fa573c..000000000
Binary files a/docs/images/autoflip_edited_example.gif and /dev/null differ
diff --git a/docs/images/autoflip_graph.png b/docs/images/autoflip_graph.png
deleted file mode 100644
index 55a647f6d..000000000
Binary files a/docs/images/autoflip_graph.png and /dev/null differ
diff --git a/docs/images/autoflip_is_required.gif b/docs/images/autoflip_is_required.gif
deleted file mode 100644
index 7db883470..000000000
Binary files a/docs/images/autoflip_is_required.gif and /dev/null differ
diff --git a/docs/images/bazel_permission.png b/docs/images/bazel_permission.png
deleted file mode 100644
index e67dd72dc..000000000
Binary files a/docs/images/bazel_permission.png and /dev/null differ
diff --git a/docs/images/box_coordinate.svg b/docs/images/box_coordinate.svg
deleted file mode 100644
index f436de896..000000000
--- a/docs/images/box_coordinate.svg
+++ /dev/null
@@ -1,3 +0,0 @@
-
-
-
diff --git a/docs/images/camera_coordinate.svg b/docs/images/camera_coordinate.svg
deleted file mode 100644
index 4cd3158ee..000000000
--- a/docs/images/camera_coordinate.svg
+++ /dev/null
@@ -1,3 +0,0 @@
-
-
-
diff --git a/docs/images/click_subgraph_handdetection.png b/docs/images/click_subgraph_handdetection.png
deleted file mode 100644
index 32cf3a1da..000000000
Binary files a/docs/images/click_subgraph_handdetection.png and /dev/null differ
diff --git a/docs/images/console_error.png b/docs/images/console_error.png
deleted file mode 100644
index 749fdf7a9..000000000
Binary files a/docs/images/console_error.png and /dev/null differ
diff --git a/docs/images/cross_platform.png b/docs/images/cross_platform.png
deleted file mode 100644
index 09dedc96a..000000000
Binary files a/docs/images/cross_platform.png and /dev/null differ
diff --git a/docs/images/cross_platform_small.png b/docs/images/cross_platform_small.png
deleted file mode 100644
index 7476327b2..000000000
Binary files a/docs/images/cross_platform_small.png and /dev/null differ
diff --git a/docs/images/cyclic_integer_sum_graph.svg b/docs/images/cyclic_integer_sum_graph.svg
deleted file mode 100644
index ac7d42a77..000000000
--- a/docs/images/cyclic_integer_sum_graph.svg
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
-
-
diff --git a/docs/images/device.png b/docs/images/device.png
deleted file mode 100644
index d911a24c2..000000000
Binary files a/docs/images/device.png and /dev/null differ
diff --git a/docs/images/editor_view.png b/docs/images/editor_view.png
deleted file mode 100644
index bfeac701f..000000000
Binary files a/docs/images/editor_view.png and /dev/null differ
diff --git a/docs/images/face_detection_desktop.png b/docs/images/face_detection_desktop.png
deleted file mode 100644
index 7f5f8ab1b..000000000
Binary files a/docs/images/face_detection_desktop.png and /dev/null differ
diff --git a/docs/images/face_geometry_metric_3d_space.gif b/docs/images/face_geometry_metric_3d_space.gif
deleted file mode 100644
index 1ecd20921..000000000
Binary files a/docs/images/face_geometry_metric_3d_space.gif and /dev/null differ
diff --git a/docs/images/face_geometry_renderer.gif b/docs/images/face_geometry_renderer.gif
deleted file mode 100644
index 1f18f765f..000000000
Binary files a/docs/images/face_geometry_renderer.gif and /dev/null differ
diff --git a/docs/images/face_mesh_ar_effects.gif b/docs/images/face_mesh_ar_effects.gif
deleted file mode 100644
index cf56ec719..000000000
Binary files a/docs/images/face_mesh_ar_effects.gif and /dev/null differ
diff --git a/docs/images/favicon.ico b/docs/images/favicon.ico
deleted file mode 100644
index 9ac07a20c..000000000
Binary files a/docs/images/favicon.ico and /dev/null differ
diff --git a/docs/images/faviconv2.ico b/docs/images/faviconv2.ico
deleted file mode 100644
index 30dce7213..000000000
Binary files a/docs/images/faviconv2.ico and /dev/null differ
diff --git a/docs/images/gpu_example_graph.png b/docs/images/gpu_example_graph.png
deleted file mode 100644
index e6f995e5a..000000000
Binary files a/docs/images/gpu_example_graph.png and /dev/null differ
diff --git a/docs/images/graph_visual.png b/docs/images/graph_visual.png
deleted file mode 100644
index 691d8df20..000000000
Binary files a/docs/images/graph_visual.png and /dev/null differ
diff --git a/docs/images/hand_tracking_desktop.png b/docs/images/hand_tracking_desktop.png
deleted file mode 100644
index 30ea34de5..000000000
Binary files a/docs/images/hand_tracking_desktop.png and /dev/null differ
diff --git a/docs/images/hello_world.png b/docs/images/hello_world.png
deleted file mode 100644
index 1005d7ffc..000000000
Binary files a/docs/images/hello_world.png and /dev/null differ
diff --git a/docs/images/iconv2.png b/docs/images/iconv2.png
deleted file mode 100644
index 74b3c7ae4..000000000
Binary files a/docs/images/iconv2.png and /dev/null differ
diff --git a/docs/images/import_mp_android_studio_project.png b/docs/images/import_mp_android_studio_project.png
deleted file mode 100644
index aa02b95ce..000000000
Binary files a/docs/images/import_mp_android_studio_project.png and /dev/null differ
diff --git a/docs/images/knift_stop_sign.gif b/docs/images/knift_stop_sign.gif
deleted file mode 100644
index a84b4aa19..000000000
Binary files a/docs/images/knift_stop_sign.gif and /dev/null differ
diff --git a/docs/images/logo.png b/docs/images/logo.png
deleted file mode 100644
index 1cca19eb2..000000000
Binary files a/docs/images/logo.png and /dev/null differ
diff --git a/docs/images/logo_horizontal_black.png b/docs/images/logo_horizontal_black.png
deleted file mode 100644
index 89f708fd0..000000000
Binary files a/docs/images/logo_horizontal_black.png and /dev/null differ
diff --git a/docs/images/logo_horizontal_color.png b/docs/images/logo_horizontal_color.png
deleted file mode 100644
index 6779a0d2a..000000000
Binary files a/docs/images/logo_horizontal_color.png and /dev/null differ
diff --git a/docs/images/logo_horizontal_white.png b/docs/images/logo_horizontal_white.png
deleted file mode 100644
index bd0e6d9ef..000000000
Binary files a/docs/images/logo_horizontal_white.png and /dev/null differ
diff --git a/docs/images/logov2.png b/docs/images/logov2.png
deleted file mode 100644
index 74b3c7ae4..000000000
Binary files a/docs/images/logov2.png and /dev/null differ
diff --git a/docs/images/maingraph_visualizer.png b/docs/images/maingraph_visualizer.png
deleted file mode 100644
index d34865c41..000000000
Binary files a/docs/images/maingraph_visualizer.png and /dev/null differ
diff --git a/docs/images/mediapipe_small.png b/docs/images/mediapipe_small.png
deleted file mode 100644
index 368e2b651..000000000
Binary files a/docs/images/mediapipe_small.png and /dev/null differ
diff --git a/docs/images/mobile/aar_location.png b/docs/images/mobile/aar_location.png
deleted file mode 100644
index 3dde1fa18..000000000
Binary files a/docs/images/mobile/aar_location.png and /dev/null differ
diff --git a/docs/images/mobile/assets_location.png b/docs/images/mobile/assets_location.png
deleted file mode 100644
index d22dbfaa5..000000000
Binary files a/docs/images/mobile/assets_location.png and /dev/null differ
diff --git a/docs/images/mobile/bazel_hello_world_android.png b/docs/images/mobile/bazel_hello_world_android.png
deleted file mode 100644
index 758e68cb8..000000000
Binary files a/docs/images/mobile/bazel_hello_world_android.png and /dev/null differ
diff --git a/docs/images/mobile/box_tracking_subgraph.png b/docs/images/mobile/box_tracking_subgraph.png
deleted file mode 100644
index fccfc65eb..000000000
Binary files a/docs/images/mobile/box_tracking_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/edge_detection_android_gpu.gif b/docs/images/mobile/edge_detection_android_gpu.gif
deleted file mode 100644
index a78a39876..000000000
Binary files a/docs/images/mobile/edge_detection_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/edge_detection_ios_gpu.gif b/docs/images/mobile/edge_detection_ios_gpu.gif
deleted file mode 100644
index 6d1a73060..000000000
Binary files a/docs/images/mobile/edge_detection_ios_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/edge_detection_mobile_gpu.png b/docs/images/mobile/edge_detection_mobile_gpu.png
deleted file mode 100644
index a082ec1d0..000000000
Binary files a/docs/images/mobile/edge_detection_mobile_gpu.png and /dev/null differ
diff --git a/docs/images/mobile/face_detection_android_gpu.gif b/docs/images/mobile/face_detection_android_gpu.gif
deleted file mode 100644
index 75d9228b3..000000000
Binary files a/docs/images/mobile/face_detection_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/face_detection_android_gpu_small.gif b/docs/images/mobile/face_detection_android_gpu_small.gif
deleted file mode 100644
index 0476602a3..000000000
Binary files a/docs/images/mobile/face_detection_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/face_detection_mobile_cpu.png b/docs/images/mobile/face_detection_mobile_cpu.png
deleted file mode 100644
index e57caa23a..000000000
Binary files a/docs/images/mobile/face_detection_mobile_cpu.png and /dev/null differ
diff --git a/docs/images/mobile/face_detection_mobile_gpu.png b/docs/images/mobile/face_detection_mobile_gpu.png
deleted file mode 100644
index 452b1a17f..000000000
Binary files a/docs/images/mobile/face_detection_mobile_gpu.png and /dev/null differ
diff --git a/docs/images/mobile/face_landmark_front_gpu_subgraph.png b/docs/images/mobile/face_landmark_front_gpu_subgraph.png
deleted file mode 100644
index a97b3da0b..000000000
Binary files a/docs/images/mobile/face_landmark_front_gpu_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/face_mesh_android_gpu.gif b/docs/images/mobile/face_mesh_android_gpu.gif
deleted file mode 100644
index cdba62021..000000000
Binary files a/docs/images/mobile/face_mesh_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/face_mesh_android_gpu_small.gif b/docs/images/mobile/face_mesh_android_gpu_small.gif
deleted file mode 100644
index 5ab431ef5..000000000
Binary files a/docs/images/mobile/face_mesh_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/face_mesh_mobile.png b/docs/images/mobile/face_mesh_mobile.png
deleted file mode 100644
index 0a109d617..000000000
Binary files a/docs/images/mobile/face_mesh_mobile.png and /dev/null differ
diff --git a/docs/images/mobile/face_renderer_gpu_subgraph.png b/docs/images/mobile/face_renderer_gpu_subgraph.png
deleted file mode 100644
index c53d854bd..000000000
Binary files a/docs/images/mobile/face_renderer_gpu_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/hair_segmentation_android_gpu.gif b/docs/images/mobile/hair_segmentation_android_gpu.gif
deleted file mode 100644
index 565f1849a..000000000
Binary files a/docs/images/mobile/hair_segmentation_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/hair_segmentation_android_gpu_small.gif b/docs/images/mobile/hair_segmentation_android_gpu_small.gif
deleted file mode 100644
index 737ef1506..000000000
Binary files a/docs/images/mobile/hair_segmentation_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/hair_segmentation_mobile_gpu.png b/docs/images/mobile/hair_segmentation_mobile_gpu.png
deleted file mode 100644
index 2a87ee834..000000000
Binary files a/docs/images/mobile/hair_segmentation_mobile_gpu.png and /dev/null differ
diff --git a/docs/images/mobile/hand_crops.png b/docs/images/mobile/hand_crops.png
deleted file mode 100644
index 46195aab0..000000000
Binary files a/docs/images/mobile/hand_crops.png and /dev/null differ
diff --git a/docs/images/mobile/hand_detection_android_gpu.gif b/docs/images/mobile/hand_detection_android_gpu.gif
deleted file mode 100644
index 38e32becf..000000000
Binary files a/docs/images/mobile/hand_detection_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/hand_detection_android_gpu_small.gif b/docs/images/mobile/hand_detection_android_gpu_small.gif
deleted file mode 100644
index bd61268fa..000000000
Binary files a/docs/images/mobile/hand_detection_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/hand_detection_gpu_subgraph.png b/docs/images/mobile/hand_detection_gpu_subgraph.png
deleted file mode 100644
index ba1fe9786..000000000
Binary files a/docs/images/mobile/hand_detection_gpu_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/hand_detection_mobile.png b/docs/images/mobile/hand_detection_mobile.png
deleted file mode 100644
index a0a763285..000000000
Binary files a/docs/images/mobile/hand_detection_mobile.png and /dev/null differ
diff --git a/docs/images/mobile/hand_landmark_gpu_subgraph.png b/docs/images/mobile/hand_landmark_gpu_subgraph.png
deleted file mode 100644
index 2e66d18d6..000000000
Binary files a/docs/images/mobile/hand_landmark_gpu_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/hand_landmarks.png b/docs/images/mobile/hand_landmarks.png
deleted file mode 100644
index f13746a86..000000000
Binary files a/docs/images/mobile/hand_landmarks.png and /dev/null differ
diff --git a/docs/images/mobile/hand_renderer_gpu_subgraph.png b/docs/images/mobile/hand_renderer_gpu_subgraph.png
deleted file mode 100644
index a32117252..000000000
Binary files a/docs/images/mobile/hand_renderer_gpu_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/hand_tracking_3d_android_gpu.gif b/docs/images/mobile/hand_tracking_3d_android_gpu.gif
deleted file mode 100644
index 60a95d438..000000000
Binary files a/docs/images/mobile/hand_tracking_3d_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/hand_tracking_android_gpu.gif b/docs/images/mobile/hand_tracking_android_gpu.gif
deleted file mode 100644
index b40e2986b..000000000
Binary files a/docs/images/mobile/hand_tracking_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/hand_tracking_android_gpu_small.gif b/docs/images/mobile/hand_tracking_android_gpu_small.gif
deleted file mode 100644
index c657edae0..000000000
Binary files a/docs/images/mobile/hand_tracking_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/hand_tracking_mobile.png b/docs/images/mobile/hand_tracking_mobile.png
deleted file mode 100644
index fb70f5e66..000000000
Binary files a/docs/images/mobile/hand_tracking_mobile.png and /dev/null differ
diff --git a/docs/images/mobile/holistic_pipeline_example.jpg b/docs/images/mobile/holistic_pipeline_example.jpg
deleted file mode 100644
index a35b3784b..000000000
Binary files a/docs/images/mobile/holistic_pipeline_example.jpg and /dev/null differ
diff --git a/docs/images/mobile/holistic_sports_and_gestures_example.gif b/docs/images/mobile/holistic_sports_and_gestures_example.gif
deleted file mode 100644
index d579e77ab..000000000
Binary files a/docs/images/mobile/holistic_sports_and_gestures_example.gif and /dev/null differ
diff --git a/docs/images/mobile/holistic_tracking_android_gpu_small.gif b/docs/images/mobile/holistic_tracking_android_gpu_small.gif
deleted file mode 100644
index 8cf0c226f..000000000
Binary files a/docs/images/mobile/holistic_tracking_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/instant_motion_tracking_android_small.gif b/docs/images/mobile/instant_motion_tracking_android_small.gif
deleted file mode 100644
index ff6d5537f..000000000
Binary files a/docs/images/mobile/instant_motion_tracking_android_small.gif and /dev/null differ
diff --git a/docs/images/mobile/iris_tracking_android_gpu.gif b/docs/images/mobile/iris_tracking_android_gpu.gif
deleted file mode 100644
index 6214d9e5c..000000000
Binary files a/docs/images/mobile/iris_tracking_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/iris_tracking_android_gpu_small.gif b/docs/images/mobile/iris_tracking_android_gpu_small.gif
deleted file mode 100644
index 050355476..000000000
Binary files a/docs/images/mobile/iris_tracking_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/iris_tracking_depth_from_iris.gif b/docs/images/mobile/iris_tracking_depth_from_iris.gif
deleted file mode 100644
index 2bcc80ea2..000000000
Binary files a/docs/images/mobile/iris_tracking_depth_from_iris.gif and /dev/null differ
diff --git a/docs/images/mobile/iris_tracking_example.gif b/docs/images/mobile/iris_tracking_example.gif
deleted file mode 100644
index 7988f3e95..000000000
Binary files a/docs/images/mobile/iris_tracking_example.gif and /dev/null differ
diff --git a/docs/images/mobile/iris_tracking_eye_and_iris_landmarks.png b/docs/images/mobile/iris_tracking_eye_and_iris_landmarks.png
deleted file mode 100644
index 1afb56395..000000000
Binary files a/docs/images/mobile/iris_tracking_eye_and_iris_landmarks.png and /dev/null differ
diff --git a/docs/images/mobile/missing_camera_permission_android.png b/docs/images/mobile/missing_camera_permission_android.png
deleted file mode 100644
index d492d56b8..000000000
Binary files a/docs/images/mobile/missing_camera_permission_android.png and /dev/null differ
diff --git a/docs/images/mobile/multi_hand_detection_gpu_subgraph.png b/docs/images/mobile/multi_hand_detection_gpu_subgraph.png
deleted file mode 100644
index 6105283b2..000000000
Binary files a/docs/images/mobile/multi_hand_detection_gpu_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/multi_hand_landmark_subgraph.png b/docs/images/mobile/multi_hand_landmark_subgraph.png
deleted file mode 100644
index 93f02bc42..000000000
Binary files a/docs/images/mobile/multi_hand_landmark_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/multi_hand_renderer_gpu_subgraph.png b/docs/images/mobile/multi_hand_renderer_gpu_subgraph.png
deleted file mode 100644
index 7da438e3f..000000000
Binary files a/docs/images/mobile/multi_hand_renderer_gpu_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/multi_hand_tracking_3d_android_gpu.gif b/docs/images/mobile/multi_hand_tracking_3d_android_gpu.gif
deleted file mode 100644
index 6aae8abca..000000000
Binary files a/docs/images/mobile/multi_hand_tracking_3d_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/multi_hand_tracking_3d_android_gpu_small.gif b/docs/images/mobile/multi_hand_tracking_3d_android_gpu_small.gif
deleted file mode 100644
index 24c101829..000000000
Binary files a/docs/images/mobile/multi_hand_tracking_3d_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/multi_hand_tracking_android_gpu.gif b/docs/images/mobile/multi_hand_tracking_android_gpu.gif
deleted file mode 100644
index 1e20dd082..000000000
Binary files a/docs/images/mobile/multi_hand_tracking_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/multi_hand_tracking_mobile.png b/docs/images/mobile/multi_hand_tracking_mobile.png
deleted file mode 100644
index b9eb410f3..000000000
Binary files a/docs/images/mobile/multi_hand_tracking_mobile.png and /dev/null differ
diff --git a/docs/images/mobile/object_detection_3d_android_gpu.png b/docs/images/mobile/object_detection_3d_android_gpu.png
deleted file mode 100644
index 4b0372d16..000000000
Binary files a/docs/images/mobile/object_detection_3d_android_gpu.png and /dev/null differ
diff --git a/docs/images/mobile/object_detection_android_cpu.gif b/docs/images/mobile/object_detection_android_cpu.gif
deleted file mode 100644
index 66c07d6ca..000000000
Binary files a/docs/images/mobile/object_detection_android_cpu.gif and /dev/null differ
diff --git a/docs/images/mobile/object_detection_android_gpu.gif b/docs/images/mobile/object_detection_android_gpu.gif
deleted file mode 100644
index 25e75f862..000000000
Binary files a/docs/images/mobile/object_detection_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/object_detection_android_gpu_small.gif b/docs/images/mobile/object_detection_android_gpu_small.gif
deleted file mode 100644
index db55678ba..000000000
Binary files a/docs/images/mobile/object_detection_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/object_detection_gpu_subgraph.png b/docs/images/mobile/object_detection_gpu_subgraph.png
deleted file mode 100644
index 8c4a24f95..000000000
Binary files a/docs/images/mobile/object_detection_gpu_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/object_detection_mobile_cpu.png b/docs/images/mobile/object_detection_mobile_cpu.png
deleted file mode 100644
index 48d7fb88e..000000000
Binary files a/docs/images/mobile/object_detection_mobile_cpu.png and /dev/null differ
diff --git a/docs/images/mobile/object_detection_mobile_gpu.png b/docs/images/mobile/object_detection_mobile_gpu.png
deleted file mode 100644
index 3f9ee6926..000000000
Binary files a/docs/images/mobile/object_detection_mobile_gpu.png and /dev/null differ
diff --git a/docs/images/mobile/object_tracking_android_gpu.gif b/docs/images/mobile/object_tracking_android_gpu.gif
deleted file mode 100644
index ed6f84ce7..000000000
Binary files a/docs/images/mobile/object_tracking_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/object_tracking_android_gpu_detection_only.gif b/docs/images/mobile/object_tracking_android_gpu_detection_only.gif
deleted file mode 100644
index b2c68520e..000000000
Binary files a/docs/images/mobile/object_tracking_android_gpu_detection_only.gif and /dev/null differ
diff --git a/docs/images/mobile/object_tracking_android_gpu_small.gif b/docs/images/mobile/object_tracking_android_gpu_small.gif
deleted file mode 100644
index db070efa2..000000000
Binary files a/docs/images/mobile/object_tracking_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/object_tracking_mobile_gpu.png b/docs/images/mobile/object_tracking_mobile_gpu.png
deleted file mode 100644
index de09514c1..000000000
Binary files a/docs/images/mobile/object_tracking_mobile_gpu.png and /dev/null differ
diff --git a/docs/images/mobile/object_tracking_renderer_gpu_subgraph.png b/docs/images/mobile/object_tracking_renderer_gpu_subgraph.png
deleted file mode 100644
index b164643a6..000000000
Binary files a/docs/images/mobile/object_tracking_renderer_gpu_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/object_tracking_subgraph.png b/docs/images/mobile/object_tracking_subgraph.png
deleted file mode 100644
index 8b7aa2143..000000000
Binary files a/docs/images/mobile/object_tracking_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/objectron_camera_android_gpu.gif b/docs/images/mobile/objectron_camera_android_gpu.gif
deleted file mode 100644
index 2ac32104d..000000000
Binary files a/docs/images/mobile/objectron_camera_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/objectron_chair_android_gpu.gif b/docs/images/mobile/objectron_chair_android_gpu.gif
deleted file mode 100644
index d2e0ef671..000000000
Binary files a/docs/images/mobile/objectron_chair_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/objectron_chair_android_gpu_small.gif b/docs/images/mobile/objectron_chair_android_gpu_small.gif
deleted file mode 100644
index 919bc0335..000000000
Binary files a/docs/images/mobile/objectron_chair_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/objectron_cup_android_gpu.gif b/docs/images/mobile/objectron_cup_android_gpu.gif
deleted file mode 100644
index 6b49e8f17..000000000
Binary files a/docs/images/mobile/objectron_cup_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/objectron_detection_subgraph.png b/docs/images/mobile/objectron_detection_subgraph.png
deleted file mode 100644
index 4d3bbc422..000000000
Binary files a/docs/images/mobile/objectron_detection_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/objectron_shoe_android_gpu.gif b/docs/images/mobile/objectron_shoe_android_gpu.gif
deleted file mode 100644
index ad0ae3697..000000000
Binary files a/docs/images/mobile/objectron_shoe_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/objectron_shoe_android_gpu_small.gif b/docs/images/mobile/objectron_shoe_android_gpu_small.gif
deleted file mode 100644
index 611f85dbe..000000000
Binary files a/docs/images/mobile/objectron_shoe_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/objectron_tracking_subgraph.png b/docs/images/mobile/objectron_tracking_subgraph.png
deleted file mode 100644
index 34296a502..000000000
Binary files a/docs/images/mobile/objectron_tracking_subgraph.png and /dev/null differ
diff --git a/docs/images/mobile/pose_classification_pairwise_distances.png b/docs/images/mobile/pose_classification_pairwise_distances.png
deleted file mode 100644
index 1aa2206df..000000000
Binary files a/docs/images/mobile/pose_classification_pairwise_distances.png and /dev/null differ
diff --git a/docs/images/mobile/pose_classification_pushups_and_squats.gif b/docs/images/mobile/pose_classification_pushups_and_squats.gif
deleted file mode 100644
index fe75f3bca..000000000
Binary files a/docs/images/mobile/pose_classification_pushups_and_squats.gif and /dev/null differ
diff --git a/docs/images/mobile/pose_classification_pushups_un_and_down_samples.jpg b/docs/images/mobile/pose_classification_pushups_un_and_down_samples.jpg
deleted file mode 100644
index 269e1b86b..000000000
Binary files a/docs/images/mobile/pose_classification_pushups_un_and_down_samples.jpg and /dev/null differ
diff --git a/docs/images/mobile/pose_segmentation.mp4 b/docs/images/mobile/pose_segmentation.mp4
deleted file mode 100644
index e0a68da70..000000000
Binary files a/docs/images/mobile/pose_segmentation.mp4 and /dev/null differ
diff --git a/docs/images/mobile/pose_tracking_android_gpu.gif b/docs/images/mobile/pose_tracking_android_gpu.gif
deleted file mode 100644
index deff2f02e..000000000
Binary files a/docs/images/mobile/pose_tracking_android_gpu.gif and /dev/null differ
diff --git a/docs/images/mobile/pose_tracking_android_gpu_small.gif b/docs/images/mobile/pose_tracking_android_gpu_small.gif
deleted file mode 100644
index 9d3ec1522..000000000
Binary files a/docs/images/mobile/pose_tracking_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/pose_tracking_detector_vitruvian_man.png b/docs/images/mobile/pose_tracking_detector_vitruvian_man.png
deleted file mode 100644
index ca25a5063..000000000
Binary files a/docs/images/mobile/pose_tracking_detector_vitruvian_man.png and /dev/null differ
diff --git a/docs/images/mobile/pose_tracking_example.gif b/docs/images/mobile/pose_tracking_example.gif
deleted file mode 100644
index e88f12f11..000000000
Binary files a/docs/images/mobile/pose_tracking_example.gif and /dev/null differ
diff --git a/docs/images/mobile/pose_tracking_full_body_landmarks.png b/docs/images/mobile/pose_tracking_full_body_landmarks.png
deleted file mode 100644
index 89530d9e4..000000000
Binary files a/docs/images/mobile/pose_tracking_full_body_landmarks.png and /dev/null differ
diff --git a/docs/images/mobile/pose_tracking_pck_chart.png b/docs/images/mobile/pose_tracking_pck_chart.png
deleted file mode 100644
index 1fa4bf97d..000000000
Binary files a/docs/images/mobile/pose_tracking_pck_chart.png and /dev/null differ
diff --git a/docs/images/mobile/pose_tracking_upper_body_landmarks.png b/docs/images/mobile/pose_tracking_upper_body_landmarks.png
deleted file mode 100644
index e2e964ec1..000000000
Binary files a/docs/images/mobile/pose_tracking_upper_body_landmarks.png and /dev/null differ
diff --git a/docs/images/mobile/pose_world_landmarks.mp4 b/docs/images/mobile/pose_world_landmarks.mp4
deleted file mode 100644
index 4a5bf3016..000000000
Binary files a/docs/images/mobile/pose_world_landmarks.mp4 and /dev/null differ
diff --git a/docs/images/mobile/renderer_gpu.png b/docs/images/mobile/renderer_gpu.png
deleted file mode 100644
index 9b062b9b1..000000000
Binary files a/docs/images/mobile/renderer_gpu.png and /dev/null differ
diff --git a/docs/images/mobile/template_matching_android_cpu.gif b/docs/images/mobile/template_matching_android_cpu.gif
deleted file mode 100644
index 9aa0229e7..000000000
Binary files a/docs/images/mobile/template_matching_android_cpu.gif and /dev/null differ
diff --git a/docs/images/mobile/template_matching_android_cpu_small.gif b/docs/images/mobile/template_matching_android_cpu_small.gif
deleted file mode 100644
index 68f64aea6..000000000
Binary files a/docs/images/mobile/template_matching_android_cpu_small.gif and /dev/null differ
diff --git a/docs/images/mobile/template_matching_mobile_graph.png b/docs/images/mobile/template_matching_mobile_graph.png
deleted file mode 100644
index 3e8c2b5d1..000000000
Binary files a/docs/images/mobile/template_matching_mobile_graph.png and /dev/null differ
diff --git a/docs/images/mobile/template_matching_mobile_template.jpg b/docs/images/mobile/template_matching_mobile_template.jpg
deleted file mode 100644
index 2843efdf6..000000000
Binary files a/docs/images/mobile/template_matching_mobile_template.jpg and /dev/null differ
diff --git a/docs/images/multi_hand_tracking_android_gpu.gif b/docs/images/multi_hand_tracking_android_gpu.gif
deleted file mode 100644
index 2cc920c86..000000000
Binary files a/docs/images/multi_hand_tracking_android_gpu.gif and /dev/null differ
diff --git a/docs/images/multi_hand_tracking_android_gpu_small.gif b/docs/images/multi_hand_tracking_android_gpu_small.gif
deleted file mode 100644
index 572b3658f..000000000
Binary files a/docs/images/multi_hand_tracking_android_gpu_small.gif and /dev/null differ
diff --git a/docs/images/multi_hand_tracking_desktop.png b/docs/images/multi_hand_tracking_desktop.png
deleted file mode 100644
index 5f84ab2f8..000000000
Binary files a/docs/images/multi_hand_tracking_desktop.png and /dev/null differ
diff --git a/docs/images/ndc_coordinate.svg b/docs/images/ndc_coordinate.svg
deleted file mode 100644
index 038660fd4..000000000
--- a/docs/images/ndc_coordinate.svg
+++ /dev/null
@@ -1,3 +0,0 @@
-
-
-
diff --git a/docs/images/object_detection_desktop_tensorflow.png b/docs/images/object_detection_desktop_tensorflow.png
deleted file mode 100644
index 50d7597f1..000000000
Binary files a/docs/images/object_detection_desktop_tensorflow.png and /dev/null differ
diff --git a/docs/images/object_detection_desktop_tflite.png b/docs/images/object_detection_desktop_tflite.png
deleted file mode 100644
index 27963d13d..000000000
Binary files a/docs/images/object_detection_desktop_tflite.png and /dev/null differ
diff --git a/docs/images/objectron_2stage_network_architecture.png b/docs/images/objectron_2stage_network_architecture.png
deleted file mode 100644
index 591f31f64..000000000
Binary files a/docs/images/objectron_2stage_network_architecture.png and /dev/null differ
diff --git a/docs/images/objectron_data_annotation.gif b/docs/images/objectron_data_annotation.gif
deleted file mode 100644
index 6466f7735..000000000
Binary files a/docs/images/objectron_data_annotation.gif and /dev/null differ
diff --git a/docs/images/objectron_example_results.png b/docs/images/objectron_example_results.png
deleted file mode 100644
index 977da33cc..000000000
Binary files a/docs/images/objectron_example_results.png and /dev/null differ
diff --git a/docs/images/objectron_network_architecture.png b/docs/images/objectron_network_architecture.png
deleted file mode 100644
index 2f0b6d9b2..000000000
Binary files a/docs/images/objectron_network_architecture.png and /dev/null differ
diff --git a/docs/images/objectron_sample_network_results.png b/docs/images/objectron_sample_network_results.png
deleted file mode 100644
index e3ae90f8a..000000000
Binary files a/docs/images/objectron_sample_network_results.png and /dev/null differ
diff --git a/docs/images/objectron_synthetic_data_generation.gif b/docs/images/objectron_synthetic_data_generation.gif
deleted file mode 100644
index 77705cca3..000000000
Binary files a/docs/images/objectron_synthetic_data_generation.gif and /dev/null differ
diff --git a/docs/images/open_source.png b/docs/images/open_source.png
deleted file mode 100644
index f337c8748..000000000
Binary files a/docs/images/open_source.png and /dev/null differ
diff --git a/docs/images/open_source_small.png b/docs/images/open_source_small.png
deleted file mode 100644
index c64ca50d3..000000000
Binary files a/docs/images/open_source_small.png and /dev/null differ
diff --git a/docs/images/packet_cloner_calculator.png b/docs/images/packet_cloner_calculator.png
deleted file mode 100644
index f2c2102ff..000000000
Binary files a/docs/images/packet_cloner_calculator.png and /dev/null differ
diff --git a/docs/images/ready_to_use.png b/docs/images/ready_to_use.png
deleted file mode 100644
index fbccbe830..000000000
Binary files a/docs/images/ready_to_use.png and /dev/null differ
diff --git a/docs/images/ready_to_use_small.png b/docs/images/ready_to_use_small.png
deleted file mode 100644
index 5091faaf6..000000000
Binary files a/docs/images/ready_to_use_small.png and /dev/null differ
diff --git a/docs/images/realtime_face_detection.gif b/docs/images/realtime_face_detection.gif
deleted file mode 100644
index a517a68d2..000000000
Binary files a/docs/images/realtime_face_detection.gif and /dev/null differ
diff --git a/docs/images/run_android_solution_app.png b/docs/images/run_android_solution_app.png
deleted file mode 100644
index aa21f3c24..000000000
Binary files a/docs/images/run_android_solution_app.png and /dev/null differ
diff --git a/docs/images/run_create_win_symlinks.png b/docs/images/run_create_win_symlinks.png
deleted file mode 100644
index 69b94b75f..000000000
Binary files a/docs/images/run_create_win_symlinks.png and /dev/null differ
diff --git a/docs/images/selfie_segmentation_web.mp4 b/docs/images/selfie_segmentation_web.mp4
deleted file mode 100644
index d9e62838e..000000000
Binary files a/docs/images/selfie_segmentation_web.mp4 and /dev/null differ
diff --git a/docs/images/side_packet.png b/docs/images/side_packet.png
deleted file mode 100644
index 5155835c0..000000000
Binary files a/docs/images/side_packet.png and /dev/null differ
diff --git a/docs/images/side_packet_code.png b/docs/images/side_packet_code.png
deleted file mode 100644
index 88a610305..000000000
Binary files a/docs/images/side_packet_code.png and /dev/null differ
diff --git a/docs/images/special_nodes.png b/docs/images/special_nodes.png
deleted file mode 100644
index bcb7763c0..000000000
Binary files a/docs/images/special_nodes.png and /dev/null differ
diff --git a/docs/images/special_nodes_code.png b/docs/images/special_nodes_code.png
deleted file mode 100644
index 148c54a3b..000000000
Binary files a/docs/images/special_nodes_code.png and /dev/null differ
diff --git a/docs/images/startup_screen.png b/docs/images/startup_screen.png
deleted file mode 100644
index a841ee759..000000000
Binary files a/docs/images/startup_screen.png and /dev/null differ
diff --git a/docs/images/stream_code.png b/docs/images/stream_code.png
deleted file mode 100644
index eabcbfe3f..000000000
Binary files a/docs/images/stream_code.png and /dev/null differ
diff --git a/docs/images/stream_ui.png b/docs/images/stream_ui.png
deleted file mode 100644
index 553e75143..000000000
Binary files a/docs/images/stream_ui.png and /dev/null differ
diff --git a/docs/images/upload_2pbtxt.png b/docs/images/upload_2pbtxt.png
deleted file mode 100644
index 02a079ae8..000000000
Binary files a/docs/images/upload_2pbtxt.png and /dev/null differ
diff --git a/docs/images/upload_button.png b/docs/images/upload_button.png
deleted file mode 100644
index 086f8379b..000000000
Binary files a/docs/images/upload_button.png and /dev/null differ
diff --git a/docs/images/upload_graph_button.png b/docs/images/upload_graph_button.png
deleted file mode 100644
index 9cbf31a8e..000000000
Binary files a/docs/images/upload_graph_button.png and /dev/null differ
diff --git a/docs/images/visualizer/ios_download_container.png b/docs/images/visualizer/ios_download_container.png
deleted file mode 100644
index 375b5410f..000000000
Binary files a/docs/images/visualizer/ios_download_container.png and /dev/null differ
diff --git a/docs/images/visualizer/ios_window_devices.png b/docs/images/visualizer/ios_window_devices.png
deleted file mode 100644
index c778afeaa..000000000
Binary files a/docs/images/visualizer/ios_window_devices.png and /dev/null differ
diff --git a/docs/images/visualizer/viz_chart_view.png b/docs/images/visualizer/viz_chart_view.png
deleted file mode 100644
index f18061397..000000000
Binary files a/docs/images/visualizer/viz_chart_view.png and /dev/null differ
diff --git a/docs/images/visualizer/viz_click_upload.png b/docs/images/visualizer/viz_click_upload.png
deleted file mode 100644
index c2f0ab127..000000000
Binary files a/docs/images/visualizer/viz_click_upload.png and /dev/null differ
diff --git a/docs/images/visualizer/viz_click_upload_trace_file.png b/docs/images/visualizer/viz_click_upload_trace_file.png
deleted file mode 100644
index d1ba8a223..000000000
Binary files a/docs/images/visualizer/viz_click_upload_trace_file.png and /dev/null differ
diff --git a/docs/images/visualizer_runner.png b/docs/images/visualizer_runner.png
deleted file mode 100644
index 5224a0949..000000000
Binary files a/docs/images/visualizer_runner.png and /dev/null differ
diff --git a/docs/images/web_effect.gif b/docs/images/web_effect.gif
deleted file mode 100644
index dac8e236b..000000000
Binary files a/docs/images/web_effect.gif and /dev/null differ
diff --git a/docs/images/web_segmentation.gif b/docs/images/web_segmentation.gif
deleted file mode 100644
index 516a07d6c..000000000
Binary files a/docs/images/web_segmentation.gif and /dev/null differ
diff --git a/docs/solutions/autoflip.md b/docs/solutions/autoflip.md
index 676abcae8..820478dca 100644
--- a/docs/solutions/autoflip.md
+++ b/docs/solutions/autoflip.md
@@ -27,7 +27,7 @@ to arbitrary aspect ratios.
For overall context on AutoFlip, please read this
[Google AI Blog](https://ai.googleblog.com/2020/02/autoflip-open-source-framework-for.html).
-
+
## Building
@@ -61,7 +61,7 @@ command above accordingly to run AutoFlip against the videos.
## MediaPipe Graph
-
+
To visualize the graph as shown above, copy the text specification of the graph
below and paste it into [MediaPipe Visualizer](https://viz.mediapipe.dev).
@@ -297,7 +297,7 @@ the required features cannot be all covered (for example, when they are too
spread out in the video), AutoFlip will apply a padding effect to cover as much
salient content as possible. See an illustration below.
-
+
### Stable vs Tracking Camera Motion
diff --git a/docs/solutions/box_tracking.md b/docs/solutions/box_tracking.md
index b84a015d1..0f65fbfca 100644
--- a/docs/solutions/box_tracking.md
+++ b/docs/solutions/box_tracking.md
@@ -80,7 +80,7 @@ frame (e.g., [MediaPipe Object Detection](./object_detection.md)):
* Object localization is temporally consistent with the help of tracking,
meaning less jitter is observable across frames.
- |
+ |
:----------------------------------------------------------------------------------: |
*Fig 1. Box tracking paired with ML-based object detection.* |
diff --git a/docs/solutions/face_detection.md b/docs/solutions/face_detection.md
index 4eccf17f5..9a56024ce 100644
--- a/docs/solutions/face_detection.md
+++ b/docs/solutions/face_detection.md
@@ -37,7 +37,7 @@ improved tie resolution strategy alternative to non-maximum suppression. For
more information about BlazeFace, please see the [Resources](#resources)
section.
-
+
## Solution APIs
diff --git a/docs/solutions/face_mesh.md b/docs/solutions/face_mesh.md
index ec43fb4ef..24ee760fc 100644
--- a/docs/solutions/face_mesh.md
+++ b/docs/solutions/face_mesh.md
@@ -38,7 +38,7 @@ lightweight statistical analysis method called
employed to drive a robust, performant and portable logic. The analysis runs on
CPU and has a minimal speed/memory footprint on top of the ML model inference.
- |
+ |
:-------------------------------------------------------------: |
*Fig 1. AR effects utilizing the 3D facial surface.* |
@@ -107,7 +107,7 @@ angle and occlusions.
You can find more information about the face landmark model in this
[paper](https://arxiv.org/abs/1907.06724).
- |
+ |
:------------------------------------------------------------------------: |
*Fig 2. Face landmarks: the red box indicates the cropped area as input to the landmark model, the red dots represent the 468 landmarks in 3D, and the green lines connecting landmarks illustrate the contours around the eyes, eyebrows, lips and the entire face.* |
@@ -124,7 +124,7 @@ The attention mesh model can be selected in the Solution APIs via the
[refine_landmarks](#refine_landmarks) option. You can also find more information
about the model in this [paper](https://arxiv.org/abs/2006.10962).
- |
+ |
:---------------------------------------------------------------------------: |
*Fig 3. Attention Mesh: Overview of model architecture.* |
@@ -161,7 +161,7 @@ coordinates back into the Metric 3D space. The *virtual camera parameters* can
be set freely, however for better results it is advised to set them as close to
the *real physical camera parameters* as possible.
- |
+ |
:-------------------------------------------------------------------------------: |
*Fig 4. A visualization of multiple key elements in the Metric 3D space.* |
@@ -225,7 +225,7 @@ hiding invisible elements behind the face surface.
The effect renderer is implemented as a MediaPipe
[calculator](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/effect_renderer_calculator.cc).
-|  |
+|  |
| :---------------------------------------------------------------------: |
| *Fig 5. An example of face effects rendered by the Face Transform Effect Renderer.* |
diff --git a/docs/solutions/hair_segmentation.md b/docs/solutions/hair_segmentation.md
index 9dd997b95..e94a4c79d 100644
--- a/docs/solutions/hair_segmentation.md
+++ b/docs/solutions/hair_segmentation.md
@@ -18,7 +18,7 @@ nav_order: 8
---
-
+
## Example Apps
@@ -58,7 +58,7 @@ processed all locally in real-time and never leaves your device. Please see
[MediaPipe on the Web](https://developers.googleblog.com/2020/01/mediapipe-on-web.html)
in Google Developers Blog for details.
-
+
## Resources
diff --git a/docs/solutions/hands.md b/docs/solutions/hands.md
index d73e32598..d3c245b76 100644
--- a/docs/solutions/hands.md
+++ b/docs/solutions/hands.md
@@ -38,7 +38,7 @@ hand perception functionality to the wider research and development community
will result in an emergence of creative use cases, stimulating new applications
and new research avenues.
- |
+ |
:------------------------------------------------------------------------------------: |
*Fig 1. Tracked 3D hand landmarks are represented by dots in different shades, with the brighter ones denoting landmarks closer to the camera.* |
@@ -91,9 +91,9 @@ To detect initial hand locations, we designed a
mobile real-time uses in a manner similar to the face detection model in
[MediaPipe Face Mesh](./face_mesh.md). Detecting hands is a decidedly complex
task: our
-[lite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_lite.tflite)
+[lite model](https://storage.googleapis.com/mediapipe-assets/palm_detection_lite.tflite)
and
-[full model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_full.tflite)
+[full model](https://storage.googleapis.com/mediapipe-assets/palm_detection_full.tflite)
have to work across a variety of hand sizes with a large scale span (~20x)
relative to the image frame and be able to detect occluded and self-occluded
hands. Whereas faces have high contrast patterns, e.g., in the eye and mouth
@@ -122,7 +122,7 @@ just 86.22%.
### Hand Landmark Model
After the palm detection over the whole image our subsequent hand landmark
-[model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_full.tflite)
+[model](https://storage.googleapis.com/mediapipe-assets/hand_landmark_full.tflite)
performs precise keypoint localization of 21 3D hand-knuckle coordinates inside
the detected hand regions via regression, that is direct coordinate prediction.
The model learns a consistent internal hand pose representation and is robust
@@ -135,11 +135,11 @@ and provide additional supervision on the nature of hand geometry, we also
render a high-quality synthetic hand model over various backgrounds and map it
to the corresponding 3D coordinates.
- |
+ |
:--------------------------------------------------------: |
*Fig 2. 21 hand landmarks.* |
- |
+ |
:-------------------------------------------------------------------------: |
*Fig 3. Top: Aligned hand crops passed to the tracking network with ground truth annotation. Bottom: Rendered synthetic hand images with ground truth annotation.* |
diff --git a/docs/solutions/holistic.md b/docs/solutions/holistic.md
index d0ab0b801..8c552834e 100644
--- a/docs/solutions/holistic.md
+++ b/docs/solutions/holistic.md
@@ -29,7 +29,7 @@ solutions for these tasks. Combining them all in real-time into a semantically
consistent end-to-end solution is a uniquely difficult problem requiring
simultaneous inference of multiple, dependent neural networks.
- |
+ |
:----------------------------------------------------------------------------------------------------: |
*Fig 1. Example of MediaPipe Holistic.* |
@@ -54,7 +54,7 @@ full-resolution input frame to these ROIs and apply task-specific face and hand
models to estimate their corresponding landmarks. Finally, we merge all
landmarks with those of the pose model to yield the full 540+ landmarks.
- |
+ |
:------------------------------------------------------------------------------: |
*Fig 2. MediaPipe Holistic Pipeline Overview.* |
diff --git a/docs/solutions/instant_motion_tracking.md b/docs/solutions/instant_motion_tracking.md
index 9fea7ec1c..6bdbe5e02 100644
--- a/docs/solutions/instant_motion_tracking.md
+++ b/docs/solutions/instant_motion_tracking.md
@@ -30,7 +30,7 @@ platforms without initialization or calibration. It is built upon the
Tracking, you can easily place virtual 2D and 3D content on static or moving
surfaces, allowing them to seamlessly interact with the real-world environment.
- |
+ |
:-----------------------------------------------------------------------: |
*Fig 1. Instant Motion Tracking is used to augment the world with a 3D sticker.* |
diff --git a/docs/solutions/iris.md b/docs/solutions/iris.md
index af71c895f..1d36f74ca 100644
--- a/docs/solutions/iris.md
+++ b/docs/solutions/iris.md
@@ -43,7 +43,7 @@ of the MediaPipe framework, MediaPipe Iris can run on most modern
[mobile phones](#mobile), [desktops/laptops](#desktop) and even on the
[web](#web).
- |
+ |
:------------------------------------------------------------------------: |
*Fig 1. Example of MediaPipe Iris: eyelid (red) and iris (blue) contours.* |
@@ -102,7 +102,7 @@ The iris model takes an image patch of the eye region and estimates both the eye
landmarks (along the eyelid) and iris landmarks (along ths iris contour). You
can find more details in this [paper](https://arxiv.org/abs/2006.11341).
- |
+ |
:----------------------------------------------------------------------------------------------------: |
*Fig 2. Eye landmarks (red) and iris landmarks (green).* |
@@ -115,7 +115,7 @@ human eye remains roughly constant at 11.7±0.5 mm across a wide population,
along with some simple geometric arguments. For more details please refer to our
[Google AI Blog post](https://ai.googleblog.com/2020/08/mediapipe-iris-real-time-iris-tracking.html).
- |
+ |
:--------------------------------------------------------------------------------------------: |
*Fig 3. (Left) MediaPipe Iris predicting metric distance in cm on a Pixel 2 from iris tracking without use of a depth sensor. (Right) Ground-truth depth.* |
@@ -200,7 +200,7 @@ never leaves your device. Please see
[MediaPipe on the Web](https://developers.googleblog.com/2020/01/mediapipe-on-web.html)
in Google Developers Blog for details.
-
+
* [MediaPipe Iris](https://viz.mediapipe.dev/demo/iris_tracking)
* [MediaPipe Iris: Depth-from-Iris](https://viz.mediapipe.dev/demo/iris_depth)
diff --git a/docs/solutions/knift.md b/docs/solutions/knift.md
index b008f1496..f2ec398ff 100644
--- a/docs/solutions/knift.md
+++ b/docs/solutions/knift.md
@@ -23,7 +23,7 @@ nav_order: 13
MediaPipe KNIFT is a template-based feature matching solution using KNIFT
(Keypoint Neural Invariant Feature Transform).
- |
+ |
:-----------------------------------------------------------------------: |
*Fig 1. Matching a real Stop Sign with a Stop Sign template using KNIFT.* |
@@ -56,7 +56,7 @@ For more information, please see
[MediaPipe KNIFT: Template-based feature matching](https://developers.googleblog.com/2020/04/mediapipe-knift-template-based-feature-matching.html)
in Google Developers Blog.
- |
+ |
:-------------------------------------------------------------------------------------: |
*Fig 2. Matching US dollar bills using KNIFT.* |
@@ -70,7 +70,7 @@ pre-computed from the 3 template images (of US dollar bills) shown below. If
you'd like to use your own template images, see
[Matching Your Own Template Images](#matching-your-own-template-images).
-
+
Please first see general instructions for
[Android](../getting_started/android.md) on how to build MediaPipe examples.
diff --git a/docs/solutions/models.md b/docs/solutions/models.md
index b2f59a9c8..18bcf0c8b 100644
--- a/docs/solutions/models.md
+++ b/docs/solutions/models.md
@@ -15,14 +15,14 @@ nav_order: 30
### [Face Detection](https://google.github.io/mediapipe/solutions/face_detection)
* Short-range model (best for faces within 2 meters from the camera):
- [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_short_range.tflite),
+ [TFLite model](https://storage.googleapis.com/mediapipe-assets/face_detection_short_range.tflite),
[TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite),
[Model card](https://mediapipe.page.link/blazeface-mc)
* Full-range model (dense, best for faces within 5 meters from the camera):
- [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_full_range.tflite),
+ [TFLite model](https://storage.googleapis.com/mediapipe-assets/face_detection_full_range.tflite),
[Model card](https://mediapipe.page.link/blazeface-back-mc)
* Full-range model (sparse, best for faces within 5 meters from the camera):
- [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite),
+ [TFLite model](https://storage.googleapis.com/mediapipe-assets/face_detection_full_range_sparse.tflite),
[Model card](https://mediapipe.page.link/blazeface-back-sparse-mc)
Full-range dense and sparse models have the same quality in terms of
@@ -39,77 +39,77 @@ one over the other.
### [Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh)
* Face landmark model:
- [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark.tflite),
+ [TFLite model](https://storage.googleapis.com/mediapipe-assets/face_landmark.tflite),
[TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
* Face landmark model w/ attention (aka Attention Mesh):
- [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_with_attention.tflite)
+ [TFLite model](https://storage.googleapis.com/mediapipe-assets/face_landmark_with_attention.tflite)
* [Model card](https://mediapipe.page.link/facemesh-mc),
[Model card (w/ attention)](https://mediapipe.page.link/attentionmesh-mc)
### [Iris](https://google.github.io/mediapipe/solutions/iris)
* Iris landmark model:
- [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/iris_landmark/iris_landmark.tflite)
+ [TFLite model](https://storage.googleapis.com/mediapipe-assets/iris_landmark.tflite)
* [Model card](https://mediapipe.page.link/iris-mc)
### [Hands](https://google.github.io/mediapipe/solutions/hands)
* Palm detection model:
- [TFLite model (lite)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_lite.tflite),
- [TFLite model (full)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/palm_detection/palm_detection_full.tflite),
+ [TFLite model (lite)](https://storage.googleapis.com/mediapipe-assets/palm_detection_lite.tflite),
+ [TFLite model (full)](https://storage.googleapis.com/mediapipe-assets/palm_detection_full.tflite),
[TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
* Hand landmark model:
- [TFLite model (lite)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_lite.tflite),
- [TFLite model (full)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_full.tflite),
+ [TFLite model (lite)](https://storage.googleapis.com/mediapipe-assets/hand_landmark_lite.tflite),
+ [TFLite model (full)](https://storage.googleapis.com/mediapipe-assets/hand_landmark_full.tflite),
[TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
* [Model card](https://mediapipe.page.link/handmc)
### [Pose](https://google.github.io/mediapipe/solutions/pose)
* Pose detection model:
- [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_detection/pose_detection.tflite)
+ [TFLite model](https://storage.googleapis.com/mediapipe-assets/pose_detection.tflite)
* Pose landmark model:
- [TFLite model (lite)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_lite.tflite),
- [TFLite model (full)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_full.tflite),
- [TFLite model (heavy)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/pose_landmark/pose_landmark_heavy.tflite)
+ [TFLite model (lite)](https://storage.googleapis.com/mediapipe-assets/pose_landmark_lite.tflite),
+ [TFLite model (full)](https://storage.googleapis.com/mediapipe-assets/pose_landmark_full.tflite),
+ [TFLite model (heavy)](https://storage.googleapis.com/mediapipe-assets/pose_landmark_heavy.tflite)
* [Model card](https://mediapipe.page.link/blazepose-mc)
### [Holistic](https://google.github.io/mediapipe/solutions/holistic)
* Hand recrop model:
- [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/modules/holistic_landmark/hand_recrop.tflite)
+ [TFLite model](https://storage.googleapis.com/mediapipe-assets/hand_recrop.tflite)
### [Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation)
-* [TFLite model (general)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/selfie_segmentation/selfie_segmentation.tflite)
-* [TFLite model (landscape)](https://github.com/google/mediapipe/tree/master/mediapipe/modules/selfie_segmentation/selfie_segmentation_landscape.tflite)
+* [TFLite model (general)](https://storage.googleapis.com/mediapipe-assets/selfie_segmentation.tflite)
+* [TFLite model (landscape)](https://storage.googleapis.com/mediapipe-assets/selfie_segmentation_landscape.tflite)
* [Model card](https://mediapipe.page.link/selfiesegmentation-mc)
### [Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation)
-* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
+* [TFLite model](https://storage.googleapis.com/mediapipe-assets/hair_segmentation.tflite)
* [Model card](https://mediapipe.page.link/hairsegmentation-mc)
### [Object Detection](https://google.github.io/mediapipe/solutions/object_detection)
-* [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
+* [TFLite model](https://storage.googleapis.com/mediapipe-assets/ssdlite_object_detection.tflite)
* [TFLite model quantized for EdgeTPU/Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite)
* [TensorFlow model](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model)
* [Model information](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md)
### [Objectron](https://google.github.io/mediapipe/solutions/objectron)
-* [TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_sneakers.tflite)
-* [TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_chair.tflite)
-* [TFLite model for cameras](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_camera.tflite)
-* [TFLite model for cups](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_cup.tflite)
-* [Single-stage TFLite model for shoes](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_sneakers_1stage.tflite)
-* [Single-stage TFLite model for chairs](https://github.com/google/mediapipe/tree/master/mediapipe/modules/objectron/object_detection_3d_chair_1stage.tflite)
+* [TFLite model for shoes](https://storage.googleapis.com/mediapipe-assets/object_detection_3d_sneakers.tflite)
+* [TFLite model for chairs](https://storage.googleapis.com/mediapipe-assets/object_detection_3d_chair.tflite)
+* [TFLite model for cameras](https://storage.googleapis.com/mediapipe-assets/object_detection_3d_camera.tflite)
+* [TFLite model for cups](https://storage.googleapis.com/mediapipe-assets/object_detection_3d_cup.tflite)
+* [Single-stage TFLite model for shoes](https://storage.googleapis.com/mediapipe-assets/object_detection_3d_sneakers_1stage.tflite)
+* [Single-stage TFLite model for chairs](https://storage.googleapis.com/mediapipe-assets/object_detection_3d_chair_1stage.tflite)
* [Model card](https://mediapipe.page.link/objectron-mc)
### [KNIFT](https://google.github.io/mediapipe/solutions/knift)
-* [TFLite model for up to 200 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float.tflite)
-* [TFLite model for up to 400 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_400.tflite)
-* [TFLite model for up to 1000 keypoints](https://github.com/google/mediapipe/tree/master/mediapipe/models/knift_float_1k.tflite)
+* [TFLite model for up to 200 keypoints](https://storage.googleapis.com/mediapipe-assets/knift_float.tflite)
+* [TFLite model for up to 400 keypoints](https://storage.googleapis.com/mediapipe-assets/knift_float_400.tflite)
+* [TFLite model for up to 1000 keypoints](https://storage.googleapis.com/mediapipe-assets/knift_float_1k.tflite)
* [Model card](https://mediapipe.page.link/knift-mc)
diff --git a/docs/solutions/object_detection.md b/docs/solutions/object_detection.md
index c60e44921..7ae5e9aff 100644
--- a/docs/solutions/object_detection.md
+++ b/docs/solutions/object_detection.md
@@ -18,7 +18,7 @@ nav_order: 9
---
-
+
## Example Apps
@@ -75,7 +75,7 @@ on how to build MediaPipe examples.
* With a TFLite Model
This uses the same
- [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/ssdlite_object_detection.tflite)
+ [TFLite model](https://storage.googleapis.com/mediapipe-assets/ssdlite_object_detection.tflite)
(see also
[model info](https://github.com/google/mediapipe/tree/master/mediapipe/models/object_detection_saved_model/README.md))
as in [Live Camera Input](#live-camera-input) above. The pipeline is
diff --git a/docs/solutions/objectron.md b/docs/solutions/objectron.md
index 23cf7c179..10483e499 100644
--- a/docs/solutions/objectron.md
+++ b/docs/solutions/objectron.md
@@ -24,7 +24,7 @@ MediaPipe Objectron is a mobile real-time 3D object detection solution for
everyday objects. It detects objects in 2D images, and estimates their poses
through a machine learning (ML) model, trained on the [Objectron dataset](https://github.com/google-research-datasets/Objectron).
- |  |  | 
+ |  |  | 
:--------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------: | :------------------------------------------------------------------------------------: | :------------------------------------------------------------------------------:
*Fig 1a. Shoe Objectron* | *Fig 1b. Chair Objectron* | *Fig 1c. Camera Objectron* | *Fig 1d. Cup Objectron*
@@ -39,7 +39,7 @@ mature and has been widely used in the industry, 3D object detection from 2D
imagery is a challenging problem, due to the lack of data and diversity of
appearances and shapes of objects within a category.
- |
+ |
:-----------------------------------------------------------------------: |
*Fig 2. Objectron example results.* |
@@ -68,7 +68,7 @@ object in a single frame and propagate its location to all frames using the
ground truth camera pose information from the AR session data, which makes the
procedure highly efficient.
-|  |
+|  |
| :--------------------------------------------------------------------------: |
| *Fig 3. Real-world data annotation for 3D object detection. (Right) 3D bounding boxes are annotated in the 3D world with detected surfaces and point clouds. (Left) Projections of annotated 3D bounding boxes are overlaid on top of video frames making it easy to validate the annotation.* |
@@ -86,7 +86,7 @@ with rendered objects that respect the scene geometry and fit seamlessly into
real backgrounds. By combining real-world data and AR synthetic data, we are
able to increase the accuracy by about 10%.
- |
+ |
:-------------------------------------------------------------------------------------------: |
*Fig 4. An example of AR synthetic data generation. The virtual white-brown cereal box is rendered into the real scene, next to the real blue book.* |
@@ -107,7 +107,7 @@ takes the image crop and estimates the 3D bounding box. At the same time, it
also computes the 2D crop of the object for the next frame, such that the object
detector does not need to run every frame.
- |
+ |
:----------------------------------------------------------------------------------------: |
*Fig 5. Network architecture and post-processing for two-stage 3D object detection.* |
@@ -119,7 +119,7 @@ mobile GPU.
### Single-stage Pipeline
- |
+ |
:---------------------------------------------------------------------------------: |
*Fig 6. Network architecture and post-processing for single-stage 3D object detection.* |
@@ -144,7 +144,7 @@ object dimensions. Given the 3D bounding box, we can easily compute pose and
size of the object. The model is light enough to run real-time on mobile devices
(at 26 FPS on an Adreno 650 mobile GPU).
- |
+ |
:-------------------------------------------------------------------------------------: |
*Fig 7. Sample results of our network — (Left) original 2D image with estimated bounding boxes, (Middle) object detection by Gaussian distribution, (Right) predicted segmentation mask.* |
@@ -573,7 +573,7 @@ Each object has its object coordinate frame. We use the below object coordinate
definition, with `+x` pointing right, `+y` pointing up and `+z` pointing front,
origin is at the center of the 3D bounding box.
-
+
### Camera Coordinate
@@ -582,7 +582,7 @@ regard to the camera coordinate frame. In this API we use the below camera
coordinate definition, with `+x` pointing right, `+y` pointing up and `-z`
pointing to the scene.
-
+
To work with box landmarks, one can first derive landmark coordinates in object
frame by scaling a origin centered unit box with `scale`, then transform to
@@ -599,7 +599,7 @@ In this API we use
as an intermediate space when projecting points from 3D to 2D. In NDC space,
`x`, `y` are confined to `[-1, 1]`.
-
+
By default the camera parameters `(fx, fy)` and `(px, py)` are defined in NDC
space. Given `(X, Y, Z)` of 3D points in camera coordinate, one can project 3D
diff --git a/docs/solutions/pose.md b/docs/solutions/pose.md
index 1c9d6a669..8c57c033e 100644
--- a/docs/solutions/pose.md
+++ b/docs/solutions/pose.md
@@ -40,7 +40,7 @@ environments for inference, whereas our method achieves real-time performance on
most modern [mobile phones](#mobile), [desktops/laptops](#desktop), in
[python](#python-solution-api) and even on the [web](#javascript-solution-api).
- |
+ |
:----------------------------------------------------------------------: |
*Fig 1. Example of MediaPipe Pose for pose tracking.* |
@@ -94,7 +94,7 @@ BlazePose GHUM Lite
[AlphaPose ResNet50](https://github.com/MVIG-SJTU/AlphaPose) | 63.4 | **96.0** | 57.8 | **95.5** | 63.4 | **96.0**
[Apple Vision](https://developer.apple.com/documentation/vision/detecting_human_body_poses_in_images) | 32.8 | **82.7** | 36.4 | **91.4** | 44.5 | **88.6**
- |
+ |
:--------------------------------------------------------------------------: |
*Fig 2. Quality evaluation in [`PCK@0.2`].* |
@@ -121,7 +121,7 @@ predict the midpoint of a person's hips, the radius of a circle circumscribing
the whole person, and the incline angle of the line connecting the shoulder and
hip midpoints.
- |
+ |
:----------------------------------------------------------------------------------------------------: |
*Fig 3. Vitruvian man aligned via two virtual keypoints predicted by BlazePose detector in addition to the face bounding box.* |
@@ -130,7 +130,7 @@ hip midpoints.
The landmark model in MediaPipe Pose predicts the location of 33 pose landmarks
(see figure below).
- |
+ |
:----------------------------------------------------------------------------------------------: |
*Fig 4. 33 pose landmarks.* |
diff --git a/docs/solutions/pose_classification.md b/docs/solutions/pose_classification.md
index 21f87a95d..38cb4f80a 100644
--- a/docs/solutions/pose_classification.md
+++ b/docs/solutions/pose_classification.md
@@ -31,7 +31,7 @@ demo within
Push-ups and squats are used for demonstration purposes as the most common
exercises.
- |
+ |
:--------------------------------------------------------------------------------------------------------: |
*Fig 1. Pose classification and repetition counting with MediaPipe Pose.* |
@@ -58,7 +58,7 @@ exercise (e.g., "up" and "down" positions for push-ups). It's important that
collected samples cover different camera angles, environment conditions, body
shapes, and exercise variations.
- |
+ |
:--------------------------------------------------------------------------------------------------------------------------: |
*Fig 2. Two terminal states of push-ups.* |
@@ -90,7 +90,7 @@ ankle and hip, and two wrists. Since the algorithm relies on distances, all
poses are normalized to have the same torso size and vertical torso orientation
before the conversion.
- |
+ |
:--------------------------------------------------------------------------------------------------------: |
*Fig 3. Main pairwise distances used for the pose feature vector.* |
diff --git a/docs/tools/tracing_and_profiling.md b/docs/tools/tracing_and_profiling.md
index 2d712461f..861564c99 100644
--- a/docs/tools/tracing_and_profiling.md
+++ b/docs/tools/tracing_and_profiling.md
@@ -70,7 +70,7 @@ MediaPipe will emit data into a pre-specified directory:
* On iOS, this can be reached through XCode. Select "Window/Devices and
Simulators" and select the "Devices" tab.
- 
+ 
You can open the Download Container. Logs will be located in `application
container/.xcappdata/AppData/Documents/`
@@ -78,7 +78,7 @@ MediaPipe will emit data into a pre-specified directory:
right click and select 'Show Package Contents' in Finder. Logs
will be located in 'AppData/Documents/'
- 
+ 
Log files are written to `\.binarypb` where, by default,
`\` is equal to `mediapipe_trace_` (the entire path and file
@@ -176,11 +176,11 @@ Trace logs can be analyzed from within the visualizer.
2. Click on the "Upload" button in the upper right.
- 
+ 
3. Click on "Upload trace file".
- 
+ 
A sample trace file has been generated for you:
[sample_trace_binary.pb](../data/visualizer/sample_trace.binarypb)
@@ -191,7 +191,7 @@ Trace logs can be analyzed from within the visualizer.
5. A chart view will appear. All of your calculators will appear along the left
with profiling information listed along the top.
- 
+ 
Click on a header to alternately sort that column in ascending or descending
order. You can also scroll horizontally and vertically within the control to
diff --git a/docs/tools/visualizer.md b/docs/tools/visualizer.md
index 9324576a2..5ed2de2d2 100644
--- a/docs/tools/visualizer.md
+++ b/docs/tools/visualizer.md
@@ -21,7 +21,7 @@ that is available online.
through a graph configuration that is pasted into the graph editor or
uploaded. The user can visualize and troubleshoot a graph they have created.
- 
+ 
## Working within the Editor
@@ -29,12 +29,12 @@ Getting Started:
The graph can be modified by adding and editing code in the Editor view.
-
+
* Pressing the "New" button in the upper right corner will clear any existing
code in the Editor window.
- 
+ 
* Pressing the "Upload" button will prompt the user to select a local PBTXT
file, which will overwrite the current code within the editor.
@@ -43,7 +43,7 @@ The graph can be modified by adding and editing code in the Editor view.
* Errors and informational messages will appear in the Feedback window.
- 
+ 
## Understanding the Graph
@@ -53,24 +53,24 @@ The visualizer graph shows the connections between calculator nodes.
enter the top of any calculator receiving the stream. (Notice the use of the
key, "input_stream" and "output_stream").
- 
+ 
- 
+ 
* Sidepackets work the same, except that they exit a node on the right and
enter on the left. (Notice the use of the key, "input_side_packet" and
"output_side_packet").
- 
+ 
- 
+ 
* There are special nodes that represent inputs and outputs to the graph and
can supply either side packets or streams.
- 
+ 
- 
+ 
## Visualizing Subgraphs
@@ -91,16 +91,16 @@ To visualize them:
* In the MediaPipe visualizer, click on the upload graph button and select the
2 pbtxt files to visualize (main graph and its associated subgraph).
- 
+ 
- 
+ 
* There will be 2 additional tabs. The main graph tab is
`hand_detection_mobile.pbtxt`.
- 
+ 
* Clicking on the `HandDetection` node in purple redirects the view to the
`hand_detection_gpu.pbtxt` tab.
- 
+ 
diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD
index e741ebad4..b28a3573a 100644
--- a/mediapipe/calculators/core/BUILD
+++ b/mediapipe/calculators/core/BUILD
@@ -151,6 +151,16 @@ mediapipe_proto_library(
],
)
+mediapipe_proto_library(
+ name = "get_vector_item_calculator_proto",
+ srcs = ["get_vector_item_calculator.proto"],
+ visibility = ["//visibility:public"],
+ deps = [
+ "//mediapipe/framework:calculator_options_proto",
+ "//mediapipe/framework:calculator_proto",
+ ],
+)
+
cc_library(
name = "add_header_calculator",
srcs = ["add_header_calculator.cc"],
@@ -561,6 +571,7 @@ cc_test(
name = "packet_cloner_calculator_test",
srcs = ["packet_cloner_calculator_test.cc"],
deps = [
+ ":gate_calculator",
":packet_cloner_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
@@ -1281,6 +1292,7 @@ cc_library(
hdrs = ["get_vector_item_calculator.h"],
visibility = ["//visibility:public"],
deps = [
+ ":get_vector_item_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/framework/api2:node",
@@ -1293,6 +1305,20 @@ cc_library(
alwayslink = 1,
)
+cc_test(
+ name = "get_vector_item_calculator_test",
+ srcs = ["get_vector_item_calculator_test.cc"],
+ deps = [
+ ":get_vector_item_calculator",
+ "//mediapipe/framework:calculator_framework",
+ "//mediapipe/framework:calculator_runner",
+ "//mediapipe/framework/port:gtest_main",
+ "//mediapipe/framework/port:parse_text_proto",
+ "@com_google_absl//absl/strings:str_format",
+ "@com_google_googletest//:gtest_main",
+ ],
+)
+
cc_library(
name = "vector_size_calculator",
srcs = ["vector_size_calculator.cc"],
@@ -1307,3 +1333,54 @@ cc_library(
],
alwayslink = 1,
)
+
+cc_library(
+ name = "packet_sequencer_calculator",
+ srcs = ["packet_sequencer_calculator.cc"],
+ visibility = [
+ "//visibility:public",
+ ],
+ deps = [
+ "//mediapipe/framework:calculator_framework",
+ "//mediapipe/framework/api2:contract",
+ "//mediapipe/framework/api2:node",
+ "//mediapipe/framework/api2:packet",
+ "//mediapipe/framework/api2:port",
+ "//mediapipe/framework/port:status",
+ "//mediapipe/framework/stream_handler:immediate_input_stream_handler",
+ ],
+ alwayslink = 1,
+)
+
+cc_test(
+ name = "packet_sequencer_calculator_test",
+ srcs = ["packet_sequencer_calculator_test.cc"],
+ deps = [
+ ":packet_sequencer_calculator",
+ "//mediapipe/calculators/core:pass_through_calculator",
+ "//mediapipe/framework:calculator_cc_proto",
+ "//mediapipe/framework:calculator_framework",
+ "//mediapipe/framework:subgraph",
+ "//mediapipe/framework/port:gtest_main",
+ "//mediapipe/framework/port:logging",
+ "//mediapipe/framework/port:parse_text_proto",
+ "//mediapipe/framework/port:ret_check",
+ "//mediapipe/framework/port:status",
+ "@com_google_absl//absl/strings",
+ ],
+)
+
+cc_library(
+ name = "merge_to_vector_calculator",
+ srcs = ["merge_to_vector_calculator.cc"],
+ hdrs = ["merge_to_vector_calculator.h"],
+ visibility = ["//visibility:public"],
+ deps = [
+ "//mediapipe/framework:calculator_framework",
+ "//mediapipe/framework/api2:node",
+ "//mediapipe/framework/api2:port",
+ "//mediapipe/framework/formats:image",
+ "@com_google_absl//absl/status",
+ ],
+ alwayslink = 1,
+)
diff --git a/mediapipe/calculators/core/clip_vector_size_calculator.proto b/mediapipe/calculators/core/clip_vector_size_calculator.proto
index 6044f77c8..5dea660d6 100644
--- a/mediapipe/calculators/core/clip_vector_size_calculator.proto
+++ b/mediapipe/calculators/core/clip_vector_size_calculator.proto
@@ -18,8 +18,6 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
-
message ClipVectorSizeCalculatorOptions {
extend CalculatorOptions {
optional ClipVectorSizeCalculatorOptions ext = 274674998;
diff --git a/mediapipe/calculators/core/concatenate_vector_calculator.proto b/mediapipe/calculators/core/concatenate_vector_calculator.proto
index 3753ffb5d..bddb8af95 100644
--- a/mediapipe/calculators/core/concatenate_vector_calculator.proto
+++ b/mediapipe/calculators/core/concatenate_vector_calculator.proto
@@ -18,8 +18,6 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
-
message ConcatenateVectorCalculatorOptions {
extend CalculatorOptions {
optional ConcatenateVectorCalculatorOptions ext = 259397839;
diff --git a/mediapipe/calculators/core/constant_side_packet_calculator.proto b/mediapipe/calculators/core/constant_side_packet_calculator.proto
index 4ad497b8e..ec12d0115 100644
--- a/mediapipe/calculators/core/constant_side_packet_calculator.proto
+++ b/mediapipe/calculators/core/constant_side_packet_calculator.proto
@@ -20,8 +20,6 @@ import "mediapipe/framework/calculator.proto";
import "mediapipe/framework/formats/classification.proto";
import "mediapipe/framework/formats/landmark.proto";
-option objc_class_prefix = "MediaPipe";
-
message ConstantSidePacketCalculatorOptions {
extend CalculatorOptions {
optional ConstantSidePacketCalculatorOptions ext = 291214597;
diff --git a/mediapipe/calculators/core/dequantize_byte_array_calculator.proto b/mediapipe/calculators/core/dequantize_byte_array_calculator.proto
index 3af8e11ef..3032dbf48 100644
--- a/mediapipe/calculators/core/dequantize_byte_array_calculator.proto
+++ b/mediapipe/calculators/core/dequantize_byte_array_calculator.proto
@@ -18,8 +18,6 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
-
message DequantizeByteArrayCalculatorOptions {
extend CalculatorOptions {
optional DequantizeByteArrayCalculatorOptions ext = 272316343;
diff --git a/mediapipe/calculators/core/flow_limiter_calculator.proto b/mediapipe/calculators/core/flow_limiter_calculator.proto
index a3a71a294..e969abf32 100644
--- a/mediapipe/calculators/core/flow_limiter_calculator.proto
+++ b/mediapipe/calculators/core/flow_limiter_calculator.proto
@@ -18,7 +18,8 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
+option java_package = "com.google.mediapipe.calculator.proto";
+option java_outer_classname = "FlowLimiterCalculatorProto";
message FlowLimiterCalculatorOptions {
extend mediapipe.CalculatorOptions {
diff --git a/mediapipe/calculators/core/gate_calculator.proto b/mediapipe/calculators/core/gate_calculator.proto
index 32402bf28..b7d597a63 100644
--- a/mediapipe/calculators/core/gate_calculator.proto
+++ b/mediapipe/calculators/core/gate_calculator.proto
@@ -18,8 +18,6 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
-
message GateCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional GateCalculatorOptions ext = 261754847;
diff --git a/mediapipe/calculators/core/get_vector_item_calculator.h b/mediapipe/calculators/core/get_vector_item_calculator.h
index 21009a30b..be89aa3a3 100644
--- a/mediapipe/calculators/core/get_vector_item_calculator.h
+++ b/mediapipe/calculators/core/get_vector_item_calculator.h
@@ -17,22 +17,24 @@
#include
+#include "mediapipe/calculators/core/get_vector_item_calculator.pb.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
namespace api2 {
-// A calcutlator to return an item from the vector by its index.
+// A calculator to return an item from the vector by its index.
+// Item index can be specified through INDEX stream and/or calculator options.
+// INDEX stream takes precedence over options.
//
// Inputs:
// VECTOR - std::vector
// Vector to take an item from.
-// INDEX - int
+// INDEX [OPTIONAL] - int
// Index of the item to return.
//
// Outputs:
@@ -45,26 +47,47 @@ namespace api2 {
// input_stream: "VECTOR:vector"
// input_stream: "INDEX:index"
// input_stream: "ITEM:item"
+// options {
+// [mediapipe.GetVectorItemCalculatorOptions.ext] {
+// item_index: 5
+// }
+// }
// }
//
template
class GetVectorItemCalculator : public Node {
public:
static constexpr Input> kIn{"VECTOR"};
- static constexpr Input kIdx{"INDEX"};
+ static constexpr Input::Optional kIdx{"INDEX"};
static constexpr Output kOut{"ITEM"};
MEDIAPIPE_NODE_CONTRACT(kIn, kIdx, kOut);
+ absl::Status Open(CalculatorContext* cc) final {
+ auto& options = cc->Options();
+ RET_CHECK(kIdx(cc).IsConnected() || options.has_item_index());
+ return absl::OkStatus();
+ }
+
absl::Status Process(CalculatorContext* cc) final {
- if (kIn(cc).IsEmpty() || kIdx(cc).IsEmpty()) {
+ if (kIn(cc).IsEmpty()) {
return absl::OkStatus();
}
const std::vector& items = kIn(cc).Get();
- const int idx = kIdx(cc).Get();
+ const auto& options =
+ cc->Options();
- RET_CHECK_LT(idx, items.size());
+ int idx = 0;
+ if (kIdx(cc).IsConnected() && !kIdx(cc).IsEmpty()) {
+ idx = kIdx(cc).Get();
+ } else if (options.has_item_index()) {
+ idx = options.item_index();
+ } else {
+ return absl::OkStatus();
+ }
+
+ RET_CHECK(idx >= 0 && idx < items.size());
kOut(cc).Send(items[idx]);
return absl::OkStatus();
diff --git a/mediapipe/calculators/core/get_vector_item_calculator.proto b/mediapipe/calculators/core/get_vector_item_calculator.proto
new file mode 100644
index 000000000..c406283e4
--- /dev/null
+++ b/mediapipe/calculators/core/get_vector_item_calculator.proto
@@ -0,0 +1,29 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message GetVectorItemCalculatorOptions {
+ extend mediapipe.CalculatorOptions {
+ optional GetVectorItemCalculatorOptions ext = 463538543;
+ }
+
+ // Index of vector item to get. INDEX input stream can be used instead, or to
+ // override.
+ optional int32 item_index = 1;
+}
diff --git a/mediapipe/calculators/core/get_vector_item_calculator_test.cc b/mediapipe/calculators/core/get_vector_item_calculator_test.cc
new file mode 100644
index 000000000..f2f788382
--- /dev/null
+++ b/mediapipe/calculators/core/get_vector_item_calculator_test.cc
@@ -0,0 +1,230 @@
+#include "mediapipe/calculators/core/get_vector_item_calculator.h"
+
+#include
+#include
+#include
+
+#include "absl/strings/str_format.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+MATCHER_P(IntPacket, value, "") {
+ return testing::Value(arg.template Get(), testing::Eq(value));
+}
+
+MATCHER_P(TimestampValue, value, "") {
+ return testing::Value(arg.Timestamp(), testing::Eq(Timestamp(value)));
+}
+
+using TestGetIntVectorItemCalculator = api2::GetVectorItemCalculator;
+MEDIAPIPE_REGISTER_NODE(TestGetIntVectorItemCalculator);
+
+CalculatorRunner MakeRunnerWithStream() {
+ return CalculatorRunner(R"(
+ calculator: "TestGetIntVectorItemCalculator"
+ input_stream: "VECTOR:vector_stream"
+ input_stream: "INDEX:index_stream"
+ output_stream: "ITEM:item_stream"
+ )");
+}
+
+CalculatorRunner MakeRunnerWithOptions(int set_index) {
+ return CalculatorRunner(absl::StrFormat(R"(
+ calculator: "TestGetIntVectorItemCalculator"
+ input_stream: "VECTOR:vector_stream"
+ output_stream: "ITEM:item_stream"
+ options {
+ [mediapipe.GetVectorItemCalculatorOptions.ext] {
+ item_index: %d
+ }
+ }
+ )",
+ set_index));
+}
+
+void AddInputVector(CalculatorRunner& runner, const std::vector& inputs,
+ int timestamp) {
+ runner.MutableInputs()->Tag("VECTOR").packets.push_back(
+ MakePacket>(inputs).At(Timestamp(timestamp)));
+}
+
+void AddInputIndex(CalculatorRunner& runner, int index, int timestamp) {
+ runner.MutableInputs()->Tag("INDEX").packets.push_back(
+ MakePacket(index).At(Timestamp(timestamp)));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, EmptyIndexStreamNoOutput) {
+ CalculatorRunner runner = MakeRunnerWithStream();
+ const std::vector inputs = {1, 2, 3};
+
+ AddInputVector(runner, inputs, 1);
+ MP_ASSERT_OK(runner.Run());
+
+ const std::vector& outputs = runner.Outputs().Tag("ITEM").packets;
+ EXPECT_EQ(0, outputs.size());
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, SuccessfulExtractionIndexStream) {
+ CalculatorRunner runner = MakeRunnerWithStream();
+ const std::vector inputs = {1, 2, 3};
+ const int index = 1;
+
+ AddInputVector(runner, inputs, 1);
+ AddInputIndex(runner, index, 1);
+ MP_ASSERT_OK(runner.Run());
+
+ const std::vector& outputs = runner.Outputs().Tag("ITEM").packets;
+ EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(inputs[index])));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, SuccessfulExtractionIndexProto) {
+ const int index = 2;
+ CalculatorRunner runner = MakeRunnerWithOptions(index);
+ const std::vector inputs = {1, 2, 3};
+
+ AddInputVector(runner, inputs, 1);
+ MP_ASSERT_OK(runner.Run());
+
+ const std::vector& outputs = runner.Outputs().Tag("ITEM").packets;
+ EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(inputs[index])));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, StreamIsPreferred) {
+ CalculatorRunner runner(R"(
+ calculator: "TestGetIntVectorItemCalculator"
+ input_stream: "VECTOR:vector_stream"
+ input_stream: "INDEX:index_stream"
+ output_stream: "ITEM:item_stream"
+ options {
+ [mediapipe.GetVectorItemCalculatorOptions.ext] {
+ item_index: 2
+ }
+ }
+ )");
+ const std::vector inputs = {1, 2, 3};
+ const int stream_index = 0;
+
+ AddInputVector(runner, inputs, 1);
+ AddInputIndex(runner, stream_index, 1);
+ MP_ASSERT_OK(runner.Run());
+
+ const std::vector& outputs = runner.Outputs().Tag("ITEM").packets;
+ EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(inputs[stream_index])));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, NoStreamNorOptionsExpectFail) {
+ CalculatorRunner runner(R"(
+ calculator: "TestGetIntVectorItemCalculator"
+ input_stream: "VECTOR:vector_stream"
+ output_stream: "ITEM:item_stream"
+ )");
+
+ absl::Status status = runner.Run();
+ ASSERT_FALSE(status.ok());
+ EXPECT_THAT(
+ status.message(),
+ testing::HasSubstr("kIdx(cc).IsConnected() || options.has_item_index()"));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, StreamIndexBoundsCheckFail1) {
+ CalculatorRunner runner = MakeRunnerWithStream();
+ const std::vector inputs = {1, 2, 3};
+ const int try_index = -1;
+
+ AddInputVector(runner, inputs, 1);
+ AddInputIndex(runner, try_index, 1);
+
+ absl::Status status = runner.Run();
+ ASSERT_FALSE(status.ok());
+ EXPECT_THAT(status.message(),
+ testing::HasSubstr("idx >= 0 && idx < items.size()"));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, StreamIndexBoundsCheckFail2) {
+ CalculatorRunner runner = MakeRunnerWithStream();
+ const std::vector inputs = {1, 2, 3};
+ const int try_index = 3;
+
+ AddInputVector(runner, inputs, 1);
+ AddInputIndex(runner, try_index, 1);
+
+ absl::Status status = runner.Run();
+ ASSERT_FALSE(status.ok());
+ EXPECT_THAT(status.message(),
+ testing::HasSubstr("idx >= 0 && idx < items.size()"));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail1) {
+ const int try_index = -1;
+ CalculatorRunner runner = MakeRunnerWithOptions(try_index);
+ const std::vector inputs = {1, 2, 3};
+
+ AddInputVector(runner, inputs, 1);
+
+ absl::Status status = runner.Run();
+ ASSERT_FALSE(status.ok());
+ EXPECT_THAT(status.message(),
+ testing::HasSubstr("idx >= 0 && idx < items.size()"));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, OptionsIndexBoundsCheckFail2) {
+ const int try_index = 3;
+ CalculatorRunner runner = MakeRunnerWithOptions(try_index);
+ const std::vector inputs = {1, 2, 3};
+
+ AddInputVector(runner, inputs, 1);
+
+ absl::Status status = runner.Run();
+ ASSERT_FALSE(status.ok());
+ EXPECT_THAT(status.message(),
+ testing::HasSubstr("idx >= 0 && idx < items.size()"));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, IndexStreamTwoTimestamps) {
+ CalculatorRunner runner = MakeRunnerWithStream();
+
+ {
+ const std::vector inputs = {1, 2, 3};
+ const int index = 1;
+ AddInputVector(runner, inputs, 1);
+ AddInputIndex(runner, index, 1);
+ }
+ {
+ const std::vector inputs = {5, 6, 7, 8};
+ const int index = 3;
+ AddInputVector(runner, inputs, 2);
+ AddInputIndex(runner, index, 2);
+ }
+ MP_ASSERT_OK(runner.Run());
+
+ const std::vector& outputs = runner.Outputs().Tag("ITEM").packets;
+ EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(2), IntPacket(8)));
+ EXPECT_THAT(outputs,
+ testing::ElementsAre(TimestampValue(1), TimestampValue(2)));
+}
+
+TEST(TestGetIntVectorItemCalculatorTest, IndexOptionsTwoTimestamps) {
+ const int static_index = 2;
+ CalculatorRunner runner = MakeRunnerWithOptions(static_index);
+
+ {
+ const std::vector inputs = {1, 2, 3};
+ AddInputVector(runner, inputs, 1);
+ }
+ {
+ const std::vector inputs = {5, 6, 7, 8};
+ AddInputVector(runner, inputs, 2);
+ }
+ MP_ASSERT_OK(runner.Run());
+
+ const std::vector& outputs = runner.Outputs().Tag("ITEM").packets;
+ EXPECT_THAT(outputs, testing::ElementsAre(IntPacket(3), IntPacket(7)));
+ EXPECT_THAT(outputs,
+ testing::ElementsAre(TimestampValue(1), TimestampValue(2)));
+}
+
+} // namespace mediapipe
diff --git a/mediapipe/calculators/core/graph_profile_calculator.proto b/mediapipe/calculators/core/graph_profile_calculator.proto
index 2bcc480c8..88b405e5d 100644
--- a/mediapipe/calculators/core/graph_profile_calculator.proto
+++ b/mediapipe/calculators/core/graph_profile_calculator.proto
@@ -18,8 +18,6 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
-
message GraphProfileCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional GraphProfileCalculatorOptions ext = 367481815;
diff --git a/mediapipe/calculators/core/merge_to_vector_calculator.cc b/mediapipe/calculators/core/merge_to_vector_calculator.cc
new file mode 100644
index 000000000..cca64bc9a
--- /dev/null
+++ b/mediapipe/calculators/core/merge_to_vector_calculator.cc
@@ -0,0 +1,27 @@
+/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "mediapipe/calculators/core/merge_to_vector_calculator.h"
+
+#include "mediapipe/framework/formats/image.h"
+
+namespace mediapipe {
+namespace api2 {
+
+typedef MergeToVectorCalculator MergeImagesToVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(MergeImagesToVectorCalculator);
+
+} // namespace api2
+} // namespace mediapipe
diff --git a/mediapipe/calculators/core/merge_to_vector_calculator.h b/mediapipe/calculators/core/merge_to_vector_calculator.h
new file mode 100644
index 000000000..bed616695
--- /dev/null
+++ b/mediapipe/calculators/core/merge_to_vector_calculator.h
@@ -0,0 +1,58 @@
+/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_MERGE_TO_VECTOR_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_MERGE_TO_VECTOR_CALCULATOR_H_
+
+#include
+#include
+#include
+#include
+
+#include "absl/status/status.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+
+namespace mediapipe {
+namespace api2 {
+
+template
+class MergeToVectorCalculator : public Node {
+ public:
+ static constexpr typename Input::Multiple kIn{""};
+ static constexpr Output> kOut{""};
+
+ MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
+
+ static absl::Status UpdateContract(CalculatorContract* cc) {
+ RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream";
+ return absl::OkStatus();
+ }
+
+ absl::Status Process(CalculatorContext* cc) {
+ const int input_num = kIn(cc).Count();
+ std::vector output_vector(input_num);
+ std::transform(kIn(cc).begin(), kIn(cc).end(), output_vector.begin(),
+ [](const auto& elem) -> T { return elem.Get(); });
+ kOut(cc).Send(output_vector);
+ return absl::OkStatus();
+ }
+};
+
+} // namespace api2
+} // namespace mediapipe
+
+#endif // MEDIAPIPE_CALCULATORS_CORE_MERGE_TO_VECTOR_CALCULATOR_H_
diff --git a/mediapipe/calculators/core/packet_cloner_calculator.cc b/mediapipe/calculators/core/packet_cloner_calculator.cc
index cc3e0ba2f..3709a9c67 100644
--- a/mediapipe/calculators/core/packet_cloner_calculator.cc
+++ b/mediapipe/calculators/core/packet_cloner_calculator.cc
@@ -58,6 +58,7 @@ namespace mediapipe {
class PacketClonerCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc) {
+ cc->SetProcessTimestampBounds(true);
const Ids ids = GetIds(*cc);
for (const auto& in_out : ids.inputs_outputs) {
auto& input = cc->Inputs().Get(in_out.in);
@@ -101,30 +102,30 @@ class PacketClonerCalculator : public CalculatorBase {
}
}
+ bool has_all_inputs = HasAllInputs();
// Output according to the TICK signal.
- if (!cc->Inputs().Get(ids_.tick_id).IsEmpty()) {
- if (output_only_when_all_inputs_received_) {
- // Return if one of the input is null.
- for (int i = 0; i < ids_.inputs_outputs.size(); ++i) {
- if (current_[i].IsEmpty()) {
- if (output_empty_packets_before_all_inputs_received_) {
- SetAllNextTimestampBounds(cc);
- }
- return absl::OkStatus();
- }
- }
- }
+ if (!cc->Inputs().Get(ids_.tick_id).IsEmpty() &&
+ (has_all_inputs || !output_only_when_all_inputs_received_)) {
// Output each stream.
for (int i = 0; i < ids_.inputs_outputs.size(); ++i) {
auto& output = cc->Outputs().Get(ids_.inputs_outputs[i].out);
if (!current_[i].IsEmpty()) {
- output.AddPacket(current_[i].At(cc->InputTimestamp()));
- } else {
- output.SetNextTimestampBound(
- cc->InputTimestamp().NextAllowedInStream());
+ output.AddPacket(current_[i].At(
+ cc->Inputs().Get(ids_.tick_id).Value().Timestamp()));
}
}
}
+
+ // Set timestamp bounds according to the TICK signal.
+ bool tick_updated = cc->Inputs().Get(ids_.tick_id).Value().Timestamp() ==
+ cc->InputTimestamp();
+ bool producing_output = has_all_inputs ||
+ output_empty_packets_before_all_inputs_received_ ||
+ !output_only_when_all_inputs_received_;
+ if (tick_updated && producing_output) {
+ SetAllNextTimestampBounds(cc);
+ }
+
return absl::OkStatus();
}
@@ -165,6 +166,15 @@ class PacketClonerCalculator : public CalculatorBase {
}
}
+ bool HasAllInputs() {
+ for (int i = 0; i < ids_.inputs_outputs.size(); ++i) {
+ if (current_[i].IsEmpty()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
std::vector current_;
Ids ids_;
bool output_only_when_all_inputs_received_;
diff --git a/mediapipe/calculators/core/packet_cloner_calculator.proto b/mediapipe/calculators/core/packet_cloner_calculator.proto
index 82bfa9c7a..29768d1e9 100644
--- a/mediapipe/calculators/core/packet_cloner_calculator.proto
+++ b/mediapipe/calculators/core/packet_cloner_calculator.proto
@@ -18,8 +18,6 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
-
message PacketClonerCalculatorOptions {
extend CalculatorOptions {
optional PacketClonerCalculatorOptions ext = 258872085;
diff --git a/mediapipe/calculators/core/packet_cloner_calculator_test.cc b/mediapipe/calculators/core/packet_cloner_calculator_test.cc
index becb70072..8397608df 100644
--- a/mediapipe/calculators/core/packet_cloner_calculator_test.cc
+++ b/mediapipe/calculators/core/packet_cloner_calculator_test.cc
@@ -33,6 +33,7 @@ namespace {
using ::testing::ElementsAre;
using ::testing::Eq;
+using ::testing::IsTrue;
using ::testing::Value;
MATCHER_P2(IntPacket, value, ts, "") {
@@ -45,6 +46,11 @@ MATCHER_P2(FloatPacket, value, ts, "") {
Value(arg.Timestamp(), Eq(Timestamp(ts)));
}
+MATCHER_P(EmptyPacket, ts, "") {
+ return Value(arg.IsEmpty(), IsTrue()) &&
+ Value(arg.Timestamp(), Eq(Timestamp(ts)));
+}
+
template
absl::Status SendPacket(const std::string& input_name, T value, int ts,
CalculatorGraph& graph) {
@@ -342,6 +348,105 @@ TEST_P(PacketClonerCalculatorTest,
FloatPacket(40.0f, 40000))));
}
+class PacketClonerCalculatorGatedInputTest : public ::testing::Test {
+ protected:
+ void SetUp() override {
+ CalculatorGraphConfig graph_config =
+ ParseTextProtoOrDie([&]() {
+ return R"pb(
+ input_stream: 'input'
+ input_stream: 'input_enabled'
+ input_stream: 'tick'
+ input_stream: 'tick_enabled'
+ node {
+ calculator: 'GateCalculator'
+ input_stream: 'tick'
+ input_stream: 'ALLOW:tick_enabled'
+ output_stream: 'tick_gated'
+ }
+ node {
+ calculator: 'GateCalculator'
+ input_stream: 'input'
+ input_stream: 'ALLOW:input_enabled'
+ output_stream: 'input_gated'
+ }
+ node {
+ calculator: 'PacketClonerCalculator'
+ input_stream: 'input_gated'
+ input_stream: 'TICK:tick_gated'
+ output_stream: 'output'
+ })pb";
+ }());
+
+ MP_ASSERT_OK(graph.Initialize(graph_config, {}));
+ MP_ASSERT_OK(graph.ObserveOutputStream(
+ "output",
+ [this](Packet const& packet) {
+ output.push_back(packet);
+ return absl::OkStatus();
+ },
+ true));
+ MP_ASSERT_OK(graph.StartRun({}));
+ }
+
+ CalculatorGraph graph;
+ std::vector output;
+};
+
+TEST_F(PacketClonerCalculatorGatedInputTest,
+ PropagatesTimestampBoundsWithEmptyInput) {
+ MP_ASSERT_OK(SendPacket("tick_enabled", false, /*ts=*/100, graph));
+ MP_ASSERT_OK(SendPacket("tick", 0, /*ts=*/100, graph));
+
+ MP_ASSERT_OK(SendPacket("input_enabled", false, /*ts=*/200, graph));
+ MP_ASSERT_OK(SendPacket("input", 1, /*ts=*/200, graph));
+
+ MP_ASSERT_OK(graph.WaitUntilIdle());
+
+ EXPECT_THAT(output, ElementsAre(EmptyPacket(100)));
+}
+
+TEST_F(PacketClonerCalculatorGatedInputTest,
+ PropagatesTimestampBoundsWithInput) {
+ MP_ASSERT_OK(SendPacket("input_enabled", true, /*ts=*/100, graph));
+ MP_ASSERT_OK(SendPacket("input", 1, /*ts=*/100, graph));
+
+ MP_ASSERT_OK(SendPacket("tick_enabled", true, /*ts=*/100, graph));
+ MP_ASSERT_OK(SendPacket("tick", 0, /*ts=*/100, graph));
+
+ MP_ASSERT_OK(SendPacket("tick_enabled", false, /*ts=*/110, graph));
+ MP_ASSERT_OK(SendPacket("tick", 0, /*ts=*/110, graph));
+
+ MP_ASSERT_OK(SendPacket("input_enabled", false, /*ts=*/200, graph));
+ MP_ASSERT_OK(SendPacket("input", 2, /*ts=*/200, graph));
+
+ MP_ASSERT_OK(graph.WaitUntilIdle());
+
+ EXPECT_THAT(output, ElementsAre(IntPacket(1, 100), EmptyPacket(110)));
+}
+
+TEST_F(PacketClonerCalculatorGatedInputTest,
+ PropagatesTimestampBoundsFromTick) {
+ MP_ASSERT_OK(SendPacket("input_enabled", true, /*ts=*/100, graph));
+ MP_ASSERT_OK(SendPacket("input", 1, /*ts=*/100, graph));
+
+ MP_ASSERT_OK(SendPacket("tick_enabled", true, /*ts=*/100, graph));
+ MP_ASSERT_OK(SendPacket("tick", 0, /*ts=*/100, graph));
+
+ MP_ASSERT_OK(SendPacket("input_enabled", true, /*ts=*/110, graph));
+ MP_ASSERT_OK(SendPacket("input", 2, /*ts=*/110, graph));
+
+ MP_ASSERT_OK(SendPacket("tick_enabled", false, /*ts=*/200, graph));
+ MP_ASSERT_OK(SendPacket("tick", 0, /*ts=*/200, graph));
+
+ MP_ASSERT_OK(SendPacket("input_enabled", false, /*ts=*/200, graph));
+ MP_ASSERT_OK(SendPacket("input", 2, /*ts=*/200, graph));
+
+ MP_ASSERT_OK(graph.WaitUntilIdle());
+
+ EXPECT_THAT(output, ElementsAre(IntPacket(1, 100), EmptyPacket(200)));
+}
+
INSTANTIATE_TEST_SUITE_P(PacketClonerCalculator, PacketClonerCalculatorTest,
testing::ValuesIn({Params{.use_tick_tag = false},
Params{.use_tick_tag = true}}));
diff --git a/mediapipe/calculators/core/packet_resampler_calculator.proto b/mediapipe/calculators/core/packet_resampler_calculator.proto
index f7ca47023..29ca8082a 100644
--- a/mediapipe/calculators/core/packet_resampler_calculator.proto
+++ b/mediapipe/calculators/core/packet_resampler_calculator.proto
@@ -18,8 +18,6 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
-
message PacketResamplerCalculatorOptions {
extend CalculatorOptions {
optional PacketResamplerCalculatorOptions ext = 95743844;
diff --git a/mediapipe/calculators/core/packet_sequencer_calculator.cc b/mediapipe/calculators/core/packet_sequencer_calculator.cc
new file mode 100644
index 000000000..815e85e35
--- /dev/null
+++ b/mediapipe/calculators/core/packet_sequencer_calculator.cc
@@ -0,0 +1,103 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include
+#include
+
+#include "mediapipe/framework/api2/contract.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/packet.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+namespace api2 {
+
+// This calculator assigns a timestamp to each "INPUT" packet reflecting
+// the most recent "TICK" timestamp.
+//
+// Each "TICK" timestamp is propagated as a settled "OUTPUT" timestamp.
+// This allows "TICK" packets to be processed right away.
+// When an "INPUT" packet arrives, it is sent to the "OUTPUT" stream with
+// the next unsettled "OUTPUT" timestamp, which is normally one greater than
+// the most recent "TICK" timestamp.
+//
+// If a "TICK" packet and an "INPUT" packet arrive together, the "OUTPUT"
+// packet timestamp is derived from the previous "TICK" timestamp,
+// and the new "OUTPUT" bound is derived from the current "TICK" timestamp.
+// This allows the current "INPUT" packet to cover the current "TICK" timestamp.
+//
+// Example config:
+// node {
+// calculator: "PacketSequencerCalculator"
+// input_stream: "INPUT:switch_selection"
+// input_stream: "TICK:input_image"
+// input_stream: "TICK:input_audio"
+// output_stream: "OUTPUT:switch_selection_timed"
+// }
+//
+class PacketSequencerCalculator : public Node {
+ public:
+ static constexpr Input::Multiple kInput{"INPUT"};
+ static constexpr Input::Multiple kTick{"TICK"};
+ static constexpr Output::Multiple kOutput{"OUTPUT"};
+
+ MEDIAPIPE_NODE_CONTRACT(kInput, kTick, kOutput,
+ StreamHandler("ImmediateInputStreamHandler"),
+ TimestampChange::Arbitrary());
+
+ static absl::Status UpdateContract(CalculatorContract* cc) {
+ RET_CHECK_EQ(kInput(cc).Count(), kOutput(cc).Count());
+ return absl::OkStatus();
+ }
+
+ absl::Status Process(CalculatorContext* cc) final {
+ // Pass through any input packets at the output stream bound.
+ for (int i = 0; i < kInput(cc).Count(); ++i) {
+ Timestamp stream_bound = kOutput(cc)[i].NextTimestampBound();
+ const PacketBase input_packet = kInput(cc)[i].packet();
+ if (!input_packet.IsEmpty()) {
+ Timestamp output_ts = std::max(Timestamp::Min(), stream_bound);
+ kOutput(cc)[i].Send(input_packet.At(output_ts));
+ }
+ }
+
+ // Find the new tick timestamp, if any.
+ Timestamp tick_ts = Timestamp::Min();
+ for (int i = 0; i < kTick(cc).Count(); ++i) {
+ const PacketBase& tick_packet = kTick(cc)[i].packet();
+ // For either an input packet or an empty input stream,
+ // the packet timestamp indicates the latest "settled timestamp",
+ // and when it arrives it equals the InputTimestamp().
+ if (tick_packet.timestamp() == cc->InputTimestamp()) {
+ tick_ts = std::max(tick_ts, tick_packet.timestamp());
+ break;
+ }
+ }
+
+ // Advance all output stream bounds past the tick timestamp.
+ for (int i = 0; i < kInput(cc).Count(); ++i) {
+ Timestamp stream_bound = kOutput(cc)[i].NextTimestampBound();
+ if (tick_ts >= stream_bound) {
+ kOutput(cc)[i].SetNextTimestampBound(tick_ts.NextAllowedInStream());
+ }
+ }
+ return absl::OkStatus();
+ }
+};
+
+MEDIAPIPE_REGISTER_NODE(PacketSequencerCalculator);
+} // namespace api2
+} // namespace mediapipe
diff --git a/mediapipe/calculators/core/packet_sequencer_calculator_test.cc b/mediapipe/calculators/core/packet_sequencer_calculator_test.cc
new file mode 100644
index 000000000..c08e6bb12
--- /dev/null
+++ b/mediapipe/calculators/core/packet_sequencer_calculator_test.cc
@@ -0,0 +1,118 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include
+#include
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+// Returns a CalculatorGraph to run a single calculator.
+CalculatorGraph BuildCalculatorGraph(CalculatorGraphConfig::Node node_config) {
+ CalculatorGraphConfig config;
+ *config.add_node() = node_config;
+ *config.mutable_input_stream() = node_config.input_stream();
+ *config.mutable_output_stream() = node_config.output_stream();
+ *config.mutable_input_side_packet() = node_config.input_side_packet();
+ *config.mutable_output_side_packet() = node_config.output_side_packet();
+ return CalculatorGraph(config);
+}
+
+// Creates a string packet.
+Packet pack(std::string data, int timestamp) {
+ return MakePacket(data).At(Timestamp(timestamp));
+}
+
+// Tests showing packet timestamp synchronization through
+// PacketSequencerCalculator.
+class PacketSequencerCalculatorTest : public ::testing::Test {
+ protected:
+ PacketSequencerCalculatorTest() {}
+ ~PacketSequencerCalculatorTest() override {}
+ void SetUp() override {}
+ void TearDown() override {}
+
+ // Defines a PacketSequencerCalculator CalculatorGraphConfig::Node.
+ CalculatorGraphConfig::Node BuildNodeConfig() {
+ CalculatorGraphConfig::Node result;
+ *result.mutable_calculator() = "PacketSequencerCalculator";
+ *result.add_input_stream() = "INPUT:select";
+ *result.add_input_stream() = "TICK:0:frame";
+ *result.add_input_stream() = "TICK:1:mask";
+ *result.add_output_stream() = "OUTPUT:select_timed";
+ return result;
+ }
+};
+
+// Shows the PacketSequencerCalculator is available.
+TEST_F(PacketSequencerCalculatorTest, IsRegistered) {
+ EXPECT_TRUE(
+ CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator"));
+}
+
+// Shows how control packets recieve timestamps before and after frame packets
+// have arrived.
+TEST_F(PacketSequencerCalculatorTest, ChannelEarly) {
+ CalculatorGraphConfig::Node node_config = BuildNodeConfig();
+ CalculatorGraph graph = BuildCalculatorGraph(node_config);
+ std::vector outputs;
+ MP_ASSERT_OK(graph.ObserveOutputStream("select_timed", [&](const Packet& p) {
+ outputs.push_back(p);
+ return absl::OkStatus();
+ }));
+ MP_ASSERT_OK(graph.StartRun({}));
+
+ // Some control packets arrive.
+ MP_ASSERT_OK(graph.AddPacketToInputStream("select", pack("p0_t10", 10)));
+ MP_ASSERT_OK(graph.AddPacketToInputStream("select", pack("p0_t20", 20)));
+ MP_ASSERT_OK(graph.WaitUntilIdle());
+
+ // The control packets are assigned low timestamps.
+ ASSERT_EQ(outputs.size(), 2);
+ EXPECT_EQ(outputs[0].Get(), "p0_t10");
+ EXPECT_EQ(outputs[0].Timestamp(), Timestamp::Min());
+ EXPECT_EQ(outputs[1].Timestamp(), Timestamp::Min() + 1);
+
+ // Some frame packets arrive.
+ MP_ASSERT_OK(graph.AddPacketToInputStream("mask", pack("p2_t10", 10)));
+ MP_ASSERT_OK(graph.AddPacketToInputStream("frame", pack("p1_t20", 20)));
+ MP_ASSERT_OK(graph.WaitUntilIdle());
+
+ // Some more control packets arrive.
+ MP_ASSERT_OK(graph.AddPacketToInputStream("select", pack("p0_t30", 30)));
+ MP_ASSERT_OK(graph.AddPacketToInputStream("select", pack("p0_t40", 40)));
+ MP_ASSERT_OK(graph.WaitUntilIdle());
+
+ // New control packets are assigned timestamps following Timestamp(20).
+ ASSERT_EQ(outputs.size(), 4);
+ EXPECT_EQ(outputs[2].Get(), "p0_t30");
+ EXPECT_EQ(outputs[2].Timestamp(), Timestamp(21));
+ EXPECT_EQ(outputs[3].Timestamp(), Timestamp(22));
+
+ MP_ASSERT_OK(graph.CloseAllPacketSources());
+ MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+} // namespace
+} // namespace mediapipe
diff --git a/mediapipe/calculators/core/packet_thinner_calculator.proto b/mediapipe/calculators/core/packet_thinner_calculator.proto
index 6c69f3afd..34fd9bc32 100644
--- a/mediapipe/calculators/core/packet_thinner_calculator.proto
+++ b/mediapipe/calculators/core/packet_thinner_calculator.proto
@@ -18,8 +18,6 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
-
message PacketThinnerCalculatorOptions {
extend CalculatorOptions {
optional PacketThinnerCalculatorOptions ext = 288533508;
diff --git a/mediapipe/calculators/core/quantize_float_vector_calculator.proto b/mediapipe/calculators/core/quantize_float_vector_calculator.proto
index 0ccc3c0d9..3f6cfda21 100644
--- a/mediapipe/calculators/core/quantize_float_vector_calculator.proto
+++ b/mediapipe/calculators/core/quantize_float_vector_calculator.proto
@@ -18,8 +18,6 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
-
message QuantizeFloatVectorCalculatorOptions {
extend CalculatorOptions {
optional QuantizeFloatVectorCalculatorOptions ext = 259848061;
diff --git a/mediapipe/calculators/core/sequence_shift_calculator.proto b/mediapipe/calculators/core/sequence_shift_calculator.proto
index cdcd284ca..15b111d71 100644
--- a/mediapipe/calculators/core/sequence_shift_calculator.proto
+++ b/mediapipe/calculators/core/sequence_shift_calculator.proto
@@ -18,8 +18,6 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
-
message SequenceShiftCalculatorOptions {
extend CalculatorOptions {
optional SequenceShiftCalculatorOptions ext = 107633927;
diff --git a/mediapipe/calculators/core/split_vector_calculator.proto b/mediapipe/calculators/core/split_vector_calculator.proto
index 40301f88b..53acbb7bf 100644
--- a/mediapipe/calculators/core/split_vector_calculator.proto
+++ b/mediapipe/calculators/core/split_vector_calculator.proto
@@ -18,8 +18,6 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
-option objc_class_prefix = "MediaPipe";
-
// A Range {begin, end} specifies beginning ane ending indices to splice a
// vector. A vector v is spliced to have elements v[begin:(end-1)], i.e., with
// begin index inclusive and end index exclusive.
diff --git a/mediapipe/calculators/image/scale_image_calculator.cc b/mediapipe/calculators/image/scale_image_calculator.cc
index 2870e0022..518e7cb64 100644
--- a/mediapipe/calculators/image/scale_image_calculator.cc
+++ b/mediapipe/calculators/image/scale_image_calculator.cc
@@ -573,8 +573,13 @@ absl::Status ScaleImageCalculator::Process(CalculatorContext* cc) {
// ImageFrame immediately, before cropping and scaling. Investigate how to
// make color space conversion more efficient when cropping or scaling is
// also needed.
- image_frame_util::YUVImageToImageFrame(*yuv_image, &converted_image_frame,
- options_.use_bt709());
+ if (options_.use_bt709() || yuv_image->fourcc() == libyuv::FOURCC_ANY) {
+ image_frame_util::YUVImageToImageFrame(
+ *yuv_image, &converted_image_frame, options_.use_bt709());
+ } else {
+ image_frame_util::YUVImageToImageFrameFromFormat(
+ *yuv_image, &converted_image_frame);
+ }
image_frame = &converted_image_frame;
} else if (output_format_ == ImageFormat::YCBCR420P) {
RET_CHECK(row_start_ == 0 && col_start_ == 0 &&
diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD
index e8659356b..99a698e4b 100644
--- a/mediapipe/calculators/tensor/BUILD
+++ b/mediapipe/calculators/tensor/BUILD
@@ -153,11 +153,12 @@ cc_library(
tags = ["nomac"], # config problem with cpuinfo via TF
visibility = ["//visibility:public"],
deps = [
+ ":inference_calculator_cc_proto",
":inference_calculator_interface",
+ "//mediapipe/framework:calculator_context",
"//mediapipe/gpu:gl_calculator_helper",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
- "@org_tensorflow//tensorflow/lite:framework_stable",
"@org_tensorflow//tensorflow/lite/delegates/gpu:gl_delegate",
],
alwayslink = 1,
@@ -172,6 +173,7 @@ cc_library(
":inference_calculator_interface",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
+ "@com_google_absl//absl/status:statusor",
"//mediapipe/framework/deps:file_path",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/util/tflite:tflite_gpu_runner",
@@ -231,6 +233,7 @@ cc_library(
deps = [
":inference_calculator_interface",
"@com_google_absl//absl/memory",
+ "@com_google_absl//absl/status",
"@org_tensorflow//tensorflow/lite/delegates/xnnpack:xnnpack_delegate",
"@org_tensorflow//tensorflow/lite:framework_stable",
"@org_tensorflow//tensorflow/lite/c:c_api_types",
@@ -636,6 +639,7 @@ cc_library(
":image_to_tensor_calculator_cc_proto",
":image_to_tensor_converter",
":image_to_tensor_utils",
+ ":loose_headers",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
@@ -990,3 +994,58 @@ cc_library(
}),
alwayslink = 1,
)
+
+cc_library(
+ name = "tensors_dequantization_calculator",
+ srcs = ["tensors_dequantization_calculator.cc"],
+ copts = select({
+ "//mediapipe:apple": [
+ "-x objective-c++",
+ "-fobjc-arc", # enable reference-counting
+ ],
+ "//conditions:default": [],
+ }),
+ visibility = ["//visibility:public"],
+ deps = [
+ "//mediapipe/framework:calculator_context",
+ "//mediapipe/framework:calculator_framework",
+ "//mediapipe/framework/api2:node",
+ "//mediapipe/framework/api2:port",
+ "//mediapipe/framework/formats:tensor",
+ "//mediapipe/framework/port:ret_check",
+ "@com_google_absl//absl/status",
+ "@com_google_absl//absl/strings",
+ ],
+ alwayslink = 1,
+)
+
+# For a more maintainable build this target should not exist and the headers
+# should be split into the existing cc_library targets, but this change was
+# automatically done so that we can remove long standing issues and complexity
+# in the build system. It's up to the OWNERS of this package to get rid of it or
+# not. The use of the textual_hdrs attribute is discouraged, use hdrs instead.
+# Here it is used to avoid header parsing errors in packages where the feature
+# parse_headers was enabled since loose headers were not being parsed.
+cc_library(
+ name = "loose_headers",
+ tags = ["avoid_dep"],
+ textual_hdrs = [
+ "image_to_tensor_converter_gl_buffer.h",
+ "image_to_tensor_converter_gl_texture.h",
+ ],
+ visibility = [":__pkg__"],
+)
+
+cc_test(
+ name = "tensors_dequantization_calculator_test",
+ srcs = ["tensors_dequantization_calculator_test.cc"],
+ deps = [
+ ":tensors_dequantization_calculator",
+ "//mediapipe/framework:calculator_framework",
+ "//mediapipe/framework:calculator_runner",
+ "//mediapipe/framework/formats:tensor",
+ "//mediapipe/framework/port:gtest_main",
+ "//mediapipe/framework/port:parse_text_proto",
+ "@com_google_absl//absl/status",
+ ],
+)
diff --git a/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc b/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc
index 12820ed16..474d6cf17 100644
--- a/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc
+++ b/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc
@@ -56,14 +56,14 @@ namespace api2 {
// previous output.
//
// The calculator has two running modes:
-// Streaming mode: when "streaming_mode" is set to true in the calculator
+// Streaming mode: when "stream_mode" is set to true in the calculator
// options, the calculator treats the input audio stream as a continuous
// stream. Thus, any samples that are not consumed in the previous runs will
// be cached in a global sample buffer. The audio data resampled from the
// current raw audio input will be appended to the global sample buffer.
// The calculator will process the global sample buffer and output as many
// tensors as possible.
-// Non-streaming mode: when "streaming_mode" is set to false in the calculator
+// Non-streaming mode: when "stream_mode" is set to false in the calculator
// options, the calculators treats the packets in the input audio stream as
// a batch of unrelated audio buffers. In each Process() call, the input
// buffer will be frist resampled, and framed as fixed-sized, possibly
@@ -104,7 +104,7 @@ namespace api2 {
// num_samples: 512
// num_overlapping_samples: 64
// target_sample_rate: 16000
-// streaming_mode: true # or false
+// stream_mode: true # or false
// }
// }
// }
@@ -136,7 +136,7 @@ class AudioToTensorCalculator : public Node {
// The number of samples per channel to advance after the current frame is
// processed.
int frame_step_;
- bool streaming_mode_;
+ bool stream_mode_;
bool check_inconsistent_timestamps_;
Timestamp initial_timestamp_ = Timestamp::Unstarted();
int64 cumulative_input_samples_ = 0;
@@ -151,8 +151,9 @@ class AudioToTensorCalculator : public Node {
Matrix sample_buffer_;
int processed_buffer_cols_ = 0;
- absl::Status ProcessStreamingData(CalculatorContext* cc);
- absl::Status ProcessNonStreamingData(CalculatorContext* cc);
+ absl::Status ProcessStreamingData(CalculatorContext* cc, const Matrix& input);
+ absl::Status ProcessNonStreamingData(CalculatorContext* cc,
+ const Matrix& input);
absl::Status SetupStreamingResampler(double input_sample_rate_);
void AppendToSampleBuffer(Matrix buffer_to_append);
@@ -172,7 +173,7 @@ absl::Status AudioToTensorCalculator::UpdateContract(CalculatorContract* cc) {
"AudioToTensorCalculatorOptions must specifiy "
"`num_channels`, `num_samples`, and `target_sample_rate`.");
}
- if (options.streaming_mode()) {
+ if (options.stream_mode()) {
// Explicitly disables tiemstamp offset to disallow the timestamp bound
// from the input streams to be propagated to the output streams.
// In the streaming mode, the output timestamp bound is based on
@@ -196,8 +197,8 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
frame_step_ = num_samples_;
}
target_sample_rate_ = options.target_sample_rate();
- streaming_mode_ = options.streaming_mode();
- if (streaming_mode_) {
+ stream_mode_ = options.stream_mode();
+ if (stream_mode_) {
check_inconsistent_timestamps_ = options.check_inconsistent_timestamps();
sample_buffer_.resize(num_channels_, Eigen::NoChange);
}
@@ -210,7 +211,7 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
mediapipe::TimeSeriesHeader input_header;
MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
kAudioIn(cc).Header(), &input_header));
- if (streaming_mode_) {
+ if (stream_mode_) {
MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
} else {
source_sample_rate_ = input_header.sample_rate();
@@ -223,7 +224,7 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
if (cc->InputTimestamp() == Timestamp::PreStream()) {
double current_source_sample_rate = kAudioSampleRateIn(cc).Get();
if (cc->Options()
- .streaming_mode()) {
+ .stream_mode()) {
return SetupStreamingResampler(current_source_sample_rate);
} else {
source_sample_rate_ = current_source_sample_rate;
@@ -232,21 +233,28 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
}
// Sanity checks.
const auto& input_frame = kAudioIn(cc).Get();
- if (input_frame.rows() != num_channels_) {
+ const bool channels_match = input_frame.rows() == num_channels_;
+ // The special case of `num_channels_ == 1` is automatic mixdown to mono.
+ const bool mono_output = num_channels_ == 1;
+ if (!mono_output && !channels_match) {
return absl::InvalidArgumentError(absl::StrFormat(
"Audio input has %d channel(s) but the model requires %d channel(s).",
input_frame.rows(), num_channels_));
}
- if (num_channels_ > 1 && input_frame.IsRowMajor) {
+ if (!mono_output && input_frame.IsRowMajor) {
return absl::InvalidArgumentError(
"The audio data should be stored in column-major.");
}
- return streaming_mode_ ? ProcessStreamingData(cc)
- : ProcessNonStreamingData(cc);
+ CHECK(channels_match || mono_output);
+ const Matrix& input = channels_match ? input_frame
+ // Mono mixdown.
+ : input_frame.colwise().mean();
+ return stream_mode_ ? ProcessStreamingData(cc, input)
+ : ProcessNonStreamingData(cc, input);
}
absl::Status AudioToTensorCalculator::Close(CalculatorContext* cc) {
- if (!streaming_mode_) {
+ if (!stream_mode_) {
return absl::OkStatus();
}
if (resampler_) {
@@ -258,8 +266,8 @@ absl::Status AudioToTensorCalculator::Close(CalculatorContext* cc) {
}
absl::Status AudioToTensorCalculator::ProcessStreamingData(
- CalculatorContext* cc) {
- const auto& input_buffer = kAudioIn(cc).Get();
+ CalculatorContext* cc, const Matrix& input) {
+ const auto& input_buffer = input;
if (initial_timestamp_ == Timestamp::Unstarted()) {
initial_timestamp_ = cc->InputTimestamp();
next_output_timestamp_ = initial_timestamp_;
@@ -303,10 +311,10 @@ absl::Status AudioToTensorCalculator::ProcessStreamingData(
}
absl::Status AudioToTensorCalculator::ProcessNonStreamingData(
- CalculatorContext* cc) {
+ CalculatorContext* cc, const Matrix& input) {
initial_timestamp_ = cc->InputTimestamp();
next_output_timestamp_ = initial_timestamp_;
- const auto& input_frame = kAudioIn(cc).Get();
+ const auto& input_frame = input;
double source_sample_rate = kAudioSampleRateIn(cc).GetOr(source_sample_rate_);
if (source_sample_rate != -1 && source_sample_rate != target_sample_rate_) {
@@ -362,7 +370,7 @@ absl::Status AudioToTensorCalculator::OutputTensors(const Matrix& buffer,
CalculatorContext* cc) {
int next_frame_first_col = 0;
std::vector timestamps;
- while ((!streaming_mode_ || !should_flush) &&
+ while ((!stream_mode_ || !should_flush) &&
next_frame_first_col + num_samples_ <= buffer.cols()) {
ASSIGN_OR_RETURN(auto output_tensor, ConvertToTensor(buffer.block(
0, next_frame_first_col,
@@ -383,7 +391,7 @@ absl::Status AudioToTensorCalculator::OutputTensors(const Matrix& buffer,
// Timestamp::Max() will be emitted. In the non-streaming mode, each
// Process() invocation will process the entire buffer completely.
Timestamp timestamp =
- streaming_mode_ ? Timestamp::Max() : next_output_timestamp_;
+ stream_mode_ ? Timestamp::Max() : next_output_timestamp_;
timestamps.push_back(timestamp);
kTensorsOut(cc).Send(std::move(output_tensor), timestamp);
}
diff --git a/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto b/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto
index c63991fc3..2090fbb81 100644
--- a/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto
+++ b/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto
@@ -24,6 +24,7 @@ message AudioToTensorCalculatorOptions {
}
// The required number of channels the output audio tensor has.
+ // If set to 1, multichannel signals will be automatically mixed down to mono.
optional int64 num_channels = 1;
// The required number of samples per channel the output audio tensor has.
@@ -38,7 +39,7 @@ message AudioToTensorCalculatorOptions {
// Whether to treat the input audio stream as a continous stream or a batch
// of unrelated audio buffers.
- optional bool streaming_mode = 5 [default = true];
+ optional bool stream_mode = 5 [default = true];
// Set to false to disable checks for jitter in timestamp values. Useful with
// live audio input.
diff --git a/mediapipe/calculators/tensor/audio_to_tensor_calculator_test.cc b/mediapipe/calculators/tensor/audio_to_tensor_calculator_test.cc
index 1b8cb9c8d..c2062134d 100644
--- a/mediapipe/calculators/tensor/audio_to_tensor_calculator_test.cc
+++ b/mediapipe/calculators/tensor/audio_to_tensor_calculator_test.cc
@@ -63,7 +63,10 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
protected:
void SetUp() override {}
void Run(int num_samples, int num_overlapping_samples,
- double resampling_factor, const Matrix& input_matrix) {
+ double resampling_factor, const Matrix& input_matrix,
+ int num_channels_override = 0) {
+ const int num_channels = num_channels_override == 0 ? input_matrix.rows()
+ : num_channels_override;
double input_sample_rate = 10000;
double target_sample_rate = input_sample_rate * resampling_factor;
auto graph_config = ParseTextProtoOrDie(
@@ -84,12 +87,12 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
num_samples: $1
num_overlapping_samples: $2
target_sample_rate: $3
- streaming_mode: false
+ stream_mode: false
}
}
}
)",
- /*$0=*/input_matrix.rows(),
+ /*$0=*/num_channels,
/*$1=*/num_samples, /*$2=*/num_overlapping_samples,
/*$3=*/target_sample_rate));
tool::AddVectorSink("tensors", &graph_config, &tensors_packets_);
@@ -114,20 +117,21 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
}
void CheckTensorsOutputPackets(const Matrix& expected_matrix,
- int sample_offset, int num_tensors_per_input) {
+ int sample_offset, int num_tensors_per_input,
+ bool mono = false) {
ASSERT_EQ(num_iterations_ * num_tensors_per_input, tensors_packets_.size());
for (int i = 0; i < num_iterations_; ++i) {
for (int j = 0; j < num_tensors_per_input; ++j) {
CheckTensorsOutputPacket(
expected_matrix, tensors_packets_[i * num_tensors_per_input + j],
- /*sample_offset*/ sample_offset * j, /*index=*/j);
+ /*sample_offset=*/sample_offset * j, /*index=*/j, /*mono=*/mono);
}
}
}
void CheckTensorsOutputPacket(const Matrix& expected_matrix,
const Packet& packet, int sample_offset,
- int index) {
+ int index, bool mono = false) {
MP_ASSERT_OK(packet.ValidateAsType>());
ASSERT_EQ(1, packet.Get>().size());
const Tensor& output_tensor = packet.Get>()[0];
@@ -137,7 +141,11 @@ class AudioToTensorCalculatorNonStreamingModeTest : public ::testing::Test {
for (int i = 0; i < num_values; ++i) {
if (i + sample_offset >= expected_matrix.size()) {
EXPECT_FLOAT_EQ(output_floats[i], 0);
+ } else if (mono) {
+ EXPECT_FLOAT_EQ(output_floats[i],
+ expected_matrix.coeff(0, i + sample_offset));
} else {
+ // Stereo.
EXPECT_FLOAT_EQ(output_floats[i],
expected_matrix.coeff((i + sample_offset) % 2,
(i + sample_offset) / 2))
@@ -209,6 +217,17 @@ TEST_F(AudioToTensorCalculatorNonStreamingModeTest, TensorsWithZeroPadding) {
CloseGraph();
}
+TEST_F(AudioToTensorCalculatorNonStreamingModeTest, Mixdown) {
+ auto input_matrix = CreateTestMatrix(2, 8, 0);
+ Run(/*num_samples=*/4, /*num_overlapping_samples=*/2,
+ /*resampling_factor=*/1.0f, *input_matrix, /*num_channels_override=*/1);
+ const Matrix& mono_matrix = input_matrix->colwise().mean();
+ CheckTensorsOutputPackets(mono_matrix, /*sample_offset=*/2,
+ /*num_tensors_per_input=*/4, /*mono=*/true);
+ CheckTimestampsOutputPackets({0, 200, 400, 600});
+ CloseGraph();
+}
+
TEST_F(AudioToTensorCalculatorNonStreamingModeTest, Downsampling) {
auto input_matrix = CreateTestMatrix(2, 1024, 0);
Run(/*num_samples=*/256, /*num_overlapping_samples=*/0,
@@ -299,7 +318,7 @@ class AudioToTensorCalculatorStreamingModeTest : public ::testing::Test {
num_samples: $0
num_overlapping_samples: $1
target_sample_rate: $2
- streaming_mode:true
+ stream_mode:true
}
}
}
diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc
index cd854beee..afd260347 100644
--- a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc
+++ b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc
@@ -348,14 +348,13 @@ class ImageToTensorCalculator : public Node {
CreateImageToGlBufferTensorConverter(
cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
#else
- // Check whether the underlying storage object is a GL texture.
- if (image.GetGpuBuffer()
- .internal_storage()) {
+ if (!gpu_converter_) {
ASSIGN_OR_RETURN(
gpu_converter_,
CreateImageToGlTextureTensorConverter(
cc, DoesGpuInputStartAtBottom(), GetBorderMode()));
- } else {
+ }
+ if (!gpu_converter_) {
return absl::UnimplementedError(
"ImageToTensorConverter for the input GPU image is unavailable.");
}
diff --git a/mediapipe/calculators/tensor/inference_calculator.cc b/mediapipe/calculators/tensor/inference_calculator.cc
index e2c5c9006..365f9f082 100644
--- a/mediapipe/calculators/tensor/inference_calculator.cc
+++ b/mediapipe/calculators/tensor/inference_calculator.cc
@@ -19,6 +19,7 @@
#include
#include
+#include "absl/status/status.h"
#include "absl/strings/string_view.h"
#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
#include "mediapipe/framework/api2/packet.h"
diff --git a/mediapipe/calculators/tensor/inference_calculator.h b/mediapipe/calculators/tensor/inference_calculator.h
index 52425dd06..8e1c32e48 100644
--- a/mediapipe/calculators/tensor/inference_calculator.h
+++ b/mediapipe/calculators/tensor/inference_calculator.h
@@ -20,18 +20,13 @@
#include
#include
-#include "absl/memory/memory.h"
#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/tensor.h"
-#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/util/tflite/tflite_model_loader.h"
#include "tensorflow/lite/core/api/op_resolver.h"
-#include "tensorflow/lite/error_reporter.h"
-#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/kernels/register.h"
-#include "tensorflow/lite/model.h"
namespace mediapipe {
namespace api2 {
@@ -119,10 +114,10 @@ class InferenceCalculator : public NodeIntf {
using TfLiteDelegatePtr =
std::unique_ptr>;
- absl::StatusOr> GetModelAsPacket(
+ static absl::StatusOr> GetModelAsPacket(
CalculatorContext* cc);
- absl::StatusOr> GetOpResolverAsPacket(
+ static absl::StatusOr> GetOpResolverAsPacket(
CalculatorContext* cc);
};
diff --git a/mediapipe/calculators/tensor/inference_calculator_cpu.cc b/mediapipe/calculators/tensor/inference_calculator_cpu.cc
index 8ac8ce31f..f330f99c2 100644
--- a/mediapipe/calculators/tensor/inference_calculator_cpu.cc
+++ b/mediapipe/calculators/tensor/inference_calculator_cpu.cc
@@ -12,13 +12,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include
#include
#include
#include
#include
#include "absl/memory/memory.h"
+#include "absl/status/status.h"
#include "mediapipe/calculators/tensor/inference_calculator.h"
+#include "tensorflow/lite/interpreter.h"
#include "tensorflow/lite/interpreter_builder.h"
#if defined(MEDIAPIPE_ANDROID)
#include "tensorflow/lite/delegates/nnapi/nnapi_delegate.h"
@@ -63,9 +66,9 @@ int GetXnnpackNumThreads(
}
template
-void CopyTensorBuffer(const Tensor& input_tensor,
- tflite::Interpreter* interpreter,
- int input_tensor_index) {
+void CopyTensorBufferToInterpreter(const Tensor& input_tensor,
+ tflite::Interpreter* interpreter,
+ int input_tensor_index) {
auto input_tensor_view = input_tensor.GetCpuReadView();
auto input_tensor_buffer = input_tensor_view.buffer();
T* local_tensor_buffer =
@@ -73,6 +76,18 @@ void CopyTensorBuffer(const Tensor& input_tensor,
std::memcpy(local_tensor_buffer, input_tensor_buffer, input_tensor.bytes());
}
+template
+void CopyTensorBufferFromInterpreter(tflite::Interpreter* interpreter,
+ int output_tensor_index,
+ Tensor* output_tensor) {
+ auto output_tensor_view = output_tensor->GetCpuWriteView();
+ auto output_tensor_buffer = output_tensor_view.buffer();
+ T* local_tensor_buffer =
+ interpreter->typed_output_tensor(output_tensor_index);
+ std::memcpy(output_tensor_buffer, local_tensor_buffer,
+ output_tensor->bytes());
+}
+
} // namespace
class InferenceCalculatorCpuImpl
@@ -99,7 +114,7 @@ class InferenceCalculatorCpuImpl
absl::Status InferenceCalculatorCpuImpl::UpdateContract(
CalculatorContract* cc) {
- const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
+ const auto& options = cc->Options();
RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
<< "Either model as side packet or model path in options is required.";
@@ -118,20 +133,32 @@ absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
RET_CHECK(!input_tensors.empty());
auto output_tensors = absl::make_unique>();
+ if (input_tensor_type_ == kTfLiteNoType) {
+ input_tensor_type_ = interpreter_->tensor(interpreter_->inputs()[0])->type;
+ }
+
// Read CPU input into tensors.
for (int i = 0; i < input_tensors.size(); ++i) {
switch (input_tensor_type_) {
case TfLiteType::kTfLiteFloat16:
case TfLiteType::kTfLiteFloat32: {
- CopyTensorBuffer(input_tensors[i], interpreter_.get(), i);
+ CopyTensorBufferToInterpreter(input_tensors[i],
+ interpreter_.get(), i);
break;
}
case TfLiteType::kTfLiteUInt8: {
- CopyTensorBuffer(input_tensors[i], interpreter_.get(), i);
+ CopyTensorBufferToInterpreter(input_tensors[i],
+ interpreter_.get(), i);
break;
}
case TfLiteType::kTfLiteInt8: {
- CopyTensorBuffer(input_tensors[i], interpreter_.get(), i);
+ CopyTensorBufferToInterpreter(input_tensors[i],
+ interpreter_.get(), i);
+ break;
+ }
+ case TfLiteType::kTfLiteInt32: {
+ CopyTensorBufferToInterpreter(input_tensors[i],
+ interpreter_.get(), i);
break;
}
default:
@@ -148,13 +175,41 @@ absl::Status InferenceCalculatorCpuImpl::Process(CalculatorContext* cc) {
output_tensors->reserve(tensor_indexes.size());
for (int i = 0; i < tensor_indexes.size(); ++i) {
TfLiteTensor* tensor = interpreter_->tensor(tensor_indexes[i]);
- output_tensors->emplace_back(
- Tensor::ElementType::kFloat32,
- Tensor::Shape{std::vector{
- tensor->dims->data, tensor->dims->data + tensor->dims->size}});
- auto cpu_view = output_tensors->back().GetCpuWriteView();
- std::memcpy(cpu_view.buffer(), tensor->data.f,
- output_tensors->back().bytes());
+ Tensor::Shape shape{std::vector{
+ tensor->dims->data, tensor->dims->data + tensor->dims->size}};
+ switch (tensor->type) {
+ case TfLiteType::kTfLiteFloat16:
+ case TfLiteType::kTfLiteFloat32:
+ output_tensors->emplace_back(Tensor::ElementType::kFloat32, shape);
+ CopyTensorBufferFromInterpreter(interpreter_.get(), i,
+ &output_tensors->back());
+ break;
+ case TfLiteType::kTfLiteUInt8:
+ output_tensors->emplace_back(
+ Tensor::ElementType::kUInt8, shape,
+ Tensor::QuantizationParameters{tensor->params.scale,
+ tensor->params.zero_point});
+ CopyTensorBufferFromInterpreter(interpreter_.get(), i,
+ &output_tensors->back());
+ break;
+ case TfLiteType::kTfLiteInt8:
+ output_tensors->emplace_back(
+ Tensor::ElementType::kInt8, shape,
+ Tensor::QuantizationParameters{tensor->params.scale,
+ tensor->params.zero_point});
+ CopyTensorBufferFromInterpreter(interpreter_.get(), i,
+ &output_tensors->back());
+ break;
+ case TfLiteType::kTfLiteInt32:
+ output_tensors->emplace_back(Tensor::ElementType::kInt32, shape);
+ CopyTensorBufferFromInterpreter(interpreter_.get(), i,
+ &output_tensors->back());
+ break;
+ default:
+ return absl::InvalidArgumentError(
+ absl::StrCat("Unsupported output tensor type:",
+ TfLiteTypeGetName(tensor->type)));
+ }
}
kOutTensors(cc).Send(std::move(output_tensors));
return absl::OkStatus();
@@ -188,7 +243,6 @@ absl::Status InferenceCalculatorCpuImpl::InitInterpreter(
absl::Status InferenceCalculatorCpuImpl::AllocateTensors() {
RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
- input_tensor_type_ = interpreter_->tensor(interpreter_->inputs()[0])->type;
return absl::OkStatus();
}
@@ -198,13 +252,14 @@ absl::Status InferenceCalculatorCpuImpl::LoadDelegate(
cc->Options();
auto opts_delegate = calculator_opts.delegate();
if (!kDelegate(cc).IsEmpty()) {
- mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
- kDelegate(cc).Get();
- CHECK(input_side_packet_delegate.has_tflite() ||
- input_side_packet_delegate.has_xnnpack() ||
- input_side_packet_delegate.has_nnapi() ||
- input_side_packet_delegate.delegate_case() ==
- mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
+ const mediapipe::InferenceCalculatorOptions::Delegate&
+ input_side_packet_delegate = kDelegate(cc).Get();
+ RET_CHECK(
+ input_side_packet_delegate.has_tflite() ||
+ input_side_packet_delegate.has_xnnpack() ||
+ input_side_packet_delegate.has_nnapi() ||
+ input_side_packet_delegate.delegate_case() ==
+ mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
<< "inference_calculator_cpu only supports delegate input side packet "
<< "for TFLite, XNNPack and Nnapi";
opts_delegate.MergeFrom(input_side_packet_delegate);
diff --git a/mediapipe/calculators/tensor/inference_calculator_gl.cc b/mediapipe/calculators/tensor/inference_calculator_gl.cc
index 55cb80c3a..1f3768ee0 100644
--- a/mediapipe/calculators/tensor/inference_calculator_gl.cc
+++ b/mediapipe/calculators/tensor/inference_calculator_gl.cc
@@ -15,11 +15,14 @@
#include
#include
#include
+#include
#include
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "mediapipe/calculators/tensor/inference_calculator.h"
+#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
+#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "tensorflow/lite/delegates/gpu/gl_delegate.h"
@@ -36,111 +39,64 @@ class InferenceCalculatorGlImpl
absl::Status Close(CalculatorContext* cc) override;
private:
- absl::Status LoadModel(CalculatorContext* cc);
- absl::Status LoadDelegate(CalculatorContext* cc);
- absl::Status LoadDelegateAndAllocateTensors(CalculatorContext* cc);
+ // Helper class that wraps everything related to GPU inference acceleration.
+ class GpuInferenceRunner {
+ public:
+ ~GpuInferenceRunner();
- // TfLite requires us to keep the model alive as long as the interpreter is.
- Packet model_packet_;
+ absl::Status Init(CalculatorContext* cc,
+ const mediapipe::InferenceCalculatorOptions::Delegate&
+ delegate_options);
+ absl::Status LoadModel(CalculatorContext* cc);
+ absl::Status LoadDelegate(
+ CalculatorContext* cc,
+ const mediapipe::InferenceCalculatorOptions::Delegate&
+ delegate_options);
+ absl::Status LoadDelegateAndAllocateTensors(
+ CalculatorContext* cc,
+ const mediapipe::InferenceCalculatorOptions::Delegate&
+ delegate_options);
+ absl::Status Process(CalculatorContext* cc,
+ const std::vector& input_tensors,
+ std::vector& output_tensors);
- mediapipe::GlCalculatorHelper gpu_helper_;
- bool allow_precision_loss_ = false;
+ private:
+ // TfLite requires us to keep the model alive as long as the interpreter is.
+ Packet model_packet_;
+ mediapipe::GlCalculatorHelper gpu_helper_;
+ TfLiteDelegatePtr delegate_;
+ std::unique_ptr interpreter_;
+ std::vector> gpu_buffers_in_;
+ std::vector> gpu_buffers_out_;
+ size_t output_size_ = 0;
+ };
- TfLiteDelegatePtr delegate_;
- std::unique_ptr interpreter_;
-
- std::vector output_shapes_;
- std::vector> gpu_buffers_in_;
- std::vector> gpu_buffers_out_;
+ std::unique_ptr gpu_inference_runner_;
};
-absl::Status InferenceCalculatorGlImpl::UpdateContract(CalculatorContract* cc) {
- const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
- RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
- << "Either model as side packet or model path in options is required.";
-
- return mediapipe::GlCalculatorHelper::UpdateContract(cc);
-}
-
-absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
- const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
- mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
- if (!kDelegate(cc).IsEmpty()) {
- mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
- kDelegate(cc).Get();
- CHECK(input_side_packet_delegate.has_gpu() ||
- input_side_packet_delegate.delegate_case() ==
- mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
- << "inference_calculator_gl only supports delegate input side packet "
- << "for Gpu";
- delegate.MergeFrom(input_side_packet_delegate);
- }
-
- MP_RETURN_IF_ERROR(LoadModel(cc));
- MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
- return gpu_helper_.RunInGlContext([this, &cc]() -> ::mediapipe::Status {
- return LoadDelegateAndAllocateTensors(cc);
- });
-}
-
-absl::Status InferenceCalculatorGlImpl::Process(CalculatorContext* cc) {
- if (kInTensors(cc).IsEmpty()) {
- return absl::OkStatus();
- }
- const auto& input_tensors = *kInTensors(cc);
- RET_CHECK(!input_tensors.empty());
- auto output_tensors = absl::make_unique>();
-
- MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
- [this, &input_tensors]() -> ::mediapipe::Status {
- // Explicitly copy input.
- for (int i = 0; i < input_tensors.size(); ++i) {
- glBindBuffer(GL_COPY_READ_BUFFER,
- input_tensors[i].GetOpenGlBufferReadView().name());
- glBindBuffer(GL_COPY_WRITE_BUFFER,
- gpu_buffers_in_[i]->GetOpenGlBufferWriteView().name());
- glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
- input_tensors[i].bytes());
- }
- return absl::OkStatus();
- }));
-
- // Run inference.
- RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
-
- MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
- [this, &output_tensors]() -> ::mediapipe::Status {
- output_tensors->reserve(output_shapes_.size());
- for (int i = 0; i < output_shapes_.size(); ++i) {
- const auto& t = gpu_buffers_out_[i];
- output_tensors->emplace_back(Tensor::ElementType::kFloat32,
- gpu_buffers_out_[i]->shape());
- auto read_view = t->GetOpenGlBufferReadView();
- glBindBuffer(GL_COPY_READ_BUFFER, read_view.name());
- auto write_view = output_tensors->back().GetOpenGlBufferWriteView();
- glBindBuffer(GL_COPY_WRITE_BUFFER, write_view.name());
- glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
- t->bytes());
- }
- return absl::OkStatus();
- }));
-
- kOutTensors(cc).Send(std::move(output_tensors));
- return absl::OkStatus();
-}
-
-absl::Status InferenceCalculatorGlImpl::Close(CalculatorContext* cc) {
- return gpu_helper_.RunInGlContext([this]() -> absl::Status {
+InferenceCalculatorGlImpl::GpuInferenceRunner::~GpuInferenceRunner() {
+ gpu_helper_.RunInGlContext([this]() {
gpu_buffers_in_.clear();
gpu_buffers_out_.clear();
// Delegate must outlive the interpreter, hence the order is important.
interpreter_ = nullptr;
delegate_ = nullptr;
- return absl::OkStatus();
});
}
-absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
+absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::Init(
+ CalculatorContext* cc,
+ const mediapipe::InferenceCalculatorOptions::Delegate& delegate_options) {
+ MP_RETURN_IF_ERROR(LoadModel(cc));
+ MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
+ return gpu_helper_.RunInGlContext(
+ [this, &cc, &delegate_options]() -> absl::Status {
+ return LoadDelegateAndAllocateTensors(cc, delegate_options);
+ });
+}
+
+absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadModel(
+ CalculatorContext* cc) {
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
const auto& model = *model_packet_.Get();
if (kSideInOpResolver(cc).IsConnected()) {
@@ -160,9 +116,11 @@ absl::Status InferenceCalculatorGlImpl::LoadModel(CalculatorContext* cc) {
return absl::OkStatus();
}
-absl::Status InferenceCalculatorGlImpl::LoadDelegateAndAllocateTensors(
- CalculatorContext* cc) {
- MP_RETURN_IF_ERROR(LoadDelegate(cc));
+absl::Status
+InferenceCalculatorGlImpl::GpuInferenceRunner::LoadDelegateAndAllocateTensors(
+ CalculatorContext* cc,
+ const mediapipe::InferenceCalculatorOptions::Delegate& delegate_options) {
+ MP_RETURN_IF_ERROR(LoadDelegate(cc, delegate_options));
// AllocateTensors() can be called only after ModifyGraphWithDelegate.
RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
@@ -173,11 +131,16 @@ absl::Status InferenceCalculatorGlImpl::LoadDelegateAndAllocateTensors(
return absl::OkStatus();
}
-absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) {
+absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::LoadDelegate(
+ CalculatorContext* cc,
+ const mediapipe::InferenceCalculatorOptions::Delegate& delegate_options) {
// Configure and create the delegate.
TfLiteGpuDelegateOptions options = TfLiteGpuDelegateOptionsDefault();
options.compile_options.precision_loss_allowed =
- allow_precision_loss_ ? 1 : 0;
+ (delegate_options.has_gpu() &&
+ delegate_options.gpu().allow_precision_loss())
+ ? 1
+ : 0;
options.compile_options.preferred_gl_object_type =
TFLITE_GL_OBJECT_TYPE_FASTEST;
options.compile_options.dynamic_batch_enabled = 0;
@@ -202,9 +165,9 @@ absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) {
interpreter_->SetAllowBufferHandleOutput(true);
// Get output image sizes.
const auto& output_indices = interpreter_->outputs();
- output_shapes_.resize(output_indices.size());
+ output_size_ = output_indices.size();
// Create and bind output buffers.
- for (int i = 0; i < output_shapes_.size(); ++i) {
+ for (int i = 0; i < output_size_; ++i) {
const TfLiteTensor* tensor = interpreter_->tensor(output_indices[i]);
gpu_buffers_out_.emplace_back(absl::make_unique(
Tensor::ElementType::kFloat32,
@@ -224,5 +187,89 @@ absl::Status InferenceCalculatorGlImpl::LoadDelegate(CalculatorContext* cc) {
return absl::OkStatus();
}
+absl::Status InferenceCalculatorGlImpl::GpuInferenceRunner::Process(
+ CalculatorContext* cc, const std::vector& input_tensors,
+ std::vector& output_tensors) {
+ return gpu_helper_.RunInGlContext(
+ [this, &input_tensors, &output_tensors]() -> absl::Status {
+ // Explicitly copy input.
+ for (int i = 0; i < input_tensors.size(); ++i) {
+ glBindBuffer(GL_COPY_READ_BUFFER,
+ input_tensors[i].GetOpenGlBufferReadView().name());
+ glBindBuffer(GL_COPY_WRITE_BUFFER,
+ gpu_buffers_in_[i]->GetOpenGlBufferWriteView().name());
+ glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
+ input_tensors[i].bytes());
+ }
+
+ // Run inference.
+ RET_CHECK_EQ(interpreter_->Invoke(), kTfLiteOk);
+
+ output_tensors.reserve(output_size_);
+ for (int i = 0; i < output_size_; ++i) {
+ const auto& t = gpu_buffers_out_[i];
+ output_tensors.emplace_back(Tensor::ElementType::kFloat32,
+ gpu_buffers_out_[i]->shape());
+ auto read_view = t->GetOpenGlBufferReadView();
+ glBindBuffer(GL_COPY_READ_BUFFER, read_view.name());
+ auto write_view = output_tensors.back().GetOpenGlBufferWriteView();
+ glBindBuffer(GL_COPY_WRITE_BUFFER, write_view.name());
+ glCopyBufferSubData(GL_COPY_READ_BUFFER, GL_COPY_WRITE_BUFFER, 0, 0,
+ t->bytes());
+ }
+
+ return absl::OkStatus();
+ });
+}
+
+absl::Status InferenceCalculatorGlImpl::UpdateContract(CalculatorContract* cc) {
+ const auto& options = cc->Options();
+ RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
+ << "Either model as side packet or model path in options is required.";
+
+ return mediapipe::GlCalculatorHelper::UpdateContract(cc);
+}
+
+absl::Status InferenceCalculatorGlImpl::Open(CalculatorContext* cc) {
+ const auto& options = cc->Options();
+ mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
+ if (!kDelegate(cc).IsEmpty()) {
+ const mediapipe::InferenceCalculatorOptions::Delegate&
+ input_side_packet_delegate = kDelegate(cc).Get();
+ RET_CHECK(
+ (input_side_packet_delegate.has_gpu() &&
+ !input_side_packet_delegate.gpu().use_advanced_gpu_api()) ||
+ input_side_packet_delegate.delegate_case() ==
+ mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
+ << "inference_calculator_gl only supports delegate input side packet "
+ << "for Gpu (non advanced)";
+ delegate.MergeFrom(input_side_packet_delegate);
+ }
+
+ gpu_inference_runner_ = std::make_unique();
+ return gpu_inference_runner_->Init(cc, delegate);
+}
+
+absl::Status InferenceCalculatorGlImpl::Process(CalculatorContext* cc) {
+ if (kInTensors(cc).IsEmpty()) {
+ return absl::OkStatus();
+ }
+
+ const auto& input_tensors = *kInTensors(cc);
+ RET_CHECK(!input_tensors.empty());
+ auto output_tensors = absl::make_unique>();
+
+ MP_RETURN_IF_ERROR(
+ gpu_inference_runner_->Process(cc, input_tensors, *output_tensors));
+
+ kOutTensors(cc).Send(std::move(output_tensors));
+ return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorGlImpl::Close(CalculatorContext* cc) {
+ gpu_inference_runner_ = nullptr;
+ return absl::OkStatus();
+}
+
} // namespace api2
} // namespace mediapipe
diff --git a/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc b/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc
index cdadc4e61..7e11ee072 100644
--- a/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc
+++ b/mediapipe/calculators/tensor/inference_calculator_gl_advanced.cc
@@ -15,10 +15,12 @@
#include
#include
#include
+#include
#include
#include "absl/memory/memory.h"
#include "absl/status/status.h"
+#include "absl/status/statusor.h"
#include "mediapipe/calculators/tensor/inference_calculator.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/util/tflite/tflite_gpu_runner.h"
@@ -28,7 +30,7 @@
#include "mediapipe/util/android/file/base/file.h"
#include "mediapipe/util/android/file/base/filesystem.h"
#include "mediapipe/util/android/file/base/helpers.h"
-#endif // ANDROID
+#endif // MEDIAPIPE_ANDROID
namespace mediapipe {
namespace api2 {
@@ -56,85 +58,71 @@ class InferenceCalculatorGlAdvancedImpl
absl::Status Close(CalculatorContext* cc) override;
private:
- absl::Status ReadGpuCaches();
- absl::Status SaveGpuCaches();
- absl::Status InitTFLiteGPURunner(CalculatorContext* cc);
+ // Helper class that saves binary data to disk, or read from disk.
+ class OnDiskCacheHelper {
+ public:
+ absl::Status Init(
+ const mediapipe::InferenceCalculatorOptions& options,
+ const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
+ gpu_delegate_options);
+ absl::Status ReadGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
+ absl::Status SaveGpuCaches(tflite::gpu::TFLiteGPURunner* gpu_runner) const;
- // TfLite requires us to keep the model alive as long as the interpreter is.
- Packet model_packet_;
+ private:
+ bool use_kernel_caching_ = false;
+ std::string cached_kernel_filename_;
+ bool use_serialized_model_ = false;
+ std::string serialized_model_path_;
+ };
- mediapipe::GlCalculatorHelper gpu_helper_;
- std::unique_ptr tflite_gpu_runner_;
- bool allow_precision_loss_ = false;
- mediapipe::InferenceCalculatorOptions::Delegate::Gpu::Api
- tflite_gpu_runner_api_;
- mediapipe::InferenceCalculatorOptions::Delegate::Gpu::InferenceUsage
- tflite_gpu_runner_usage_;
+ // Helper class that wraps everything related to GPU inference acceleration.
+ class GpuInferenceRunner {
+ public:
+ absl::Status Init(
+ CalculatorContext* cc,
+ const mediapipe::InferenceCalculatorOptions::Delegate& delegate);
- std::vector output_shapes_;
+ absl::StatusOr> Process(
+ const std::vector& input_tensors);
- bool use_kernel_caching_ = false;
- std::string cached_kernel_filename_;
- bool use_serialized_model_ = false;
- std::string serialized_model_path_;
+ absl::Status Close();
+
+ private:
+ absl::Status InitTFLiteGPURunner(
+ CalculatorContext* cc,
+ const mediapipe::InferenceCalculatorOptions::Delegate& delegate);
+
+ // TfLite requires us to keep the model alive as long as the interpreter is.
+ Packet model_packet_;
+
+ mediapipe::GlCalculatorHelper gpu_helper_;
+ std::unique_ptr tflite_gpu_runner_;
+
+ std::vector output_shapes_;
+
+ OnDiskCacheHelper on_disk_cache_helper_;
+ };
+
+ std::unique_ptr gpu_inference_runner_;
};
-absl::Status InferenceCalculatorGlAdvancedImpl::UpdateContract(
- CalculatorContract* cc) {
- const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
- RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
- << "Either model as side packet or model path in options is required.";
-
- MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
- return absl::OkStatus();
-}
-
-absl::Status InferenceCalculatorGlAdvancedImpl::Open(CalculatorContext* cc) {
- const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
- mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
- if (!kDelegate(cc).IsEmpty()) {
- mediapipe::InferenceCalculatorOptions::Delegate input_side_packet_delegate =
- kDelegate(cc).Get();
- CHECK(input_side_packet_delegate.has_gpu() ||
- input_side_packet_delegate.delegate_case() ==
- mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
- << "inference_calculator_gl_advanced only supports delegate input side "
- "packet for Gpu";
- delegate.MergeFrom(input_side_packet_delegate);
- }
- allow_precision_loss_ = delegate.gpu().allow_precision_loss();
- tflite_gpu_runner_api_ = delegate.gpu().api();
- tflite_gpu_runner_usage_ = delegate.gpu().usage();
- use_kernel_caching_ = delegate.gpu().has_cached_kernel_path();
- use_serialized_model_ = delegate.gpu().has_serialized_model_dir() &&
- delegate.gpu().has_model_token();
-
- if (use_kernel_caching_) {
-#ifdef MEDIAPIPE_ANDROID
- cached_kernel_filename_ = delegate.gpu().cached_kernel_path() +
- mediapipe::File::Basename(options.model_path()) +
- ".ker";
-#endif // MEDIAPIPE_ANDROID
- }
- if (use_serialized_model_) {
-#ifdef MEDIAPIPE_ANDROID
- serialized_model_path_ = mediapipe::file::JoinPath(
- delegate.gpu().serialized_model_dir(), delegate.gpu().model_token());
-#endif // MEDIAPIPE_ANDROID
- }
-
+absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Init(
+ CalculatorContext* cc,
+ const mediapipe::InferenceCalculatorOptions::Delegate& delegate) {
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc));
- return gpu_helper_.RunInGlContext(
- [this, &cc]() -> absl::Status { return InitTFLiteGPURunner(cc); });
+
+ const auto& options = cc->Options();
+ MP_RETURN_IF_ERROR(on_disk_cache_helper_.Init(options, delegate.gpu()));
+
+ return gpu_helper_.RunInGlContext([this, &cc, &delegate]() -> absl::Status {
+ return InitTFLiteGPURunner(cc, delegate);
+ });
}
-absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
- if (kInTensors(cc).IsEmpty()) {
- return absl::OkStatus();
- }
- const auto& input_tensors = *kInTensors(cc);
- RET_CHECK(!input_tensors.empty());
- auto output_tensors = absl::make_unique>();
+absl::StatusOr>
+InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Process(
+ const std::vector& input_tensors) {
+ std::vector output_tensors;
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext(
[this, &input_tensors, &output_tensors]() -> absl::Status {
@@ -142,90 +130,46 @@ absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToInputTensor(
input_tensors[i].GetOpenGlBufferReadView().name(), i));
}
- output_tensors->reserve(output_shapes_.size());
+ output_tensors.reserve(output_shapes_.size());
for (int i = 0; i < output_shapes_.size(); ++i) {
- output_tensors->emplace_back(Tensor::ElementType::kFloat32,
- output_shapes_[i]);
+ output_tensors.emplace_back(Tensor::ElementType::kFloat32,
+ output_shapes_[i]);
MP_RETURN_IF_ERROR(tflite_gpu_runner_->BindSSBOToOutputTensor(
- output_tensors->back().GetOpenGlBufferWriteView().name(), i));
+ output_tensors.back().GetOpenGlBufferWriteView().name(), i));
}
- return absl::OkStatus();
+ // Run inference.
+ return tflite_gpu_runner_->Invoke();
}));
- // Run inference.
- MP_RETURN_IF_ERROR(tflite_gpu_runner_->Invoke());
- kOutTensors(cc).Send(std::move(output_tensors));
- return absl::OkStatus();
+ return output_tensors;
}
-absl::Status InferenceCalculatorGlAdvancedImpl::SaveGpuCaches() {
-#ifdef MEDIAPIPE_ANDROID
- if (use_kernel_caching_) {
- // Save kernel file.
- auto kernel_cache = absl::make_unique>(
- tflite_gpu_runner_->GetSerializedBinaryCache());
- std::string cache_str(kernel_cache->begin(), kernel_cache->end());
- MP_RETURN_IF_ERROR(
- mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
- }
- if (use_serialized_model_) {
- // Save serialized model file.
- ASSIGN_OR_RETURN(std::vector serialized_model_vec,
- tflite_gpu_runner_->GetSerializedModel());
- absl::string_view serialized_model(
- reinterpret_cast(serialized_model_vec.data()),
- serialized_model_vec.size());
- MP_RETURN_IF_ERROR(
- mediapipe::file::SetContents(serialized_model_path_, serialized_model));
- }
-#endif // MEDIAPIPE_ANDROID
- return absl::OkStatus();
-}
-
-absl::Status InferenceCalculatorGlAdvancedImpl::Close(CalculatorContext* cc) {
- MP_RETURN_IF_ERROR(SaveGpuCaches());
+absl::Status InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::Close() {
+ MP_RETURN_IF_ERROR(
+ on_disk_cache_helper_.SaveGpuCaches(tflite_gpu_runner_.get()));
return gpu_helper_.RunInGlContext([this]() -> absl::Status {
tflite_gpu_runner_.reset();
return absl::OkStatus();
});
}
-absl::Status InferenceCalculatorGlAdvancedImpl::ReadGpuCaches() {
-#ifdef MEDIAPIPE_ANDROID
- if (use_kernel_caching_ && File::Exists(cached_kernel_filename_)) {
- // Load pre-compiled kernel file.
- std::string cache_str;
- MP_RETURN_IF_ERROR(
- mediapipe::file::GetContents(cached_kernel_filename_, &cache_str));
- std::vector cache_vec(cache_str.begin(), cache_str.end());
- tflite_gpu_runner_->SetSerializedBinaryCache(std::move(cache_vec));
- }
- if (use_serialized_model_ && File::Exists(serialized_model_path_)) {
- // Load serialized model file.
- std::string serialized_model_str;
- MP_RETURN_IF_ERROR(
- file::GetContents(serialized_model_path_, &serialized_model_str));
- std::vector serialized_model_vec(serialized_model_str.begin(),
- serialized_model_str.end());
- tflite_gpu_runner_->SetSerializedModel(std::move(serialized_model_vec));
- }
-#endif // MEDIAPIPE_ANDROID
- return absl::OkStatus();
-}
-
-absl::Status InferenceCalculatorGlAdvancedImpl::InitTFLiteGPURunner(
- CalculatorContext* cc) {
+absl::Status
+InferenceCalculatorGlAdvancedImpl::GpuInferenceRunner::InitTFLiteGPURunner(
+ CalculatorContext* cc,
+ const mediapipe::InferenceCalculatorOptions::Delegate& delegate) {
ASSIGN_OR_RETURN(model_packet_, GetModelAsPacket(cc));
const auto& model = *model_packet_.Get();
+ bool allow_precision_loss = delegate.gpu().allow_precision_loss();
+
// Create runner
tflite::gpu::InferenceOptions options;
- options.priority1 = allow_precision_loss_
+ options.priority1 = allow_precision_loss
? tflite::gpu::InferencePriority::MIN_LATENCY
: tflite::gpu::InferencePriority::MAX_PRECISION;
options.priority2 = tflite::gpu::InferencePriority::AUTO;
options.priority3 = tflite::gpu::InferencePriority::AUTO;
- switch (tflite_gpu_runner_usage_) {
+ switch (delegate.gpu().usage()) {
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::
FAST_SINGLE_ANSWER: {
options.usage = tflite::gpu::InferenceUsage::FAST_SINGLE_ANSWER;
@@ -241,7 +185,7 @@ absl::Status InferenceCalculatorGlAdvancedImpl::InitTFLiteGPURunner(
}
}
tflite_gpu_runner_ = std::make_unique(options);
- switch (tflite_gpu_runner_api_) {
+ switch (delegate.gpu().api()) {
case mediapipe::InferenceCalculatorOptions::Delegate::Gpu::ANY: {
// Do not need to force any specific API.
break;
@@ -277,9 +221,148 @@ absl::Status InferenceCalculatorGlAdvancedImpl::InitTFLiteGPURunner(
tflite_gpu_runner_->GetOutputShapes()[i].c};
}
- MP_RETURN_IF_ERROR(ReadGpuCaches());
+ MP_RETURN_IF_ERROR(
+ on_disk_cache_helper_.ReadGpuCaches(tflite_gpu_runner_.get()));
return tflite_gpu_runner_->Build();
}
+#if defined(MEDIAPIPE_ANDROID)
+absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
+ const mediapipe::InferenceCalculatorOptions& options,
+ const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
+ gpu_delegate_options) {
+ use_kernel_caching_ = gpu_delegate_options.has_cached_kernel_path();
+ use_serialized_model_ = gpu_delegate_options.has_serialized_model_dir() &&
+ gpu_delegate_options.has_model_token();
+
+ if (use_kernel_caching_) {
+ cached_kernel_filename_ = gpu_delegate_options.cached_kernel_path() +
+ mediapipe::File::Basename(options.model_path()) +
+ ".ker";
+ }
+ if (use_serialized_model_) {
+ serialized_model_path_ =
+ mediapipe::file::JoinPath(gpu_delegate_options.serialized_model_dir(),
+ gpu_delegate_options.model_token());
+ }
+ return absl::OkStatus();
+}
+
+absl::Status
+InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
+ tflite::gpu::TFLiteGPURunner* gpu_runner) const {
+ if (use_kernel_caching_) {
+ // Save kernel file.
+ auto kernel_cache = absl::make_unique>(
+ gpu_runner->GetSerializedBinaryCache());
+ std::string cache_str(kernel_cache->begin(), kernel_cache->end());
+ MP_RETURN_IF_ERROR(
+ mediapipe::file::SetContents(cached_kernel_filename_, cache_str));
+ }
+ if (use_serialized_model_) {
+ // Save serialized model file.
+ ASSIGN_OR_RETURN(std::vector serialized_model_vec,
+ gpu_runner->GetSerializedModel());
+ absl::string_view serialized_model(
+ reinterpret_cast(serialized_model_vec.data()),
+ serialized_model_vec.size());
+ MP_RETURN_IF_ERROR(
+ mediapipe::file::SetContents(serialized_model_path_, serialized_model));
+ }
+ return absl::OkStatus();
+}
+
+absl::Status
+InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::ReadGpuCaches(
+ tflite::gpu::TFLiteGPURunner* gpu_runner) const {
+ if (use_kernel_caching_ && File::Exists(cached_kernel_filename_)) {
+ // Load pre-compiled kernel file.
+ std::string cache_str;
+ MP_RETURN_IF_ERROR(
+ mediapipe::file::GetContents(cached_kernel_filename_, &cache_str));
+ std::vector cache_vec(cache_str.begin(), cache_str.end());
+ gpu_runner->SetSerializedBinaryCache(std::move(cache_vec));
+ }
+ if (use_serialized_model_ && File::Exists(serialized_model_path_)) {
+ // Load serialized model file.
+ std::string serialized_model_str;
+ MP_RETURN_IF_ERROR(
+ file::GetContents(serialized_model_path_, &serialized_model_str));
+ std::vector serialized_model_vec(serialized_model_str.begin(),
+ serialized_model_str.end());
+ gpu_runner->SetSerializedModel(std::move(serialized_model_vec));
+ }
+ return absl::OkStatus();
+}
+#else
+absl::Status InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::Init(
+ const mediapipe::InferenceCalculatorOptions& options,
+ const mediapipe::InferenceCalculatorOptions::Delegate::Gpu&
+ gpu_delegate_options) {
+ return absl::OkStatus();
+}
+
+absl::Status
+InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::ReadGpuCaches(
+ tflite::gpu::TFLiteGPURunner* gpu_runner) const {
+ return absl::OkStatus();
+}
+
+absl::Status
+InferenceCalculatorGlAdvancedImpl::OnDiskCacheHelper::SaveGpuCaches(
+ tflite::gpu::TFLiteGPURunner* gpu_runner) const {
+ return absl::OkStatus();
+}
+#endif // MEDIAPIPE_ANDROID
+
+absl::Status InferenceCalculatorGlAdvancedImpl::UpdateContract(
+ CalculatorContract* cc) {
+ const auto& options = cc->Options();
+ RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
+ << "Either model as side packet or model path in options is required.";
+
+ MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
+ return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorGlAdvancedImpl::Open(CalculatorContext* cc) {
+ const auto& options = cc->Options();
+ mediapipe::InferenceCalculatorOptions::Delegate delegate = options.delegate();
+ if (!kDelegate(cc).IsEmpty()) {
+ const mediapipe::InferenceCalculatorOptions::Delegate&
+ input_side_packet_delegate = kDelegate(cc).Get();
+ RET_CHECK(
+ input_side_packet_delegate.has_gpu() ||
+ input_side_packet_delegate.delegate_case() ==
+ mediapipe::InferenceCalculatorOptions::Delegate::DELEGATE_NOT_SET)
+ << "inference_calculator_gl_advanced only supports gpu delegate "
+ "configuration through side packet.";
+ delegate.MergeFrom(input_side_packet_delegate);
+ }
+
+ gpu_inference_runner_ = std::make_unique();
+ return gpu_inference_runner_->Init(cc, delegate);
+}
+
+absl::Status InferenceCalculatorGlAdvancedImpl::Process(CalculatorContext* cc) {
+ if (kInTensors(cc).IsEmpty()) {
+ return absl::OkStatus();
+ }
+
+ const auto& input_tensors = *kInTensors(cc);
+ RET_CHECK(!input_tensors.empty());
+ auto output_tensors = absl::make_unique>();
+
+ ASSIGN_OR_RETURN(*output_tensors,
+ gpu_inference_runner_->Process(input_tensors));
+
+ kOutTensors(cc).Send(std::move(output_tensors));
+ return absl::OkStatus();
+}
+
+absl::Status InferenceCalculatorGlAdvancedImpl::Close(CalculatorContext* cc) {
+ return gpu_inference_runner_->Close();
+}
+
} // namespace api2
} // namespace mediapipe
diff --git a/mediapipe/calculators/tensor/inference_calculator_metal.cc b/mediapipe/calculators/tensor/inference_calculator_metal.cc
index ae0a5e38d..ff8ebe149 100644
--- a/mediapipe/calculators/tensor/inference_calculator_metal.cc
+++ b/mediapipe/calculators/tensor/inference_calculator_metal.cc
@@ -116,7 +116,9 @@ class InferenceCalculatorMetalImpl
absl::Status InferenceCalculatorMetalImpl::UpdateContract(
CalculatorContract* cc) {
- const auto& options = cc->Options<::mediapipe::InferenceCalculatorOptions>();
+ RET_CHECK(!kDelegate(cc).IsConnected())
+ << "Delegate configuration through side packet is not supported.";
+ const auto& options = cc->Options();
RET_CHECK(!options.model_path().empty() ^ kSideInModel(cc).IsConnected())
<< "Either model as side packet or model path in options is required.";
diff --git a/mediapipe/calculators/tensor/inference_calculator_test.cc b/mediapipe/calculators/tensor/inference_calculator_test.cc
index fe96c0662..3662af391 100644
--- a/mediapipe/calculators/tensor/inference_calculator_test.cc
+++ b/mediapipe/calculators/tensor/inference_calculator_test.cc
@@ -16,13 +16,17 @@
#include
#include
+#include "absl/log/check.h"
+#include "absl/strings/str_cat.h"
#include "absl/strings/str_replace.h"
#include "absl/strings/string_view.h"
#include "mediapipe/calculators/tensor/inference_calculator.pb.h"
+#include "mediapipe/calculators/tensor/inference_calculator_test_base.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/benchmark.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
@@ -118,9 +122,11 @@ void RunGraphThenClose(CalculatorGraph& graph, std::vector input_vec) {
MP_ASSERT_OK(graph.StartRun({}));
// Push the tensor into the graph.
- MP_ASSERT_OK(graph.AddPacketToInputStream(
- "tensor_in",
- MakePacket>(std::move(input_vec)).At(Timestamp(0))));
+ if (!input_vec.empty()) {
+ MP_ASSERT_OK(graph.AddPacketToInputStream(
+ "tensor_in", MakePacket>(std::move(input_vec))
+ .At(Timestamp(0))));
+ }
// Wait until the calculator done processing.
MP_ASSERT_OK(graph.WaitUntilIdle());
@@ -174,5 +180,13 @@ TEST(InferenceCalculatorTest, ModelAsInputSidePacketSmokeTest) {
DoSmokeTest(kGraphWithModelAsInputSidePacket);
}
+void BM_InitializeCalculator(benchmark::State& state) {
+ mediapipe::InferenceCalculatorOptions::Delegate delegate;
+ delegate.mutable_tflite();
+ RunBenchmarkCalculatorInitialization(state, delegate);
+}
+
+BENCHMARK(BM_InitializeCalculator);
+
} // namespace
} // namespace mediapipe
diff --git a/mediapipe/calculators/tensor/landmarks_to_tensor_calculator.cc b/mediapipe/calculators/tensor/landmarks_to_tensor_calculator.cc
index 8f9323818..3a4655154 100644
--- a/mediapipe/calculators/tensor/landmarks_to_tensor_calculator.cc
+++ b/mediapipe/calculators/tensor/landmarks_to_tensor_calculator.cc
@@ -15,6 +15,8 @@
#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h"
#include
+#include
+#include
#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.pb.h"
#include "mediapipe/framework/api2/node.h"
@@ -28,8 +30,25 @@ namespace api2 {
namespace {
+// Returns the scale attribute should be multiplied by.
+float GetAttributeScale(
+ const LandmarksToTensorCalculatorOptions::Attribute& attribute,
+ const std::pair& image_size) {
+ switch (attribute) {
+ case LandmarksToTensorCalculatorOptions::X:
+ case LandmarksToTensorCalculatorOptions::Z:
+ return image_size.first;
+ case LandmarksToTensorCalculatorOptions::Y:
+ return image_size.second;
+ case LandmarksToTensorCalculatorOptions::VISIBILITY:
+ case LandmarksToTensorCalculatorOptions::PRESENCE:
+ return 1.0f;
+ }
+}
+
+template
float GetAttribute(
- const Landmark& landmark,
+ const LandmarkType& landmark,
const LandmarksToTensorCalculatorOptions::Attribute& attribute) {
switch (attribute) {
case LandmarksToTensorCalculatorOptions::X:
@@ -45,6 +64,33 @@ float GetAttribute(
}
}
+template
+Tensor ConvertLandmarksToTensor(
+ const LandmarksT& landmarks, const std::vector& attribute_scales,
+ const LandmarksToTensorCalculatorOptions& options) {
+ // Determine tensor shape.
+ const int n_landmarks = landmarks.landmark_size();
+ const int n_attributes = options.attributes_size();
+ auto tensor_shape = options.flatten()
+ ? Tensor::Shape{1, n_landmarks * n_attributes}
+ : Tensor::Shape{1, n_landmarks, n_attributes};
+
+ // Create empty tesnor.
+ Tensor tensor(Tensor::ElementType::kFloat32, tensor_shape);
+ auto* buffer = tensor.GetCpuWriteView().buffer();
+
+ // Fill tensor with landmark attributes.
+ for (int i = 0; i < n_landmarks; ++i) {
+ for (int j = 0; j < n_attributes; ++j) {
+ float value = GetAttribute(landmarks.landmark(i), options.attributes(j));
+ float scale = attribute_scales[j];
+ buffer[i * n_attributes + j] = value * scale;
+ }
+ }
+
+ return tensor;
+}
+
} // namespace
class LandmarksToTensorCalculatorImpl
@@ -54,39 +100,52 @@ class LandmarksToTensorCalculatorImpl
options_ = cc->Options();
RET_CHECK(options_.attributes_size() > 0)
<< "At least one attribute must be specified";
+
+ RET_CHECK(kInLandmarkList(cc).IsConnected() ^
+ kInNormLandmarkList(cc).IsConnected())
+ << "Exactly one landmarks input should be provided";
+ RET_CHECK_EQ(kInNormLandmarkList(cc).IsConnected(),
+ kImageSize(cc).IsConnected())
+ << "Image size should be provided only for normalized landmarks";
+
return absl::OkStatus();
}
absl::Status Process(CalculatorContext* cc) override {
- if (kInLandmarkList(cc).IsEmpty()) {
- return absl::OkStatus();
- }
-
- // Get input landmarks.
- const auto& in_landmarks = *kInLandmarkList(cc);
-
- // Determine tensor shape.
- const int n_landmarks = in_landmarks.landmark_size();
- const int n_attributes = options_.attributes_size();
- auto tensor_shape = options_.flatten()
- ? Tensor::Shape{1, n_landmarks * n_attributes}
- : Tensor::Shape{1, n_landmarks, n_attributes};
-
- // Create empty tesnor.
- Tensor tensor(Tensor::ElementType::kFloat32, tensor_shape);
- auto* buffer = tensor.GetCpuWriteView().buffer();
-
- // Fill tensor with landmark attributes.
- for (int i = 0; i < n_landmarks; ++i) {
- for (int j = 0; j < n_attributes; ++j) {
- buffer[i * n_attributes + j] =
- GetAttribute(in_landmarks.landmark(i), options_.attributes(j));
+ // Get attribute scales depending on whether landmarks are normalized or
+ // not.
+ std::vector attribute_scales;
+ if (kInLandmarkList(cc).IsConnected()) {
+ for (int j = 0; j < options_.attributes_size(); ++j) {
+ attribute_scales.push_back(1.0f);
+ }
+ } else {
+ RET_CHECK(!kImageSize(cc).IsEmpty());
+ auto image_size = kImageSize(cc).Get();
+ for (int j = 0; j < options_.attributes_size(); ++j) {
+ attribute_scales.push_back(
+ GetAttributeScale(options_.attributes(j), image_size));
}
}
- // Return vector with a single tensor.
+ // Convert landmarks to tensor.
auto result = std::vector();
- result.push_back(std::move(tensor));
+ if (kInLandmarkList(cc).IsConnected()) {
+ if (kInLandmarkList(cc).IsEmpty()) {
+ return absl::OkStatus();
+ }
+ Tensor tensor = ConvertLandmarksToTensor(kInLandmarkList(cc).Get(),
+ attribute_scales, options_);
+ result.push_back(std::move(tensor));
+ } else {
+ if (kInNormLandmarkList(cc).IsEmpty()) {
+ return absl::OkStatus();
+ }
+ Tensor tensor = ConvertLandmarksToTensor(kInNormLandmarkList(cc).Get(),
+ attribute_scales, options_);
+ result.push_back(std::move(tensor));
+ }
+
kOutTensors(cc).Send(std::move(result));
return absl::OkStatus();
diff --git a/mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h b/mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h
index 662f1b05f..6b9b90be5 100644
--- a/mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h
+++ b/mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h
@@ -28,8 +28,12 @@ namespace api2 {
// A calculator for converting landmars into a Tensor.
//
// Input:
-// LANDMARKS - LandmarkList
+// LANDMARKS (optional) - LandmarkList
// Landmarks to be converted into a Tensor.
+// NORM_LANDMARKS (optional) - NormalizedLandmarkList.
+// Normalized landmarks to be converted into a Tensor.
+// IMAGE_SIZE (optional) - std::pair
+// Image size to scale NORM_LANDMARKS.
//
// Output:
// TENSORS - std::vector
@@ -49,10 +53,15 @@ namespace api2 {
// }
class LandmarksToTensorCalculator : public NodeIntf {
public:
- static constexpr Input::Optional kInLandmarkList{"LANDMARKS"};
+ static constexpr Input::Optional kInLandmarkList{
+ "LANDMARKS"};
+ static constexpr Input::Optional
+ kInNormLandmarkList{"NORM_LANDMARKS"};
+ static constexpr Input>::Optional kImageSize{
+ "IMAGE_SIZE"};
static constexpr Output> kOutTensors{"TENSORS"};
MEDIAPIPE_NODE_INTERFACE(LandmarksToTensorCalculator, kInLandmarkList,
- kOutTensors);
+ kInNormLandmarkList, kImageSize, kOutTensors);
};
} // namespace api2
diff --git a/mediapipe/calculators/tensor/landmarks_to_tensor_calculator_test.cc b/mediapipe/calculators/tensor/landmarks_to_tensor_calculator_test.cc
index dfda71b55..6ef1e4190 100644
--- a/mediapipe/calculators/tensor/landmarks_to_tensor_calculator_test.cc
+++ b/mediapipe/calculators/tensor/landmarks_to_tensor_calculator_test.cc
@@ -40,6 +40,20 @@ void RunLandmarks(mediapipe::CalculatorRunner* runner,
MP_ASSERT_OK(runner->Run());
}
+void RunNormLandmarks(mediapipe::CalculatorRunner* runner,
+ const NormalizedLandmarkList& landmarks,
+ const std::pair image_size) {
+ runner->MutableInputs()
+ ->Tag("NORM_LANDMARKS")
+ .packets.push_back(
+ MakePacket(landmarks).At(Timestamp(0)));
+ runner->MutableInputs()
+ ->Tag("IMAGE_SIZE")
+ .packets.push_back(
+ MakePacket>(image_size).At(Timestamp(0)));
+ MP_ASSERT_OK(runner->Run());
+}
+
const Tensor& GetOutputTensor(mediapipe::CalculatorRunner* runner) {
const auto& output_packets = runner->Outputs().Tag("TENSORS").packets;
EXPECT_EQ(output_packets.size(), 1);
@@ -151,5 +165,34 @@ TEST(LandmarksToTensorCalculatorTest, XYZAttributes_Flatten) {
{1.0f, 2.0f, 3.0f, 6.0f, 7.0f, 8.0f});
}
+TEST(LandmarksToTensorCalculatorTest, NormalizedLandmarks) {
+ mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"pb(
+ calculator: "LandmarksToTensorCalculator"
+ input_stream: "NORM_LANDMARKS:landmarks"
+ input_stream: "IMAGE_SIZE:image_size"
+ output_stream: "TENSORS:tensors"
+ options: {
+ [mediapipe.LandmarksToTensorCalculatorOptions.ext] {
+ attributes: [ X, Y, Z, VISIBILITY, PRESENCE ]
+ }
+ }
+ )pb"));
+
+ NormalizedLandmarkList landmarks;
+ auto* landmark1 = landmarks.add_landmark();
+ landmark1->set_x(0.1f);
+ landmark1->set_y(0.5f);
+ landmark1->set_z(1.0f);
+ landmark1->set_visibility(4.0f);
+ landmark1->set_presence(5.0f);
+
+ std::pair image_size{200, 100};
+
+ RunNormLandmarks(&runner, landmarks, image_size);
+ const auto& tensor = GetOutputTensor(&runner);
+ ValidateTensor(tensor, /*expected_shape=*/{1, 1, 5}, /*expected_values=*/
+ {20.0f, 50.0f, 200.0f, 4.0f, 5.0f});
+}
+
} // namespace
} // namespace mediapipe
diff --git a/mediapipe/calculators/tensor/tensors_dequantization_calculator.cc b/mediapipe/calculators/tensor/tensors_dequantization_calculator.cc
new file mode 100644
index 000000000..0b7e6f082
--- /dev/null
+++ b/mediapipe/calculators/tensor/tensors_dequantization_calculator.cc
@@ -0,0 +1,102 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include
+#include
+
+#include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/api2/node.h"
+#include "mediapipe/framework/api2/port.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/ret_check.h"
+
+namespace mediapipe {
+namespace api2 {
+namespace {
+
+template
+void Dequantize(const Tensor& input, Tensor* output) {
+ auto input_view = input.GetCpuReadView();
+ auto input_buffer = input_view.buffer();
+ auto output_view = output->GetCpuWriteView();
+ auto output_buffer = output_view.buffer();
+ for (int i = 0; i < input.shape().num_elements(); ++i) {
+ output_buffer[i] = input.quantization_parameters().scale *
+ (static_cast(input_buffer[i]) -
+ input.quantization_parameters().zero_point);
+ }
+}
+
+} // namespace
+
+// Performs dequantization using the quantization parameters from the input
+// UInt8 or Int8 tensors. Each element of the input tensors is converted using:
+//
+// output = quantization_parameters.scale *
+// (input - quantization_parameters.zero_point)
+//
+// Input:
+// TENSORS - Vector of quantized Tensors of type kUint8 or kInt8.
+// Output:
+// TENSORS - Vector of dequantized Tensors of type kFloat32.
+//
+// Usage example:
+// node {
+// calculator: "TensorsDequantizationCalculator"
+// input_stream: "TENSORS:quantized_tensors"
+// output_stream: "TENSORS:dequantized_tensors"
+// }
+class TensorsDequantizationCalculator : public Node {
+ public:
+ static constexpr Input> kInTensors{"TENSORS"};
+ static constexpr Output> kOutTensors{"TENSORS"};
+ MEDIAPIPE_NODE_CONTRACT(kInTensors, kOutTensors);
+
+ absl::Status Process(CalculatorContext* cc) override;
+};
+
+absl::Status TensorsDequantizationCalculator::Process(CalculatorContext* cc) {
+ if (kInTensors(cc).IsEmpty()) {
+ return absl::OkStatus();
+ }
+ const auto& input_tensors = *kInTensors(cc);
+ RET_CHECK(!input_tensors.empty());
+ auto output_tensors = std::make_unique>();
+ output_tensors->reserve(input_tensors.size());
+ for (const auto& input_tensor : input_tensors) {
+ output_tensors->emplace_back(Tensor::ElementType::kFloat32,
+ input_tensor.shape());
+ switch (input_tensor.element_type()) {
+ case Tensor::ElementType::kUInt8:
+ Dequantize(input_tensor, &output_tensors->back());
+ break;
+ case Tensor::ElementType::kInt8:
+ Dequantize(input_tensor, &output_tensors->back());
+ break;
+ default:
+ return absl::InvalidArgumentError(absl::StrCat(
+ "Unsupported input tensor type: ", input_tensor.element_type()));
+ }
+ }
+ kOutTensors(cc).Send(std::move(output_tensors));
+ return absl::OkStatus();
+}
+
+MEDIAPIPE_REGISTER_NODE(TensorsDequantizationCalculator);
+
+} // namespace api2
+} // namespace mediapipe
diff --git a/mediapipe/calculators/tensor/tensors_dequantization_calculator_test.cc b/mediapipe/calculators/tensor/tensors_dequantization_calculator_test.cc
new file mode 100644
index 000000000..fd41cc763
--- /dev/null
+++ b/mediapipe/calculators/tensor/tensors_dequantization_calculator_test.cc
@@ -0,0 +1,128 @@
+// Copyright 2022 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include
+#include
+#include
+
+#include "absl/status/status.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+namespace {
+
+using ::mediapipe::ParseTextProtoOrDie;
+using ::testing::HasSubstr;
+using Node = ::mediapipe::CalculatorGraphConfig::Node;
+
+constexpr char kCalculatorConfig[] = R"pb(
+ calculator: "TensorsDequantizationCalculator"
+ input_stream: "TENSORS:input"
+ output_stream: "TENSORS:output"
+)pb";
+
+// Compares the provided tensor contents with the expected values.
+void ValidateResult(const Tensor& actual, const std::vector& expected) {
+ EXPECT_EQ(actual.element_type(), Tensor::ElementType::kFloat32);
+ EXPECT_EQ(expected.size(), actual.shape().num_elements());
+ auto view = actual.GetCpuReadView();
+ auto buffer = view.buffer();
+ for (int i = 0; i < expected.size(); ++i) {
+ EXPECT_FLOAT_EQ(expected[i], buffer[i]);
+ }
+}
+
+class TensorsDequantizationCalculatorTest : public ::testing::Test {
+ protected:
+ TensorsDequantizationCalculatorTest()
+ : runner_(ParseTextProtoOrDie(kCalculatorConfig)) {}
+
+ template
+ void PushTensor(Tensor::ElementType type, std::vector tensor,
+ std::optional
+ quantization_params = std::nullopt) {
+ auto tensors = std::make_unique>();
+ if (quantization_params.has_value()) {
+ tensors->emplace_back(type,
+ Tensor::Shape{static_cast(tensor.size())},
+ quantization_params.value());
+ } else {
+ tensors->emplace_back(type,
+ Tensor::Shape{static_cast(tensor.size())});
+ }
+ auto view = tensors->back().GetCpuWriteView();
+ auto buffer = view.buffer();
+ std::copy(tensor.begin(), tensor.end(), buffer);
+ runner_.MutableInputs()->Tag("TENSORS").packets.push_back(
+ Adopt(tensors.release()).At(Timestamp(0)));
+ }
+
+ const Tensor& GetOutput() {
+ return runner_.Outputs()
+ .Get("TENSORS", 0)
+ .packets[0]
+ .Get>()[0];
+ }
+
+ CalculatorRunner runner_;
+};
+
+TEST_F(TensorsDequantizationCalculatorTest, FailsWithFloatTensors) {
+ std::vector tensor = {0, 1};
+ PushTensor(Tensor::ElementType::kFloat32, tensor);
+
+ auto status = runner_.Run();
+
+ EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument);
+ EXPECT_THAT(status.message(), HasSubstr("Unsupported input tensor type"));
+}
+
+TEST_F(TensorsDequantizationCalculatorTest, FailsWithInt32Tensors) {
+ std::vector tensor = {0, 1};
+ PushTensor(Tensor::ElementType::kInt32, tensor);
+
+ auto status = runner_.Run();
+
+ EXPECT_EQ(status.code(), absl::StatusCode::kInvalidArgument);
+ EXPECT_THAT(status.message(), HasSubstr("Unsupported input tensor type"));
+}
+
+TEST_F(TensorsDequantizationCalculatorTest, SucceedsWithUInt8Tensors) {
+ std::vector tensor = {0, 127, 255};
+ PushTensor(Tensor::ElementType::kUInt8, tensor,
+ Tensor::QuantizationParameters{1.0f / 127, 127});
+
+ MP_ASSERT_OK(runner_.Run());
+
+ ValidateResult(GetOutput(), {-1, 0, 1.007874});
+}
+
+TEST_F(TensorsDequantizationCalculatorTest, SucceedsWithInt8Tensors) {
+ std::vector tensor = {-128, 0, 127};
+ PushTensor(Tensor::ElementType::kInt8, tensor,
+ Tensor::QuantizationParameters{1.0f / 127, 0});
+
+ MP_ASSERT_OK(runner_.Run());
+
+ ValidateResult(GetOutput(), {-1.007874, 0, 1});
+}
+
+} // namespace
+} // namespace mediapipe
diff --git a/mediapipe/calculators/tensor/tensors_to_classification_calculator.cc b/mediapipe/calculators/tensor/tensors_to_classification_calculator.cc
index 2f7354958..5bfc00ed7 100644
--- a/mediapipe/calculators/tensor/tensors_to_classification_calculator.cc
+++ b/mediapipe/calculators/tensor/tensors_to_classification_calculator.cc
@@ -165,6 +165,7 @@ absl::Status TensorsToClassificationCalculator::Open(CalculatorContext* cc) {
absl::Status TensorsToClassificationCalculator::Process(CalculatorContext* cc) {
const auto& input_tensors = *kInTensors(cc);
RET_CHECK_EQ(input_tensors.size(), 1);
+ RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
int num_classes = input_tensors[0].shape().num_elements();
diff --git a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc
index b1babaffb..11c1341d4 100644
--- a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc
+++ b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc
@@ -287,7 +287,11 @@ absl::Status TensorsToDetectionsCalculator::Process(CalculatorContext* cc) {
}
}
}
- const int num_input_tensors = kInTensors(cc)->size();
+ const auto& input_tensors = *kInTensors(cc);
+ for (const auto& tensor : input_tensors) {
+ RET_CHECK(tensor.element_type() == Tensor::ElementType::kFloat32);
+ }
+ const int num_input_tensors = input_tensors.size();
if (!scores_tensor_index_is_set_) {
if (num_input_tensors == 2 ||
num_input_tensors == kNumInputTensorsWithAnchors) {
diff --git a/mediapipe/calculators/tensor/tensors_to_floats_calculator.cc b/mediapipe/calculators/tensor/tensors_to_floats_calculator.cc
index 5ec3b4dea..b359953f5 100644
--- a/mediapipe/calculators/tensor/tensors_to_floats_calculator.cc
+++ b/mediapipe/calculators/tensor/tensors_to_floats_calculator.cc
@@ -76,6 +76,7 @@ absl::Status TensorsToFloatsCalculator::Open(CalculatorContext* cc) {
absl::Status TensorsToFloatsCalculator::Process(CalculatorContext* cc) {
const auto& input_tensors = *kInTensors(cc);
RET_CHECK(!input_tensors.empty());
+ RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
// TODO: Add option to specify which tensor to take from.
auto view = input_tensors[0].GetCpuReadView();
auto raw_floats = view.buffer();
diff --git a/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc b/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc
index 8e4066bee..a1cc4e202 100644
--- a/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc
+++ b/mediapipe/calculators/tensor/tensors_to_landmarks_calculator.cc
@@ -139,6 +139,7 @@ absl::Status TensorsToLandmarksCalculator::Process(CalculatorContext* cc) {
bool flip_vertically = kFlipVertically(cc).GetOr(options_.flip_vertically());
const auto& input_tensors = *kInTensors(cc);
+ RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
int num_values = input_tensors[0].shape().num_elements();
const int num_dimensions = num_values / num_landmarks_;
CHECK_GT(num_dimensions, 0);
diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc
index 21f983894..172f70880 100644
--- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc
+++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc
@@ -116,8 +116,9 @@ using ::tflite::gpu::gl::GlShader;
//
// Inputs:
// One of the following TENSORS tags:
-// TENSORS: Vector of Tensor,
-// The tensor dimensions are specified in this calculator's options.
+// TENSORS: Vector of Tensors of type kFloat32. Only the first tensor will be
+// used. The tensor dimensions are specified in this calculator's
+// options.
// OUTPUT_SIZE(optional): std::pair,
// If provided, the size to upscale mask to.
//
@@ -261,6 +262,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
// Validate tensor channels and activation type.
{
RET_CHECK(!input_tensors.empty());
+ RET_CHECK(input_tensors[0].element_type() == Tensor::ElementType::kFloat32);
ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
int tensor_channels = std::get<2>(hwc);
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD
index a57e1b202..4037d89ce 100644
--- a/mediapipe/calculators/tensorflow/BUILD
+++ b/mediapipe/calculators/tensorflow/BUILD
@@ -13,7 +13,7 @@
# limitations under the License.
#
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
licenses(["notice"])
@@ -88,6 +88,13 @@ proto_library(
deps = ["//mediapipe/framework:calculator_proto"],
)
+proto_library(
+ name = "tensor_to_vector_int_calculator_options_proto",
+ srcs = ["tensor_to_vector_int_calculator_options.proto"],
+ visibility = ["//visibility:public"],
+ deps = ["//mediapipe/framework:calculator_proto"],
+)
+
proto_library(
name = "tensor_to_vector_string_calculator_options_proto",
srcs = ["tensor_to_vector_string_calculator_options.proto"],
@@ -95,10 +102,12 @@ proto_library(
deps = ["//mediapipe/framework:calculator_proto"],
)
-proto_library(
+mediapipe_proto_library(
name = "unpack_media_sequence_calculator_proto",
srcs = ["unpack_media_sequence_calculator.proto"],
- visibility = ["//visibility:public"],
+ visibility = [
+ "//visibility:public",
+ ],
deps = [
"//mediapipe/calculators/core:packet_resampler_calculator_proto",
"//mediapipe/framework:calculator_proto",
@@ -166,17 +175,6 @@ mediapipe_cc_proto_library(
deps = [":object_detection_tensors_to_detections_calculator_proto"],
)
-mediapipe_cc_proto_library(
- name = "pack_media_sequence_calculator_cc_proto",
- srcs = ["pack_media_sequence_calculator.proto"],
- cc_deps = [
- "//mediapipe/framework:calculator_cc_proto",
- "@org_tensorflow//tensorflow/core:protos_all_cc",
- ],
- visibility = ["//visibility:public"],
- deps = [":pack_media_sequence_calculator_proto"],
-)
-
mediapipe_cc_proto_library(
name = "tensorflow_inference_calculator_cc_proto",
srcs = ["tensorflow_inference_calculator.proto"],
@@ -264,6 +262,14 @@ mediapipe_cc_proto_library(
deps = [":tensor_to_vector_float_calculator_options_proto"],
)
+mediapipe_cc_proto_library(
+ name = "tensor_to_vector_int_calculator_options_cc_proto",
+ srcs = ["tensor_to_vector_int_calculator_options.proto"],
+ cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+ visibility = ["//visibility:public"],
+ deps = [":tensor_to_vector_int_calculator_options_proto"],
+)
+
mediapipe_cc_proto_library(
name = "tensor_to_vector_string_calculator_options_cc_proto",
srcs = ["tensor_to_vector_string_calculator_options.proto"],
@@ -272,18 +278,6 @@ mediapipe_cc_proto_library(
deps = [":tensor_to_vector_string_calculator_options_proto"],
)
-mediapipe_cc_proto_library(
- name = "unpack_media_sequence_calculator_cc_proto",
- srcs = ["unpack_media_sequence_calculator.proto"],
- cc_deps = [
- "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
- "//mediapipe/framework:calculator_cc_proto",
- "//mediapipe/util:audio_decoder_cc_proto",
- ],
- visibility = ["//visibility:public"],
- deps = [":unpack_media_sequence_calculator_proto"],
-)
-
mediapipe_cc_proto_library(
name = "vector_int_to_tensor_calculator_options_cc_proto",
srcs = ["vector_int_to_tensor_calculator_options.proto"],
@@ -420,8 +414,9 @@ cc_library(
"//mediapipe/calculators/image:opencv_image_encoder_calculator_cc_proto",
"//mediapipe/calculators/tensorflow:pack_media_sequence_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
- "//mediapipe/framework/formats:detection_cc_proto",
+ "//mediapipe/framework/formats:detection_cc_proto", # build_cleaner: keep
"//mediapipe/framework/formats:location",
+ "//mediapipe/framework/formats:location_opencv",
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
@@ -458,6 +453,7 @@ cc_library(
deps = [
":tensorflow_session",
":tensorflow_inference_calculator_cc_proto",
+ "@com_google_absl//absl/log:check",
"//mediapipe/framework:timestamp",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/memory",
@@ -722,6 +718,28 @@ cc_library(
alwayslink = 1,
)
+cc_library(
+ name = "tensor_to_vector_int_calculator",
+ srcs = ["tensor_to_vector_int_calculator.cc"],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":tensor_to_vector_int_calculator_options_cc_proto",
+ "@com_google_absl//absl/base:core_headers",
+ "//mediapipe/framework/port:integral_types",
+ "//mediapipe/framework:calculator_framework",
+ "//mediapipe/framework/port:status",
+ "//mediapipe/framework/port:ret_check",
+ ] + select({
+ "//conditions:default": [
+ "@org_tensorflow//tensorflow/core:framework",
+ ],
+ "//mediapipe:android": [
+ "@org_tensorflow//tensorflow/core:portable_tensorflow_lib_lite",
+ ],
+ }),
+ alwayslink = 1,
+)
+
cc_library(
name = "tensor_to_vector_string_calculator",
srcs = ["tensor_to_vector_string_calculator.cc"],
@@ -916,6 +934,7 @@ cc_test(
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/formats:location",
+ "//mediapipe/framework/formats:location_opencv",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/util/sequence:media_sequence",
@@ -1106,6 +1125,20 @@ cc_test(
],
)
+cc_test(
+ name = "tensor_to_vector_int_calculator_test",
+ srcs = ["tensor_to_vector_int_calculator_test.cc"],
+ deps = [
+ ":tensor_to_vector_int_calculator",
+ ":tensor_to_vector_int_calculator_options_cc_proto",
+ "//mediapipe/framework:calculator_framework",
+ "//mediapipe/framework:calculator_runner",
+ "//mediapipe/framework/port:gtest_main",
+ "@org_tensorflow//tensorflow/core:framework",
+ "@org_tensorflow//tensorflow/core:protos_all_cc",
+ ],
+)
+
cc_test(
name = "tensor_to_vector_string_calculator_test",
srcs = ["tensor_to_vector_string_calculator_test.cc"],
diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc
index 7e58329f0..3f7525c99 100644
--- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc
+++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator.cc
@@ -22,6 +22,7 @@
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/location.h"
+#include "mediapipe/framework/formats/location_opencv.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
#include "mediapipe/framework/port/ret_check.h"
@@ -37,6 +38,7 @@ const char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
const char kImageTag[] = "IMAGE";
const char kFloatContextFeaturePrefixTag[] = "FLOAT_CONTEXT_FEATURE_";
const char kFloatFeaturePrefixTag[] = "FLOAT_FEATURE_";
+const char kIntFeaturePrefixTag[] = "INT_FEATURE_";
const char kBytesFeaturePrefixTag[] = "BYTES_FEATURE_";
const char kForwardFlowEncodedTag[] = "FORWARD_FLOW_ENCODED";
const char kBBoxTag[] = "BBOX";
@@ -88,7 +90,7 @@ namespace mpms = mediapipe::mediasequence;
// }
namespace {
uint8 ConvertFloatToByte(const float float_value) {
- float clamped_value = MathUtil::Clamp(0.0f, 1.0f, float_value);
+ float clamped_value = std::clamp(0.0f, 1.0f, float_value);
return static_cast(clamped_value * 255.0 + .5f);
}
} // namespace
@@ -154,6 +156,9 @@ class PackMediaSequenceCalculator : public CalculatorBase {
if (absl::StartsWith(tag, kFloatFeaturePrefixTag)) {
cc->Inputs().Tag(tag).Set>();
}
+ if (absl::StartsWith(tag, kIntFeaturePrefixTag)) {
+ cc->Inputs().Tag(tag).Set>();
+ }
if (absl::StartsWith(tag, kBytesFeaturePrefixTag)) {
cc->Inputs().Tag(tag).Set>();
}
@@ -235,6 +240,12 @@ class PackMediaSequenceCalculator : public CalculatorBase {
mpms::ClearFeatureFloats(key, sequence_.get());
mpms::ClearFeatureTimestamp(key, sequence_.get());
}
+ if (absl::StartsWith(tag, kIntFeaturePrefixTag)) {
+ std::string key = tag.substr(
+ sizeof(kIntFeaturePrefixTag) / sizeof(*kIntFeaturePrefixTag) - 1);
+ mpms::ClearFeatureInts(key, sequence_.get());
+ mpms::ClearFeatureTimestamp(key, sequence_.get());
+ }
if (absl::StartsWith(tag, kBytesFeaturePrefixTag)) {
std::string key = tag.substr(sizeof(kBytesFeaturePrefixTag) /
sizeof(*kBytesFeaturePrefixTag) -
@@ -416,6 +427,16 @@ class PackMediaSequenceCalculator : public CalculatorBase {
cc->Inputs().Tag(tag).Get>(),
sequence_.get());
}
+ if (absl::StartsWith(tag, kIntFeaturePrefixTag) &&
+ !cc->Inputs().Tag(tag).IsEmpty()) {
+ std::string key = tag.substr(
+ sizeof(kIntFeaturePrefixTag) / sizeof(*kIntFeaturePrefixTag) - 1);
+ mpms::AddFeatureTimestamp(key, cc->InputTimestamp().Value(),
+ sequence_.get());
+ mpms::AddFeatureInts(key,
+ cc->Inputs().Tag(tag).Get>(),
+ sequence_.get());
+ }
if (absl::StartsWith(tag, kBytesFeaturePrefixTag) &&
!cc->Inputs().Tag(tag).IsEmpty()) {
std::string key = tag.substr(sizeof(kBytesFeaturePrefixTag) /
@@ -508,7 +529,7 @@ class PackMediaSequenceCalculator : public CalculatorBase {
RET_CHECK(!already_has_mask)
<< "We currently only support adding one mask per timestamp. "
<< sequence_->DebugString();
- auto mask_mat_ptr = Location(detection.location_data()).GetCvMask();
+ auto mask_mat_ptr = GetCvMask(Location(detection.location_data()));
std::vector bytes;
RET_CHECK(cv::imencode(".png", *mask_mat_ptr, bytes, {}));
diff --git a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc
index b39a0bac0..4e74b06df 100644
--- a/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc
+++ b/mediapipe/calculators/tensorflow/pack_media_sequence_calculator_test.cc
@@ -25,6 +25,7 @@
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/location.h"
+#include "mediapipe/framework/formats/location_opencv.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
@@ -56,6 +57,8 @@ constexpr char kFloatContextFeatureOtherTag[] = "FLOAT_CONTEXT_FEATURE_OTHER";
constexpr char kFloatContextFeatureTestTag[] = "FLOAT_CONTEXT_FEATURE_TEST";
constexpr char kFloatFeatureOtherTag[] = "FLOAT_FEATURE_OTHER";
constexpr char kFloatFeatureTestTag[] = "FLOAT_FEATURE_TEST";
+constexpr char kIntFeatureOtherTag[] = "INT_FEATURE_OTHER";
+constexpr char kIntFeatureTestTag[] = "INT_FEATURE_TEST";
constexpr char kImagePrefixTag[] = "IMAGE_PREFIX";
constexpr char kSequenceExampleTag[] = "SEQUENCE_EXAMPLE";
constexpr char kImageTag[] = "IMAGE";
@@ -217,6 +220,50 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoFloatLists) {
}
}
+TEST_F(PackMediaSequenceCalculatorTest, PacksTwoIntLists) {
+ SetUpCalculator({"INT_FEATURE_TEST:test", "INT_FEATURE_OTHER:test2"}, {},
+ false, true);
+ auto input_sequence = ::absl::make_unique();
+
+ int num_timesteps = 2;
+ for (int i = 0; i < num_timesteps; ++i) {
+ auto vi_ptr = ::absl::make_unique>(2, 2 << i);
+ runner_->MutableInputs()
+ ->Tag(kIntFeatureTestTag)
+ .packets.push_back(Adopt(vi_ptr.release()).At(Timestamp(i)));
+ vi_ptr = ::absl::make_unique>(2, 2 << i);
+ runner_->MutableInputs()
+ ->Tag(kIntFeatureOtherTag)
+ .packets.push_back(Adopt(vi_ptr.release()).At(Timestamp(i)));
+ }
+
+ runner_->MutableSidePackets()->Tag(kSequenceExampleTag) =
+ Adopt(input_sequence.release());
+
+ MP_ASSERT_OK(runner_->Run());
+
+ const std::vector& output_packets =
+ runner_->Outputs().Tag(kSequenceExampleTag).packets;
+ ASSERT_EQ(1, output_packets.size());
+ const tf::SequenceExample& output_sequence =
+ output_packets[0].Get();
+
+ ASSERT_EQ(num_timesteps,
+ mpms::GetFeatureTimestampSize("TEST", output_sequence));
+ ASSERT_EQ(num_timesteps, mpms::GetFeatureIntsSize("TEST", output_sequence));
+ ASSERT_EQ(num_timesteps,
+ mpms::GetFeatureTimestampSize("OTHER", output_sequence));
+ ASSERT_EQ(num_timesteps, mpms::GetFeatureIntsSize("OTHER", output_sequence));
+ for (int i = 0; i < num_timesteps; ++i) {
+ ASSERT_EQ(i, mpms::GetFeatureTimestampAt("TEST", output_sequence, i));
+ ASSERT_THAT(mpms::GetFeatureIntsAt("TEST", output_sequence, i),
+ ::testing::ElementsAreArray(std::vector(2, 2 << i)));
+ ASSERT_EQ(i, mpms::GetFeatureTimestampAt("OTHER", output_sequence, i));
+ ASSERT_THAT(mpms::GetFeatureIntsAt("OTHER", output_sequence, i),
+ ::testing::ElementsAreArray(std::vector(2, 2 << i)));
+ }
+}
+
TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBytesLists) {
SetUpCalculator({"BYTES_FEATURE_TEST:test", "BYTES_FEATURE_OTHER:test2"}, {},
false, true);
@@ -434,7 +481,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoBBoxDetections) {
detection.add_label("mask");
detection.add_score(1.0);
cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0));
- Location::CreateCvMaskLocation(image).ConvertToProto(
+ mediapipe::CreateCvMaskLocation(image).ConvertToProto(
detection.mutable_location_data());
detections->push_back(detection);
@@ -513,7 +560,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithoutImageDims) {
detection.add_label("mask");
detection.add_score(1.0);
cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0));
- Location::CreateCvMaskLocation(image).ConvertToProto(
+ mediapipe::CreateCvMaskLocation(image).ConvertToProto(
detection.mutable_location_data());
detections->push_back(detection);
@@ -561,7 +608,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksBBoxWithImages) {
detection.add_label("mask");
detection.add_score(1.0);
cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0));
- Location::CreateCvMaskLocation(image).ConvertToProto(
+ mediapipe::CreateCvMaskLocation(image).ConvertToProto(
detection.mutable_location_data());
detections->push_back(detection);
@@ -677,7 +724,7 @@ TEST_F(PackMediaSequenceCalculatorTest, PacksTwoMaskDetections) {
detection.add_label("mask");
detection.add_score(1.0);
cv::Mat image(2, 3, CV_8UC1, cv::Scalar(0));
- Location::CreateCvMaskLocation(image).ConvertToProto(
+ mediapipe::CreateCvMaskLocation(image).ConvertToProto(
detection.mutable_location_data());
detections->push_back(detection);
diff --git a/mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator.cc b/mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator.cc
new file mode 100644
index 000000000..2f4ff28cf
--- /dev/null
+++ b/mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator.cc
@@ -0,0 +1,151 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Calculator converts from one-dimensional Tensor of DT_FLOAT to vector
+// OR from (batched) two-dimensional Tensor of DT_FLOAT to vector.
+
+#include
+
+#include "absl/base/integral_types.h"
+#include "mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/status.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+
+namespace mediapipe {
+
+namespace tf = ::tensorflow;
+
+class TensorToVectorIntCalculator : public CalculatorBase {
+ public:
+ static absl::Status GetContract(CalculatorContract* cc);
+
+ absl::Status Open(CalculatorContext* cc) override;
+ absl::Status Process(CalculatorContext* cc) override;
+
+ private:
+ void TokenizeVector(std::vector* vector) const;
+
+ TensorToVectorIntCalculatorOptions options_;
+};
+REGISTER_CALCULATOR(TensorToVectorIntCalculator);
+
+absl::Status TensorToVectorIntCalculator::GetContract(CalculatorContract* cc) {
+ // Start with only one input packet.
+ RET_CHECK_EQ(cc->Inputs().NumEntries(), 1)
+ << "Only one input stream is supported.";
+ cc->Inputs().Index(0).Set(
+ // Input Tensor
+ );
+ RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
+ << "Only one output stream is supported.";
+ const auto& options = cc->Options();
+ if (options.tensor_is_2d()) {
+ RET_CHECK(!options.flatten_nd());
+ cc->Outputs().Index(0).Set