diff --git a/.bazelrc b/.bazelrc
index 724dd23fd..44bc3d0a1 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -87,6 +87,9 @@ build:ios_fat --config=ios
build:ios_fat --ios_multi_cpus=armv7,arm64
build:ios_fat --watchos_cpus=armv7k
+build:ios_sim_fat --config=ios
+build:ios_sim_fat --ios_multi_cpus=x86_64,sim_arm64
+
build:darwin_x86_64 --apple_platform_type=macos
build:darwin_x86_64 --macos_minimum_os=10.12
build:darwin_x86_64 --cpu=darwin_x86_64
diff --git a/.github/ISSUE_TEMPLATE/task_issue_template.yaml b/.github/ISSUE_TEMPLATE/00-task-issue-template.yaml
similarity index 96%
rename from .github/ISSUE_TEMPLATE/task_issue_template.yaml
rename to .github/ISSUE_TEMPLATE/00-task-issue-template.yaml
index da3ec013e..d6130edb6 100644
--- a/.github/ISSUE_TEMPLATE/task_issue_template.yaml
+++ b/.github/ISSUE_TEMPLATE/00-task-issue-template.yaml
@@ -40,18 +40,16 @@ body:
label: Programming Language and version (e.g. C++, Python, Java)
validations:
required: true
- - type: textarea
+ - type: input
id: current_model
attributes:
label: Describe the actual behavior
- render: shell
validations:
required: true
- - type: textarea
+ - type: input
id: expected_model
attributes:
label: Describe the expected behaviour
- render: shell
validations:
required: true
- type: textarea
diff --git a/.github/ISSUE_TEMPLATE/model_maker_issue_template.yaml b/.github/ISSUE_TEMPLATE/11-model-maker-issue-template.yaml
similarity index 96%
rename from .github/ISSUE_TEMPLATE/model_maker_issue_template.yaml
rename to .github/ISSUE_TEMPLATE/11-model-maker-issue-template.yaml
index 23fa08245..7a6d92152 100644
--- a/.github/ISSUE_TEMPLATE/model_maker_issue_template.yaml
+++ b/.github/ISSUE_TEMPLATE/11-model-maker-issue-template.yaml
@@ -41,18 +41,16 @@ body:
label: Task name (e.g. Image classification, Gesture recognition etc.)
validations:
required: true
- - type: textarea
+ - type: input
id: current_model
attributes:
label: Describe the actual behavior
- render: shell
validations:
required: true
- - type: textarea
+ - type: input
id: expected_model
attributes:
label: Describe the expected behaviour
- render: shell
validations:
required: true
- type: textarea
diff --git a/.github/ISSUE_TEMPLATE/studio_issue_template.yaml b/.github/ISSUE_TEMPLATE/12-studio-issue-template.yaml
similarity index 96%
rename from .github/ISSUE_TEMPLATE/studio_issue_template.yaml
rename to .github/ISSUE_TEMPLATE/12-studio-issue-template.yaml
index 645a9e4e4..ffaa315f9 100644
--- a/.github/ISSUE_TEMPLATE/studio_issue_template.yaml
+++ b/.github/ISSUE_TEMPLATE/12-studio-issue-template.yaml
@@ -31,18 +31,16 @@ body:
label: URL that shows the problem
validations:
required: false
- - type: textarea
+ - type: input
id: current_model
attributes:
label: Describe the actual behavior
- render: shell
validations:
required: false
- - type: textarea
+ - type: input
id: expected_model
attributes:
label: Describe the expected behaviour
- render: shell
validations:
required: false
- type: textarea
diff --git a/.github/ISSUE_TEMPLATE/feature_request_issue_template.yaml b/.github/ISSUE_TEMPLATE/14-feature-request-issue-template.yaml
similarity index 89%
rename from .github/ISSUE_TEMPLATE/feature_request_issue_template.yaml
rename to .github/ISSUE_TEMPLATE/14-feature-request-issue-template.yaml
index d707f09cc..c170891fb 100644
--- a/.github/ISSUE_TEMPLATE/feature_request_issue_template.yaml
+++ b/.github/ISSUE_TEMPLATE/14-feature-request-issue-template.yaml
@@ -28,37 +28,33 @@ body:
- 'No'
validations:
required: false
- - type: textarea
+ - type: input
id: behaviour
attributes:
label: Describe the feature and the current behaviour/state
- render: shell
validations:
required: true
- - type: textarea
+ - type: input
id: api_change
attributes:
label: Will this change the current API? How?
- render: shell
validations:
required: false
- - type: textarea
+ - type: input
id: benifit
attributes:
label: Who will benefit with this feature?
validations:
required: false
- - type: textarea
+ - type: input
id: use_case
attributes:
label: Please specify the use cases for this feature
- render: shell
validations:
required: true
- - type: textarea
+ - type: input
id: info_other
attributes:
label: Any Other info
- render: shell
validations:
required: false
diff --git a/.github/ISSUE_TEMPLATE/build.install_issue_template.yaml b/.github/ISSUE_TEMPLATE/15-build-install-issue-template.yaml
similarity index 98%
rename from .github/ISSUE_TEMPLATE/build.install_issue_template.yaml
rename to .github/ISSUE_TEMPLATE/15-build-install-issue-template.yaml
index 20cf9723d..ded9d09a6 100644
--- a/.github/ISSUE_TEMPLATE/build.install_issue_template.yaml
+++ b/.github/ISSUE_TEMPLATE/15-build-install-issue-template.yaml
@@ -87,14 +87,13 @@ body:
placeholder:
validations:
required: false
- - type: textarea
+ - type: input
id: what-happened
attributes:
label: Describe the problem
description: Provide the exact sequence of commands / steps that you executed before running into the [problem](https://google.github.io/mediapipe/getting_started/getting_started.html)
placeholder: Tell us what you see!
value: "A bug happened!"
- render: shell
validations:
required: true
- type: textarea
diff --git a/.github/ISSUE_TEMPLATE/bug_issue_template.yaml b/.github/ISSUE_TEMPLATE/16-bug-issue-template.yaml
similarity index 97%
rename from .github/ISSUE_TEMPLATE/bug_issue_template.yaml
rename to .github/ISSUE_TEMPLATE/16-bug-issue-template.yaml
index e997958ae..efa925b44 100644
--- a/.github/ISSUE_TEMPLATE/bug_issue_template.yaml
+++ b/.github/ISSUE_TEMPLATE/16-bug-issue-template.yaml
@@ -80,18 +80,16 @@ body:
label: Xcode & Tulsi version (if issue is related to building for iOS)
validations:
required: false
- - type: textarea
+ - type: input
id: current_model
attributes:
label: Describe the actual behavior
- render: shell
validations:
required: true
- - type: textarea
+ - type: input
id: expected_model
attributes:
label: Describe the expected behaviour
- render: shell
validations:
required: true
- type: textarea
diff --git a/.github/ISSUE_TEMPLATE/Documentation_issue_template.yaml b/.github/ISSUE_TEMPLATE/17-documentation-issue-template.yaml
similarity index 100%
rename from .github/ISSUE_TEMPLATE/Documentation_issue_template.yaml
rename to .github/ISSUE_TEMPLATE/17-documentation-issue-template.yaml
diff --git a/.github/ISSUE_TEMPLATE/Solution(Legacy_issue_template).yaml b/.github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml
similarity index 97%
rename from .github/ISSUE_TEMPLATE/Solution(Legacy_issue_template).yaml
rename to .github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml
index 26c59737b..acb0f5b89 100644
--- a/.github/ISSUE_TEMPLATE/Solution(Legacy_issue_template).yaml
+++ b/.github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml
@@ -48,18 +48,16 @@ body:
placeholder: e.g. C++, Python, Java
validations:
required: false
- - type: textarea
+ - type: input
id: current_model
attributes:
label: Describe the actual behavior
- render: shell
validations:
required: false
- - type: textarea
+ - type: input
id: expected_model
attributes:
label: Describe the expected behaviour
- render: shell
validations:
required: false
- type: textarea
diff --git a/.github/ISSUE_TEMPLATE/50-other-issues.md b/.github/ISSUE_TEMPLATE/19-other-issues.md
similarity index 100%
rename from .github/ISSUE_TEMPLATE/50-other-issues.md
rename to .github/ISSUE_TEMPLATE/19-other-issues.md
diff --git a/.github/stale.yml b/.github/stale.yml
deleted file mode 100644
index 03c67d0f6..000000000
--- a/.github/stale.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2021 The MediaPipe Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-#
-# This file was assembled from multiple pieces, whose use is documented
-# throughout. Please refer to the TensorFlow dockerfiles documentation
-# for more information.
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 7
-# Number of days of inactivity before a stale Issue or Pull Request is closed
-daysUntilClose: 7
-# Only issues or pull requests with all of these labels are checked if stale. Defaults to `[]` (disabled)
-onlyLabels:
- - stat:awaiting response
-# Comment to post when marking as stale. Set to `false` to disable
-markComment: >
- This issue has been automatically marked as stale because it has not had
- recent activity. It will be closed if no further activity occurs. Thank you.
-# Comment to post when removing the stale label. Set to `false` to disable
-unmarkComment: false
-closeComment: >
- Closing as stale. Please reopen if you'd like to work on this further.
diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml
new file mode 100644
index 000000000..c5e15bce6
--- /dev/null
+++ b/.github/workflows/stale.yaml
@@ -0,0 +1,66 @@
+# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# This workflow alerts and then closes the stale issues/PRs after specific time
+# You can adjust the behavior by modifying this file.
+# For more information, see:
+# https://github.com/actions/stale
+
+name: 'Close stale issues and PRs'
+"on":
+ schedule:
+ - cron: "30 1 * * *"
+permissions:
+ contents: read
+ issues: write
+ pull-requests: write
+jobs:
+ stale:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: 'actions/stale@v7'
+ with:
+ # Comma separated list of labels that can be assigned to issues to exclude them from being marked as stale.
+ exempt-issue-labels: 'override-stale'
+ # Comma separated list of labels that can be assigned to PRs to exclude them from being marked as stale.
+ exempt-pr-labels: "override-stale"
+ # Limit the No. of API calls in one run default value is 30.
+ operations-per-run: 500
+ # Prevent to remove stale label when PRs or issues are updated.
+ remove-stale-when-updated: false
+ # comment on issue if not active for more then 7 days.
+ stale-issue-message: 'This issue has been marked stale because it has no recent activity since 7 days. It will be closed if no further activity occurs. Thank you.'
+ # comment on PR if not active for more then 14 days.
+ stale-pr-message: 'This PR has been marked stale because it has no recent activity since 14 days. It will be closed if no further activity occurs. Thank you.'
+ # comment on issue if stale for more then 7 days.
+ close-issue-message: This issue was closed due to lack of activity after being marked stale for past 7 days.
+ # comment on PR if stale for more then 14 days.
+ close-pr-message: This PR was closed due to lack of activity after being marked stale for past 14 days.
+ # Number of days of inactivity before an Issue Request becomes stale
+ days-before-issue-stale: 7
+ # Number of days of inactivity before a stale Issue is closed
+ days-before-issue-close: 7
+ # reason for closed the issue default value is not_planned
+ close-issue-reason: completed
+ # Number of days of inactivity before a stale PR is closed
+ days-before-pr-close: 14
+ # Number of days of inactivity before an PR Request becomes stale
+ days-before-pr-stale: 14
+ # Check for label to stale or close the issue/PR
+ any-of-labels: 'stat:awaiting response'
+ # override stale to stalled for PR
+ stale-pr-label: 'stale'
+ # override stale to stalled for Issue
+ stale-issue-label: "stale"
diff --git a/README.md b/README.md
index a82c88ab1..e4f5dd182 100644
--- a/README.md
+++ b/README.md
@@ -1,99 +1,121 @@
---
-layout: default
+layout: forward
+target: https://developers.google.com/mediapipe
title: Home
nav_order: 1
---
-
-
----
-**Attention:** *Thanks for your interest in MediaPipe! We have moved to
+**Attention:** *We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
-*This notice and web page will be removed on June 1, 2023.*
+
-----
+**Attention**: MediaPipe Solutions Preview is an early release. [Learn
+more](https://developers.google.com/mediapipe/solutions/about#notice).
-
-
-
+**On-device machine learning for everyone**
---------------------------------------------------------------------------------
+Delight your customers with innovative machine learning features. MediaPipe
+contains everything that you need to customize and deploy to mobile (Android,
+iOS), web, desktop, edge devices, and IoT, effortlessly.
-## Live ML anywhere
+* [See demos](https://goo.gle/mediapipe-studio)
+* [Learn more](https://developers.google.com/mediapipe/solutions)
-[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
-ML solutions for live and streaming media.
+## Get started
- | 
-:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
-***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
- | 
-***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
+You can get started with MediaPipe Solutions by by checking out any of the
+developer guides for
+[vision](https://developers.google.com/mediapipe/solutions/vision/object_detector),
+[text](https://developers.google.com/mediapipe/solutions/text/text_classifier),
+and
+[audio](https://developers.google.com/mediapipe/solutions/audio/audio_classifier)
+tasks. If you need help setting up a development environment for use with
+MediaPipe Tasks, check out the setup guides for
+[Android](https://developers.google.com/mediapipe/solutions/setup_android), [web
+apps](https://developers.google.com/mediapipe/solutions/setup_web), and
+[Python](https://developers.google.com/mediapipe/solutions/setup_python).
-----
+## Solutions
-## ML solutions in MediaPipe
+MediaPipe Solutions provides a suite of libraries and tools for you to quickly
+apply artificial intelligence (AI) and machine learning (ML) techniques in your
+applications. You can plug these solutions into your applications immediately,
+customize them to your needs, and use them across multiple development
+platforms. MediaPipe Solutions is part of the MediaPipe [open source
+project](https://github.com/google/mediapipe), so you can further customize the
+solutions code to meet your application needs.
-Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
-:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :------:
-[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/holistic)
+These libraries and resources provide the core functionality for each MediaPipe
+Solution:
-Hair Segmentation | Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
-:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
-[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
+* **MediaPipe Tasks**: Cross-platform APIs and libraries for deploying
+ solutions. [Learn
+ more](https://developers.google.com/mediapipe/solutions/tasks).
+* **MediaPipe models**: Pre-trained, ready-to-run models for use with each
+ solution.
-
-
+These tools let you customize and evaluate solutions:
-[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
-:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
-[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
-[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
-[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
-[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
-[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
-[Holistic](https://google.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ |
-[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ |
-[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | |
-[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
-[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
-[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
-[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
-[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
-[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
-[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
+* **MediaPipe Model Maker**: Customize models for solutions with your data.
+ [Learn more](https://developers.google.com/mediapipe/solutions/model_maker).
+* **MediaPipe Studio**: Visualize, evaluate, and benchmark solutions in your
+ browser. [Learn
+ more](https://developers.google.com/mediapipe/solutions/studio).
-See also
-[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
-for ML models released in MediaPipe.
+### Legacy solutions
-## Getting started
+We have ended support for [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
+as of March 1, 2023. All other MediaPipe Legacy Solutions will be upgraded to
+a new MediaPipe Solution. See the [Solutions guide](https://developers.google.com/mediapipe/solutions/guide#legacy)
+for details. The [code repository](https://github.com/google/mediapipe/tree/master/mediapipe)
+and prebuilt binaries for all MediaPipe Legacy Solutions will continue to be
+provided on an as-is basis.
-To start using MediaPipe
-[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few
-lines code, see example code and demos in
-[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and
-[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript).
+For more on the legacy solutions, see the [documentation](https://github.com/google/mediapipe/tree/master/docs/solutions).
-To use MediaPipe in C++, Android and iOS, which allow further customization of
-the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as
-building your own, learn how to
-[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and
-start building example applications in
-[C++](https://google.github.io/mediapipe/getting_started/cpp),
-[Android](https://google.github.io/mediapipe/getting_started/android) and
-[iOS](https://google.github.io/mediapipe/getting_started/ios).
+## Framework
-The source code is hosted in the
-[MediaPipe Github repository](https://github.com/google/mediapipe), and you can
-run code search using
-[Google Open Source Code Search](https://cs.opensource.google/mediapipe/mediapipe).
+To start using MediaPipe Framework, [install MediaPipe
+Framework](https://developers.google.com/mediapipe/framework/getting_started/install)
+and start building example applications in C++, Android, and iOS.
-## Publications
+[MediaPipe Framework](https://developers.google.com/mediapipe/framework) is the
+low-level component used to build efficient on-device machine learning
+pipelines, similar to the premade MediaPipe Solutions.
+
+Before using MediaPipe Framework, familiarize yourself with the following key
+[Framework
+concepts](https://developers.google.com/mediapipe/framework/framework_concepts/overview.md):
+
+* [Packets](https://developers.google.com/mediapipe/framework/framework_concepts/packets.md)
+* [Graphs](https://developers.google.com/mediapipe/framework/framework_concepts/graphs.md)
+* [Calculators](https://developers.google.com/mediapipe/framework/framework_concepts/calculators.md)
+
+## Community
+
+* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe
+ users.
+* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
+ community discussion around MediaPipe.
+* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
+ curated list of awesome MediaPipe related frameworks, libraries and
+ software.
+
+## Contributing
+
+We welcome contributions. Please follow these
+[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
+
+We use GitHub issues for tracking requests and bugs. Please post questions to
+the MediaPipe Stack Overflow with a `mediapipe` tag.
+
+## Resources
+
+### Publications
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
in Google Developers Blog
@@ -102,7 +124,8 @@ run code search using
* [SignAll SDK: Sign language interface using MediaPipe is now available for
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
in Google Developers Blog
-* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
+* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on
+ Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
in Google AI Blog
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
in Google AI Blog
@@ -130,43 +153,6 @@ run code search using
in Google AI Blog
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
-## Videos
+### Videos
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
-
-## Events
-
-* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
-* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
-* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
-* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
-* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
-* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
-* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop,
- Seoul, Korea ICCV
- 2019](https://research.google.com/youtube8m/workshop2019/index.html)
-* [AI DevWorld 2019, 10 Oct 2019, San Jose, CA](https://aidevworld.com)
-* [Google Industry Workshop at ICIP 2019, 24 Sept 2019, Taipei, Taiwan](http://2019.ieeeicip.org/?action=page4&id=14#Google)
- ([presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5))
-* [Open sourced at CVPR 2019, 17~20 June, Long Beach, CA](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe)
-
-## Community
-
-* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
- curated list of awesome MediaPipe related frameworks, libraries and software
-* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
-* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
- community discussion around MediaPipe
-
-## Alpha disclaimer
-
-MediaPipe is currently in alpha at v0.7. We may be still making breaking API
-changes and expect to get to stable APIs by v1.0.
-
-## Contributing
-
-We welcome contributions. Please follow these
-[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
-
-We use GitHub issues for tracking requests and bugs. Please post questions to
-the MediaPipe Stack Overflow with a `mediapipe` tag.
diff --git a/WORKSPACE b/WORKSPACE
index 760898185..5341b094a 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -45,12 +45,13 @@ http_archive(
)
http_archive(
- name = "rules_foreign_cc",
- strip_prefix = "rules_foreign_cc-0.1.0",
- url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
+ name = "rules_foreign_cc",
+ sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
+ strip_prefix = "rules_foreign_cc-0.9.0",
+ url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",
)
-load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
+load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
rules_foreign_cc_dependencies()
@@ -72,12 +73,9 @@ http_archive(
http_archive(
name = "zlib",
build_file = "@//third_party:zlib.BUILD",
- sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
- strip_prefix = "zlib-1.2.11",
- urls = [
- "http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
- "http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15
- ],
+ sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30",
+ strip_prefix = "zlib-1.2.13",
+ url = "http://zlib.net/fossils/zlib-1.2.13.tar.gz",
patches = [
"@//third_party:zlib.diff",
],
@@ -156,22 +154,22 @@ http_archive(
# 2020-08-21
http_archive(
name = "com_github_glog_glog",
- strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
- sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
+ strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
+ sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
urls = [
- "https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
+ "https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
],
)
http_archive(
name = "com_github_glog_glog_no_gflags",
- strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
- sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
+ strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
+ sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
build_file = "@//third_party:glog_no_gflags.BUILD",
urls = [
- "https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
+ "https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
],
patches = [
- "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff",
+ "@//third_party:com_github_glog_glog.diff",
],
patch_args = [
"-p1",
@@ -266,10 +264,10 @@ http_archive(
http_archive(
name = "com_googlesource_code_re2",
- sha256 = "e06b718c129f4019d6e7aa8b7631bee38d3d450dd980246bfaf493eb7db67868",
- strip_prefix = "re2-fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8",
+ sha256 = "ef516fb84824a597c4d5d0d6d330daedb18363b5a99eda87d027e6bdd9cba299",
+ strip_prefix = "re2-03da4fc0857c285e3a26782f6bc8931c4c950df4",
urls = [
- "https://github.com/google/re2/archive/fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8.tar.gz",
+ "https://github.com/google/re2/archive/03da4fc0857c285e3a26782f6bc8931c4c950df4.tar.gz",
],
)
@@ -375,6 +373,22 @@ http_archive(
url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
)
+# Building an opencv.xcframework from the OpenCV 4.5.3 sources is necessary for
+# MediaPipe iOS Task Libraries to be supported on arm64(M1) Macs. An
+# `opencv.xcframework` archive has not been released and it is recommended to
+# build the same from source using a script provided in OpenCV 4.5.0 upwards.
+# OpenCV is fixed to version to 4.5.3 since swift support can only be disabled
+# from 4.5.3 upwards. This is needed to avoid errors when the library is linked
+# in Xcode. Swift support will be added in when the final binary MediaPipe iOS
+# Task libraries are built.
+http_archive(
+ name = "ios_opencv_source",
+ sha256 = "a61e7a4618d353140c857f25843f39b2abe5f451b018aab1604ef0bc34cd23d5",
+ build_file = "@//third_party:opencv_ios_source.BUILD",
+ type = "zip",
+ url = "https://github.com/opencv/opencv/archive/refs/tags/4.5.3.zip",
+)
+
http_archive(
name = "stblib",
strip_prefix = "stb-b42009b3b9d4ca35bc703f5310eedc74f584be58",
@@ -468,9 +482,10 @@ http_archive(
)
# TensorFlow repo should always go after the other external dependencies.
-# TF on 2023-04-12.
-_TENSORFLOW_GIT_COMMIT = "d712c0c9e24519cc8cd3720279666720d1000eee"
-_TENSORFLOW_SHA256 = "ba98de6ea5f720071246691a1536ecd5e1b1763033e8c82a1e721a06d3dfd4c1"
+# TF on 2023-07-26.
+_TENSORFLOW_GIT_COMMIT = "e92261fd4cec0b726692081c4d2966b75abf31dd"
+# curl -L https://github.com/tensorflow/tensorflow/archive/.tar.gz | shasum -a 256
+_TENSORFLOW_SHA256 = "478a229bd4ec70a5b568ac23b5ea013d9fca46a47d6c43e30365a0412b9febf4"
http_archive(
name = "org_tensorflow",
urls = [
@@ -478,6 +493,7 @@ http_archive(
],
patches = [
"@//third_party:org_tensorflow_compatibility_fixes.diff",
+ "@//third_party:org_tensorflow_system_python.diff",
# Diff is generated with a script, don't update it manually.
"@//third_party:org_tensorflow_custom_ops.diff",
],
diff --git a/docs/build_java_api_docs.py b/docs/build_java_api_docs.py
index b13e8d1df..c30426557 100644
--- a/docs/build_java_api_docs.py
+++ b/docs/build_java_api_docs.py
@@ -1,4 +1,4 @@
-# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
# ==============================================================================
"""Generate Java reference docs for MediaPipe."""
import pathlib
+import shutil
from absl import app
from absl import flags
@@ -41,7 +42,9 @@ def main(_) -> None:
mp_root = pathlib.Path(__file__)
while (mp_root := mp_root.parent).name != 'mediapipe':
# Find the nearest `mediapipe` dir.
- pass
+ if not mp_root.name:
+ # We've hit the filesystem root - abort.
+ raise FileNotFoundError('"mediapipe" root not found')
# Find the root from which all packages are relative.
root = mp_root.parent
@@ -51,6 +54,14 @@ def main(_) -> None:
if (mp_root / 'mediapipe').exists():
mp_root = mp_root / 'mediapipe'
+ # We need to copy this into the tasks dir to ensure we don't leave broken
+ # links in the generated docs.
+ old_api_dir = 'java/com/google/mediapipe/framework/image'
+ shutil.copytree(
+ mp_root / old_api_dir,
+ mp_root / 'tasks' / old_api_dir,
+ dirs_exist_ok=True)
+
gen_java.gen_java_docs(
package='com.google.mediapipe',
source_path=mp_root / 'tasks/java',
diff --git a/docs/build_model_maker_api_docs.py b/docs/build_model_maker_api_docs.py
index 7732b7d56..377536c33 100644
--- a/docs/build_model_maker_api_docs.py
+++ b/docs/build_model_maker_api_docs.py
@@ -1,4 +1,4 @@
-# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/docs/build_py_api_docs.py b/docs/build_py_api_docs.py
index 02eb04074..10b799320 100644
--- a/docs/build_py_api_docs.py
+++ b/docs/build_py_api_docs.py
@@ -1,4 +1,4 @@
-# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/docs/getting_started/hello_world_cpp.md b/docs/getting_started/hello_world_cpp.md
index 7c8f9be3e..f0c7ff0f9 100644
--- a/docs/getting_started/hello_world_cpp.md
+++ b/docs/getting_started/hello_world_cpp.md
@@ -50,7 +50,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.*
3. The [`hello world`] example uses a simple MediaPipe graph in the
`PrintHelloWorld()` function, defined in a [`CalculatorGraphConfig`] proto.
- ```C++
+ ```c++
absl::Status PrintHelloWorld() {
// Configures a simple graph, which concatenates 2 PassThroughCalculators.
CalculatorGraphConfig config = ParseTextProtoOrDie(R"(
@@ -126,7 +126,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.*
```c++
mediapipe::Packet packet;
while (poller.Next(&packet)) {
- LOG(INFO) << packet.Get();
+ ABSL_LOG(INFO) << packet.Get();
}
```
diff --git a/docs/getting_started/hello_world_ios.md b/docs/getting_started/hello_world_ios.md
index 4be097646..118b9a05b 100644
--- a/docs/getting_started/hello_world_ios.md
+++ b/docs/getting_started/hello_world_ios.md
@@ -138,7 +138,7 @@ Create a `BUILD` file in the `$APPLICATION_PATH` and add the following build
rules:
```
-MIN_IOS_VERSION = "11.0"
+MIN_IOS_VERSION = "12.0"
load(
"@build_bazel_rules_apple//apple:ios.bzl",
diff --git a/docs/index.md b/docs/index.md
index a82c88ab1..e4f5dd182 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -1,99 +1,121 @@
---
-layout: default
+layout: forward
+target: https://developers.google.com/mediapipe
title: Home
nav_order: 1
---
-
-
----
-**Attention:** *Thanks for your interest in MediaPipe! We have moved to
+**Attention:** *We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
-*This notice and web page will be removed on June 1, 2023.*
+
-----
+**Attention**: MediaPipe Solutions Preview is an early release. [Learn
+more](https://developers.google.com/mediapipe/solutions/about#notice).
-
-
-
+**On-device machine learning for everyone**
---------------------------------------------------------------------------------
+Delight your customers with innovative machine learning features. MediaPipe
+contains everything that you need to customize and deploy to mobile (Android,
+iOS), web, desktop, edge devices, and IoT, effortlessly.
-## Live ML anywhere
+* [See demos](https://goo.gle/mediapipe-studio)
+* [Learn more](https://developers.google.com/mediapipe/solutions)
-[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
-ML solutions for live and streaming media.
+## Get started
- | 
-:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
-***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
- | 
-***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
+You can get started with MediaPipe Solutions by by checking out any of the
+developer guides for
+[vision](https://developers.google.com/mediapipe/solutions/vision/object_detector),
+[text](https://developers.google.com/mediapipe/solutions/text/text_classifier),
+and
+[audio](https://developers.google.com/mediapipe/solutions/audio/audio_classifier)
+tasks. If you need help setting up a development environment for use with
+MediaPipe Tasks, check out the setup guides for
+[Android](https://developers.google.com/mediapipe/solutions/setup_android), [web
+apps](https://developers.google.com/mediapipe/solutions/setup_web), and
+[Python](https://developers.google.com/mediapipe/solutions/setup_python).
-----
+## Solutions
-## ML solutions in MediaPipe
+MediaPipe Solutions provides a suite of libraries and tools for you to quickly
+apply artificial intelligence (AI) and machine learning (ML) techniques in your
+applications. You can plug these solutions into your applications immediately,
+customize them to your needs, and use them across multiple development
+platforms. MediaPipe Solutions is part of the MediaPipe [open source
+project](https://github.com/google/mediapipe), so you can further customize the
+solutions code to meet your application needs.
-Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
-:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :------:
-[](https://google.github.io/mediapipe/solutions/face_detection) | [](https://google.github.io/mediapipe/solutions/face_mesh) | [](https://google.github.io/mediapipe/solutions/iris) | [](https://google.github.io/mediapipe/solutions/hands) | [](https://google.github.io/mediapipe/solutions/pose) | [](https://google.github.io/mediapipe/solutions/holistic)
+These libraries and resources provide the core functionality for each MediaPipe
+Solution:
-Hair Segmentation | Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
-:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
-[](https://google.github.io/mediapipe/solutions/hair_segmentation) | [](https://google.github.io/mediapipe/solutions/object_detection) | [](https://google.github.io/mediapipe/solutions/box_tracking) | [](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [](https://google.github.io/mediapipe/solutions/objectron) | [](https://google.github.io/mediapipe/solutions/knift)
+* **MediaPipe Tasks**: Cross-platform APIs and libraries for deploying
+ solutions. [Learn
+ more](https://developers.google.com/mediapipe/solutions/tasks).
+* **MediaPipe models**: Pre-trained, ready-to-run models for use with each
+ solution.
-
-
+These tools let you customize and evaluate solutions:
-[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
-:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
-[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
-[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
-[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
-[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
-[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
-[Holistic](https://google.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ |
-[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ |
-[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | |
-[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
-[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
-[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
-[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
-[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
-[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
-[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
-[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
+* **MediaPipe Model Maker**: Customize models for solutions with your data.
+ [Learn more](https://developers.google.com/mediapipe/solutions/model_maker).
+* **MediaPipe Studio**: Visualize, evaluate, and benchmark solutions in your
+ browser. [Learn
+ more](https://developers.google.com/mediapipe/solutions/studio).
-See also
-[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
-for ML models released in MediaPipe.
+### Legacy solutions
-## Getting started
+We have ended support for [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
+as of March 1, 2023. All other MediaPipe Legacy Solutions will be upgraded to
+a new MediaPipe Solution. See the [Solutions guide](https://developers.google.com/mediapipe/solutions/guide#legacy)
+for details. The [code repository](https://github.com/google/mediapipe/tree/master/mediapipe)
+and prebuilt binaries for all MediaPipe Legacy Solutions will continue to be
+provided on an as-is basis.
-To start using MediaPipe
-[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few
-lines code, see example code and demos in
-[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and
-[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript).
+For more on the legacy solutions, see the [documentation](https://github.com/google/mediapipe/tree/master/docs/solutions).
-To use MediaPipe in C++, Android and iOS, which allow further customization of
-the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as
-building your own, learn how to
-[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and
-start building example applications in
-[C++](https://google.github.io/mediapipe/getting_started/cpp),
-[Android](https://google.github.io/mediapipe/getting_started/android) and
-[iOS](https://google.github.io/mediapipe/getting_started/ios).
+## Framework
-The source code is hosted in the
-[MediaPipe Github repository](https://github.com/google/mediapipe), and you can
-run code search using
-[Google Open Source Code Search](https://cs.opensource.google/mediapipe/mediapipe).
+To start using MediaPipe Framework, [install MediaPipe
+Framework](https://developers.google.com/mediapipe/framework/getting_started/install)
+and start building example applications in C++, Android, and iOS.
-## Publications
+[MediaPipe Framework](https://developers.google.com/mediapipe/framework) is the
+low-level component used to build efficient on-device machine learning
+pipelines, similar to the premade MediaPipe Solutions.
+
+Before using MediaPipe Framework, familiarize yourself with the following key
+[Framework
+concepts](https://developers.google.com/mediapipe/framework/framework_concepts/overview.md):
+
+* [Packets](https://developers.google.com/mediapipe/framework/framework_concepts/packets.md)
+* [Graphs](https://developers.google.com/mediapipe/framework/framework_concepts/graphs.md)
+* [Calculators](https://developers.google.com/mediapipe/framework/framework_concepts/calculators.md)
+
+## Community
+
+* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe
+ users.
+* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
+ community discussion around MediaPipe.
+* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
+ curated list of awesome MediaPipe related frameworks, libraries and
+ software.
+
+## Contributing
+
+We welcome contributions. Please follow these
+[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
+
+We use GitHub issues for tracking requests and bugs. Please post questions to
+the MediaPipe Stack Overflow with a `mediapipe` tag.
+
+## Resources
+
+### Publications
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
in Google Developers Blog
@@ -102,7 +124,8 @@ run code search using
* [SignAll SDK: Sign language interface using MediaPipe is now available for
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
in Google Developers Blog
-* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
+* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on
+ Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
in Google AI Blog
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
in Google AI Blog
@@ -130,43 +153,6 @@ run code search using
in Google AI Blog
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
-## Videos
+### Videos
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
-
-## Events
-
-* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
-* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
-* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
-* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
-* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
-* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
-* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop,
- Seoul, Korea ICCV
- 2019](https://research.google.com/youtube8m/workshop2019/index.html)
-* [AI DevWorld 2019, 10 Oct 2019, San Jose, CA](https://aidevworld.com)
-* [Google Industry Workshop at ICIP 2019, 24 Sept 2019, Taipei, Taiwan](http://2019.ieeeicip.org/?action=page4&id=14#Google)
- ([presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5))
-* [Open sourced at CVPR 2019, 17~20 June, Long Beach, CA](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe)
-
-## Community
-
-* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
- curated list of awesome MediaPipe related frameworks, libraries and software
-* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
-* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
- community discussion around MediaPipe
-
-## Alpha disclaimer
-
-MediaPipe is currently in alpha at v0.7. We may be still making breaking API
-changes and expect to get to stable APIs by v1.0.
-
-## Contributing
-
-We welcome contributions. Please follow these
-[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
-
-We use GitHub issues for tracking requests and bugs. Please post questions to
-the MediaPipe Stack Overflow with a `mediapipe` tag.
diff --git a/docs/solutions/face_detection.md b/docs/solutions/face_detection.md
index f060d062c..93f239c37 100644
--- a/docs/solutions/face_detection.md
+++ b/docs/solutions/face_detection.md
@@ -20,9 +20,9 @@ nav_order: 1
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_detector)
site.*
----
diff --git a/docs/solutions/face_mesh.md b/docs/solutions/face_mesh.md
index ab34ba401..a859bafaa 100644
--- a/docs/solutions/face_mesh.md
+++ b/docs/solutions/face_mesh.md
@@ -20,9 +20,9 @@ nav_order: 2
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
site.*
----
diff --git a/docs/solutions/iris.md b/docs/solutions/iris.md
index eab3dedf6..c0af4342c 100644
--- a/docs/solutions/iris.md
+++ b/docs/solutions/iris.md
@@ -20,9 +20,9 @@ nav_order: 3
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
site.*
----
diff --git a/docs/solutions/pose.md b/docs/solutions/pose.md
index b6f9408ec..09c313b5e 100644
--- a/docs/solutions/pose.md
+++ b/docs/solutions/pose.md
@@ -22,9 +22,9 @@ nav_order: 5
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
-[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
+[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker)
site.*
----
diff --git a/docs/solutions/pose_classification.md b/docs/solutions/pose_classification.md
index 8420e2d7c..091b0c998 100644
--- a/docs/solutions/pose_classification.md
+++ b/docs/solutions/pose_classification.md
@@ -21,7 +21,7 @@ nav_order: 1
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
-As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
+As of May 10, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
site.*
diff --git a/docs/solutions/solutions.md b/docs/solutions/solutions.md
index 7bc32d169..10551b7c9 100644
--- a/docs/solutions/solutions.md
+++ b/docs/solutions/solutions.md
@@ -1,5 +1,6 @@
---
-layout: default
+layout: forward
+target: https://developers.google.com/mediapipe/solutions/guide#legacy
title: MediaPipe Legacy Solutions
nav_order: 3
has_children: true
@@ -13,8 +14,7 @@ has_toc: false
{:toc}
---
-**Attention:** *Thank you for your interest in MediaPipe Solutions. We have
-ended support for
+**Attention:** *We have ended support for
[these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
as of March 1, 2023. All other
[MediaPipe Legacy Solutions will be upgraded](https://developers.google.com/mediapipe/solutions/guide#legacy)
@@ -25,14 +25,6 @@ be provided on an as-is basis. We encourage you to check out the new MediaPipe
Solutions at:
[https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)*
-*This notice and web page will be removed on June 1, 2023.*
-
-----
-
-
-
-
-
----
MediaPipe offers open source cross-platform, customizable ML solutions for live
diff --git a/mediapipe/BUILD b/mediapipe/BUILD
index 3187c0cf7..432ed18f6 100644
--- a/mediapipe/BUILD
+++ b/mediapipe/BUILD
@@ -14,81 +14,155 @@
licenses(["notice"]) # Apache 2.0
-# Note: yes, these need to use "//external:android/crosstool", not
-# @androidndk//:default_crosstool.
+load("@mediapipe//mediapipe:platforms.bzl", "config_setting_and_platform")
+# Generic Android
config_setting(
name = "android",
- values = {"crosstool_top": "//external:android/crosstool"},
+ constraint_values = [
+ "@platforms//os:android",
+ ],
visibility = ["//visibility:public"],
)
-config_setting(
+# Android x86 32-bit.
+config_setting_and_platform(
name = "android_x86",
- values = {
- "crosstool_top": "//external:android/crosstool",
- "cpu": "x86",
- },
+ constraint_values = [
+ "@platforms//os:android",
+ "@platforms//cpu:x86_32",
+ ],
visibility = ["//visibility:public"],
)
-config_setting(
+# Android x86 64-bit.
+config_setting_and_platform(
name = "android_x86_64",
- values = {
- "crosstool_top": "//external:android/crosstool",
- "cpu": "x86_64",
- },
+ constraint_values = [
+ "@platforms//os:android",
+ "@platforms//cpu:x86_64",
+ ],
visibility = ["//visibility:public"],
)
-config_setting(
- name = "android_armeabi",
- values = {
- "crosstool_top": "//external:android/crosstool",
- "cpu": "armeabi",
- },
- visibility = ["//visibility:public"],
-)
-
-config_setting(
+# Android ARMv7.
+config_setting_and_platform(
name = "android_arm",
- values = {
- "crosstool_top": "//external:android/crosstool",
- "cpu": "armeabi-v7a",
- },
+ constraint_values = [
+ "@platforms//os:android",
+ "@platforms//cpu:armv7",
+ ],
visibility = ["//visibility:public"],
)
-config_setting(
+# Android ARM64.
+config_setting_and_platform(
name = "android_arm64",
- values = {
- "crosstool_top": "//external:android/crosstool",
- "cpu": "arm64-v8a",
- },
+ constraint_values = [
+ "@platforms//os:android",
+ "@platforms//cpu:arm64",
+ ],
visibility = ["//visibility:public"],
)
-# Note: this cannot just match "apple_platform_type": "macos" because that option
-# defaults to "macos" even when building on Linux!
-alias(
+# Generic MacOS.
+config_setting(
name = "macos",
- actual = select({
- ":macos_i386": ":macos_i386",
- ":macos_x86_64": ":macos_x86_64",
- ":macos_arm64": ":macos_arm64",
- "//conditions:default": ":macos_i386", # Arbitrarily chosen from above.
- }),
+ constraint_values = [
+ "@platforms//os:macos",
+ ],
visibility = ["//visibility:public"],
)
-# Note: this also matches on crosstool_top so that it does not produce ambiguous
-# selectors when used together with "android".
+# MacOS x86 64-bit.
+config_setting_and_platform(
+ name = "macos_x86_64",
+ constraint_values = [
+ "@platforms//os:macos",
+ "@platforms//cpu:x86_64",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+# MacOS ARM64.
+config_setting_and_platform(
+ name = "macos_arm64",
+ constraint_values = [
+ "@platforms//os:macos",
+ "@platforms//cpu:arm64",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+# Generic iOS.
config_setting(
name = "ios",
- values = {
- "crosstool_top": "@bazel_tools//tools/cpp:toolchain",
- "apple_platform_type": "ios",
- },
+ constraint_values = [
+ "@platforms//os:ios",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+# iOS device ARM32.
+config_setting_and_platform(
+ name = "ios_armv7",
+ constraint_values = [
+ "@platforms//os:ios",
+ "@platforms//cpu:arm",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+# iOS device ARM64.
+config_setting_and_platform(
+ name = "ios_arm64",
+ constraint_values = [
+ "@platforms//os:ios",
+ "@platforms//cpu:arm64",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+# iOS device ARM64E.
+config_setting_and_platform(
+ name = "ios_arm64e",
+ constraint_values = [
+ "@platforms//os:ios",
+ "@platforms//cpu:arm64e",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+# iOS simulator x86 32-bit.
+config_setting_and_platform(
+ name = "ios_i386",
+ constraint_values = [
+ "@platforms//os:ios",
+ "@platforms//cpu:x86_32",
+ "@build_bazel_apple_support//constraints:simulator",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+# iOS simulator x86 64-bit.
+config_setting_and_platform(
+ name = "ios_x86_64",
+ constraint_values = [
+ "@platforms//os:ios",
+ "@platforms//cpu:x86_64",
+ "@build_bazel_apple_support//constraints:simulator",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+# iOS simulator ARM64.
+config_setting_and_platform(
+ name = "ios_sim_arm64",
+ constraint_values = [
+ "@platforms//os:ios",
+ "@platforms//cpu:arm64",
+ "@build_bazel_apple_support//constraints:simulator",
+ ],
visibility = ["//visibility:public"],
)
@@ -102,51 +176,24 @@ alias(
visibility = ["//visibility:public"],
)
-config_setting(
- name = "macos_i386",
- values = {
- "apple_platform_type": "macos",
- "cpu": "darwin",
- },
- visibility = ["//visibility:public"],
-)
-
-config_setting(
- name = "macos_x86_64",
- values = {
- "apple_platform_type": "macos",
- "cpu": "darwin_x86_64",
- },
- visibility = ["//visibility:public"],
-)
-
-config_setting(
- name = "macos_arm64",
- values = {
- "apple_platform_type": "macos",
- "cpu": "darwin_arm64",
- },
- visibility = ["//visibility:public"],
-)
-
-[
- config_setting(
- name = arch,
- values = {"cpu": arch},
- visibility = ["//visibility:public"],
- )
- for arch in [
- "ios_i386",
- "ios_x86_64",
- "ios_armv7",
- "ios_arm64",
- "ios_arm64e",
- ]
-]
-
-config_setting(
+# Windows 64-bit.
+config_setting_and_platform(
name = "windows",
- values = {"cpu": "x64_windows"},
+ constraint_values = [
+ "@platforms//os:windows",
+ "@platforms//cpu:x86_64",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+# Linux 64-bit.
+config_setting_and_platform(
+ name = "linux",
+ constraint_values = [
+ "@platforms//os:linux",
+ "@platforms//cpu:x86_64",
+ ],
+ visibility = ["//visibility:public"],
)
exports_files(
diff --git a/mediapipe/calculators/audio/BUILD b/mediapipe/calculators/audio/BUILD
index 4a8f0f598..c12583e5b 100644
--- a/mediapipe/calculators/audio/BUILD
+++ b/mediapipe/calculators/audio/BUILD
@@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Placeholder: load py_proto_library
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
licenses(["notice"])
@@ -145,6 +146,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_util",
+ "@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings",
"@com_google_audio_tools//audio/dsp/mfcc",
"@eigen_archive//:eigen3",
@@ -163,8 +165,9 @@ cc_library(
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:integral_types",
- "//mediapipe/framework/port:logging",
"//mediapipe/util:time_series_util",
+ "@com_google_absl//absl/log:absl_check",
+ "@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
"@com_google_audio_tools//audio/dsp:resampler",
"@com_google_audio_tools//audio/dsp:resampler_q",
@@ -185,6 +188,7 @@ cc_library(
"//mediapipe/framework/port:core_proto",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_util",
+ "@com_google_absl//absl/log:absl_check",
],
alwayslink = 1,
)
@@ -219,13 +223,12 @@ cc_library(
deps = [
":time_series_framer_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
+ "//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:time_series_header_cc_proto",
- "//mediapipe/framework/port:integral_types",
- "//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
- "//mediapipe/framework/port:status",
"//mediapipe/util:time_series_util",
+ "@com_google_absl//absl/log:absl_check",
"@com_google_audio_tools//audio/dsp:window_functions",
"@eigen_archive//:eigen3",
],
@@ -296,6 +299,7 @@ cc_test(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_test_util",
+ "@com_google_absl//absl/log:absl_log",
"@com_google_audio_tools//audio/dsp:number_util",
"@eigen_archive//:eigen3",
],
@@ -319,6 +323,21 @@ cc_test(
],
)
+cc_binary(
+ name = "time_series_framer_calculator_benchmark",
+ srcs = ["time_series_framer_calculator_benchmark.cc"],
+ deps = [
+ ":time_series_framer_calculator",
+ ":time_series_framer_calculator_cc_proto",
+ "//mediapipe/framework:calculator_framework",
+ "//mediapipe/framework:packet",
+ "//mediapipe/framework/formats:matrix",
+ "//mediapipe/framework/formats:time_series_header_cc_proto",
+ "@com_google_absl//absl/log:absl_check",
+ "@com_google_benchmark//:benchmark",
+ ],
+)
+
cc_test(
name = "time_series_framer_calculator_test",
srcs = ["time_series_framer_calculator_test.cc"],
@@ -333,6 +352,7 @@ cc_test(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_test_util",
+ "@com_google_absl//absl/log:absl_log",
"@com_google_audio_tools//audio/dsp:window_functions",
"@eigen_archive//:eigen3",
],
diff --git a/mediapipe/calculators/audio/mfcc_mel_calculators.cc b/mediapipe/calculators/audio/mfcc_mel_calculators.cc
index a63b9d6ea..ec936c844 100644
--- a/mediapipe/calculators/audio/mfcc_mel_calculators.cc
+++ b/mediapipe/calculators/audio/mfcc_mel_calculators.cc
@@ -23,6 +23,7 @@
#include
#include "Eigen/Core"
+#include "absl/log/absl_check.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "absl/strings/substitute.h"
@@ -138,7 +139,7 @@ absl::Status FramewiseTransformCalculatorBase::Process(CalculatorContext* cc) {
TransformFrame(input_frame, &output_frame);
// Copy output from vector to Eigen::Vector.
- CHECK_EQ(output_frame.size(), num_output_channels_);
+ ABSL_CHECK_EQ(output_frame.size(), num_output_channels_);
Eigen::Map output_frame_map(&output_frame[0],
output_frame.size(), 1);
output->col(frame) = output_frame_map.cast();
diff --git a/mediapipe/calculators/audio/rational_factor_resample_calculator.cc b/mediapipe/calculators/audio/rational_factor_resample_calculator.cc
index 1a4210c30..e01bf5269 100644
--- a/mediapipe/calculators/audio/rational_factor_resample_calculator.cc
+++ b/mediapipe/calculators/audio/rational_factor_resample_calculator.cc
@@ -16,6 +16,8 @@
#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
+#include "absl/log/absl_check.h"
+#include "absl/log/absl_log.h"
#include "audio/dsp/resampler_q.h"
using audio_dsp::Resampler;
@@ -45,9 +47,9 @@ void CopyVectorToChannel(const std::vector& vec, Matrix* matrix,
if (matrix->cols() == 0) {
matrix->resize(matrix->rows(), vec.size());
} else {
- CHECK_EQ(vec.size(), matrix->cols());
+ ABSL_CHECK_EQ(vec.size(), matrix->cols());
}
- CHECK_LT(channel, matrix->rows());
+ ABSL_CHECK_LT(channel, matrix->rows());
matrix->row(channel) =
Eigen::Map(vec.data(), vec.size());
}
@@ -77,7 +79,7 @@ absl::Status RationalFactorResampleCalculator::Open(CalculatorContext* cc) {
r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_,
resample_options);
if (!r) {
- LOG(ERROR) << "Failed to initialize resampler.";
+ ABSL_LOG(ERROR) << "Failed to initialize resampler.";
return absl::UnknownError("Failed to initialize resampler.");
}
}
diff --git a/mediapipe/calculators/audio/rational_factor_resample_calculator.h b/mediapipe/calculators/audio/rational_factor_resample_calculator.h
index 325886dc7..2c9df30b4 100644
--- a/mediapipe/calculators/audio/rational_factor_resample_calculator.h
+++ b/mediapipe/calculators/audio/rational_factor_resample_calculator.h
@@ -27,7 +27,6 @@
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/time_series_header.pb.h"
#include "mediapipe/framework/port/integral_types.h"
-#include "mediapipe/framework/port/logging.h"
#include "mediapipe/util/time_series_util.h"
namespace mediapipe {
diff --git a/mediapipe/calculators/audio/spectrogram_calculator.cc b/mediapipe/calculators/audio/spectrogram_calculator.cc
index 939e721ab..7f6528ec1 100644
--- a/mediapipe/calculators/audio/spectrogram_calculator.cc
+++ b/mediapipe/calculators/audio/spectrogram_calculator.cc
@@ -210,6 +210,23 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
// Factor to convert ln(SQUARED_MAGNITUDE) to deciBels = 10.0/ln(10.0).
const float SpectrogramCalculator::kLnSquaredMagnitudeToDb = 4.342944819032518;
+namespace {
+std::unique_ptr MakeWindowFun(
+ const SpectrogramCalculatorOptions::WindowType window_type) {
+ switch (window_type) {
+ // The cosine window and square root of Hann are equivalent.
+ case SpectrogramCalculatorOptions::COSINE:
+ case SpectrogramCalculatorOptions::SQRT_HANN:
+ return std::make_unique();
+ case SpectrogramCalculatorOptions::HANN:
+ return std::make_unique();
+ case SpectrogramCalculatorOptions::HAMMING:
+ return std::make_unique();
+ }
+ return nullptr;
+}
+} // namespace
+
absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
SpectrogramCalculatorOptions spectrogram_options =
cc->Options();
@@ -266,28 +283,14 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
output_scale_ = spectrogram_options.output_scale();
- std::vector window;
- switch (spectrogram_options.window_type()) {
- case SpectrogramCalculatorOptions::COSINE:
- audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
- &window);
- break;
- case SpectrogramCalculatorOptions::HANN:
- audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
- &window);
- break;
- case SpectrogramCalculatorOptions::HAMMING:
- audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
- &window);
- break;
- case SpectrogramCalculatorOptions::SQRT_HANN: {
- audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
- &window);
- absl::c_transform(window, window.begin(),
- [](double x) { return std::sqrt(x); });
- break;
- }
+ auto window_fun = MakeWindowFun(spectrogram_options.window_type());
+ if (window_fun == nullptr) {
+ return absl::Status(absl::StatusCode::kInvalidArgument,
+ absl::StrCat("Invalid window type ",
+ spectrogram_options.window_type()));
}
+ std::vector window;
+ window_fun->GetPeriodicSamples(frame_duration_samples_, &window);
// Propagate settings down to the actual Spectrogram object.
spectrogram_generators_.clear();
@@ -433,9 +436,9 @@ absl::Status SpectrogramCalculator::ProcessVectorToOutput(
absl::Status SpectrogramCalculator::ProcessVector(const Matrix& input_stream,
CalculatorContext* cc) {
switch (output_type_) {
- // These blocks deliberately ignore clang-format to preserve the
- // "silhouette" of the different cases.
- // clang-format off
+ // These blocks deliberately ignore clang-format to preserve the
+ // "silhouette" of the different cases.
+ // clang-format off
case SpectrogramCalculatorOptions::COMPLEX: {
return ProcessVectorToOutput(
input_stream,
diff --git a/mediapipe/calculators/audio/spectrogram_calculator.proto b/mediapipe/calculators/audio/spectrogram_calculator.proto
index ddfca1d1c..d8bca3f76 100644
--- a/mediapipe/calculators/audio/spectrogram_calculator.proto
+++ b/mediapipe/calculators/audio/spectrogram_calculator.proto
@@ -68,7 +68,7 @@ message SpectrogramCalculatorOptions {
HANN = 0;
HAMMING = 1;
COSINE = 2;
- SQRT_HANN = 4;
+ SQRT_HANN = 4; // Alias of COSINE.
}
optional WindowType window_type = 6 [default = HANN];
diff --git a/mediapipe/calculators/audio/spectrogram_calculator_test.cc b/mediapipe/calculators/audio/spectrogram_calculator_test.cc
index b35f30583..14cd74a3c 100644
--- a/mediapipe/calculators/audio/spectrogram_calculator_test.cc
+++ b/mediapipe/calculators/audio/spectrogram_calculator_test.cc
@@ -22,6 +22,7 @@
#include
#include "Eigen/Core"
+#include "absl/log/absl_log.h"
#include "audio/dsp/number_util.h"
#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -882,11 +883,11 @@ void BM_ProcessDC(benchmark::State& state) {
const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0);
const Matrix& output_matrix = output.packets[0].Get();
- LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x"
- << output_matrix.cols();
- LOG(INFO) << "First values=" << output_matrix(0, 0) << ", "
- << output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", "
- << output_matrix(3, 0);
+ ABSL_LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x"
+ << output_matrix.cols();
+ ABSL_LOG(INFO) << "First values=" << output_matrix(0, 0) << ", "
+ << output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", "
+ << output_matrix(3, 0);
}
BENCHMARK(BM_ProcessDC);
diff --git a/mediapipe/calculators/audio/stabilized_log_calculator.cc b/mediapipe/calculators/audio/stabilized_log_calculator.cc
index 0c697a196..a7de6a37c 100644
--- a/mediapipe/calculators/audio/stabilized_log_calculator.cc
+++ b/mediapipe/calculators/audio/stabilized_log_calculator.cc
@@ -18,6 +18,7 @@
#include
#include
+#include "absl/log/absl_check.h"
#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h"
@@ -59,7 +60,7 @@ class StabilizedLogCalculator : public CalculatorBase {
output_scale_ = stabilized_log_calculator_options.output_scale();
check_nonnegativity_ =
stabilized_log_calculator_options.check_nonnegativity();
- CHECK_GE(stabilizer_, 0.0)
+ ABSL_CHECK_GE(stabilizer_, 0.0)
<< "stabilizer must be >= 0.0, received a value of " << stabilizer_;
// If the input packets have a header, propagate the header to the output.
diff --git a/mediapipe/calculators/audio/time_series_framer_calculator.cc b/mediapipe/calculators/audio/time_series_framer_calculator.cc
index a200b898a..d8cda5149 100644
--- a/mediapipe/calculators/audio/time_series_framer_calculator.cc
+++ b/mediapipe/calculators/audio/time_series_framer_calculator.cc
@@ -15,19 +15,17 @@
// Defines TimeSeriesFramerCalculator.
#include
-#include
-#include
-#include
+#include
#include "Eigen/Core"
+#include "absl/log/absl_check.h"
#include "audio/dsp/window_functions.h"
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/time_series_header.pb.h"
-#include "mediapipe/framework/port/integral_types.h"
-#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/timestamp.h"
#include "mediapipe/util/time_series_util.h"
namespace mediapipe {
@@ -88,11 +86,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
absl::Status Close(CalculatorContext* cc) override;
private:
- // Adds input data to the internal buffer.
- void EnqueueInput(CalculatorContext* cc);
- // Constructs and emits framed output packets.
- void FrameOutput(CalculatorContext* cc);
-
Timestamp CurrentOutputTimestamp() {
if (use_local_timestamp_) {
return current_timestamp_;
@@ -106,21 +99,13 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
Timestamp::kTimestampUnitsPerSecond);
}
- // Returns the timestamp of a sample on a base, which is usually the time
- // stamp of a packet.
- Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
- int64_t number_of_samples) {
- return timestamp_base + round(number_of_samples / sample_rate_ *
- Timestamp::kTimestampUnitsPerSecond);
- }
-
// The number of input samples to advance after the current output frame is
// emitted.
int next_frame_step_samples() const {
// All numbers are in input samples.
const int64_t current_output_frame_start = static_cast(
round(cumulative_output_frames_ * average_frame_step_samples_));
- CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
+ ABSL_CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
const int64_t next_output_frame_start = static_cast(
round((cumulative_output_frames_ + 1) * average_frame_step_samples_));
return next_output_frame_start - current_output_frame_start;
@@ -142,61 +127,174 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
Timestamp initial_input_timestamp_;
// The current timestamp is updated along with the incoming packets.
Timestamp current_timestamp_;
- int num_channels_;
- // Each entry in this deque consists of a single sample, i.e. a
- // single column vector, and its timestamp.
- std::deque> sample_buffer_;
+ // Samples are buffered in a vector of sample blocks.
+ class SampleBlockBuffer {
+ public:
+ // Initializes the buffer.
+ void Init(double sample_rate, int num_channels) {
+ ts_units_per_sample_ = Timestamp::kTimestampUnitsPerSecond / sample_rate;
+ num_channels_ = num_channels;
+ num_samples_ = 0;
+ first_block_offset_ = 0;
+ }
+
+ // Number of channels, equal to the number of rows in each Matrix.
+ int num_channels() const { return num_channels_; }
+ // Total number of available samples over all blocks.
+ int num_samples() const { return num_samples_; }
+
+ // Pushes a new block of samples on the back of the buffer with `timestamp`
+ // being the input timestamp of the packet containing the Matrix.
+ void Push(const Matrix& samples, Timestamp timestamp);
+ // Copies `count` samples from the front of the buffer. If there are fewer
+ // samples than this, the result is zero padded to have `count` samples.
+ // The timestamp of the last copied sample is written to *last_timestamp.
+ // This output is used below to update `current_timestamp_`, which is only
+ // used when `use_local_timestamp` is true.
+ Matrix CopySamples(int count, Timestamp* last_timestamp) const;
+ // Drops `count` samples from the front of the buffer. If `count` exceeds
+ // `num_samples()`, the buffer is emptied. Returns how many samples were
+ // dropped.
+ int DropSamples(int count);
+
+ private:
+ struct Block {
+ // Matrix of num_channels rows by num_samples columns, a block of possibly
+ // multiple samples.
+ Matrix samples;
+ // Timestamp of the first sample in the Block. This comes from the input
+ // packet's timestamp that contains this Matrix.
+ Timestamp timestamp;
+
+ Block() : timestamp(Timestamp::Unstarted()) {}
+ Block(const Matrix& samples, Timestamp timestamp)
+ : samples(samples), timestamp(timestamp) {}
+ int num_samples() const { return samples.cols(); }
+ };
+ std::vector blocks_;
+ // Number of timestamp units per sample. Used to compute timestamps as
+ // nth sample timestamp = base_timestamp + round(ts_units_per_sample_ * n).
+ double ts_units_per_sample_;
+ // Number of rows in each Matrix.
+ int num_channels_;
+ // The total number of samples over all blocks, equal to
+ // (sum_i blocks_[i].num_samples()) - first_block_offset_.
+ int num_samples_;
+ // The number of samples in the first block that have been discarded. This
+ // way we can cheaply represent "partially discarding" a block.
+ int first_block_offset_;
+ } sample_buffer_;
bool use_window_;
- Matrix window_;
+ Eigen::RowVectorXf window_;
bool use_local_timestamp_;
};
REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
-void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
- const Matrix& input_frame = cc->Inputs().Index(0).Get();
-
- for (int i = 0; i < input_frame.cols(); ++i) {
- sample_buffer_.emplace_back(std::make_pair(
- input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
- }
+void TimeSeriesFramerCalculator::SampleBlockBuffer::Push(const Matrix& samples,
+ Timestamp timestamp) {
+ num_samples_ += samples.cols();
+ blocks_.emplace_back(samples, timestamp);
}
-void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
- while (sample_buffer_.size() >=
+Matrix TimeSeriesFramerCalculator::SampleBlockBuffer::CopySamples(
+ int count, Timestamp* last_timestamp) const {
+ Matrix copied(num_channels_, count);
+
+ if (!blocks_.empty()) {
+ int num_copied = 0;
+ // First block has an offset for samples that have been discarded.
+ int offset = first_block_offset_;
+ int n;
+ Timestamp last_block_ts;
+ int last_sample_index;
+
+ for (auto it = blocks_.begin(); it != blocks_.end() && count > 0; ++it) {
+ n = std::min(it->num_samples() - offset, count);
+ // Copy `n` samples from the next block.
+ copied.middleCols(num_copied, n) = it->samples.middleCols(offset, n);
+ count -= n;
+ num_copied += n;
+ last_block_ts = it->timestamp;
+ last_sample_index = offset + n - 1;
+ offset = 0; // No samples have been discarded in subsequent blocks.
+ }
+
+ // Compute the timestamp of the last copied sample.
+ *last_timestamp =
+ last_block_ts + std::round(ts_units_per_sample_ * last_sample_index);
+ }
+
+ if (count > 0) {
+ copied.rightCols(count).setZero(); // Zero pad if needed.
+ }
+
+ return copied;
+}
+
+int TimeSeriesFramerCalculator::SampleBlockBuffer::DropSamples(int count) {
+ if (blocks_.empty()) {
+ return 0;
+ }
+
+ auto block_it = blocks_.begin();
+ if (first_block_offset_ + count < block_it->num_samples()) {
+ // `count` is less than the remaining samples in the first block.
+ first_block_offset_ += count;
+ num_samples_ -= count;
+ return count;
+ }
+
+ int num_samples_dropped = block_it->num_samples() - first_block_offset_;
+ count -= num_samples_dropped;
+ first_block_offset_ = 0;
+
+ for (++block_it; block_it != blocks_.end(); ++block_it) {
+ if (block_it->num_samples() > count) {
+ break;
+ }
+ num_samples_dropped += block_it->num_samples();
+ count -= block_it->num_samples();
+ }
+
+ blocks_.erase(blocks_.begin(), block_it); // Drop whole blocks.
+ if (!blocks_.empty()) {
+ first_block_offset_ = count; // Drop part of the next block.
+ num_samples_dropped += count;
+ }
+
+ num_samples_ -= num_samples_dropped;
+ return num_samples_dropped;
+}
+
+absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
+ if (initial_input_timestamp_ == Timestamp::Unstarted()) {
+ initial_input_timestamp_ = cc->InputTimestamp();
+ current_timestamp_ = initial_input_timestamp_;
+ }
+
+ // Add input data to the internal buffer.
+ sample_buffer_.Push(cc->Inputs().Index(0).Get(),
+ cc->InputTimestamp());
+
+ // Construct and emit framed output packets.
+ while (sample_buffer_.num_samples() >=
frame_duration_samples_ + samples_still_to_drop_) {
- while (samples_still_to_drop_ > 0) {
- sample_buffer_.pop_front();
- --samples_still_to_drop_;
- }
+ sample_buffer_.DropSamples(samples_still_to_drop_);
+ Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
+ ¤t_timestamp_);
const int frame_step_samples = next_frame_step_samples();
- std::unique_ptr output_frame(
- new Matrix(num_channels_, frame_duration_samples_));
- for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
- ++i) {
- output_frame->col(i) = sample_buffer_.front().first;
- current_timestamp_ = sample_buffer_.front().second;
- sample_buffer_.pop_front();
- }
- const int frame_overlap_samples =
- frame_duration_samples_ - frame_step_samples;
- if (frame_overlap_samples > 0) {
- for (int i = 0; i < frame_overlap_samples; ++i) {
- output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
- current_timestamp_ = sample_buffer_[i].second;
- }
- } else {
- samples_still_to_drop_ = -frame_overlap_samples;
- }
+ samples_still_to_drop_ = frame_step_samples;
if (use_window_) {
- *output_frame = (output_frame->array() * window_.array()).matrix();
+ // Apply the window to each row of output_frame.
+ output_frame.array().rowwise() *= window_.array();
}
- cc->Outputs().Index(0).Add(output_frame.release(),
- CurrentOutputTimestamp());
+ cc->Outputs().Index(0).AddPacket(MakePacket(std::move(output_frame))
+ .At(CurrentOutputTimestamp()));
++cumulative_output_frames_;
cumulative_completed_samples_ += frame_step_samples;
}
@@ -206,35 +304,18 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
// fact to enable packet queueing optimizations.
cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp());
}
-}
-
-absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
- if (initial_input_timestamp_ == Timestamp::Unstarted()) {
- initial_input_timestamp_ = cc->InputTimestamp();
- current_timestamp_ = initial_input_timestamp_;
- }
-
- EnqueueInput(cc);
- FrameOutput(cc);
return absl::OkStatus();
}
absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
- while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
- sample_buffer_.pop_front();
- --samples_still_to_drop_;
- }
- if (!sample_buffer_.empty() && pad_final_packet_) {
- std::unique_ptr output_frame(new Matrix);
- output_frame->setZero(num_channels_, frame_duration_samples_);
- for (int i = 0; i < sample_buffer_.size(); ++i) {
- output_frame->col(i) = sample_buffer_[i].first;
- current_timestamp_ = sample_buffer_[i].second;
- }
+ sample_buffer_.DropSamples(samples_still_to_drop_);
- cc->Outputs().Index(0).Add(output_frame.release(),
- CurrentOutputTimestamp());
+ if (sample_buffer_.num_samples() > 0 && pad_final_packet_) {
+ Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
+ ¤t_timestamp_);
+ cc->Outputs().Index(0).AddPacket(MakePacket(std::move(output_frame))
+ .At(CurrentOutputTimestamp()));
}
return absl::OkStatus();
@@ -258,7 +339,7 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
cc->Inputs().Index(0).Header(), &input_header));
sample_rate_ = input_header.sample_rate();
- num_channels_ = input_header.num_channels();
+ sample_buffer_.Init(sample_rate_, input_header.num_channels());
frame_duration_samples_ = time_series_util::SecondsToSamples(
framer_options.frame_duration_seconds(), sample_rate_);
RET_CHECK_GT(frame_duration_samples_, 0)
@@ -312,9 +393,8 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
}
if (use_window_) {
- window_ = Matrix::Ones(num_channels_, 1) *
- Eigen::Map(window_vector.data(), 1,
- frame_duration_samples_)
+ window_ = Eigen::Map(window_vector.data(),
+ frame_duration_samples_)
.cast();
}
use_local_timestamp_ = framer_options.use_local_timestamp();
diff --git a/mediapipe/calculators/audio/time_series_framer_calculator_benchmark.cc b/mediapipe/calculators/audio/time_series_framer_calculator_benchmark.cc
new file mode 100644
index 000000000..6eada1ad3
--- /dev/null
+++ b/mediapipe/calculators/audio/time_series_framer_calculator_benchmark.cc
@@ -0,0 +1,93 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Benchmark for TimeSeriesFramerCalculator.
+#include
+#include
+#include
+
+#include "absl/log/absl_check.h"
+#include "benchmark/benchmark.h"
+#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/matrix.h"
+#include "mediapipe/framework/formats/time_series_header.pb.h"
+#include "mediapipe/framework/packet.h"
+
+using ::mediapipe::Matrix;
+
+void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
+ constexpr float kSampleRate = 32000.0;
+ constexpr int kNumChannels = 2;
+ constexpr int kFrameDurationSeconds = 5.0;
+ std::mt19937 rng(0 /*seed*/);
+ // Input around a half second's worth of samples at a time.
+ std::uniform_int_distribution input_size_dist(15000, 17000);
+ // Generate a pool of random blocks of samples up front.
+ std::vector sample_pool;
+ sample_pool.reserve(20);
+ for (int i = 0; i < 20; ++i) {
+ sample_pool.push_back(Matrix::Random(kNumChannels, input_size_dist(rng)));
+ }
+ std::uniform_int_distribution pool_index_dist(0, sample_pool.size() - 1);
+
+ mediapipe::CalculatorGraphConfig config;
+ config.add_input_stream("input");
+ config.add_output_stream("output");
+ auto* node = config.add_node();
+ node->set_calculator("TimeSeriesFramerCalculator");
+ node->add_input_stream("input");
+ node->add_output_stream("output");
+ mediapipe::TimeSeriesFramerCalculatorOptions* options =
+ node->mutable_options()->MutableExtension(
+ mediapipe::TimeSeriesFramerCalculatorOptions::ext);
+ options->set_frame_duration_seconds(kFrameDurationSeconds);
+
+ for (auto _ : state) {
+ state.PauseTiming(); // Pause benchmark timing.
+
+ // Prepare input packets of random blocks of samples.
+ std::vector input_packets;
+ input_packets.reserve(32);
+ float t = 0;
+ for (int i = 0; i < 32; ++i) {
+ auto samples =
+ std::make_unique(sample_pool[pool_index_dist(rng)]);
+ const int num_samples = samples->cols();
+ input_packets.push_back(mediapipe::Adopt(samples.release())
+ .At(mediapipe::Timestamp::FromSeconds(t)));
+ t += num_samples / kSampleRate;
+ }
+ // Initialize graph.
+ mediapipe::CalculatorGraph graph;
+ ABSL_CHECK_OK(graph.Initialize(config));
+ // Prepare input header.
+ auto header = std::make_unique();
+ header->set_sample_rate(kSampleRate);
+ header->set_num_channels(kNumChannels);
+
+ state.ResumeTiming(); // Resume benchmark timing.
+
+ ABSL_CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}}));
+ for (auto& packet : input_packets) {
+ ABSL_CHECK_OK(graph.AddPacketToInputStream("input", packet));
+ }
+ ABSL_CHECK(!graph.HasError());
+ ABSL_CHECK_OK(graph.CloseAllInputStreams());
+ ABSL_CHECK_OK(graph.WaitUntilIdle());
+ }
+}
+BENCHMARK(BM_TimeSeriesFramerCalculator);
+
+BENCHMARK_MAIN();
diff --git a/mediapipe/calculators/audio/time_series_framer_calculator_test.cc b/mediapipe/calculators/audio/time_series_framer_calculator_test.cc
index 72e9c88f7..fe42ecb12 100644
--- a/mediapipe/calculators/audio/time_series_framer_calculator_test.cc
+++ b/mediapipe/calculators/audio/time_series_framer_calculator_test.cc
@@ -19,6 +19,7 @@
#include
#include "Eigen/Core"
+#include "absl/log/absl_log.h"
#include "audio/dsp/window_functions.h"
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -186,11 +187,12 @@ class TimeSeriesFramerCalculatorTest
const int num_unique_output_samples =
round((output().packets.size() - 1) * frame_step_samples) +
frame_duration_samples;
- LOG(INFO) << "packets.size()=" << output().packets.size()
- << " frame_duration_samples=" << frame_duration_samples
- << " frame_step_samples=" << frame_step_samples
- << " num_input_samples_=" << num_input_samples_
- << " num_unique_output_samples=" << num_unique_output_samples;
+ ABSL_LOG(INFO) << "packets.size()=" << output().packets.size()
+ << " frame_duration_samples=" << frame_duration_samples
+ << " frame_step_samples=" << frame_step_samples
+ << " num_input_samples_=" << num_input_samples_
+ << " num_unique_output_samples="
+ << num_unique_output_samples;
const int num_padding_samples =
num_unique_output_samples - num_input_samples_;
if (options_.pad_final_packet()) {
diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD
index d5ba6c74f..02efc84ea 100644
--- a/mediapipe/calculators/core/BUILD
+++ b/mediapipe/calculators/core/BUILD
@@ -117,6 +117,7 @@ mediapipe_proto_library(
"//mediapipe/framework:calculator_proto",
"//mediapipe/framework/formats:classification_proto",
"//mediapipe/framework/formats:landmark_proto",
+ "//mediapipe/framework/formats:matrix_data_proto",
"//mediapipe/framework/formats:time_series_header_proto",
],
)
@@ -192,17 +193,19 @@ cc_library(
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_contract",
"//mediapipe/framework:calculator_framework",
- "//mediapipe/framework:collection_item_id",
"//mediapipe/framework:packet",
"//mediapipe/framework/formats:detection_cc_proto",
+ "//mediapipe/framework/formats:image",
+ "//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
- "//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
+ "//mediapipe/gpu:gpu_buffer",
"@com_google_absl//absl/memory",
+ "@com_google_absl//absl/status",
],
alwayslink = 1,
)
@@ -215,18 +218,18 @@ cc_library(
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_contract",
"//mediapipe/framework:calculator_framework",
- "//mediapipe/framework:collection_item_id",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:image",
+ "//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
- "//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
- "//mediapipe/framework/port:status",
+ "//mediapipe/gpu:gpu_buffer",
"//mediapipe/util:render_data_cc_proto",
+ "@com_google_absl//absl/status",
"@org_tensorflow//tensorflow/lite:framework",
],
alwayslink = 1,
@@ -287,6 +290,7 @@ cc_library(
"//mediapipe/framework/api2:node",
"//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:classification_cc_proto",
+ "//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:integral_types",
@@ -295,8 +299,7 @@ cc_library(
"//mediapipe/util:render_data_cc_proto",
"@org_tensorflow//tensorflow/lite:framework",
] + select({
- "//mediapipe/gpu:disable_gpu": [],
- "//mediapipe:ios": [],
+ ":ios_or_disable_gpu": [],
"//conditions:default": [
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
],
@@ -378,17 +381,6 @@ cc_library(
alwayslink = 1,
)
-cc_library(
- name = "clip_detection_vector_size_calculator",
- srcs = ["clip_detection_vector_size_calculator.cc"],
- deps = [
- ":clip_vector_size_calculator",
- "//mediapipe/framework:calculator_framework",
- "//mediapipe/framework/formats:detection_cc_proto",
- ],
- alwayslink = 1,
-)
-
cc_test(
name = "clip_vector_size_calculator_test",
srcs = ["clip_vector_size_calculator_test.cc"],
@@ -590,6 +582,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util",
+ "@com_google_absl//absl/log:absl_check",
],
alwayslink = 1,
)
@@ -605,6 +598,7 @@ cc_test(
"//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
+ "@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings",
],
)
@@ -637,6 +631,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
+ "@com_google_absl//absl/log:absl_log",
],
alwayslink = 1,
)
@@ -784,10 +779,11 @@ cc_library(
"//mediapipe/framework/deps:random",
"//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:integral_types",
- "//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util",
+ "@com_google_absl//absl/log:absl_check",
+ "@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
],
alwayslink = 1,
@@ -843,6 +839,7 @@ cc_test(
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/tool:validate_type",
+ "@com_google_absl//absl/log:absl_check",
"@eigen_archive//:eigen3",
],
)
@@ -904,6 +901,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
+ "//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
@@ -1029,6 +1027,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/port:status",
+ "@com_google_absl//absl/log:absl_log",
],
alwayslink = 1,
)
@@ -1067,6 +1066,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
+ "@com_google_absl//absl/log:absl_log",
],
)
@@ -1113,6 +1113,7 @@ cc_library(
"//mediapipe/framework/api2:node",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
+ "@com_google_absl//absl/log:absl_log",
],
alwayslink = 1,
)
@@ -1136,6 +1137,7 @@ cc_library(
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
+ "//mediapipe/framework/api2:node",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
@@ -1164,6 +1166,7 @@ cc_library(
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
+ "//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
@@ -1238,6 +1241,7 @@ cc_library(
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
+ "//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
diff --git a/mediapipe/calculators/core/begin_loop_calculator.cc b/mediapipe/calculators/core/begin_loop_calculator.cc
index 441c66937..d030bbbde 100644
--- a/mediapipe/calculators/core/begin_loop_calculator.cc
+++ b/mediapipe/calculators/core/begin_loop_calculator.cc
@@ -17,10 +17,13 @@
#include
#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/gpu/gpu_buffer.h"
namespace mediapipe {
@@ -60,4 +63,22 @@ REGISTER_CALCULATOR(BeginLoopUint64tCalculator);
typedef BeginLoopCalculator> BeginLoopTensorCalculator;
REGISTER_CALCULATOR(BeginLoopTensorCalculator);
+// A calculator to process std::vector.
+typedef BeginLoopCalculator>
+ BeginLoopImageFrameCalculator;
+REGISTER_CALCULATOR(BeginLoopImageFrameCalculator);
+
+// A calculator to process std::vector.
+typedef BeginLoopCalculator>
+ BeginLoopGpuBufferCalculator;
+REGISTER_CALCULATOR(BeginLoopGpuBufferCalculator);
+
+// A calculator to process std::vector.
+typedef BeginLoopCalculator> BeginLoopImageCalculator;
+REGISTER_CALCULATOR(BeginLoopImageCalculator);
+
+// A calculator to process std::vector.
+typedef BeginLoopCalculator> BeginLoopFloatCalculator;
+REGISTER_CALCULATOR(BeginLoopFloatCalculator);
+
} // namespace mediapipe
diff --git a/mediapipe/calculators/core/begin_loop_calculator.h b/mediapipe/calculators/core/begin_loop_calculator.h
index 81fff39da..c0b3022d4 100644
--- a/mediapipe/calculators/core/begin_loop_calculator.h
+++ b/mediapipe/calculators/core/begin_loop_calculator.h
@@ -15,47 +15,57 @@
#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
+#include "absl/status/status.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/packet.h"
-#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
-#include "mediapipe/framework/port/status_macros.h"
namespace mediapipe {
// Calculator for implementing loops on iterable collections inside a MediaPipe
-// graph.
+// graph. Assume InputIterT is an iterable for type InputT, and OutputIterT is
+// an iterable for type OutputT, e.g. vector and vector.
+// First, instantiate specializations in the loop calculators' implementations
+// if missing:
+// BeginLoopInputTCalculator = BeginLoopCalculator
+// EndLoopOutputTCalculator = EndLoopCalculator
+// Then, the following graph transforms an item of type InputIterT to an
+// OutputIterT by applying InputToOutputConverter to every element:
//
-// It is designed to be used like:
-//
-// node {
-// calculator: "BeginLoopWithIterableCalculator"
-// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
-// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
-// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
+// node { # Type @timestamp
+// calculator: "BeginLoopInputTCalculator"
+// input_stream: "ITERABLE:input_iterable" # InputIterT @iterable_ts
+// input_stream: "CLONE:extra_input" # ExtraT @extra_ts
+// output_stream: "ITEM:input_iterator" # InputT @loop_internal_ts
+// output_stream: "CLONE:cloned_extra_input" # ExtraT @loop_internal_ts
+// output_stream: "BATCH_END:iterable_ts" # Timestamp @loop_internal_ts
// }
//
// node {
-// calculator: "ElementToBlaConverterSubgraph"
-// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
-// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
+// calculator: "InputToOutputConverter"
+// input_stream: "INPUT:input_iterator" # InputT @loop_internal_ts
+// input_stream: "EXTRA:cloned_extra_input" # ExtraT @loop_internal_ts
+// output_stream: "OUTPUT:output_iterator" # OutputT @loop_internal_ts
// }
//
// node {
-// calculator: "EndLoopWithOutputCalculator"
-// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
-// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
-// output_stream: "ITERABLE:aggregated_result" # IterableU @ext_ts
+// calculator: "EndLoopOutputTCalculator"
+// input_stream: "ITEM:output_iterator" # OutputT @loop_internal_ts
+// input_stream: "BATCH_END:iterable_ts" # Timestamp @loop_internal_ts
+// output_stream: "ITERABLE:output_iterable" # OutputIterT @iterable_ts
// }
//
+// The resulting 'output_iterable' has the same timestamp as 'input_iterable'.
+// The output packets of this calculator are part of the loop body and have
+// loop-internal timestamps that are unrelated to the input iterator timestamp.
+//
// Input streams tagged with "CLONE" are cloned to the corresponding output
-// streams at loop timestamps. This ensures that a MediaPipe graph or sub-graph
-// can run multiple times, once per element in the "ITERABLE" for each pakcet
-// clone of the packets in the "CLONE" input streams.
+// streams at loop-internal timestamps. This ensures that a MediaPipe graph or
+// sub-graph can run multiple times, once per element in the "ITERABLE" for each
+// packet clone of the packets in the "CLONE" input streams. Think of CLONEd
+// inputs as loop-wide constants.
template
class BeginLoopCalculator : public CalculatorBase {
using ItemT = typename IterableT::value_type;
diff --git a/mediapipe/calculators/core/concatenate_vector_calculator.cc b/mediapipe/calculators/core/concatenate_vector_calculator.cc
index 0079aa98d..53b3debf1 100644
--- a/mediapipe/calculators/core/concatenate_vector_calculator.cc
+++ b/mediapipe/calculators/core/concatenate_vector_calculator.cc
@@ -17,6 +17,7 @@
#include
#include "mediapipe/framework/formats/classification.pb.h"
+#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/integral_types.h"
@@ -55,6 +56,10 @@ MEDIAPIPE_REGISTER_NODE(ConcatenateUInt64VectorCalculator);
typedef ConcatenateVectorCalculator ConcatenateBoolVectorCalculator;
MEDIAPIPE_REGISTER_NODE(ConcatenateBoolVectorCalculator);
+typedef ConcatenateVectorCalculator
+ ConcatenateStringVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(ConcatenateStringVectorCalculator);
+
// Example config:
// node {
// calculator: "ConcatenateTfLiteTensorVectorCalculator"
@@ -100,4 +105,7 @@ typedef ConcatenateVectorCalculator
ConcatenateRenderDataVectorCalculator;
MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator);
+typedef ConcatenateVectorCalculator
+ ConcatenateImageVectorCalculator;
+MEDIAPIPE_REGISTER_NODE(ConcatenateImageVectorCalculator);
} // namespace mediapipe
diff --git a/mediapipe/calculators/core/concatenate_vector_calculator_test.cc b/mediapipe/calculators/core/concatenate_vector_calculator_test.cc
index 5510b98a3..3fccf58fd 100644
--- a/mediapipe/calculators/core/concatenate_vector_calculator_test.cc
+++ b/mediapipe/calculators/core/concatenate_vector_calculator_test.cc
@@ -30,13 +30,15 @@ namespace mediapipe {
typedef ConcatenateVectorCalculator TestConcatenateIntVectorCalculator;
MEDIAPIPE_REGISTER_NODE(TestConcatenateIntVectorCalculator);
-void AddInputVector(int index, const std::vector& input, int64_t timestamp,
+template
+void AddInputVector(int index, const std::vector& input, int64_t timestamp,
CalculatorRunner* runner) {
runner->MutableInputs()->Index(index).packets.push_back(
- MakePacket>(input).At(Timestamp(timestamp)));
+ MakePacket>(input).At(Timestamp(timestamp)));
}
-void AddInputVectors(const std::vector>& inputs,
+template
+void AddInputVectors(const std::vector>& inputs,
int64_t timestamp, CalculatorRunner* runner) {
for (int i = 0; i < inputs.size(); ++i) {
AddInputVector(i, inputs[i], timestamp, runner);
@@ -382,6 +384,23 @@ TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) {
EXPECT_EQ(0, outputs.size());
}
+TEST(ConcatenateStringVectorCalculatorTest, OneTimestamp) {
+ CalculatorRunner runner("ConcatenateStringVectorCalculator",
+ /*options_string=*/"", /*num_inputs=*/3,
+ /*num_outputs=*/1, /*num_side_packets=*/0);
+
+ std::vector> inputs = {
+ {"a", "b"}, {"c"}, {"d", "e", "f"}};
+ AddInputVectors(inputs, /*timestamp=*/1, &runner);
+ MP_ASSERT_OK(runner.Run());
+
+ const std::vector& outputs = runner.Outputs().Index(0).packets;
+ EXPECT_EQ(1, outputs.size());
+ EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+ std::vector expected_vector = {"a", "b", "c", "d", "e", "f"};
+ EXPECT_EQ(expected_vector, outputs[0].Get>());
+}
+
typedef ConcatenateVectorCalculator>
TestConcatenateUniqueIntPtrCalculator;
MEDIAPIPE_REGISTER_NODE(TestConcatenateUniqueIntPtrCalculator);
diff --git a/mediapipe/calculators/core/constant_side_packet_calculator.cc b/mediapipe/calculators/core/constant_side_packet_calculator.cc
index 509f7e9dd..8762c9874 100644
--- a/mediapipe/calculators/core/constant_side_packet_calculator.cc
+++ b/mediapipe/calculators/core/constant_side_packet_calculator.cc
@@ -19,6 +19,7 @@
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix_data.pb.h"
#include "mediapipe/framework/formats/time_series_header.pb.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/integral_types.h"
@@ -85,8 +86,12 @@ class ConstantSidePacketCalculator : public CalculatorBase {
packet.Set();
} else if (packet_options.has_double_value()) {
packet.Set();
+ } else if (packet_options.has_matrix_data_value()) {
+ packet.Set();
} else if (packet_options.has_time_series_header_value()) {
packet.Set();
+ } else if (packet_options.has_int64_value()) {
+ packet.Set();
} else {
return absl::InvalidArgumentError(
"None of supported values were specified in options.");
@@ -121,9 +126,13 @@ class ConstantSidePacketCalculator : public CalculatorBase {
MakePacket(packet_options.landmark_list_value()));
} else if (packet_options.has_double_value()) {
packet.Set(MakePacket(packet_options.double_value()));
+ } else if (packet_options.has_matrix_data_value()) {
+ packet.Set(MakePacket(packet_options.matrix_data_value()));
} else if (packet_options.has_time_series_header_value()) {
packet.Set(MakePacket(
packet_options.time_series_header_value()));
+ } else if (packet_options.has_int64_value()) {
+ packet.Set(MakePacket(packet_options.int64_value()));
} else {
return absl::InvalidArgumentError(
"None of supported values were specified in options.");
diff --git a/mediapipe/calculators/core/constant_side_packet_calculator.proto b/mediapipe/calculators/core/constant_side_packet_calculator.proto
index 78a773a6c..0d53175fc 100644
--- a/mediapipe/calculators/core/constant_side_packet_calculator.proto
+++ b/mediapipe/calculators/core/constant_side_packet_calculator.proto
@@ -19,6 +19,7 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
import "mediapipe/framework/formats/classification.proto";
import "mediapipe/framework/formats/landmark.proto";
+import "mediapipe/framework/formats/matrix_data.proto";
import "mediapipe/framework/formats/time_series_header.proto";
message ConstantSidePacketCalculatorOptions {
@@ -29,14 +30,16 @@ message ConstantSidePacketCalculatorOptions {
message ConstantSidePacket {
oneof value {
int32 int_value = 1;
+ uint64 uint64_value = 5;
+ int64 int64_value = 11;
float float_value = 2;
+ double double_value = 9;
bool bool_value = 3;
string string_value = 4;
- uint64 uint64_value = 5;
ClassificationList classification_list_value = 6;
LandmarkList landmark_list_value = 7;
- double double_value = 9;
TimeSeriesHeader time_series_header_value = 10;
+ MatrixData matrix_data_value = 12;
}
}
diff --git a/mediapipe/calculators/core/constant_side_packet_calculator_test.cc b/mediapipe/calculators/core/constant_side_packet_calculator_test.cc
index a7ff808f4..6e8c0ec33 100644
--- a/mediapipe/calculators/core/constant_side_packet_calculator_test.cc
+++ b/mediapipe/calculators/core/constant_side_packet_calculator_test.cc
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include
#include
#include "absl/strings/string_view.h"
@@ -58,6 +59,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
DoTestSingleSidePacket("{ bool_value: true }", true);
DoTestSingleSidePacket(R"({ string_value: "str" })", "str");
+ DoTestSingleSidePacket("{ int64_value: 63 }", 63);
}
TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {
diff --git a/mediapipe/calculators/core/end_loop_calculator.cc b/mediapipe/calculators/core/end_loop_calculator.cc
index b3b889ecd..94f7ee22e 100644
--- a/mediapipe/calculators/core/end_loop_calculator.cc
+++ b/mediapipe/calculators/core/end_loop_calculator.cc
@@ -14,15 +14,19 @@
#include "mediapipe/calculators/core/end_loop_calculator.h"
+#include
+#include
#include
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/image.h"
+#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/formats/tensor.h"
+#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/util/render_data.pb.h"
#include "tensorflow/lite/interpreter.h"
@@ -68,8 +72,22 @@ REGISTER_CALCULATOR(EndLoopMatrixCalculator);
typedef EndLoopCalculator> EndLoopTensorCalculator;
REGISTER_CALCULATOR(EndLoopTensorCalculator);
+typedef EndLoopCalculator> EndLoopImageFrameCalculator;
+REGISTER_CALCULATOR(EndLoopImageFrameCalculator);
+
+typedef EndLoopCalculator> EndLoopGpuBufferCalculator;
+REGISTER_CALCULATOR(EndLoopGpuBufferCalculator);
+
typedef EndLoopCalculator>
EndLoopImageCalculator;
REGISTER_CALCULATOR(EndLoopImageCalculator);
+typedef EndLoopCalculator>>
+ EndLoopAffineMatrixCalculator;
+REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator);
+
+typedef EndLoopCalculator>>
+ EndLoopImageSizeCalculator;
+REGISTER_CALCULATOR(EndLoopImageSizeCalculator);
+
} // namespace mediapipe
diff --git a/mediapipe/calculators/core/end_loop_calculator.h b/mediapipe/calculators/core/end_loop_calculator.h
index 2598194e6..1e258f046 100644
--- a/mediapipe/calculators/core/end_loop_calculator.h
+++ b/mediapipe/calculators/core/end_loop_calculator.h
@@ -17,13 +17,11 @@
#include
+#include "absl/status/status.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/collection_item_id.h"
-#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/ret_check.h"
-#include "mediapipe/framework/port/status.h"
namespace mediapipe {
@@ -33,27 +31,7 @@ namespace mediapipe {
// from the "BATCH_END" tagged input stream, it emits the aggregated results
// at the original timestamp contained in the "BATCH_END" input stream.
//
-// It is designed to be used like:
-//
-// node {
-// calculator: "BeginLoopWithIterableCalculator"
-// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
-// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
-// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
-// }
-//
-// node {
-// calculator: "ElementToBlaConverterSubgraph"
-// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
-// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
-// }
-//
-// node {
-// calculator: "EndLoopWithOutputCalculator"
-// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
-// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
-// output_stream: "ITERABLE:aggregated_result" # IterableU @ext_ts
-// }
+// See BeginLoopCalculator for a usage example.
template
class EndLoopCalculator : public CalculatorBase {
using ItemT = typename IterableT::value_type;
@@ -79,7 +57,7 @@ class EndLoopCalculator : public CalculatorBase {
}
// Try to consume the item and move it into the collection. If the items
// are not consumable, then try to copy them instead. If the items are
- // not copiable, then an error will be returned.
+ // not copyable, then an error will be returned.
auto item_ptr_or = cc->Inputs().Tag("ITEM").Value().Consume();
if (item_ptr_or.ok()) {
input_stream_collection_->push_back(std::move(*item_ptr_or.value()));
diff --git a/mediapipe/calculators/core/flow_limiter_calculator.cc b/mediapipe/calculators/core/flow_limiter_calculator.cc
index 5b08f3af5..46e5bf6a3 100644
--- a/mediapipe/calculators/core/flow_limiter_calculator.cc
+++ b/mediapipe/calculators/core/flow_limiter_calculator.cc
@@ -42,7 +42,7 @@ constexpr char kOptionsTag[] = "OPTIONS";
//
// Increasing `max_in_flight` to 2 or more can yield the better throughput
// when the graph exhibits a high degree of pipeline parallelism. Decreasing
-// `max_in_flight` to 0 can yield a better average latency, but at the cost of
+// `max_in_queue` to 0 can yield a better average latency, but at the cost of
// lower throughput (lower framerate) due to the time during which the graph
// is idle awaiting the next input frame.
//
diff --git a/mediapipe/calculators/core/gate_calculator.cc b/mediapipe/calculators/core/gate_calculator.cc
index 448329b88..e5e87b69b 100644
--- a/mediapipe/calculators/core/gate_calculator.cc
+++ b/mediapipe/calculators/core/gate_calculator.cc
@@ -26,19 +26,15 @@ constexpr char kStateChangeTag[] = "STATE_CHANGE";
constexpr char kDisallowTag[] = "DISALLOW";
constexpr char kAllowTag[] = "ALLOW";
-enum GateState {
- GATE_UNINITIALIZED,
- GATE_ALLOW,
- GATE_DISALLOW,
-};
-
-std::string ToString(GateState state) {
+std::string ToString(GateCalculatorOptions::GateState state) {
switch (state) {
- case GATE_UNINITIALIZED:
+ case GateCalculatorOptions::UNSPECIFIED:
+ return "UNSPECIFIED";
+ case GateCalculatorOptions::GATE_UNINITIALIZED:
return "UNINITIALIZED";
- case GATE_ALLOW:
+ case GateCalculatorOptions::GATE_ALLOW:
return "ALLOW";
- case GATE_DISALLOW:
+ case GateCalculatorOptions::GATE_DISALLOW:
return "DISALLOW";
}
DLOG(FATAL) << "Unknown GateState";
@@ -153,10 +149,12 @@ class GateCalculator : public CalculatorBase {
cc->SetOffset(TimestampDiff(0));
num_data_streams_ = cc->Inputs().NumEntries("");
- last_gate_state_ = GATE_UNINITIALIZED;
- RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
+ last_gate_state_ = options.initial_gate_state();
+
+ RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
+
empty_packets_as_allow_ = options.empty_packets_as_allow();
if (!use_side_packet_for_allow_disallow_ &&
@@ -184,10 +182,12 @@ class GateCalculator : public CalculatorBase {
allow = !cc->Inputs().Tag(kDisallowTag).Get();
}
}
- const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
+ const GateCalculatorOptions::GateState new_gate_state =
+ allow ? GateCalculatorOptions::GATE_ALLOW
+ : GateCalculatorOptions::GATE_DISALLOW;
if (cc->Outputs().HasTag(kStateChangeTag)) {
- if (last_gate_state_ != GATE_UNINITIALIZED &&
+ if (last_gate_state_ != GateCalculatorOptions::GATE_UNINITIALIZED &&
last_gate_state_ != new_gate_state) {
VLOG(2) << "State transition in " << cc->NodeName() << " @ "
<< cc->InputTimestamp().Value() << " from "
@@ -223,7 +223,8 @@ class GateCalculator : public CalculatorBase {
}
private:
- GateState last_gate_state_ = GATE_UNINITIALIZED;
+ GateCalculatorOptions::GateState last_gate_state_ =
+ GateCalculatorOptions::GATE_UNINITIALIZED;
int num_data_streams_;
bool empty_packets_as_allow_;
bool use_side_packet_for_allow_disallow_ = false;
diff --git a/mediapipe/calculators/core/gate_calculator.proto b/mediapipe/calculators/core/gate_calculator.proto
index b7d597a63..4153d5f32 100644
--- a/mediapipe/calculators/core/gate_calculator.proto
+++ b/mediapipe/calculators/core/gate_calculator.proto
@@ -31,4 +31,13 @@ message GateCalculatorOptions {
// Whether to allow or disallow the input streams to pass when no
// ALLOW/DISALLOW input or side input is specified.
optional bool allow = 2 [default = false];
+
+ enum GateState {
+ UNSPECIFIED = 0;
+ GATE_UNINITIALIZED = 1;
+ GATE_ALLOW = 2;
+ GATE_DISALLOW = 3;
+ }
+
+ optional GateState initial_gate_state = 3 [default = GATE_UNINITIALIZED];
}
diff --git a/mediapipe/calculators/core/gate_calculator_test.cc b/mediapipe/calculators/core/gate_calculator_test.cc
index 192019820..0c49f1449 100644
--- a/mediapipe/calculators/core/gate_calculator_test.cc
+++ b/mediapipe/calculators/core/gate_calculator_test.cc
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "absl/log/absl_log.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/port/gtest.h"
@@ -356,18 +357,18 @@ TEST_F(GateCalculatorTest, AllowWithStateChangeNoDataStreams) {
RunTimeStepWithoutDataStream(kTimestampValue2, "ALLOW", true);
constexpr int64_t kTimestampValue3 = 45;
RunTimeStepWithoutDataStream(kTimestampValue3, "ALLOW", false);
- LOG(INFO) << "a";
+ ABSL_LOG(INFO) << "a";
const std::vector& output =
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
- LOG(INFO) << "s";
+ ABSL_LOG(INFO) << "s";
ASSERT_EQ(2, output.size());
- LOG(INFO) << "d";
+ ABSL_LOG(INFO) << "d";
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
- LOG(INFO) << "f";
+ ABSL_LOG(INFO) << "f";
EXPECT_EQ(true, output[0].Get()); // Allow.
EXPECT_EQ(false, output[1].Get()); // Disallow.
- LOG(INFO) << "g";
+ ABSL_LOG(INFO) << "g";
}
TEST_F(GateCalculatorTest, DisallowWithStateChange) {
@@ -458,5 +459,29 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
ASSERT_EQ(0, output.size());
}
+// Must detect allow value for first timestamp as a state change when the
+// initial state is set to GATE_DISALLOW.
+TEST_F(GateCalculatorTest, StateChangeTriggeredWithInitialGateStateOption) {
+ SetRunner(R"(
+ calculator: "GateCalculator"
+ input_stream: "test_input"
+ input_stream: "ALLOW:allow"
+ output_stream: "test_output"
+ output_stream: "STATE_CHANGE:state_change"
+ options: {
+ [mediapipe.GateCalculatorOptions.ext] {
+ initial_gate_state: GATE_DISALLOW
+ }
+ }
+ )");
+
+ constexpr int64_t kTimestampValue0 = 42;
+ RunTimeStep(kTimestampValue0, "ALLOW", true);
+
+ const std::vector& output =
+ runner()->Outputs().Get("STATE_CHANGE", 0).packets;
+ ASSERT_EQ(1, output.size());
+}
+
} // namespace
} // namespace mediapipe
diff --git a/mediapipe/calculators/core/get_vector_item_calculator.cc b/mediapipe/calculators/core/get_vector_item_calculator.cc
index 3306e4ff3..5dbda6d99 100644
--- a/mediapipe/calculators/core/get_vector_item_calculator.cc
+++ b/mediapipe/calculators/core/get_vector_item_calculator.cc
@@ -17,6 +17,7 @@
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
namespace mediapipe {
namespace api2 {
@@ -37,5 +38,12 @@ using GetDetectionVectorItemCalculator =
GetVectorItemCalculator;
REGISTER_CALCULATOR(GetDetectionVectorItemCalculator);
+using GetNormalizedRectVectorItemCalculator =
+ GetVectorItemCalculator;
+REGISTER_CALCULATOR(GetNormalizedRectVectorItemCalculator);
+
+using GetRectVectorItemCalculator = GetVectorItemCalculator;
+REGISTER_CALCULATOR(GetRectVectorItemCalculator);
+
} // namespace api2
} // namespace mediapipe
diff --git a/mediapipe/calculators/core/immediate_mux_calculator.cc b/mediapipe/calculators/core/immediate_mux_calculator.cc
index 0e51cda5e..05de05e40 100644
--- a/mediapipe/calculators/core/immediate_mux_calculator.cc
+++ b/mediapipe/calculators/core/immediate_mux_calculator.cc
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "absl/log/absl_log.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
@@ -78,7 +79,7 @@ absl::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) {
if (packet.Timestamp() >= cc->Outputs().Index(0).NextTimestampBound()) {
cc->Outputs().Index(0).AddPacket(packet);
} else {
- LOG_FIRST_N(WARNING, 5)
+ ABSL_LOG_FIRST_N(WARNING, 5)
<< "Dropping a packet with timestamp " << packet.Timestamp();
}
if (cc->Outputs().NumEntries() >= 2) {
diff --git a/mediapipe/calculators/core/matrix_multiply_calculator_test.cc b/mediapipe/calculators/core/matrix_multiply_calculator_test.cc
index e62ca8073..60976577a 100644
--- a/mediapipe/calculators/core/matrix_multiply_calculator_test.cc
+++ b/mediapipe/calculators/core/matrix_multiply_calculator_test.cc
@@ -16,6 +16,7 @@
#include
#include "Eigen/Core"
+#include "absl/log/absl_check.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/matrix.h"
@@ -209,7 +210,7 @@ TEST(MatrixMultiplyCalculatorTest, Multiply) {
MatrixFromTextProto(kSamplesText, &samples);
Matrix expected;
MatrixFromTextProto(kExpectedText, &expected);
- CHECK_EQ(samples.cols(), expected.cols());
+ ABSL_CHECK_EQ(samples.cols(), expected.cols());
for (int i = 0; i < samples.cols(); ++i) {
// Take a column from samples and produce a packet with just that
diff --git a/mediapipe/calculators/core/matrix_to_vector_calculator_test.cc b/mediapipe/calculators/core/matrix_to_vector_calculator_test.cc
index 1f994cbed..8b4254cbc 100644
--- a/mediapipe/calculators/core/matrix_to_vector_calculator_test.cc
+++ b/mediapipe/calculators/core/matrix_to_vector_calculator_test.cc
@@ -35,7 +35,7 @@ class MatrixToVectorCalculatorTest
void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; }
void AppendInput(const std::vector& column_major_data,
- int64 timestamp) {
+ int64_t timestamp) {
ASSERT_EQ(num_input_samples_ * num_input_channels_,
column_major_data.size());
Eigen::Map data_map(&column_major_data[0],
diff --git a/mediapipe/calculators/core/merge_calculator.cc b/mediapipe/calculators/core/merge_calculator.cc
index a283842ae..43fc3b878 100644
--- a/mediapipe/calculators/core/merge_calculator.cc
+++ b/mediapipe/calculators/core/merge_calculator.cc
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "absl/log/absl_log.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h"
@@ -53,7 +54,7 @@ class MergeCalculator : public Node {
static absl::Status UpdateContract(CalculatorContract* cc) {
RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream";
if (kIn(cc).Count() == 1) {
- LOG(WARNING)
+ ABSL_LOG(WARNING)
<< "MergeCalculator expects multiple input streams to merge but is "
"receiving only one. Make sure the calculator is configured "
"correctly or consider removing this calculator to reduce "
@@ -72,8 +73,8 @@ class MergeCalculator : public Node {
}
}
- LOG(WARNING) << "Empty input packets at timestamp "
- << cc->InputTimestamp().Value();
+ ABSL_LOG(WARNING) << "Empty input packets at timestamp "
+ << cc->InputTimestamp().Value();
return absl::OkStatus();
}
diff --git a/mediapipe/calculators/core/merge_to_vector_calculator.cc b/mediapipe/calculators/core/merge_to_vector_calculator.cc
index fd053ed2b..4bb3c8a40 100644
--- a/mediapipe/calculators/core/merge_to_vector_calculator.cc
+++ b/mediapipe/calculators/core/merge_to_vector_calculator.cc
@@ -1,4 +1,4 @@
-/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+/* Copyright 2022 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
diff --git a/mediapipe/calculators/core/merge_to_vector_calculator.h b/mediapipe/calculators/core/merge_to_vector_calculator.h
index b4f7a37c2..4ec674c05 100644
--- a/mediapipe/calculators/core/merge_to_vector_calculator.h
+++ b/mediapipe/calculators/core/merge_to_vector_calculator.h
@@ -1,4 +1,4 @@
-/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+/* Copyright 2022 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
diff --git a/mediapipe/calculators/core/packet_resampler_calculator.cc b/mediapipe/calculators/core/packet_resampler_calculator.cc
index 60ec40537..81a68f03f 100644
--- a/mediapipe/calculators/core/packet_resampler_calculator.cc
+++ b/mediapipe/calculators/core/packet_resampler_calculator.cc
@@ -16,6 +16,9 @@
#include
+#include "absl/log/absl_check.h"
+#include "absl/log/absl_log.h"
+
namespace {
// Reflect an integer against the lower and upper bound of an interval.
int64_t ReflectBetween(int64_t ts, int64_t ts_min, int64_t ts_max) {
@@ -177,7 +180,7 @@ PacketResamplerCalculator::GetSamplingStrategy(
const PacketResamplerCalculatorOptions& options) {
if (options.reproducible_sampling()) {
if (!options.jitter_with_reflection()) {
- LOG(WARNING)
+ ABSL_LOG(WARNING)
<< "reproducible_sampling enabled w/ jitter_with_reflection "
"disabled. "
<< "reproducible_sampling always uses jitter with reflection, "
@@ -200,15 +203,15 @@ PacketResamplerCalculator::GetSamplingStrategy(
Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(
int64_t index) const {
- CHECK_EQ(jitter_, 0.0);
- CHECK_NE(first_timestamp_, Timestamp::Unset());
+ ABSL_CHECK_EQ(jitter_, 0.0);
+ ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset());
return first_timestamp_ + TimestampDiffFromSeconds(index / frame_rate_);
}
int64_t PacketResamplerCalculator::TimestampToPeriodIndex(
Timestamp timestamp) const {
- CHECK_EQ(jitter_, 0.0);
- CHECK_NE(first_timestamp_, Timestamp::Unset());
+ ABSL_CHECK_EQ(jitter_, 0.0);
+ ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset());
return MathUtil::SafeRound(
(timestamp - first_timestamp_).Seconds() * frame_rate_);
}
@@ -229,13 +232,15 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) {
- LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
- "the actual value.";
+ ABSL_LOG(WARNING)
+ << "VideoHeader::frame_rate holds the target value and not "
+ "the actual value.";
}
if (calculator_->flush_last_packet_) {
- LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
- "ignored, because we are adding jitter.";
+ ABSL_LOG(WARNING)
+ << "PacketResamplerCalculatorOptions.flush_last_packet is "
+ "ignored, because we are adding jitter.";
}
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get();
@@ -254,7 +259,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
}
absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
if (!packet_reservoir_->IsEmpty()) {
- LOG(INFO) << "Emitting pack from reservoir.";
+ ABSL_LOG(INFO) << "Emitting pack from reservoir.";
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
}
return absl::OkStatus();
@@ -285,7 +290,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Process(
if (calculator_->frame_time_usec_ <
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
- LOG_FIRST_N(WARNING, 2)
+ ABSL_LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling.";
}
@@ -340,8 +345,8 @@ void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
next_output_timestamp_ = Timestamp(ReflectBetween(
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
next_output_timestamp_max_.Value()));
- CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
- CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
+ ABSL_CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
+ ABSL_CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
}
absl::Status ReproducibleJitterWithReflectionStrategy::Open(
@@ -352,13 +357,15 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Open(
if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) {
- LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
- "the actual value.";
+ ABSL_LOG(WARNING)
+ << "VideoHeader::frame_rate holds the target value and not "
+ "the actual value.";
}
if (calculator_->flush_last_packet_) {
- LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
- "ignored, because we are adding jitter.";
+ ABSL_LOG(WARNING)
+ << "PacketResamplerCalculatorOptions.flush_last_packet is "
+ "ignored, because we are adding jitter.";
}
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get();
@@ -411,7 +418,7 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Process(
// Note, if the stream is upsampling, this could lead to the same packet
// being emitted twice. Upsampling and jitter doesn't make much sense
// but does technically work.
- LOG_FIRST_N(WARNING, 2)
+ ABSL_LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling.";
}
@@ -499,13 +506,15 @@ absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) {
- LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
- "the actual value.";
+ ABSL_LOG(WARNING)
+ << "VideoHeader::frame_rate holds the target value and not "
+ "the actual value.";
}
if (calculator_->flush_last_packet_) {
- LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
- "ignored, because we are adding jitter.";
+ ABSL_LOG(WARNING)
+ << "PacketResamplerCalculatorOptions.flush_last_packet is "
+ "ignored, because we are adding jitter.";
}
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get();
@@ -555,7 +564,7 @@ absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
if (calculator_->frame_time_usec_ <
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
- LOG_FIRST_N(WARNING, 2)
+ ABSL_LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling.";
}
diff --git a/mediapipe/calculators/core/packet_resampler_calculator.h b/mediapipe/calculators/core/packet_resampler_calculator.h
index fbecdb0e7..f26dc2ca4 100644
--- a/mediapipe/calculators/core/packet_resampler_calculator.h
+++ b/mediapipe/calculators/core/packet_resampler_calculator.h
@@ -13,7 +13,6 @@
#include "mediapipe/framework/deps/random_base.h"
#include "mediapipe/framework/formats/video_stream_header.h"
#include "mediapipe/framework/port/integral_types.h"
-#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
diff --git a/mediapipe/calculators/core/packet_resampler_calculator_test.cc b/mediapipe/calculators/core/packet_resampler_calculator_test.cc
index f02da0d18..d80793da4 100644
--- a/mediapipe/calculators/core/packet_resampler_calculator_test.cc
+++ b/mediapipe/calculators/core/packet_resampler_calculator_test.cc
@@ -51,9 +51,9 @@ class SimpleRunner : public CalculatorRunner {
virtual ~SimpleRunner() {}
- void SetInput(const std::vector& timestamp_list) {
+ void SetInput(const std::vector& timestamp_list) {
MutableInputs()->Index(0).packets.clear();
- for (const int64 ts : timestamp_list) {
+ for (const int64_t ts : timestamp_list) {
MutableInputs()->Index(0).packets.push_back(
Adopt(new std::string(absl::StrCat("Frame #", ts)))
.At(Timestamp(ts)));
@@ -72,8 +72,8 @@ class SimpleRunner : public CalculatorRunner {
}
void CheckOutputTimestamps(
- const std::vector& expected_frames,
- const std::vector& expected_timestamps) const {
+ const std::vector& expected_frames,
+ const std::vector& expected_timestamps) const {
EXPECT_EQ(expected_frames.size(), Outputs().Index(0).packets.size());
EXPECT_EQ(expected_timestamps.size(), Outputs().Index(0).packets.size());
int count = 0;
@@ -112,7 +112,7 @@ MATCHER_P2(PacketAtTimestamp, payload, timestamp,
*result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
return false;
}
- int64 actual_payload = arg.template Get();
+ int64_t actual_payload = arg.template Get();
if (actual_payload != payload) {
*result_listener << "with incorrect payload = " << actual_payload;
return false;
@@ -137,18 +137,18 @@ class ReproducibleJitterWithReflectionStrategyForTesting
//
// An EXPECT will fail if sequence is less than the number requested during
// processing.
- static std::vector random_sequence;
+ static std::vector random_sequence;
protected:
- virtual uint64 GetNextRandom(uint64 n) {
+ virtual uint64_t GetNextRandom(uint64_t n) {
EXPECT_LT(sequence_index_, random_sequence.size());
return random_sequence[sequence_index_++] % n;
}
private:
- int32 sequence_index_ = 0;
+ int32_t sequence_index_ = 0;
};
-std::vector
+std::vector
ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
// PacketResamplerCalculator child class which injects a specified stream
@@ -469,7 +469,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
}
)pb"));
- for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
+ for (const int64_t ts : {0, 5000, 10010, 15001, 19990}) {
runner.MutableInputs()->Tag(kDataTag).packets.push_back(
Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts)));
}
diff --git a/mediapipe/calculators/core/packet_thinner_calculator.cc b/mediapipe/calculators/core/packet_thinner_calculator.cc
index 35cd966ea..0bc5cc16d 100644
--- a/mediapipe/calculators/core/packet_thinner_calculator.cc
+++ b/mediapipe/calculators/core/packet_thinner_calculator.cc
@@ -17,6 +17,7 @@
#include // for ceil
#include
+#include "absl/log/absl_check.h"
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -160,8 +161,8 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
thinner_type_ = options.thinner_type();
// This check enables us to assume only two thinner types exist in Process()
- CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
- thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
+ ABSL_CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
+ thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
<< "Unsupported thinner type.";
if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
@@ -177,7 +178,8 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
} else {
period_ = TimestampDiff(options.period());
}
- CHECK_LT(TimestampDiff(0), period_) << "Specified period must be positive.";
+ ABSL_CHECK_LT(TimestampDiff(0), period_)
+ << "Specified period must be positive.";
if (options.has_start_time()) {
start_time_ = Timestamp(options.start_time());
@@ -189,7 +191,7 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
end_time_ =
options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max();
- CHECK_LT(start_time_, end_time_)
+ ABSL_CHECK_LT(start_time_, end_time_)
<< "Invalid PacketThinner: start_time must be earlier than end_time";
sync_output_timestamps_ = options.sync_output_timestamps();
@@ -232,7 +234,7 @@ absl::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
// Emit any saved packets before quitting.
if (!saved_packet_.IsEmpty()) {
// Only sync thinner should have saved packets.
- CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
+ ABSL_CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
if (sync_output_timestamps_) {
cc->Outputs().Index(0).AddPacket(
saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp())));
@@ -269,7 +271,7 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess(
const Timestamp saved_sync = NearestSyncTimestamp(saved);
const Timestamp now = cc->InputTimestamp();
const Timestamp now_sync = NearestSyncTimestamp(now);
- CHECK_LE(saved_sync, now_sync);
+ ABSL_CHECK_LE(saved_sync, now_sync);
if (saved_sync == now_sync) {
// Saved Packet is in same interval as current packet.
// Replace saved packet with current if it is at least as
@@ -295,7 +297,7 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess(
}
Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
- CHECK_NE(start_time_, Timestamp::Unset())
+ ABSL_CHECK_NE(start_time_, Timestamp::Unset())
<< "Method only valid for sync thinner calculator.";
// Computation is done using int64 arithmetic. No easy way to avoid
@@ -303,12 +305,12 @@ Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
const int64_t now64 = now.Value();
const int64_t start64 = start_time_.Value();
const int64_t period64 = period_.Value();
- CHECK_LE(0, period64);
+ ABSL_CHECK_LE(0, period64);
// Round now64 to its closest interval (units of period64).
int64_t sync64 =
(now64 - start64 + period64 / 2) / period64 * period64 + start64;
- CHECK_LE(abs(now64 - sync64), period64 / 2)
+ ABSL_CHECK_LE(abs(now64 - sync64), period64 / 2)
<< "start64: " << start64 << "; now64: " << now64
<< "; sync64: " << sync64;
diff --git a/mediapipe/calculators/core/packet_thinner_calculator_test.cc b/mediapipe/calculators/core/packet_thinner_calculator_test.cc
index 09de0ca70..69c008395 100644
--- a/mediapipe/calculators/core/packet_thinner_calculator_test.cc
+++ b/mediapipe/calculators/core/packet_thinner_calculator_test.cc
@@ -16,6 +16,7 @@
#include
#include
+#include "absl/log/absl_check.h"
#include "absl/strings/str_cat.h"
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -70,7 +71,7 @@ class SimpleRunner : public CalculatorRunner {
}
double GetFrameRate() const {
- CHECK(!Outputs().Index(0).header.IsEmpty());
+ ABSL_CHECK(!Outputs().Index(0).header.IsEmpty());
return Outputs().Index(0).header.Get().frame_rate;
}
};
diff --git a/mediapipe/calculators/core/previous_loopback_calculator.cc b/mediapipe/calculators/core/previous_loopback_calculator.cc
index d67e6c061..36ee0f2d7 100644
--- a/mediapipe/calculators/core/previous_loopback_calculator.cc
+++ b/mediapipe/calculators/core/previous_loopback_calculator.cc
@@ -123,7 +123,10 @@ class PreviousLoopbackCalculator : public Node {
// However, LOOP packet is empty.
kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1);
} else {
- kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
+ // Avoids sending leftovers to a stream that's already closed.
+ if (!kPrevLoop(cc).IsClosed()) {
+ kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
+ }
}
loop_packets_.pop_front();
main_packet_specs_.pop_front();
diff --git a/mediapipe/calculators/core/previous_loopback_calculator_test.cc b/mediapipe/calculators/core/previous_loopback_calculator_test.cc
index 563417669..d8c358909 100644
--- a/mediapipe/calculators/core/previous_loopback_calculator_test.cc
+++ b/mediapipe/calculators/core/previous_loopback_calculator_test.cc
@@ -43,8 +43,8 @@ constexpr char kDisallowTag[] = "DISALLOW";
// Returns the timestamp values for a vector of Packets.
// TODO: puth this kind of test util in a common place.
-std::vector TimestampValues(const std::vector& packets) {
- std::vector result;
+std::vector TimestampValues(const std::vector& packets) {
+ std::vector result;
for (const Packet& packet : packets) {
result.push_back(packet.Timestamp().Value());
}
@@ -371,7 +371,7 @@ TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
for (int main_ts = 0; main_ts < 50; ++main_ts) {
send_packet("in", main_ts);
MP_EXPECT_OK(graph_.WaitUntilIdle());
- std::vector ts_values = TimestampValues(outputs);
+ std::vector ts_values = TimestampValues(outputs);
EXPECT_EQ(ts_values.size(), main_ts + 1);
for (int j = 0; j < main_ts + 1; ++j) {
EXPECT_EQ(ts_values[j], j);
diff --git a/mediapipe/calculators/core/sequence_shift_calculator.cc b/mediapipe/calculators/core/sequence_shift_calculator.cc
index 026048b79..5b2a73fd3 100644
--- a/mediapipe/calculators/core/sequence_shift_calculator.cc
+++ b/mediapipe/calculators/core/sequence_shift_calculator.cc
@@ -14,6 +14,7 @@
#include
+#include "absl/log/absl_log.h"
#include "mediapipe/calculators/core/sequence_shift_calculator.pb.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -101,7 +102,7 @@ void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) {
kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp()));
packet_cache_.pop_front();
} else if (emit_empty_packets_before_first_packet_) {
- LOG(FATAL) << "Not supported yet";
+ ABSL_LOG(FATAL) << "Not supported yet";
}
// Store current packet for later output.
packet_cache_.push_back(kIn(cc).packet());
diff --git a/mediapipe/calculators/core/side_packet_to_stream_calculator.cc b/mediapipe/calculators/core/side_packet_to_stream_calculator.cc
index ed89889df..311f7d815 100644
--- a/mediapipe/calculators/core/side_packet_to_stream_calculator.cc
+++ b/mediapipe/calculators/core/side_packet_to_stream_calculator.cc
@@ -121,7 +121,7 @@ absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
if (cc->Outputs().HasTag(kTagAtTimestamp)) {
RET_CHECK_EQ(num_entries + 1, cc->InputSidePackets().NumEntries())
<< "For AT_TIMESTAMP tag, 2 input side packets are required.";
- cc->InputSidePackets().Tag(kTagSideInputTimestamp).Set();
+ cc->InputSidePackets().Tag(kTagSideInputTimestamp).Set();
} else {
RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries())
<< "Same number of input side packets and output streams is required.";
@@ -178,8 +178,8 @@ absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
}
} else if (cc->Outputs().HasTag(kTagAtTimestamp)) {
- int64 timestamp =
- cc->InputSidePackets().Tag(kTagSideInputTimestamp).Get();
+ int64_t timestamp =
+ cc->InputSidePackets().Tag(kTagSideInputTimestamp).Get();
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
cc->Outputs()
.Get(output_tag_, i)
diff --git a/mediapipe/calculators/core/split_vector_calculator.cc b/mediapipe/calculators/core/split_vector_calculator.cc
index b76722de9..67fc38ce9 100644
--- a/mediapipe/calculators/core/split_vector_calculator.cc
+++ b/mediapipe/calculators/core/split_vector_calculator.cc
@@ -18,6 +18,7 @@
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h"
@@ -86,4 +87,12 @@ REGISTER_CALCULATOR(SplitUint64tVectorCalculator);
typedef SplitVectorCalculator SplitFloatVectorCalculator;
REGISTER_CALCULATOR(SplitFloatVectorCalculator);
+typedef SplitVectorCalculator
+ SplitImageVectorCalculator;
+REGISTER_CALCULATOR(SplitImageVectorCalculator);
+
+typedef SplitVectorCalculator, false>
+ SplitAffineMatrixVectorCalculator;
+REGISTER_CALCULATOR(SplitAffineMatrixVectorCalculator);
+
} // namespace mediapipe
diff --git a/mediapipe/calculators/core/stream_to_side_packet_calculator.cc b/mediapipe/calculators/core/stream_to_side_packet_calculator.cc
index 9dc25142a..72e812255 100644
--- a/mediapipe/calculators/core/stream_to_side_packet_calculator.cc
+++ b/mediapipe/calculators/core/stream_to_side_packet_calculator.cc
@@ -12,11 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/timestamp.h"
namespace mediapipe {
+namespace api2 {
// A calculator that takes a packet of an input stream and converts it to an
// output side packet. This calculator only works under the assumption that the
@@ -28,21 +30,21 @@ namespace mediapipe {
// input_stream: "stream"
// output_side_packet: "side_packet"
// }
-class StreamToSidePacketCalculator : public mediapipe::CalculatorBase {
+class StreamToSidePacketCalculator : public Node {
public:
- static absl::Status GetContract(mediapipe::CalculatorContract* cc) {
- cc->Inputs().Index(0).SetAny();
- cc->OutputSidePackets().Index(0).SetAny();
- return absl::OkStatus();
- }
+ static constexpr Input::Optional kIn{""};
+ static constexpr SideOutput> kOut{""};
+
+ MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
absl::Status Process(mediapipe::CalculatorContext* cc) override {
- mediapipe::Packet& packet = cc->Inputs().Index(0).Value();
- cc->OutputSidePackets().Index(0).Set(
- packet.At(mediapipe::Timestamp::Unset()));
+ kOut(cc).Set(
+ kIn(cc).packet().As().At(mediapipe::Timestamp::Unset()));
return absl::OkStatus();
}
};
-REGISTER_CALCULATOR(StreamToSidePacketCalculator);
+MEDIAPIPE_REGISTER_NODE(StreamToSidePacketCalculator);
+
+} // namespace api2
} // namespace mediapipe
diff --git a/mediapipe/calculators/image/BUILD b/mediapipe/calculators/image/BUILD
index 6de54189f..259012d64 100644
--- a/mediapipe/calculators/image/BUILD
+++ b/mediapipe/calculators/image/BUILD
@@ -97,6 +97,7 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:source_location",
"//mediapipe/framework/port:status",
+ "@com_google_absl//absl/log:absl_check",
],
alwayslink = 1,
)
@@ -125,6 +126,7 @@ cc_library(
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status",
+ "@com_google_absl//absl/log:absl_check",
],
alwayslink = 1,
)
@@ -135,7 +137,6 @@ cc_library(
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame_opencv",
- "//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status",
],
@@ -152,11 +153,11 @@ cc_library(
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
- "//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
+ "@com_google_absl//absl/log:absl_log",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
@@ -203,6 +204,7 @@ cc_library(
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
+ "@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings",
] + select({
"//mediapipe/gpu:disable_gpu": [],
@@ -262,9 +264,12 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:scale_mode_cc_proto",
+ "@com_google_absl//absl/status",
+ "@com_google_absl//absl/strings",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
+ "//mediapipe/gpu:gl_base_hdr",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:gl_simple_shaders",
@@ -274,6 +279,36 @@ cc_library(
alwayslink = 1,
)
+cc_test(
+ name = "image_transformation_calculator_test",
+ srcs = ["image_transformation_calculator_test.cc"],
+ data = ["//mediapipe/calculators/image/testdata:test_images"],
+ tags = [
+ "desktop_only_test",
+ ],
+ deps = [
+ ":image_transformation_calculator",
+ "//mediapipe/framework:calculator_cc_proto",
+ "//mediapipe/framework:calculator_framework",
+ "//mediapipe/framework:calculator_runner",
+ "//mediapipe/framework/deps:file_path",
+ "//mediapipe/framework/formats:image_format_cc_proto",
+ "//mediapipe/framework/formats:image_frame",
+ "//mediapipe/framework/formats:image_frame_opencv",
+ "//mediapipe/framework/port:gtest",
+ "//mediapipe/framework/port:opencv_imgcodecs",
+ "//mediapipe/framework/port:opencv_imgproc",
+ "//mediapipe/framework/port:parse_text_proto",
+ "//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
+ "//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
+ "//third_party:opencv",
+ "@com_google_absl//absl/container:flat_hash_set",
+ "@com_google_absl//absl/flags:flag",
+ "@com_google_absl//absl/strings",
+ "@com_google_googletest//:gtest_main",
+ ],
+)
+
cc_library(
name = "image_cropping_calculator",
srcs = ["image_cropping_calculator.cc"],
@@ -301,6 +336,7 @@ cc_library(
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
+ "@com_google_absl//absl/log:absl_log",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
@@ -317,6 +353,7 @@ cc_library(
cc_test(
name = "image_cropping_calculator_test",
srcs = ["image_cropping_calculator_test.cc"],
+ tags = ["not_run:arm"],
deps = [
":image_cropping_calculator",
":image_cropping_calculator_cc_proto",
@@ -396,6 +433,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
+ "@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings",
],
)
@@ -420,6 +458,8 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/util:image_frame_util",
+ "@com_google_absl//absl/log:absl_check",
+ "@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
"@libyuv",
],
@@ -625,9 +665,9 @@ cc_library(
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
- "//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
+ "@com_google_absl//absl/log:absl_log",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
@@ -650,6 +690,7 @@ cc_library(
cc_test(
name = "segmentation_smoothing_calculator_test",
srcs = ["segmentation_smoothing_calculator_test.cc"],
+ tags = ["not_run:arm"],
deps = [
":image_clone_calculator",
":image_clone_calculator_cc_proto",
@@ -664,6 +705,7 @@ cc_test(
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:parse_text_proto",
+ "@com_google_absl//absl/log:absl_log",
],
)
@@ -686,6 +728,7 @@ cc_library(
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/gpu:shader_util",
+ "@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
@@ -771,7 +814,10 @@ cc_test(
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero_interp_cubic.png",
"//mediapipe/calculators/tensor:testdata/image_to_tensor/noop_except_range.png",
],
- tags = ["desktop_only_test"],
+ tags = [
+ "desktop_only_test",
+ "not_run:arm",
+ ],
deps = [
":affine_transformation",
":image_transformation_calculator",
diff --git a/mediapipe/calculators/image/affine_transformation_runner_gl.cc b/mediapipe/calculators/image/affine_transformation_runner_gl.cc
index 006416916..0fe2d2744 100644
--- a/mediapipe/calculators/image/affine_transformation_runner_gl.cc
+++ b/mediapipe/calculators/image/affine_transformation_runner_gl.cc
@@ -20,6 +20,7 @@
#include "Eigen/Core"
#include "Eigen/Geometry"
#include "Eigen/LU"
+#include "absl/log/absl_log.h"
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
@@ -53,6 +54,10 @@ bool IsMatrixVerticalFlipNeeded(GpuOrigin::Mode gpu_origin) {
#endif // __APPLE__
case GpuOrigin::TOP_LEFT:
return false;
+ default:
+ ABSL_LOG(ERROR) << "Incorrect GpuOrigin: "
+ << static_cast(gpu_origin);
+ return true;
}
}
@@ -384,6 +389,8 @@ class GlTextureWarpAffineRunner
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, 0);
+ glFlush();
+
return absl::OkStatus();
}
diff --git a/mediapipe/calculators/image/bilateral_filter_calculator.cc b/mediapipe/calculators/image/bilateral_filter_calculator.cc
index 6bb43dc00..3d364ad93 100644
--- a/mediapipe/calculators/image/bilateral_filter_calculator.cc
+++ b/mediapipe/calculators/image/bilateral_filter_calculator.cc
@@ -15,6 +15,7 @@
#include
#include
+#include "absl/log/absl_check.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -112,7 +113,7 @@ class BilateralFilterCalculator : public CalculatorBase {
REGISTER_CALCULATOR(BilateralFilterCalculator);
absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) {
- CHECK_GE(cc->Inputs().NumEntries(), 1);
+ RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
if (cc->Inputs().HasTag(kInputFrameTag) &&
cc->Inputs().HasTag(kInputFrameTagGpu)) {
@@ -183,8 +184,8 @@ absl::Status BilateralFilterCalculator::Open(CalculatorContext* cc) {
sigma_color_ = options_.sigma_color();
sigma_space_ = options_.sigma_space();
- CHECK_GE(sigma_color_, 0.0);
- CHECK_GE(sigma_space_, 0.0);
+ ABSL_CHECK_GE(sigma_color_, 0.0);
+ ABSL_CHECK_GE(sigma_space_, 0.0);
if (!use_gpu_) sigma_color_ *= 255.0;
if (use_gpu_) {
diff --git a/mediapipe/calculators/image/color_convert_calculator.cc b/mediapipe/calculators/image/color_convert_calculator.cc
index 4781f1ea1..f8f018363 100644
--- a/mediapipe/calculators/image/color_convert_calculator.cc
+++ b/mediapipe/calculators/image/color_convert_calculator.cc
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "absl/log/absl_check.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
@@ -25,8 +26,8 @@
namespace mediapipe {
namespace {
void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
- CHECK(mat->depth() == CV_8U);
- CHECK(channel < mat->channels());
+ ABSL_CHECK(mat->depth() == CV_8U);
+ ABSL_CHECK(channel < mat->channels());
const int step = mat->channels();
for (int r = 0; r < mat->rows; ++r) {
uint8* row_ptr = mat->ptr(r);
diff --git a/mediapipe/calculators/image/image_cropping_calculator.cc b/mediapipe/calculators/image/image_cropping_calculator.cc
index 6776da7c8..9eb3e6808 100644
--- a/mediapipe/calculators/image/image_cropping_calculator.cc
+++ b/mediapipe/calculators/image/image_cropping_calculator.cc
@@ -16,6 +16,7 @@
#include
+#include "absl/log/absl_log.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/rect.pb.h"
@@ -202,8 +203,9 @@ absl::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
switch (options.border_mode()) {
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
- LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU "
- << "implementation and will fall back into BORDER_REPLICATE";
+ ABSL_LOG(WARNING)
+ << "BORDER_ZERO mode is not supported by GPU "
+ << "implementation and will fall back into BORDER_REPLICATE";
break;
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
break;
diff --git a/mediapipe/calculators/image/image_transformation_calculator.cc b/mediapipe/calculators/image/image_transformation_calculator.cc
index dbf8f7337..8c6f715a0 100644
--- a/mediapipe/calculators/image/image_transformation_calculator.cc
+++ b/mediapipe/calculators/image/image_transformation_calculator.cc
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "absl/status/status.h"
#include "mediapipe/calculators/image/image_transformation_calculator.pb.h"
#include "mediapipe/calculators/image/rotation_mode.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -27,6 +28,7 @@
#include "mediapipe/gpu/scale_mode.pb.h"
#if !MEDIAPIPE_DISABLE_GPU
+#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_quad_renderer.h"
#include "mediapipe/gpu/gl_simple_shaders.h"
@@ -60,42 +62,42 @@ constexpr char kVideoPrestreamTag[] = "VIDEO_PRESTREAM";
int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
switch (rotation) {
- case mediapipe::RotationMode_Mode_UNKNOWN:
- case mediapipe::RotationMode_Mode_ROTATION_0:
+ case mediapipe::RotationMode::UNKNOWN:
+ case mediapipe::RotationMode::ROTATION_0:
return 0;
- case mediapipe::RotationMode_Mode_ROTATION_90:
+ case mediapipe::RotationMode::ROTATION_90:
return 90;
- case mediapipe::RotationMode_Mode_ROTATION_180:
+ case mediapipe::RotationMode::ROTATION_180:
return 180;
- case mediapipe::RotationMode_Mode_ROTATION_270:
+ case mediapipe::RotationMode::ROTATION_270:
return 270;
}
}
mediapipe::RotationMode_Mode DegreesToRotationMode(int degrees) {
switch (degrees) {
case 0:
- return mediapipe::RotationMode_Mode_ROTATION_0;
+ return mediapipe::RotationMode::ROTATION_0;
case 90:
- return mediapipe::RotationMode_Mode_ROTATION_90;
+ return mediapipe::RotationMode::ROTATION_90;
case 180:
- return mediapipe::RotationMode_Mode_ROTATION_180;
+ return mediapipe::RotationMode::ROTATION_180;
case 270:
- return mediapipe::RotationMode_Mode_ROTATION_270;
+ return mediapipe::RotationMode::ROTATION_270;
default:
- return mediapipe::RotationMode_Mode_UNKNOWN;
+ return mediapipe::RotationMode::UNKNOWN;
}
}
mediapipe::ScaleMode_Mode ParseScaleMode(
mediapipe::ScaleMode_Mode scale_mode,
mediapipe::ScaleMode_Mode default_mode) {
switch (scale_mode) {
- case mediapipe::ScaleMode_Mode_DEFAULT:
+ case mediapipe::ScaleMode::DEFAULT:
return default_mode;
- case mediapipe::ScaleMode_Mode_STRETCH:
+ case mediapipe::ScaleMode::STRETCH:
return scale_mode;
- case mediapipe::ScaleMode_Mode_FIT:
+ case mediapipe::ScaleMode::FIT:
return scale_mode;
- case mediapipe::ScaleMode_Mode_FILL_AND_CROP:
+ case mediapipe::ScaleMode::FILL_AND_CROP:
return scale_mode;
default:
return default_mode;
@@ -208,6 +210,8 @@ class ImageTransformationCalculator : public CalculatorBase {
bool use_gpu_ = false;
cv::Scalar padding_color_;
+ ImageTransformationCalculatorOptions::InterpolationMode interpolation_mode_;
+
#if !MEDIAPIPE_DISABLE_GPU
GlCalculatorHelper gpu_helper_;
std::unique_ptr rgb_renderer_;
@@ -343,6 +347,11 @@ absl::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
options_.padding_color().green(),
options_.padding_color().blue());
+ interpolation_mode_ = options_.interpolation_mode();
+ if (options_.interpolation_mode() ==
+ ImageTransformationCalculatorOptions::DEFAULT) {
+ interpolation_mode_ = ImageTransformationCalculatorOptions::LINEAR;
+ }
if (use_gpu_) {
#if !MEDIAPIPE_DISABLE_GPU
// Let the helper access the GL context information.
@@ -457,26 +466,48 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
ComputeOutputDimensions(input_width, input_height, &output_width,
&output_height);
+ int opencv_interpolation_mode = cv::INTER_LINEAR;
if (output_width_ > 0 && output_height_ > 0) {
cv::Mat scaled_mat;
- if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
- int scale_flag =
- input_mat.cols > output_width_ && input_mat.rows > output_height_
- ? cv::INTER_AREA
- : cv::INTER_LINEAR;
+ if (scale_mode_ == mediapipe::ScaleMode::STRETCH) {
+ if (interpolation_mode_ == ImageTransformationCalculatorOptions::LINEAR) {
+ // Use INTER_AREA for downscaling if interpolation mode is set to
+ // LINEAR.
+ if (input_mat.cols > output_width_ && input_mat.rows > output_height_) {
+ opencv_interpolation_mode = cv::INTER_AREA;
+
+ } else {
+ opencv_interpolation_mode = cv::INTER_LINEAR;
+ }
+ } else {
+ opencv_interpolation_mode = cv::INTER_NEAREST;
+ }
cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_),
- 0, 0, scale_flag);
+ 0, 0, opencv_interpolation_mode);
} else {
const float scale =
std::min(static_cast(output_width_) / input_width,
static_cast(output_height_) / input_height);
const int target_width = std::round(input_width * scale);
const int target_height = std::round(input_height * scale);
- int scale_flag = scale < 1.0f ? cv::INTER_AREA : cv::INTER_LINEAR;
- if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
+
+ if (interpolation_mode_ == ImageTransformationCalculatorOptions::LINEAR) {
+ // Use INTER_AREA for downscaling if interpolation mode is set to
+ // LINEAR.
+ if (scale < 1.0f) {
+ opencv_interpolation_mode = cv::INTER_AREA;
+ } else {
+ opencv_interpolation_mode = cv::INTER_LINEAR;
+ }
+ } else {
+ opencv_interpolation_mode = cv::INTER_NEAREST;
+ }
+
+ if (scale_mode_ == mediapipe::ScaleMode::FIT) {
cv::Mat intermediate_mat;
cv::resize(input_mat, intermediate_mat,
- cv::Size(target_width, target_height), 0, 0, scale_flag);
+ cv::Size(target_width, target_height), 0, 0,
+ opencv_interpolation_mode);
const int top = (output_height_ - target_height) / 2;
const int bottom = output_height_ - target_height - top;
const int left = (output_width_ - target_width) / 2;
@@ -488,7 +519,7 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
padding_color_);
} else {
cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height),
- 0, 0, scale_flag);
+ 0, 0, opencv_interpolation_mode);
output_width = target_width;
output_height = target_height;
}
@@ -514,17 +545,17 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
cv::warpAffine(input_mat, rotated_mat, rotation_mat, rotated_size);
} else {
switch (rotation_) {
- case mediapipe::RotationMode_Mode_UNKNOWN:
- case mediapipe::RotationMode_Mode_ROTATION_0:
+ case mediapipe::RotationMode::UNKNOWN:
+ case mediapipe::RotationMode::ROTATION_0:
rotated_mat = input_mat;
break;
- case mediapipe::RotationMode_Mode_ROTATION_90:
+ case mediapipe::RotationMode::ROTATION_90:
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_COUNTERCLOCKWISE);
break;
- case mediapipe::RotationMode_Mode_ROTATION_180:
+ case mediapipe::RotationMode::ROTATION_180:
cv::rotate(input_mat, rotated_mat, cv::ROTATE_180);
break;
- case mediapipe::RotationMode_Mode_ROTATION_270:
+ case mediapipe::RotationMode::ROTATION_270:
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_CLOCKWISE);
break;
}
@@ -561,7 +592,7 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
ComputeOutputDimensions(input_width, input_height, &output_width,
&output_height);
- if (scale_mode_ == mediapipe::ScaleMode_Mode_FILL_AND_CROP) {
+ if (scale_mode_ == mediapipe::ScaleMode::FILL_AND_CROP) {
const float scale =
std::min(static_cast(output_width_) / input_width,
static_cast(output_height_) / input_height);
@@ -628,6 +659,12 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
glActiveTexture(GL_TEXTURE1);
glBindTexture(src1.target(), src1.name());
+ if (interpolation_mode_ == ImageTransformationCalculatorOptions::NEAREST) {
+ // TODO: revert texture params.
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
+ glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
+ }
+
MP_RETURN_IF_ERROR(renderer->GlRender(
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
rotation, flip_horizontally_, flip_vertically_,
@@ -652,8 +689,8 @@ void ImageTransformationCalculator::ComputeOutputDimensions(
if (output_width_ > 0 && output_height_ > 0) {
*output_width = output_width_;
*output_height = output_height_;
- } else if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 ||
- rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) {
+ } else if (rotation_ == mediapipe::RotationMode::ROTATION_90 ||
+ rotation_ == mediapipe::RotationMode::ROTATION_270) {
*output_width = input_height;
*output_height = input_width;
} else {
@@ -666,9 +703,9 @@ void ImageTransformationCalculator::ComputeOutputLetterboxPadding(
int input_width, int input_height, int output_width, int output_height,
std::array* padding) {
padding->fill(0.f);
- if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
- if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 ||
- rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) {
+ if (scale_mode_ == mediapipe::ScaleMode::FIT) {
+ if (rotation_ == mediapipe::RotationMode::ROTATION_90 ||
+ rotation_ == mediapipe::RotationMode::ROTATION_270) {
std::swap(input_width, input_height);
}
const float input_aspect_ratio =
diff --git a/mediapipe/calculators/image/image_transformation_calculator.proto b/mediapipe/calculators/image/image_transformation_calculator.proto
index 16f60fcbc..0e2453a46 100644
--- a/mediapipe/calculators/image/image_transformation_calculator.proto
+++ b/mediapipe/calculators/image/image_transformation_calculator.proto
@@ -54,4 +54,15 @@ message ImageTransformationCalculatorOptions {
// The color for the padding. This option is only used when the scale mode is
// FIT. Default is black. This is for CPU only.
optional Color padding_color = 8;
+
+ // Interpolation method to use. Note that on CPU when LINEAR is specified,
+ // INTER_LINEAR is used for upscaling and INTER_AREA is used for downscaling.
+ enum InterpolationMode {
+ DEFAULT = 0;
+ LINEAR = 1;
+ NEAREST = 2;
+ }
+
+ // Mode DEFAULT will use LINEAR interpolation.
+ optional InterpolationMode interpolation_mode = 9;
}
diff --git a/mediapipe/calculators/image/image_transformation_calculator_test.cc b/mediapipe/calculators/image/image_transformation_calculator_test.cc
new file mode 100644
index 000000000..48828cc70
--- /dev/null
+++ b/mediapipe/calculators/image/image_transformation_calculator_test.cc
@@ -0,0 +1,315 @@
+#include
+#include
+#include
+
+#include "absl/container/flat_hash_set.h"
+#include "absl/flags/flag.h"
+#include "absl/strings/substitute.h"
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/deps/file_path.h"
+#include "mediapipe/framework/formats/image_format.pb.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "testing/base/public/gmock.h"
+#include "testing/base/public/googletest.h"
+#include "third_party/OpenCV/core.hpp" // IWYU pragma: keep
+#include "third_party/OpenCV/core/mat.hpp"
+
+namespace mediapipe {
+
+namespace {
+
+absl::flat_hash_set computeUniqueValues(const cv::Mat& mat) {
+ // Compute the unique values in cv::Mat
+ absl::flat_hash_set unique_values;
+ for (int i = 0; i < mat.rows; i++) {
+ for (int j = 0; j < mat.cols; j++) {
+ unique_values.insert(mat.at(i, j));
+ }
+ }
+ return unique_values;
+}
+
+TEST(ImageTransformationCalculatorTest, NearestNeighborResizing) {
+ cv::Mat input_mat;
+ cv::cvtColor(cv::imread(file::JoinPath("./",
+ "/mediapipe/calculators/"
+ "image/testdata/binary_mask.png")),
+ input_mat, cv::COLOR_BGR2GRAY);
+ Packet input_image_packet = MakePacket(
+ ImageFormat::GRAY8, input_mat.size().width, input_mat.size().height);
+ input_mat.copyTo(formats::MatView(&(input_image_packet.Get())));
+
+ std::vector> output_dims{
+ {256, 333}, {512, 512}, {1024, 1024}};
+
+ for (auto& output_dim : output_dims) {
+ Packet input_output_dim_packet =
+ MakePacket>(output_dim);
+ std::vector scale_modes{"FIT", "STRETCH"};
+ for (const auto& scale_mode : scale_modes) {
+ CalculatorGraphConfig::Node node_config =
+ ParseTextProtoOrDie(
+ absl::Substitute(R"(
+ calculator: "ImageTransformationCalculator"
+ input_stream: "IMAGE:input_image"
+ input_stream: "OUTPUT_DIMENSIONS:image_size"
+ output_stream: "IMAGE:output_image"
+ options: {
+ [mediapipe.ImageTransformationCalculatorOptions.ext]: {
+ scale_mode: $0
+ interpolation_mode: NEAREST
+ }
+ })",
+ scale_mode));
+
+ CalculatorRunner runner(node_config);
+ runner.MutableInputs()->Tag("IMAGE").packets.push_back(
+ input_image_packet.At(Timestamp(0)));
+ runner.MutableInputs()
+ ->Tag("OUTPUT_DIMENSIONS")
+ .packets.push_back(input_output_dim_packet.At(Timestamp(0)));
+
+ MP_ASSERT_OK(runner.Run());
+ const auto& outputs = runner.Outputs();
+ ASSERT_EQ(outputs.NumEntries(), 1);
+ const std::vector& packets = outputs.Tag("IMAGE").packets;
+ ASSERT_EQ(packets.size(), 1);
+ const auto& result = packets[0].Get();
+ ASSERT_EQ(output_dim.first, result.Width());
+ ASSERT_EQ(output_dim.second, result.Height());
+
+ auto unique_input_values = computeUniqueValues(input_mat);
+ auto unique_output_values =
+ computeUniqueValues(formats::MatView(&result));
+ EXPECT_THAT(unique_input_values,
+ ::testing::ContainerEq(unique_output_values));
+ }
+ }
+}
+
+TEST(ImageTransformationCalculatorTest,
+ NearestNeighborResizingWorksForFloatInput) {
+ cv::Mat input_mat;
+ cv::cvtColor(cv::imread(file::JoinPath("./",
+ "/mediapipe/calculators/"
+ "image/testdata/binary_mask.png")),
+ input_mat, cv::COLOR_BGR2GRAY);
+ Packet input_image_packet = MakePacket(
+ ImageFormat::VEC32F1, input_mat.size().width, input_mat.size().height);
+ cv::Mat packet_mat_view =
+ formats::MatView(&(input_image_packet.Get()));
+ input_mat.convertTo(packet_mat_view, CV_32FC1, 1 / 255.f);
+
+ std::vector> output_dims{
+ {256, 333}, {512, 512}, {1024, 1024}};
+
+ for (auto& output_dim : output_dims) {
+ Packet input_output_dim_packet =
+ MakePacket>(output_dim);
+ std::vector scale_modes{"FIT", "STRETCH"};
+ for (const auto& scale_mode : scale_modes) {
+ CalculatorGraphConfig::Node node_config =
+ ParseTextProtoOrDie(
+ absl::Substitute(R"(
+ calculator: "ImageTransformationCalculator"
+ input_stream: "IMAGE:input_image"
+ input_stream: "OUTPUT_DIMENSIONS:image_size"
+ output_stream: "IMAGE:output_image"
+ options: {
+ [mediapipe.ImageTransformationCalculatorOptions.ext]: {
+ scale_mode: $0
+ interpolation_mode: NEAREST
+ }
+ })",
+ scale_mode));
+
+ CalculatorRunner runner(node_config);
+ runner.MutableInputs()->Tag("IMAGE").packets.push_back(
+ input_image_packet.At(Timestamp(0)));
+ runner.MutableInputs()
+ ->Tag("OUTPUT_DIMENSIONS")
+ .packets.push_back(input_output_dim_packet.At(Timestamp(0)));
+
+ MP_ASSERT_OK(runner.Run());
+ const auto& outputs = runner.Outputs();
+ ASSERT_EQ(outputs.NumEntries(), 1);
+ const std::vector& packets = outputs.Tag("IMAGE").packets;
+ ASSERT_EQ(packets.size(), 1);
+ const auto& result = packets[0].Get();
+ ASSERT_EQ(output_dim.first, result.Width());
+ ASSERT_EQ(output_dim.second, result.Height());
+
+ auto unique_input_values = computeUniqueValues(packet_mat_view);
+ auto unique_output_values =
+ computeUniqueValues(formats::MatView(&result));
+ EXPECT_THAT(unique_input_values,
+ ::testing::ContainerEq(unique_output_values));
+ }
+ }
+}
+
+TEST(ImageTransformationCalculatorTest, NearestNeighborResizingGpu) {
+ cv::Mat input_mat;
+ cv::cvtColor(cv::imread(file::JoinPath("./",
+ "/mediapipe/calculators/"
+ "image/testdata/binary_mask.png")),
+ input_mat, cv::COLOR_BGR2RGBA);
+
+ std::vector> output_dims{
+ {256, 333}, {512, 512}, {1024, 1024}};
+
+ for (auto& output_dim : output_dims) {
+ std::vector scale_modes{"FIT"}; //, "STRETCH"};
+ for (const auto& scale_mode : scale_modes) {
+ CalculatorGraphConfig graph_config =
+ ParseTextProtoOrDie(
+ absl::Substitute(R"(
+ input_stream: "input_image"
+ input_stream: "image_size"
+ output_stream: "output_image"
+
+ node {
+ calculator: "ImageFrameToGpuBufferCalculator"
+ input_stream: "input_image"
+ output_stream: "input_image_gpu"
+ }
+
+ node {
+ calculator: "ImageTransformationCalculator"
+ input_stream: "IMAGE_GPU:input_image_gpu"
+ input_stream: "OUTPUT_DIMENSIONS:image_size"
+ output_stream: "IMAGE_GPU:output_image_gpu"
+ options: {
+ [mediapipe.ImageTransformationCalculatorOptions.ext]: {
+ scale_mode: $0
+ interpolation_mode: NEAREST
+ }
+ }
+ }
+ node {
+ calculator: "GpuBufferToImageFrameCalculator"
+ input_stream: "output_image_gpu"
+ output_stream: "output_image"
+ })",
+ scale_mode));
+ ImageFrame input_image(ImageFormat::SRGBA, input_mat.size().width,
+ input_mat.size().height);
+ input_mat.copyTo(formats::MatView(&input_image));
+
+ std::vector output_image_packets;
+ tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
+
+ CalculatorGraph graph(graph_config);
+ MP_ASSERT_OK(graph.StartRun({}));
+
+ MP_ASSERT_OK(graph.AddPacketToInputStream(
+ "input_image",
+ MakePacket(std::move(input_image)).At(Timestamp(0))));
+ MP_ASSERT_OK(graph.AddPacketToInputStream(
+ "image_size",
+ MakePacket>(output_dim).At(Timestamp(0))));
+
+ MP_ASSERT_OK(graph.WaitUntilIdle());
+ ASSERT_THAT(output_image_packets, testing::SizeIs(1));
+
+ const auto& output_image = output_image_packets[0].Get();
+ ASSERT_EQ(output_dim.first, output_image.Width());
+ ASSERT_EQ(output_dim.second, output_image.Height());
+
+ auto unique_input_values = computeUniqueValues(input_mat);
+ auto unique_output_values =
+ computeUniqueValues(formats::MatView(&output_image));
+ EXPECT_THAT(unique_input_values,
+ ::testing::ContainerEq(unique_output_values));
+ }
+ }
+}
+
+TEST(ImageTransformationCalculatorTest,
+ NearestNeighborResizingWorksForFloatTexture) {
+ cv::Mat input_mat;
+ cv::cvtColor(cv::imread(file::JoinPath("./",
+ "/mediapipe/calculators/"
+ "image/testdata/binary_mask.png")),
+ input_mat, cv::COLOR_BGR2GRAY);
+ Packet input_image_packet = MakePacket(
+ ImageFormat::VEC32F1, input_mat.size().width, input_mat.size().height);
+ cv::Mat packet_mat_view =
+ formats::MatView(&(input_image_packet.Get()));
+ input_mat.convertTo(packet_mat_view, CV_32FC1, 1 / 255.f);
+
+ std::vector> output_dims{
+ {256, 333}, {512, 512}, {1024, 1024}};
+
+ for (auto& output_dim : output_dims) {
+ std::vector scale_modes{"FIT"}; //, "STRETCH"};
+ for (const auto& scale_mode : scale_modes) {
+ CalculatorGraphConfig graph_config =
+ ParseTextProtoOrDie(
+ absl::Substitute(R"(
+ input_stream: "input_image"
+ input_stream: "image_size"
+ output_stream: "output_image"
+
+ node {
+ calculator: "ImageFrameToGpuBufferCalculator"
+ input_stream: "input_image"
+ output_stream: "input_image_gpu"
+ }
+
+ node {
+ calculator: "ImageTransformationCalculator"
+ input_stream: "IMAGE_GPU:input_image_gpu"
+ input_stream: "OUTPUT_DIMENSIONS:image_size"
+ output_stream: "IMAGE_GPU:output_image_gpu"
+ options: {
+ [mediapipe.ImageTransformationCalculatorOptions.ext]: {
+ scale_mode: $0
+ interpolation_mode: NEAREST
+ }
+ }
+ }
+ node {
+ calculator: "GpuBufferToImageFrameCalculator"
+ input_stream: "output_image_gpu"
+ output_stream: "output_image"
+ })",
+ scale_mode));
+
+ std::vector output_image_packets;
+ tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
+
+ CalculatorGraph graph(graph_config);
+ MP_ASSERT_OK(graph.StartRun({}));
+
+ MP_ASSERT_OK(graph.AddPacketToInputStream(
+ "input_image", input_image_packet.At(Timestamp(0))));
+ MP_ASSERT_OK(graph.AddPacketToInputStream(
+ "image_size",
+ MakePacket>(output_dim).At(Timestamp(0))));
+
+ MP_ASSERT_OK(graph.WaitUntilIdle());
+ ASSERT_THAT(output_image_packets, testing::SizeIs(1));
+
+ const auto& output_image = output_image_packets[0].Get();
+ ASSERT_EQ(output_dim.first, output_image.Width());
+ ASSERT_EQ(output_dim.second, output_image.Height());
+
+ auto unique_input_values = computeUniqueValues(packet_mat_view);
+ auto unique_output_values =
+ computeUniqueValues(formats::MatView(&output_image));
+ EXPECT_THAT(unique_input_values,
+ ::testing::ContainerEq(unique_output_values));
+ }
+ }
+}
+
+} // namespace
+} // namespace mediapipe
diff --git a/mediapipe/calculators/image/opencv_image_encoder_calculator.cc b/mediapipe/calculators/image/opencv_image_encoder_calculator.cc
index 93ec9435f..0308b9b8c 100644
--- a/mediapipe/calculators/image/opencv_image_encoder_calculator.cc
+++ b/mediapipe/calculators/image/opencv_image_encoder_calculator.cc
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "absl/log/absl_check.h"
#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
@@ -61,7 +62,7 @@ absl::Status OpenCvImageEncoderCalculator::Open(CalculatorContext* cc) {
absl::Status OpenCvImageEncoderCalculator::Process(CalculatorContext* cc) {
const ImageFrame& image_frame = cc->Inputs().Index(0).Get();
- CHECK_EQ(1, image_frame.ByteDepth());
+ ABSL_CHECK_EQ(1, image_frame.ByteDepth());
std::unique_ptr encoded_result =
absl::make_unique();
diff --git a/mediapipe/calculators/image/scale_image_calculator.cc b/mediapipe/calculators/image/scale_image_calculator.cc
index d8a3cb93b..1d4f980fe 100644
--- a/mediapipe/calculators/image/scale_image_calculator.cc
+++ b/mediapipe/calculators/image/scale_image_calculator.cc
@@ -18,6 +18,8 @@
#include
#include
+#include "absl/log/absl_check.h"
+#include "absl/log/absl_log.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/substitute.h"
#include "libyuv/scale.h"
@@ -293,7 +295,7 @@ absl::Status ScaleImageCalculator::InitializeFrameInfo(CalculatorContext* cc) {
header->width = output_width_;
header->height = output_height_;
header->format = output_format_;
- LOG(INFO) << "OUTPUTTING HEADER on stream";
+ ABSL_LOG(INFO) << "OUTPUTTING HEADER on stream";
cc->Outputs()
.Tag("VIDEO_HEADER")
.Add(header.release(), Timestamp::PreStream());
@@ -393,10 +395,11 @@ absl::Status ScaleImageCalculator::Open(CalculatorContext* cc) {
.SetHeader(Adopt(output_header.release()));
has_header_ = true;
} else {
- LOG(WARNING) << "Stream had a VideoHeader which didn't have sufficient "
- "information. "
- "Dropping VideoHeader and trying to deduce needed "
- "information.";
+ ABSL_LOG(WARNING)
+ << "Stream had a VideoHeader which didn't have sufficient "
+ "information. "
+ "Dropping VideoHeader and trying to deduce needed "
+ "information.";
input_width_ = 0;
input_height_ = 0;
if (!options_.has_input_format()) {
@@ -507,7 +510,7 @@ absl::Status ScaleImageCalculator::ValidateImageFrame(
absl::Status ScaleImageCalculator::ValidateYUVImage(CalculatorContext* cc,
const YUVImage& yuv_image) {
- CHECK_EQ(input_format_, ImageFormat::YCBCR420P);
+ ABSL_CHECK_EQ(input_format_, ImageFormat::YCBCR420P);
if (!has_header_) {
if (input_width_ != yuv_image.width() ||
input_height_ != yuv_image.height()) {
diff --git a/mediapipe/calculators/image/scale_image_utils.cc b/mediapipe/calculators/image/scale_image_utils.cc
index 86a53ffc5..77b7c0ece 100644
--- a/mediapipe/calculators/image/scale_image_utils.cc
+++ b/mediapipe/calculators/image/scale_image_utils.cc
@@ -18,6 +18,7 @@
#include
+#include "absl/log/absl_check.h"
#include "absl/strings/str_split.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
@@ -40,10 +41,10 @@ absl::Status FindCropDimensions(int input_width, int input_height, //
const std::string& max_aspect_ratio, //
int* crop_width, int* crop_height, //
int* col_start, int* row_start) {
- CHECK(crop_width);
- CHECK(crop_height);
- CHECK(col_start);
- CHECK(row_start);
+ ABSL_CHECK(crop_width);
+ ABSL_CHECK(crop_height);
+ ABSL_CHECK(col_start);
+ ABSL_CHECK(row_start);
double min_aspect_ratio_q = 0.0;
double max_aspect_ratio_q = 0.0;
@@ -83,8 +84,8 @@ absl::Status FindCropDimensions(int input_width, int input_height, //
}
}
- CHECK_LE(*crop_width, input_width);
- CHECK_LE(*crop_height, input_height);
+ ABSL_CHECK_LE(*crop_width, input_width);
+ ABSL_CHECK_LE(*crop_height, input_height);
return absl::OkStatus();
}
@@ -96,8 +97,8 @@ absl::Status FindOutputDimensions(int input_width, //
bool preserve_aspect_ratio, //
int scale_to_multiple_of, //
int* output_width, int* output_height) {
- CHECK(output_width);
- CHECK(output_height);
+ ABSL_CHECK(output_width);
+ ABSL_CHECK(output_height);
if (target_max_area > 0 && input_width * input_height > target_max_area) {
preserve_aspect_ratio = true;
diff --git a/mediapipe/calculators/image/segmentation_smoothing_calculator.cc b/mediapipe/calculators/image/segmentation_smoothing_calculator.cc
index 81732f904..1194412a6 100644
--- a/mediapipe/calculators/image/segmentation_smoothing_calculator.cc
+++ b/mediapipe/calculators/image/segmentation_smoothing_calculator.cc
@@ -15,13 +15,13 @@
#include
#include
+#include "absl/log/absl_log.h"
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
-#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/vector.h"
@@ -110,7 +110,7 @@ REGISTER_CALCULATOR(SegmentationSmoothingCalculator);
absl::Status SegmentationSmoothingCalculator::GetContract(
CalculatorContract* cc) {
- CHECK_GE(cc->Inputs().NumEntries(), 1);
+ RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
cc->Inputs().Tag(kCurrentMaskTag).Set();
cc->Inputs().Tag(kPreviousMaskTag).Set();
@@ -273,7 +273,7 @@ absl::Status SegmentationSmoothingCalculator::RenderGpu(CalculatorContext* cc) {
const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get();
if (previous_frame.format() != current_frame.format()) {
- LOG(ERROR) << "Warning: mixing input format types. ";
+ ABSL_LOG(ERROR) << "Warning: mixing input format types. ";
}
auto previous_texture = gpu_helper_.CreateSourceTexture(previous_frame);
diff --git a/mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc b/mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc
index eeb812cb7..0f5152fc3 100644
--- a/mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc
+++ b/mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc
@@ -14,6 +14,7 @@
#include
+#include "absl/log/absl_log.h"
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
@@ -169,7 +170,7 @@ void RunTest(bool use_gpu, float mix_ratio, cv::Mat& test_result) {
}
}
} else {
- LOG(ERROR) << "invalid ratio";
+ ABSL_LOG(ERROR) << "invalid ratio";
}
}
diff --git a/mediapipe/calculators/image/set_alpha_calculator.cc b/mediapipe/calculators/image/set_alpha_calculator.cc
index e20621e8d..d451cd21c 100644
--- a/mediapipe/calculators/image/set_alpha_calculator.cc
+++ b/mediapipe/calculators/image/set_alpha_calculator.cc
@@ -14,13 +14,13 @@
#include
+#include "absl/log/absl_log.h"
#include "mediapipe/calculators/image/set_alpha_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
-#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/status.h"
@@ -142,7 +142,7 @@ class SetAlphaCalculator : public CalculatorBase {
REGISTER_CALCULATOR(SetAlphaCalculator);
absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
- CHECK_GE(cc->Inputs().NumEntries(), 1);
+ RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
bool use_gpu = false;
@@ -268,7 +268,7 @@ absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get();
const cv::Mat input_mat = formats::MatView(&input_frame);
if (!(input_mat.type() == CV_8UC3 || input_mat.type() == CV_8UC4)) {
- LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported";
+ ABSL_LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported";
}
// Setup destination image
@@ -328,7 +328,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
cc->Inputs().Tag(kInputFrameTagGpu).Get();
if (!(input_frame.format() == mediapipe::GpuBufferFormat::kBGRA32 ||
input_frame.format() == mediapipe::GpuBufferFormat::kRGB24)) {
- LOG(ERROR) << "Only RGB or RGBA input image supported";
+ ABSL_LOG(ERROR) << "Only RGB or RGBA input image supported";
}
auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);
diff --git a/mediapipe/calculators/image/testdata/BUILD b/mediapipe/calculators/image/testdata/BUILD
index da192b513..89df807c7 100644
--- a/mediapipe/calculators/image/testdata/BUILD
+++ b/mediapipe/calculators/image/testdata/BUILD
@@ -18,6 +18,7 @@ licenses(["notice"])
filegroup(
name = "test_images",
srcs = [
+ "binary_mask.png",
"dino.jpg",
"dino_quality_50.jpg",
"dino_quality_80.jpg",
diff --git a/mediapipe/calculators/image/testdata/binary_mask.png b/mediapipe/calculators/image/testdata/binary_mask.png
new file mode 100644
index 000000000..aa38e5d3f
Binary files /dev/null and b/mediapipe/calculators/image/testdata/binary_mask.png differ
diff --git a/mediapipe/calculators/image/yuv_to_image_calculator.cc b/mediapipe/calculators/image/yuv_to_image_calculator.cc
index e84eee74e..6a82877c3 100644
--- a/mediapipe/calculators/image/yuv_to_image_calculator.cc
+++ b/mediapipe/calculators/image/yuv_to_image_calculator.cc
@@ -38,7 +38,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
buf[0] = (fourcc >> 24) & 0xff;
buf[1] = (fourcc >> 16) & 0xff;
buf[2] = (fourcc >> 8) & 0xff;
- buf[3] = (fourcc)&0xff;
+ buf[3] = (fourcc) & 0xff;
buf[4] = 0;
return std::string(buf);
}
diff --git a/mediapipe/calculators/internal/BUILD b/mediapipe/calculators/internal/BUILD
index a92a2f252..a5d82e134 100644
--- a/mediapipe/calculators/internal/BUILD
+++ b/mediapipe/calculators/internal/BUILD
@@ -31,12 +31,14 @@ mediapipe_proto_library(
cc_library(
name = "callback_packet_calculator",
srcs = ["callback_packet_calculator.cc"],
+ hdrs = ["callback_packet_calculator.h"],
visibility = ["//mediapipe/framework:__subpackages__"],
deps = [
":callback_packet_calculator_cc_proto",
"//mediapipe/framework:calculator_base",
"//mediapipe/framework:calculator_registry",
"//mediapipe/framework:output_side_packet",
+ "@com_google_absl//absl/status",
],
alwayslink = 1,
)
diff --git a/mediapipe/calculators/internal/callback_packet_calculator.cc b/mediapipe/calculators/internal/callback_packet_calculator.cc
index cc153483e..aa86c0617 100644
--- a/mediapipe/calculators/internal/callback_packet_calculator.cc
+++ b/mediapipe/calculators/internal/callback_packet_calculator.cc
@@ -11,10 +11,12 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
+#include "mediapipe/calculators/internal/callback_packet_calculator.h"
#include
#include
+#include "absl/status/status.h"
#include "mediapipe/calculators/internal/callback_packet_calculator.pb.h" // NOLINT
#include "mediapipe/framework/calculator_base.h"
#include "mediapipe/framework/calculator_registry.h"
@@ -39,64 +41,55 @@ void DumpPostStreamPacket(Packet* post_stream_packet, const Packet& packet) {
*post_stream_packet = packet;
}
}
+
} // namespace
-// Creates a callback which takes a packet and stores it either in a
-// vector of packets or stores only the packet at PostStream timestamp.
-// The kind of callback is controlled by an option. The callback is
-// a std::function and is directly usable by CallbackCalculator.
-// Since the options for the packet generator include a serialized pointer
-// value, the resulting callback is only valid on the original machine
-// while that pointer is still alive.
-class CallbackPacketCalculator : public CalculatorBase {
- public:
- static absl::Status GetContract(CalculatorContract* cc) {
- const auto& options = cc->Options();
- switch (options.type()) {
- case CallbackPacketCalculatorOptions::VECTOR_PACKET:
- case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
- cc->OutputSidePackets()
- .Index(0)
- .Set>();
- break;
- default:
- return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
- << "Invalid type of callback to produce.";
- }
- return absl::OkStatus();
- }
-
- absl::Status Open(CalculatorContext* cc) override {
- const auto& options = cc->Options();
- void* ptr;
- if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) {
+absl::Status CallbackPacketCalculator::GetContract(CalculatorContract* cc) {
+ const auto& options = cc->Options();
+ switch (options.type()) {
+ case CallbackPacketCalculatorOptions::VECTOR_PACKET:
+ case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
+ cc->OutputSidePackets()
+ .Index(0)
+ .Set>();
+ break;
+ default:
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
- << "Stored pointer value in options is invalid.";
- }
- switch (options.type()) {
- case CallbackPacketCalculatorOptions::VECTOR_PACKET:
- cc->OutputSidePackets().Index(0).Set(
- MakePacket>(std::bind(
- &DumpToVector, reinterpret_cast*>(ptr),
- std::placeholders::_1)));
- break;
- case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
- cc->OutputSidePackets().Index(0).Set(
- MakePacket>(
- std::bind(&DumpPostStreamPacket, reinterpret_cast(ptr),
- std::placeholders::_1)));
- break;
- default:
- return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
- << "Invalid type to dump into.";
- }
- return absl::OkStatus();
+ << "Invalid type of callback to produce.";
}
+ return absl::OkStatus();
+}
- absl::Status Process(CalculatorContext* cc) override {
- return absl::OkStatus();
+absl::Status CallbackPacketCalculator::Open(CalculatorContext* cc) {
+ const auto& options = cc->Options();
+ void* ptr;
+ if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) {
+ return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+ << "Stored pointer value in options is invalid.";
}
-};
+ switch (options.type()) {
+ case CallbackPacketCalculatorOptions::VECTOR_PACKET:
+ cc->OutputSidePackets().Index(0).Set(
+ MakePacket>(std::bind(
+ &DumpToVector, reinterpret_cast*>(ptr),
+ std::placeholders::_1)));
+ break;
+ case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
+ cc->OutputSidePackets().Index(0).Set(
+ MakePacket>(
+ std::bind(&DumpPostStreamPacket, reinterpret_cast(ptr),
+ std::placeholders::_1)));
+ break;
+ default:
+ return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
+ << "Invalid type to dump into.";
+ }
+ return absl::OkStatus();
+}
+
+absl::Status CallbackPacketCalculator::Process(CalculatorContext* cc) {
+ return absl::OkStatus();
+}
REGISTER_CALCULATOR(CallbackPacketCalculator);
diff --git a/mediapipe/calculators/internal/callback_packet_calculator.h b/mediapipe/calculators/internal/callback_packet_calculator.h
new file mode 100644
index 000000000..e0b170e36
--- /dev/null
+++ b/mediapipe/calculators/internal/callback_packet_calculator.h
@@ -0,0 +1,39 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
+
+#include "absl/status/status.h"
+#include "mediapipe/framework/calculator_base.h"
+
+namespace mediapipe {
+
+// Creates a callback which takes a packet and stores it either in a
+// vector of packets or stores only the packet at PostStream timestamp.
+// The kind of callback is controlled by an option. The callback is
+// a std::function and is directly usable by CallbackCalculator.
+// Since the options for the packet generator include a serialized pointer
+// value, the resulting callback is only valid on the original machine
+// while that pointer is still alive.
+class CallbackPacketCalculator : public CalculatorBase {
+ public:
+ static absl::Status GetContract(CalculatorContract* cc);
+ absl::Status Open(CalculatorContext* cc) override;
+ absl::Status Process(CalculatorContext* cc) override;
+};
+
+} // namespace mediapipe
+
+#endif // MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD
index 5d52dda0f..e74ecffe6 100644
--- a/mediapipe/calculators/tensor/BUILD
+++ b/mediapipe/calculators/tensor/BUILD
@@ -87,6 +87,7 @@ cc_library(
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/util:time_series_util",
+ "@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
@@ -181,6 +182,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:tensor",
+ "@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/status",
],
alwayslink = 1,
@@ -198,6 +200,7 @@ cc_test(
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
+ "@com_google_absl//absl/log:absl_check",
"@org_tensorflow//tensorflow/lite/c:common",
],
)
@@ -228,7 +231,6 @@ cc_library(
"//mediapipe/tasks/metadata:metadata_schema_cc",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/status",
- "@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
],
alwayslink = 1,
@@ -280,7 +282,6 @@ cc_library(
"//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils",
"//mediapipe/tasks/metadata:metadata_schema_cc",
"@com_google_absl//absl/status",
- "@com_google_absl//absl/status:statusor",
],
alwayslink = 1,
)
@@ -394,7 +395,7 @@ mediapipe_proto_library(
# If you want to have precise control of which implementations to include (e.g. for strict binary
# size concerns), depend on those implementations directly, and do not depend on
# :inference_calculator.
-# In all cases, use "InferenceCalulator" in your graphs.
+# In all cases, use "InferenceCalculator" in your graphs.
cc_library_with_tflite(
name = "inference_calculator_interface",
srcs = ["inference_calculator.cc"],
@@ -447,6 +448,7 @@ cc_library(
"//mediapipe/framework/deps:file_path",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/util/tflite:tflite_gpu_runner",
+ "@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
@@ -476,6 +478,7 @@ cc_library(
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/objc:mediapipe_framework_ios",
"//mediapipe/util/tflite:config",
+ "@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings:str_format",
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
@@ -622,6 +625,7 @@ mediapipe_proto_library(
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
+ "//mediapipe/gpu:gpu_origin_proto",
],
)
@@ -651,10 +655,21 @@ cc_library(
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
+ "//mediapipe/framework/port:status",
+ "//mediapipe/framework/port:statusor",
+ "//mediapipe/gpu:gpu_buffer_format",
+ "//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/util:resource_util",
+ "@com_google_absl//absl/log:absl_check",
+ "@com_google_absl//absl/strings:str_format",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": ["tensor_converter_calculator_gpu_deps"],
+ }) + select({
+ "//mediapipe:apple": [
+ "//third_party/apple_frameworks:MetalKit",
+ ],
+ "//conditions:default": [],
}),
alwayslink = 1,
)
@@ -696,6 +711,7 @@ cc_test(
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
+ "//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/memory",
@@ -734,6 +750,8 @@ cc_library(
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/formats/object_detection:anchor_cc_proto",
"//mediapipe/framework/port:ret_check",
+ "@com_google_absl//absl/log:absl_check",
+ "@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span",
] + selects.with_or({
@@ -790,6 +808,7 @@ cc_library(
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
+ "@com_google_absl//absl/log:absl_check",
],
alwayslink = 1,
)
@@ -982,6 +1001,8 @@ cc_library(
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_origin_cc_proto",
+ "@com_google_absl//absl/log:absl_check",
+ "@com_google_absl//absl/log:absl_log",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [":image_to_tensor_calculator_gpu_deps"],
@@ -1052,6 +1073,7 @@ cc_test(
"testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png",
"testdata/image_to_tensor/noop_except_range.png",
],
+ tags = ["not_run:arm"],
deps = [
":image_to_tensor_calculator",
":image_to_tensor_converter",
@@ -1073,6 +1095,7 @@ cc_test(
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/util:image_test_utils",
"@com_google_absl//absl/flags:flag",
+ "@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
@@ -1200,6 +1223,7 @@ cc_library(
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util",
+ "@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
],
}),
diff --git a/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc b/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc
index 47617b375..eaf593a69 100644
--- a/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc
+++ b/mediapipe/calculators/tensor/audio_to_tensor_calculator.cc
@@ -20,6 +20,7 @@
#include
#include
+#include "absl/log/absl_check.h"
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
@@ -282,18 +283,23 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
if (options.has_volume_gain_db()) {
gain_ = pow(10, options.volume_gain_db() / 20.0);
}
- RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
- !kAudioIn(cc).Header().IsEmpty())
- << "Must either specify the time series header of the \"AUDIO\" stream "
- "or have the \"SAMPLE_RATE\" stream connected.";
- if (!kAudioIn(cc).Header().IsEmpty()) {
- mediapipe::TimeSeriesHeader input_header;
- MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
- kAudioIn(cc).Header(), &input_header));
- if (stream_mode_) {
- MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
- } else {
- source_sample_rate_ = input_header.sample_rate();
+ if (options.has_source_sample_rate()) {
+ source_sample_rate_ = options.source_sample_rate();
+ } else {
+ RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
+ !kAudioIn(cc).Header().IsEmpty())
+ << "Must either specify the time series header of the \"AUDIO\" stream "
+ "or have the \"SAMPLE_RATE\" stream connected.";
+ if (!kAudioIn(cc).Header().IsEmpty()) {
+ mediapipe::TimeSeriesHeader input_header;
+ MP_RETURN_IF_ERROR(
+ mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
+ kAudioIn(cc).Header(), &input_header));
+ if (stream_mode_) {
+ MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
+ } else {
+ source_sample_rate_ = input_header.sample_rate();
+ }
}
}
AppendZerosToSampleBuffer(padding_samples_before_);
@@ -343,7 +349,7 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
return absl::InvalidArgumentError(
"The audio data should be stored in column-major.");
}
- CHECK(channels_match || mono_output);
+ ABSL_CHECK(channels_match || mono_output);
const Matrix& input = channels_match ? input_frame
// Mono mixdown.
: input_frame.colwise().mean();
@@ -452,7 +458,7 @@ absl::Status AudioToTensorCalculator::SetupStreamingResampler(
}
void AudioToTensorCalculator::AppendZerosToSampleBuffer(int num_samples) {
- CHECK_GE(num_samples, 0); // Ensured by `UpdateContract`.
+ ABSL_CHECK_GE(num_samples, 0); // Ensured by `UpdateContract`.
if (num_samples == 0) {
return;
}
diff --git a/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto b/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto
index 5b7d61bcb..948c82a36 100644
--- a/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto
+++ b/mediapipe/calculators/tensor/audio_to_tensor_calculator.proto
@@ -85,4 +85,7 @@ message AudioToTensorCalculatorOptions {
// The volume gain, measured in dB.
// Scale the input audio amplitude by 10^(volume_gain_db/20).
optional double volume_gain_db = 12;
+
+ // The source number of samples per second (hertz) of the input audio buffers.
+ optional double source_sample_rate = 13;
}
diff --git a/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc b/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc
index b56122805..12db1493c 100644
--- a/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc
+++ b/mediapipe/calculators/tensor/bert_preprocessor_calculator.cc
@@ -22,7 +22,6 @@
#include "absl/container/flat_hash_set.h"
#include "absl/status/status.h"
-#include "absl/status/statusor.h"
#include "absl/strings/ascii.h"
#include "absl/strings/string_view.h"
#include "absl/strings/substitute.h"
@@ -244,7 +243,8 @@ std::vector BertPreprocessorCalculator::GenerateInputTensors(
input_tensors.reserve(kNumInputTensorsForBert);
for (int i = 0; i < kNumInputTensorsForBert; ++i) {
input_tensors.push_back(
- {Tensor::ElementType::kInt32, Tensor::Shape({tensor_size})});
+ {Tensor::ElementType::kInt32,
+ Tensor::Shape({1, tensor_size}, has_dynamic_input_tensors_)});
}
std::memcpy(input_tensors[input_ids_tensor_index_]
.GetCpuWriteView()
diff --git a/mediapipe/calculators/tensor/bert_preprocessor_calculator.proto b/mediapipe/calculators/tensor/bert_preprocessor_calculator.proto
index 5dc9815a1..b2dc5578f 100644
--- a/mediapipe/calculators/tensor/bert_preprocessor_calculator.proto
+++ b/mediapipe/calculators/tensor/bert_preprocessor_calculator.proto
@@ -1,4 +1,4 @@
-/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
+/* Copyright 2022 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
diff --git a/mediapipe/calculators/tensor/feedback_tensors_calculator_test.cc b/mediapipe/calculators/tensor/feedback_tensors_calculator_test.cc
index 5797cc31c..6c5e5cc4f 100644
--- a/mediapipe/calculators/tensor/feedback_tensors_calculator_test.cc
+++ b/mediapipe/calculators/tensor/feedback_tensors_calculator_test.cc
@@ -18,6 +18,7 @@
#include
#include
+#include "absl/log/absl_check.h"
#include "mediapipe/calculators/tensor/feedback_tensors_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@@ -65,7 +66,7 @@ template
Tensor MakeTensor(std::initializer_list shape,
std::initializer_list values) {
Tensor tensor(TensorElementType::value, shape);
- CHECK_EQ(values.size(), tensor.shape().num_elements())
+ ABSL_CHECK_EQ(values.size(), tensor.shape().num_elements())
<< "The size of `values` is incompatible with `shape`";
absl::c_copy(values, tensor.GetCpuWriteView().buffer());
return tensor;
diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc
index 344e12da6..26fb1d868 100644
--- a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc
+++ b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc
@@ -16,6 +16,7 @@
#include
#include
+#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
@@ -284,9 +285,9 @@ class ImageToTensorCalculator : public Node {
cc, GetBorderMode(options_.border_mode()),
GetOutputTensorType(/*uses_gpu=*/false, params_)));
#else
- LOG(FATAL) << "Cannot create image to tensor CPU converter since "
- "MEDIAPIPE_DISABLE_OPENCV is defined and "
- "MEDIAPIPE_ENABLE_HALIDE is not defined.";
+ ABSL_LOG(FATAL) << "Cannot create image to tensor CPU converter since "
+ "MEDIAPIPE_DISABLE_OPENCV is defined and "
+ "MEDIAPIPE_ENABLE_HALIDE is not defined.";
#endif // !MEDIAPIPE_DISABLE_HALIDE
}
}
diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc
index 409b8623c..7017c1e3a 100644
--- a/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc
+++ b/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc
@@ -18,6 +18,7 @@
#include
#include "absl/flags/flag.h"
+#include "absl/log/absl_check.h"
#include "absl/memory/memory.h"
#include "absl/strings/str_format.h"
#include "absl/strings/substitute.h"
@@ -205,7 +206,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
} else if (image_channels == 1) {
return ImageFormat::GRAY8;
}
- CHECK(false) << "Unsupported input image channles: " << image_channels;
+ ABSL_CHECK(false) << "Unsupported input image channles: " << image_channels;
}
Packet MakeImageFramePacket(cv::Mat input) {
diff --git a/mediapipe/calculators/tensor/image_to_tensor_converter_frame_buffer.cc b/mediapipe/calculators/tensor/image_to_tensor_converter_frame_buffer.cc
index 093f50d76..6f6f6f11c 100644
--- a/mediapipe/calculators/tensor/image_to_tensor_converter_frame_buffer.cc
+++ b/mediapipe/calculators/tensor/image_to_tensor_converter_frame_buffer.cc
@@ -95,7 +95,8 @@ absl::Status FrameBufferProcessor::Convert(const mediapipe::Image& input,
static_cast