cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into face-stylizer-new-add-tests

Browse Source
This commit is contained in:
Kinar R 2023-09-13 22:22:07 +05:30 committed by GitHub
commit 1216c0f998
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1786 changed files with 74819 additions and 13187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.bazelrc
View File

@ -87,6 +87,9 @@ build:ios_fat --config=ios
build:ios_fat --ios_multi_cpus=armv7,arm64
build:ios_fat --watchos_cpus=armv7k
build:ios_sim_fat --config=ios
build:ios_sim_fat --ios_multi_cpus=x86_64,sim_arm64
build:darwin_x86_64 --apple_platform_type=macos
build:darwin_x86_64 --macos_minimum_os=10.12
build:darwin_x86_64 --cpu=darwin_x86_64

6
.github/ISSUE_TEMPLATE/task_issue_template.yaml → .github/ISSUE_TEMPLATE/00-task-issue-template.yaml vendored
View File

@ -40,18 +40,16 @@ body:
label: Programming Language and version (e.g. C++, Python, Java)
validations:
required: true
- type: textarea
- type: input
id: current_model
attributes:
label: Describe the actual behavior
render: shell
validations:
required: true
- type: textarea
- type: input
id: expected_model
attributes:
label: Describe the expected behaviour
render: shell
validations:
required: true
- type: textarea

6
.github/ISSUE_TEMPLATE/model_maker_issue_template.yaml → .github/ISSUE_TEMPLATE/11-model-maker-issue-template.yaml vendored
View File

@ -41,18 +41,16 @@ body:
label: Task name (e.g. Image classification, Gesture recognition etc.)
validations:
required: true
- type: textarea
- type: input
id: current_model
attributes:
label: Describe the actual behavior
render: shell
validations:
required: true
- type: textarea
- type: input
id: expected_model
attributes:
label: Describe the expected behaviour
render: shell
validations:
required: true
- type: textarea

6
.github/ISSUE_TEMPLATE/studio_issue_template.yaml → .github/ISSUE_TEMPLATE/12-studio-issue-template.yaml vendored
View File

@ -31,18 +31,16 @@ body:
label: URL that shows the problem
validations:
required: false
- type: textarea
- type: input
id: current_model
attributes:
label: Describe the actual behavior
render: shell
validations:
required: false
- type: textarea
- type: input
id: expected_model
attributes:
label: Describe the expected behaviour
render: shell
validations:
required: false
- type: textarea

14
.github/ISSUE_TEMPLATE/feature_request_issue_template.yaml → .github/ISSUE_TEMPLATE/14-feature-request-issue-template.yaml vendored
View File

@ -28,37 +28,33 @@ body:
- 'No'
validations:
required: false
- type: textarea
- type: input
id: behaviour
attributes:
label: Describe the feature and the current behaviour/state
render: shell
validations:
required: true
- type: textarea
- type: input
id: api_change
attributes:
label: Will this change the current API? How?
render: shell
validations:
required: false
- type: textarea
- type: input
id: benifit
attributes:
label: Who will benefit with this feature?
validations:
required: false
- type: textarea
- type: input
id: use_case
attributes:
label: Please specify the use cases for this feature
render: shell
validations:
required: true
- type: textarea
- type: input
id: info_other
attributes:
label: Any Other info
render: shell
validations:
required: false

3
.github/ISSUE_TEMPLATE/build.install_issue_template.yaml → .github/ISSUE_TEMPLATE/15-build-install-issue-template.yaml vendored
View File

@ -87,14 +87,13 @@ body:
placeholder:
validations:
required: false
- type: textarea
- type: input
id: what-happened
attributes:
label: Describe the problem
description: Provide the exact sequence of commands / steps that you executed before running into the [problem](https://google.github.io/mediapipe/getting_started/getting_started.html)
placeholder: Tell us what you see!
value: "A bug happened!"
render: shell
validations:
required: true
- type: textarea

6
.github/ISSUE_TEMPLATE/bug_issue_template.yaml → .github/ISSUE_TEMPLATE/16-bug-issue-template.yaml vendored
View File

@ -80,18 +80,16 @@ body:
label: Xcode & Tulsi version (if issue is related to building for iOS)
validations:
required: false
- type: textarea
- type: input
id: current_model
attributes:
label: Describe the actual behavior
render: shell
validations:
required: true
- type: textarea
- type: input
id: expected_model
attributes:
label: Describe the expected behaviour
render: shell
validations:
required: true
- type: textarea

0
.github/ISSUE_TEMPLATE/Documentation_issue_template.yaml → .github/ISSUE_TEMPLATE/17-documentation-issue-template.yaml vendored
View File

6
.github/ISSUE_TEMPLATE/Solution(Legacy_issue_template).yaml → .github/ISSUE_TEMPLATE/18-solution-legacy-issue-template.yaml vendored
View File

@ -48,18 +48,16 @@ body:
placeholder: e.g. C++, Python, Java
validations:
required: false
- type: textarea
- type: input
id: current_model
attributes:
label: Describe the actual behavior
render: shell
validations:
required: false
- type: textarea
- type: input
id: expected_model
attributes:
label: Describe the expected behaviour
render: shell
validations:
required: false
- type: textarea

0
.github/ISSUE_TEMPLATE/50-other-issues.md → .github/ISSUE_TEMPLATE/19-other-issues.md vendored
View File

34
.github/stale.yml vendored
View File

@ -1,34 +0,0 @@
# Copyright 2021 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
#
# This file was assembled from multiple pieces, whose use is documented
# throughout. Please refer to the TensorFlow dockerfiles documentation
# for more information.
# Number of days of inactivity before an Issue or Pull Request becomes stale
daysUntilStale: 7
# Number of days of inactivity before a stale Issue or Pull Request is closed
daysUntilClose: 7
# Only issues or pull requests with all of these labels are checked if stale. Defaults to `[]` (disabled)
onlyLabels:
- stat:awaiting response
# Comment to post when marking as stale. Set to `false` to disable
markComment: >
This issue has been automatically marked as stale because it has not had
recent activity. It will be closed if no further activity occurs. Thank you.
# Comment to post when removing the stale label. Set to `false` to disable
unmarkComment: false
closeComment: >
Closing as stale. Please reopen if you'd like to work on this further.

66
.github/workflows/stale.yaml vendored Normal file
View File

@ -0,0 +1,66 @@
# Copyright 2023 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# This workflow alerts and then closes the stale issues/PRs after specific time
# You can adjust the behavior by modifying this file.
# For more information, see:
# https://github.com/actions/stale
name: 'Close stale issues and PRs'
"on":
schedule:
- cron: "30 1 * * *"
permissions:
contents: read
issues: write
pull-requests: write
jobs:
stale:
runs-on: ubuntu-latest
steps:
- uses: 'actions/stale@v7'
with:
# Comma separated list of labels that can be assigned to issues to exclude them from being marked as stale.
exempt-issue-labels: 'override-stale'
# Comma separated list of labels that can be assigned to PRs to exclude them from being marked as stale.
exempt-pr-labels: "override-stale"
# Limit the No. of API calls in one run default value is 30.
operations-per-run: 500
# Prevent to remove stale label when PRs or issues are updated.
remove-stale-when-updated: false
# comment on issue if not active for more then 7 days.
stale-issue-message: 'This issue has been marked stale because it has no recent activity since 7 days. It will be closed if no further activity occurs. Thank you.'
# comment on PR if not active for more then 14 days.
stale-pr-message: 'This PR has been marked stale because it has no recent activity since 14 days. It will be closed if no further activity occurs. Thank you.'
# comment on issue if stale for more then 7 days.
close-issue-message: This issue was closed due to lack of activity after being marked stale for past 7 days.
# comment on PR if stale for more then 14 days.
close-pr-message: This PR was closed due to lack of activity after being marked stale for past 14 days.
# Number of days of inactivity before an Issue Request becomes stale
days-before-issue-stale: 7
# Number of days of inactivity before a stale Issue is closed
days-before-issue-close: 7
# reason for closed the issue default value is not_planned
close-issue-reason: completed
# Number of days of inactivity before a stale PR is closed
days-before-pr-close: 14
# Number of days of inactivity before an PR Request becomes stale
days-before-pr-stale: 14
# Check for label to stale or close the issue/PR
any-of-labels: 'stat:awaiting response'
# override stale to stalled for PR
stale-pr-label: 'stale'
# override stale to stalled for Issue
stale-issue-label: "stale"

200
README.md
View File

@ -1,99 +1,121 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe
title: Home
nav_order: 1
---
![MediaPipe](https://mediapipe.dev/images/mediapipe_small.png)
----
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
**Attention:** *We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
*This notice and web page will be removed on June 1, 2023.*
![MediaPipe](https://developers.google.com/static/mediapipe/images/home/hero_01_1920.png)
----
**Attention**: MediaPipe Solutions Preview is an early release. [Learn
more](https://developers.google.com/mediapipe/solutions/about#notice).
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
**On-device machine learning for everyone**
--------------------------------------------------------------------------------
Delight your customers with innovative machine learning features. MediaPipe
contains everything that you need to customize and deploy to mobile (Android,
iOS), web, desktop, edge devices, and IoT, effortlessly.
## Live ML anywhere
* [See demos](https://goo.gle/mediapipe-studio)
* [Learn more](https://developers.google.com/mediapipe/solutions)
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
ML solutions for live and streaming media.
## Get started
![accelerated.png](https://mediapipe.dev/images/accelerated_small.png) | ![cross_platform.png](https://mediapipe.dev/images/cross_platform_small.png)
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
![ready_to_use.png](https://mediapipe.dev/images/ready_to_use_small.png) | ![open_source.png](https://mediapipe.dev/images/open_source_small.png)
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
You can get started with MediaPipe Solutions by by checking out any of the
developer guides for
[vision](https://developers.google.com/mediapipe/solutions/vision/object_detector),
[text](https://developers.google.com/mediapipe/solutions/text/text_classifier),
and
[audio](https://developers.google.com/mediapipe/solutions/audio/audio_classifier)
tasks. If you need help setting up a development environment for use with
MediaPipe Tasks, check out the setup guides for
[Android](https://developers.google.com/mediapipe/solutions/setup_android), [web
apps](https://developers.google.com/mediapipe/solutions/setup_web), and
[Python](https://developers.google.com/mediapipe/solutions/setup_python).
----
## Solutions
## ML solutions in MediaPipe
MediaPipe Solutions provides a suite of libraries and tools for you to quickly
apply artificial intelligence (AI) and machine learning (ML) techniques in your
applications. You can plug these solutions into your applications immediately,
customize them to your needs, and use them across multiple development
platforms. MediaPipe Solutions is part of the MediaPipe [open source
project](https://github.com/google/mediapipe), so you can further customize the
solutions code to meet your application needs.
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :------:
[![face_detection](https://mediapipe.dev/images/mobile/face_detection_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/face_detection) | [![face_mesh](https://mediapipe.dev/images/mobile/face_mesh_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/face_mesh) | [![iris](https://mediapipe.dev/images/mobile/iris_tracking_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/iris) | [![hand](https://mediapipe.dev/images/mobile/hand_tracking_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/hands) | [![pose](https://mediapipe.dev/images/mobile/pose_tracking_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/pose) | [![hair_segmentation](https://mediapipe.dev/images/mobile/holistic_tracking_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/holistic)
These libraries and resources provide the core functionality for each MediaPipe
Solution:
Hair Segmentation | Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
[![hair_segmentation](https://mediapipe.dev/images/mobile/hair_segmentation_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/hair_segmentation) | [![object_detection](https://mediapipe.dev/images/mobile/object_detection_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/object_detection) | [![box_tracking](https://mediapipe.dev/images/mobile/object_tracking_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/box_tracking) | [![instant_motion_tracking](https://mediapipe.dev/images/mobile/instant_motion_tracking_android_small.gif)](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [![objectron](https://mediapipe.dev/images/mobile/objectron_chair_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/objectron) | [![knift](https://mediapipe.dev/images/mobile/template_matching_android_cpu_small.gif)](https://google.github.io/mediapipe/solutions/knift)
* **MediaPipe Tasks**: Cross-platform APIs and libraries for deploying
solutions. [Learn
more](https://developers.google.com/mediapipe/solutions/tasks).
* **MediaPipe models**: Pre-trained, ready-to-run models for use with each
solution.
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
These tools let you customize and evaluate solutions:
[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
[Holistic](https://google.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ |
[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ |
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | |
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
* **MediaPipe Model Maker**: Customize models for solutions with your data.
[Learn more](https://developers.google.com/mediapipe/solutions/model_maker).
* **MediaPipe Studio**: Visualize, evaluate, and benchmark solutions in your
browser. [Learn
more](https://developers.google.com/mediapipe/solutions/studio).
See also
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
for ML models released in MediaPipe.
### Legacy solutions
## Getting started
We have ended support for [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
as of March 1, 2023. All other MediaPipe Legacy Solutions will be upgraded to
a new MediaPipe Solution. See the [Solutions guide](https://developers.google.com/mediapipe/solutions/guide#legacy)
for details. The [code repository](https://github.com/google/mediapipe/tree/master/mediapipe)
and prebuilt binaries for all MediaPipe Legacy Solutions will continue to be
provided on an as-is basis.
To start using MediaPipe
[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few
lines code, see example code and demos in
[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and
[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript).
For more on the legacy solutions, see the [documentation](https://github.com/google/mediapipe/tree/master/docs/solutions).
To use MediaPipe in C++, Android and iOS, which allow further customization of
the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as
building your own, learn how to
[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and
start building example applications in
[C++](https://google.github.io/mediapipe/getting_started/cpp),
[Android](https://google.github.io/mediapipe/getting_started/android) and
[iOS](https://google.github.io/mediapipe/getting_started/ios).
## Framework
The source code is hosted in the
[MediaPipe Github repository](https://github.com/google/mediapipe), and you can
run code search using
[Google Open Source Code Search](https://cs.opensource.google/mediapipe/mediapipe).
To start using MediaPipe Framework, [install MediaPipe
Framework](https://developers.google.com/mediapipe/framework/getting_started/install)
and start building example applications in C++, Android, and iOS.
## Publications
[MediaPipe Framework](https://developers.google.com/mediapipe/framework) is the
low-level component used to build efficient on-device machine learning
pipelines, similar to the premade MediaPipe Solutions.
Before using MediaPipe Framework, familiarize yourself with the following key
[Framework
concepts](https://developers.google.com/mediapipe/framework/framework_concepts/overview.md):
* [Packets](https://developers.google.com/mediapipe/framework/framework_concepts/packets.md)
* [Graphs](https://developers.google.com/mediapipe/framework/framework_concepts/graphs.md)
* [Calculators](https://developers.google.com/mediapipe/framework/framework_concepts/calculators.md)
## Community
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe
users.
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
community discussion around MediaPipe.
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
curated list of awesome MediaPipe related frameworks, libraries and
software.
## Contributing
We welcome contributions. Please follow these
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
We use GitHub issues for tracking requests and bugs. Please post questions to
the MediaPipe Stack Overflow with a `mediapipe` tag.
## Resources
### Publications
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
in Google Developers Blog
@ -102,7 +124,8 @@ run code search using
* [SignAll SDK: Sign language interface using MediaPipe is now available for
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
in Google Developers Blog
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on
Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
in Google AI Blog
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
in Google AI Blog
@ -130,43 +153,6 @@ run code search using
in Google AI Blog
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
## Videos
### Videos
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
## Events
* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop,
Seoul, Korea ICCV
2019](https://research.google.com/youtube8m/workshop2019/index.html)
* [AI DevWorld 2019, 10 Oct 2019, San Jose, CA](https://aidevworld.com)
* [Google Industry Workshop at ICIP 2019, 24 Sept 2019, Taipei, Taiwan](http://2019.ieeeicip.org/?action=page4&id=14#Google)
([presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5))
* [Open sourced at CVPR 2019, 17~20 June, Long Beach, CA](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe)
## Community
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
curated list of awesome MediaPipe related frameworks, libraries and software
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
community discussion around MediaPipe
## Alpha disclaimer
MediaPipe is currently in alpha at v0.7. We may be still making breaking API
changes and expect to get to stable APIs by v1.0.
## Contributing
We welcome contributions. Please follow these
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
We use GitHub issues for tracking requests and bugs. Please post questions to
the MediaPipe Stack Overflow with a `mediapipe` tag.

62
WORKSPACE
View File

@ -45,12 +45,13 @@ http_archive(
)
http_archive(
name = "rules_foreign_cc",
strip_prefix = "rules_foreign_cc-0.1.0",
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/0.1.0.zip",
name = "rules_foreign_cc",
sha256 = "2a4d07cd64b0719b39a7c12218a3e507672b82a97b98c6a89d38565894cf7c51",
strip_prefix = "rules_foreign_cc-0.9.0",
url = "https://github.com/bazelbuild/rules_foreign_cc/archive/refs/tags/0.9.0.tar.gz",
)
load("@rules_foreign_cc//:workspace_definitions.bzl", "rules_foreign_cc_dependencies")
load("@rules_foreign_cc//foreign_cc:repositories.bzl", "rules_foreign_cc_dependencies")
rules_foreign_cc_dependencies()
@ -72,12 +73,9 @@ http_archive(
http_archive(
name = "zlib",
build_file = "@//third_party:zlib.BUILD",
sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
strip_prefix = "zlib-1.2.11",
urls = [
"http://mirror.bazel.build/zlib.net/fossils/zlib-1.2.11.tar.gz",
"http://zlib.net/fossils/zlib-1.2.11.tar.gz", # 2017-01-15
],
sha256 = "b3a24de97a8fdbc835b9833169501030b8977031bcb54b3b3ac13740f846ab30",
strip_prefix = "zlib-1.2.13",
url = "http://zlib.net/fossils/zlib-1.2.13.tar.gz",
patches = [
"@//third_party:zlib.diff",
],
@ -156,22 +154,22 @@ http_archive(
# 2020-08-21
http_archive(
name = "com_github_glog_glog",
strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
urls = [
"https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
"https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
],
)
http_archive(
name = "com_github_glog_glog_no_gflags",
strip_prefix = "glog-0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6",
sha256 = "58c9b3b6aaa4dd8b836c0fd8f65d0f941441fb95e27212c5eeb9979cfd3592ab",
strip_prefix = "glog-3a0d4d22c5ae0b9a2216988411cfa6bf860cc372",
sha256 = "170d08f80210b82d95563f4723a15095eff1aad1863000e8eeb569c96a98fefb",
build_file = "@//third_party:glog_no_gflags.BUILD",
urls = [
"https://github.com/google/glog/archive/0a2e5931bd5ff22fd3bf8999eb8ce776f159cda6.zip",
"https://github.com/google/glog/archive/3a0d4d22c5ae0b9a2216988411cfa6bf860cc372.zip",
],
patches = [
"@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff",
"@//third_party:com_github_glog_glog.diff",
],
patch_args = [
"-p1",
@ -266,10 +264,10 @@ http_archive(
http_archive(
name = "com_googlesource_code_re2",
sha256 = "e06b718c129f4019d6e7aa8b7631bee38d3d450dd980246bfaf493eb7db67868",
strip_prefix = "re2-fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8",
sha256 = "ef516fb84824a597c4d5d0d6d330daedb18363b5a99eda87d027e6bdd9cba299",
strip_prefix = "re2-03da4fc0857c285e3a26782f6bc8931c4c950df4",
urls = [
"https://github.com/google/re2/archive/fe4a310131c37f9a7e7f7816fa6ce2a8b27d65a8.tar.gz",
"https://github.com/google/re2/archive/03da4fc0857c285e3a26782f6bc8931c4c950df4.tar.gz",
],
)
@ -375,6 +373,22 @@ http_archive(
url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
)
# Building an opencv.xcframework from the OpenCV 4.5.3 sources is necessary for
# MediaPipe iOS Task Libraries to be supported on arm64(M1) Macs. An
# `opencv.xcframework` archive has not been released and it is recommended to
# build the same from source using a script provided in OpenCV 4.5.0 upwards.
# OpenCV is fixed to version to 4.5.3 since swift support can only be disabled
# from 4.5.3 upwards. This is needed to avoid errors when the library is linked
# in Xcode. Swift support will be added in when the final binary MediaPipe iOS
# Task libraries are built.
http_archive(
name = "ios_opencv_source",
sha256 = "a61e7a4618d353140c857f25843f39b2abe5f451b018aab1604ef0bc34cd23d5",
build_file = "@//third_party:opencv_ios_source.BUILD",
type = "zip",
url = "https://github.com/opencv/opencv/archive/refs/tags/4.5.3.zip",
)
http_archive(
name = "stblib",
strip_prefix = "stb-b42009b3b9d4ca35bc703f5310eedc74f584be58",
@ -468,9 +482,10 @@ http_archive(
)
# TensorFlow repo should always go after the other external dependencies.
# TF on 2023-04-12.
_TENSORFLOW_GIT_COMMIT = "d712c0c9e24519cc8cd3720279666720d1000eee"
_TENSORFLOW_SHA256 = "ba98de6ea5f720071246691a1536ecd5e1b1763033e8c82a1e721a06d3dfd4c1"
# TF on 2023-07-26.
_TENSORFLOW_GIT_COMMIT = "e92261fd4cec0b726692081c4d2966b75abf31dd"
# curl -L https://github.com/tensorflow/tensorflow/archive/<TENSORFLOW_GIT_COMMIT>.tar.gz | shasum -a 256
_TENSORFLOW_SHA256 = "478a229bd4ec70a5b568ac23b5ea013d9fca46a47d6c43e30365a0412b9febf4"
http_archive(
name = "org_tensorflow",
urls = [
@ -478,6 +493,7 @@ http_archive(
],
patches = [
"@//third_party:org_tensorflow_compatibility_fixes.diff",
"@//third_party:org_tensorflow_system_python.diff",
# Diff is generated with a script, don't update it manually.
"@//third_party:org_tensorflow_custom_ops.diff",
],

15
docs/build_java_api_docs.py
View File

@ -1,4 +1,4 @@
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -14,6 +14,7 @@
# ==============================================================================
"""Generate Java reference docs for MediaPipe."""
import pathlib
import shutil
from absl import app
from absl import flags
@ -41,7 +42,9 @@ def main(_) -> None:
mp_root = pathlib.Path(__file__)
while (mp_root := mp_root.parent).name != 'mediapipe':
# Find the nearest `mediapipe` dir.
pass
if not mp_root.name:
# We've hit the filesystem root - abort.
raise FileNotFoundError('"mediapipe" root not found')
# Find the root from which all packages are relative.
root = mp_root.parent
@ -51,6 +54,14 @@ def main(_) -> None:
if (mp_root / 'mediapipe').exists():
mp_root = mp_root / 'mediapipe'
# We need to copy this into the tasks dir to ensure we don't leave broken
# links in the generated docs.
old_api_dir = 'java/com/google/mediapipe/framework/image'
shutil.copytree(
mp_root / old_api_dir,
mp_root / 'tasks' / old_api_dir,
dirs_exist_ok=True)
gen_java.gen_java_docs(
package='com.google.mediapipe',
source_path=mp_root / 'tasks/java',

2
docs/build_model_maker_api_docs.py
View File

@ -1,4 +1,4 @@
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

2
docs/build_py_api_docs.py
View File

@ -1,4 +1,4 @@
# Copyright 2022 The MediaPipe Authors. All Rights Reserved.
# Copyright 2022 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

4
docs/getting_started/hello_world_cpp.md
View File

@ -50,7 +50,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.*
3. The [`hello world`] example uses a simple MediaPipe graph in the
`PrintHelloWorld()` function, defined in a [`CalculatorGraphConfig`] proto.
```C++
```c++
absl::Status PrintHelloWorld() {
// Configures a simple graph, which concatenates 2 PassThroughCalculators.
CalculatorGraphConfig config = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
@ -126,7 +126,7 @@ as the primary developer documentation site for MediaPipe as of April 3, 2023.*
```c++
mediapipe::Packet packet;
while (poller.Next(&packet)) {
LOG(INFO) << packet.Get<string>();
ABSL_LOG(INFO) << packet.Get<string>();
}
```

2
docs/getting_started/hello_world_ios.md
View File

@ -138,7 +138,7 @@ Create a `BUILD` file in the `$APPLICATION_PATH` and add the following build
rules:
```
MIN_IOS_VERSION = "11.0"
MIN_IOS_VERSION = "12.0"
load(
"@build_bazel_rules_apple//apple:ios.bzl",

200
docs/index.md
View File

@ -1,99 +1,121 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe
title: Home
nav_order: 1
---
![MediaPipe](https://mediapipe.dev/images/mediapipe_small.png)
----
**Attention:** *Thanks for your interest in MediaPipe! We have moved to
**Attention:** *We have moved to
[https://developers.google.com/mediapipe](https://developers.google.com/mediapipe)
as the primary developer documentation site for MediaPipe as of April 3, 2023.*
*This notice and web page will be removed on June 1, 2023.*
![MediaPipe](https://developers.google.com/static/mediapipe/images/home/hero_01_1920.png)
----
**Attention**: MediaPipe Solutions Preview is an early release. [Learn
more](https://developers.google.com/mediapipe/solutions/about#notice).
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
**On-device machine learning for everyone**
--------------------------------------------------------------------------------
Delight your customers with innovative machine learning features. MediaPipe
contains everything that you need to customize and deploy to mobile (Android,
iOS), web, desktop, edge devices, and IoT, effortlessly.
## Live ML anywhere
* [See demos](https://goo.gle/mediapipe-studio)
* [Learn more](https://developers.google.com/mediapipe/solutions)
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
ML solutions for live and streaming media.
## Get started
![accelerated.png](https://mediapipe.dev/images/accelerated_small.png) | ![cross_platform.png](https://mediapipe.dev/images/cross_platform_small.png)
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
![ready_to_use.png](https://mediapipe.dev/images/ready_to_use_small.png) | ![open_source.png](https://mediapipe.dev/images/open_source_small.png)
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
You can get started with MediaPipe Solutions by by checking out any of the
developer guides for
[vision](https://developers.google.com/mediapipe/solutions/vision/object_detector),
[text](https://developers.google.com/mediapipe/solutions/text/text_classifier),
and
[audio](https://developers.google.com/mediapipe/solutions/audio/audio_classifier)
tasks. If you need help setting up a development environment for use with
MediaPipe Tasks, check out the setup guides for
[Android](https://developers.google.com/mediapipe/solutions/setup_android), [web
apps](https://developers.google.com/mediapipe/solutions/setup_web), and
[Python](https://developers.google.com/mediapipe/solutions/setup_python).
----
## Solutions
## ML solutions in MediaPipe
MediaPipe Solutions provides a suite of libraries and tools for you to quickly
apply artificial intelligence (AI) and machine learning (ML) techniques in your
applications. You can plug these solutions into your applications immediately,
customize them to your needs, and use them across multiple development
platforms. MediaPipe Solutions is part of the MediaPipe [open source
project](https://github.com/google/mediapipe), so you can further customize the
solutions code to meet your application needs.
Face Detection | Face Mesh | Iris | Hands | Pose | Holistic
:----------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------: | :------:
[![face_detection](https://mediapipe.dev/images/mobile/face_detection_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/face_detection) | [![face_mesh](https://mediapipe.dev/images/mobile/face_mesh_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/face_mesh) | [![iris](https://mediapipe.dev/images/mobile/iris_tracking_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/iris) | [![hand](https://mediapipe.dev/images/mobile/hand_tracking_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/hands) | [![pose](https://mediapipe.dev/images/mobile/pose_tracking_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/pose) | [![hair_segmentation](https://mediapipe.dev/images/mobile/holistic_tracking_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/holistic)
These libraries and resources provide the core functionality for each MediaPipe
Solution:
Hair Segmentation | Object Detection | Box Tracking | Instant Motion Tracking | Objectron | KNIFT
:-------------------------------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------: | :---------------------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------: | :---:
[![hair_segmentation](https://mediapipe.dev/images/mobile/hair_segmentation_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/hair_segmentation) | [![object_detection](https://mediapipe.dev/images/mobile/object_detection_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/object_detection) | [![box_tracking](https://mediapipe.dev/images/mobile/object_tracking_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/box_tracking) | [![instant_motion_tracking](https://mediapipe.dev/images/mobile/instant_motion_tracking_android_small.gif)](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | [![objectron](https://mediapipe.dev/images/mobile/objectron_chair_android_gpu_small.gif)](https://google.github.io/mediapipe/solutions/objectron) | [![knift](https://mediapipe.dev/images/mobile/template_matching_android_cpu_small.gif)](https://google.github.io/mediapipe/solutions/knift)
* **MediaPipe Tasks**: Cross-platform APIs and libraries for deploying
solutions. [Learn
more](https://developers.google.com/mediapipe/solutions/tasks).
* **MediaPipe models**: Pre-trained, ready-to-run models for use with each
solution.
<!-- []() in the first cell is needed to preserve table formatting in GitHub Pages. -->
<!-- Whenever this table is updated, paste a copy to solutions/solutions.md. -->
These tools let you customize and evaluate solutions:
[]() | [Android](https://google.github.io/mediapipe/getting_started/android) | [iOS](https://google.github.io/mediapipe/getting_started/ios) | [C++](https://google.github.io/mediapipe/getting_started/cpp) | [Python](https://google.github.io/mediapipe/getting_started/python) | [JS](https://google.github.io/mediapipe/getting_started/javascript) | [Coral](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
:---------------------------------------------------------------------------------------- | :-------------------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------: | :-----------------------------------------------------------: | :-----------------------------------------------------------: | :--------------------------------------------------------------------:
[Face Detection](https://google.github.io/mediapipe/solutions/face_detection) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅
[Face Mesh](https://google.github.io/mediapipe/solutions/face_mesh) | ✅ | ✅ | ✅ | ✅ | ✅ |
[Iris](https://google.github.io/mediapipe/solutions/iris) | ✅ | ✅ | ✅ | | |
[Hands](https://google.github.io/mediapipe/solutions/hands) | ✅ | ✅ | ✅ | ✅ | ✅ |
[Pose](https://google.github.io/mediapipe/solutions/pose) | ✅ | ✅ | ✅ | ✅ | ✅ |
[Holistic](https://google.github.io/mediapipe/solutions/holistic) | ✅ | ✅ | ✅ | ✅ | ✅ |
[Selfie Segmentation](https://google.github.io/mediapipe/solutions/selfie_segmentation) | ✅ | ✅ | ✅ | ✅ | ✅ |
[Hair Segmentation](https://google.github.io/mediapipe/solutions/hair_segmentation) | ✅ | | ✅ | | |
[Object Detection](https://google.github.io/mediapipe/solutions/object_detection) | ✅ | ✅ | ✅ | | | ✅
[Box Tracking](https://google.github.io/mediapipe/solutions/box_tracking) | ✅ | ✅ | ✅ | | |
[Instant Motion Tracking](https://google.github.io/mediapipe/solutions/instant_motion_tracking) | ✅ | | | | |
[Objectron](https://google.github.io/mediapipe/solutions/objectron) | ✅ | | ✅ | ✅ | ✅ |
[KNIFT](https://google.github.io/mediapipe/solutions/knift) | ✅ | | | | |
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
* **MediaPipe Model Maker**: Customize models for solutions with your data.
[Learn more](https://developers.google.com/mediapipe/solutions/model_maker).
* **MediaPipe Studio**: Visualize, evaluate, and benchmark solutions in your
browser. [Learn
more](https://developers.google.com/mediapipe/solutions/studio).
See also
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
for ML models released in MediaPipe.
### Legacy solutions
## Getting started
We have ended support for [these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
as of March 1, 2023. All other MediaPipe Legacy Solutions will be upgraded to
a new MediaPipe Solution. See the [Solutions guide](https://developers.google.com/mediapipe/solutions/guide#legacy)
for details. The [code repository](https://github.com/google/mediapipe/tree/master/mediapipe)
and prebuilt binaries for all MediaPipe Legacy Solutions will continue to be
provided on an as-is basis.
To start using MediaPipe
[solutions](https://google.github.io/mediapipe/solutions/solutions) with only a few
lines code, see example code and demos in
[MediaPipe in Python](https://google.github.io/mediapipe/getting_started/python) and
[MediaPipe in JavaScript](https://google.github.io/mediapipe/getting_started/javascript).
For more on the legacy solutions, see the [documentation](https://github.com/google/mediapipe/tree/master/docs/solutions).
To use MediaPipe in C++, Android and iOS, which allow further customization of
the [solutions](https://google.github.io/mediapipe/solutions/solutions) as well as
building your own, learn how to
[install](https://google.github.io/mediapipe/getting_started/install) MediaPipe and
start building example applications in
[C++](https://google.github.io/mediapipe/getting_started/cpp),
[Android](https://google.github.io/mediapipe/getting_started/android) and
[iOS](https://google.github.io/mediapipe/getting_started/ios).
## Framework
The source code is hosted in the
[MediaPipe Github repository](https://github.com/google/mediapipe), and you can
run code search using
[Google Open Source Code Search](https://cs.opensource.google/mediapipe/mediapipe).
To start using MediaPipe Framework, [install MediaPipe
Framework](https://developers.google.com/mediapipe/framework/getting_started/install)
and start building example applications in C++, Android, and iOS.
## Publications
[MediaPipe Framework](https://developers.google.com/mediapipe/framework) is the
low-level component used to build efficient on-device machine learning
pipelines, similar to the premade MediaPipe Solutions.
Before using MediaPipe Framework, familiarize yourself with the following key
[Framework
concepts](https://developers.google.com/mediapipe/framework/framework_concepts/overview.md):
* [Packets](https://developers.google.com/mediapipe/framework/framework_concepts/packets.md)
* [Graphs](https://developers.google.com/mediapipe/framework/framework_concepts/graphs.md)
* [Calculators](https://developers.google.com/mediapipe/framework/framework_concepts/calculators.md)
## Community
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe
users.
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
community discussion around MediaPipe.
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
curated list of awesome MediaPipe related frameworks, libraries and
software.
## Contributing
We welcome contributions. Please follow these
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
We use GitHub issues for tracking requests and bugs. Please post questions to
the MediaPipe Stack Overflow with a `mediapipe` tag.
## Resources
### Publications
* [Bringing artworks to life with AR](https://developers.googleblog.com/2021/07/bringing-artworks-to-life-with-ar.html)
in Google Developers Blog
@ -102,7 +124,8 @@ run code search using
* [SignAll SDK: Sign language interface using MediaPipe is now available for
developers](https://developers.googleblog.com/2021/04/signall-sdk-sign-language-interface-using-mediapipe-now-available.html)
in Google Developers Blog
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
* [MediaPipe Holistic - Simultaneous Face, Hand and Pose Prediction, on
Device](https://ai.googleblog.com/2020/12/mediapipe-holistic-simultaneous-face.html)
in Google AI Blog
* [Background Features in Google Meet, Powered by Web ML](https://ai.googleblog.com/2020/10/background-features-in-google-meet.html)
in Google AI Blog
@ -130,43 +153,6 @@ run code search using
in Google AI Blog
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
## Videos
### Videos
* [YouTube Channel](https://www.youtube.com/c/MediaPipe)
## Events
* [MediaPipe Seattle Meetup, Google Building Waterside, 13 Feb 2020](https://mediapipe.page.link/seattle2020)
* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
* [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop,
Seoul, Korea ICCV
2019](https://research.google.com/youtube8m/workshop2019/index.html)
* [AI DevWorld 2019, 10 Oct 2019, San Jose, CA](https://aidevworld.com)
* [Google Industry Workshop at ICIP 2019, 24 Sept 2019, Taipei, Taiwan](http://2019.ieeeicip.org/?action=page4&id=14#Google)
([presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5))
* [Open sourced at CVPR 2019, 17~20 June, Long Beach, CA](https://sites.google.com/corp/view/perception-cv4arvr/mediapipe)
## Community
* [Awesome MediaPipe](https://mediapipe.page.link/awesome-mediapipe) - A
curated list of awesome MediaPipe related frameworks, libraries and software
* [Slack community](https://mediapipe.page.link/joinslack) for MediaPipe users
* [Discuss](https://groups.google.com/forum/#!forum/mediapipe) - General
community discussion around MediaPipe
## Alpha disclaimer
MediaPipe is currently in alpha at v0.7. We may be still making breaking API
changes and expect to get to stable APIs by v1.0.
## Contributing
We welcome contributions. Please follow these
[guidelines](https://github.com/google/mediapipe/blob/master/CONTRIBUTING.md).
We use GitHub issues for tracking requests and bugs. Please post questions to
the MediaPipe Stack Overflow with a `mediapipe` tag.

4
docs/solutions/face_detection.md
View File

@ -20,9 +20,9 @@ nav_order: 1
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
As of May 10, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_detector)
site.*
----

4
docs/solutions/face_mesh.md
View File

@ -20,9 +20,9 @@ nav_order: 2
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
As of May 10, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
site.*
----

4
docs/solutions/iris.md
View File

@ -20,9 +20,9 @@ nav_order: 3
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
As of May 10, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/face_landmarker)
site.*
----

4
docs/solutions/pose.md
View File

@ -22,9 +22,9 @@ nav_order: 5
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
As of May 10, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker)
site.*
----

2
docs/solutions/pose_classification.md
View File

@ -21,7 +21,7 @@ nav_order: 1
---
**Attention:** *Thank you for your interest in MediaPipe Solutions.
As of March 1, 2023, this solution is planned to be upgraded to a new MediaPipe
As of May 10, 2023, this solution was upgraded to a new MediaPipe
Solution. For more information, see the
[MediaPipe Solutions](https://developers.google.com/mediapipe/solutions/vision/pose_landmarker/)
site.*

14
docs/solutions/solutions.md
View File

@ -1,5 +1,6 @@
---
layout: default
layout: forward
target: https://developers.google.com/mediapipe/solutions/guide#legacy
title: MediaPipe Legacy Solutions
nav_order: 3
has_children: true
@ -13,8 +14,7 @@ has_toc: false
{:toc}
---
**Attention:** *Thank you for your interest in MediaPipe Solutions. We have
ended support for
**Attention:** *We have ended support for
[these MediaPipe Legacy Solutions](https://developers.google.com/mediapipe/solutions/guide#legacy)
as of March 1, 2023. All other
[MediaPipe Legacy Solutions will be upgraded](https://developers.google.com/mediapipe/solutions/guide#legacy)
@ -25,14 +25,6 @@ be provided on an as-is basis. We encourage you to check out the new MediaPipe
Solutions at:
[https://developers.google.com/mediapipe/solutions](https://developers.google.com/mediapipe/solutions)*
*This notice and web page will be removed on June 1, 2023.*
----
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
<br><br><br><br><br><br><br><br><br><br>
----
MediaPipe offers open source cross-platform, customizable ML solutions for live

229
mediapipe/BUILD
View File

@ -14,81 +14,155 @@
licenses(["notice"]) # Apache 2.0
# Note: yes, these need to use "//external:android/crosstool", not
# @androidndk//:default_crosstool.
load("@mediapipe//mediapipe:platforms.bzl", "config_setting_and_platform")
# Generic Android
config_setting(
name = "android",
values = {"crosstool_top": "//external:android/crosstool"},
constraint_values = [
"@platforms//os:android",
],
visibility = ["//visibility:public"],
)
config_setting(
# Android x86 32-bit.
config_setting_and_platform(
name = "android_x86",
values = {
"crosstool_top": "//external:android/crosstool",
"cpu": "x86",
},
constraint_values = [
"@platforms//os:android",
"@platforms//cpu:x86_32",
],
visibility = ["//visibility:public"],
)
config_setting(
# Android x86 64-bit.
config_setting_and_platform(
name = "android_x86_64",
values = {
"crosstool_top": "//external:android/crosstool",
"cpu": "x86_64",
},
constraint_values = [
"@platforms//os:android",
"@platforms//cpu:x86_64",
],
visibility = ["//visibility:public"],
)
config_setting(
name = "android_armeabi",
values = {
"crosstool_top": "//external:android/crosstool",
"cpu": "armeabi",
},
visibility = ["//visibility:public"],
)
config_setting(
# Android ARMv7.
config_setting_and_platform(
name = "android_arm",
values = {
"crosstool_top": "//external:android/crosstool",
"cpu": "armeabi-v7a",
},
constraint_values = [
"@platforms//os:android",
"@platforms//cpu:armv7",
],
visibility = ["//visibility:public"],
)
config_setting(
# Android ARM64.
config_setting_and_platform(
name = "android_arm64",
values = {
"crosstool_top": "//external:android/crosstool",
"cpu": "arm64-v8a",
},
constraint_values = [
"@platforms//os:android",
"@platforms//cpu:arm64",
],
visibility = ["//visibility:public"],
)
# Note: this cannot just match "apple_platform_type": "macos" because that option
# defaults to "macos" even when building on Linux!
alias(
# Generic MacOS.
config_setting(
name = "macos",
actual = select({
":macos_i386": ":macos_i386",
":macos_x86_64": ":macos_x86_64",
":macos_arm64": ":macos_arm64",
"//conditions:default": ":macos_i386", # Arbitrarily chosen from above.
}),
constraint_values = [
"@platforms//os:macos",
],
visibility = ["//visibility:public"],
)
# Note: this also matches on crosstool_top so that it does not produce ambiguous
# selectors when used together with "android".
# MacOS x86 64-bit.
config_setting_and_platform(
name = "macos_x86_64",
constraint_values = [
"@platforms//os:macos",
"@platforms//cpu:x86_64",
],
visibility = ["//visibility:public"],
)
# MacOS ARM64.
config_setting_and_platform(
name = "macos_arm64",
constraint_values = [
"@platforms//os:macos",
"@platforms//cpu:arm64",
],
visibility = ["//visibility:public"],
)
# Generic iOS.
config_setting(
name = "ios",
values = {
"crosstool_top": "@bazel_tools//tools/cpp:toolchain",
"apple_platform_type": "ios",
},
constraint_values = [
"@platforms//os:ios",
],
visibility = ["//visibility:public"],
)
# iOS device ARM32.
config_setting_and_platform(
name = "ios_armv7",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:arm",
],
visibility = ["//visibility:public"],
)
# iOS device ARM64.
config_setting_and_platform(
name = "ios_arm64",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:arm64",
],
visibility = ["//visibility:public"],
)
# iOS device ARM64E.
config_setting_and_platform(
name = "ios_arm64e",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:arm64e",
],
visibility = ["//visibility:public"],
)
# iOS simulator x86 32-bit.
config_setting_and_platform(
name = "ios_i386",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:x86_32",
"@build_bazel_apple_support//constraints:simulator",
],
visibility = ["//visibility:public"],
)
# iOS simulator x86 64-bit.
config_setting_and_platform(
name = "ios_x86_64",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:x86_64",
"@build_bazel_apple_support//constraints:simulator",
],
visibility = ["//visibility:public"],
)
# iOS simulator ARM64.
config_setting_and_platform(
name = "ios_sim_arm64",
constraint_values = [
"@platforms//os:ios",
"@platforms//cpu:arm64",
"@build_bazel_apple_support//constraints:simulator",
],
visibility = ["//visibility:public"],
)
@ -102,51 +176,24 @@ alias(
visibility = ["//visibility:public"],
)
config_setting(
name = "macos_i386",
values = {
"apple_platform_type": "macos",
"cpu": "darwin",
},
visibility = ["//visibility:public"],
)
config_setting(
name = "macos_x86_64",
values = {
"apple_platform_type": "macos",
"cpu": "darwin_x86_64",
},
visibility = ["//visibility:public"],
)
config_setting(
name = "macos_arm64",
values = {
"apple_platform_type": "macos",
"cpu": "darwin_arm64",
},
visibility = ["//visibility:public"],
)
[
config_setting(
name = arch,
values = {"cpu": arch},
visibility = ["//visibility:public"],
)
for arch in [
"ios_i386",
"ios_x86_64",
"ios_armv7",
"ios_arm64",
"ios_arm64e",
]
]
config_setting(
# Windows 64-bit.
config_setting_and_platform(
name = "windows",
values = {"cpu": "x64_windows"},
constraint_values = [
"@platforms//os:windows",
"@platforms//cpu:x86_64",
],
visibility = ["//visibility:public"],
)
# Linux 64-bit.
config_setting_and_platform(
name = "linux",
constraint_values = [
"@platforms//os:linux",
"@platforms//cpu:x86_64",
],
visibility = ["//visibility:public"],
)
exports_files(

28
mediapipe/calculators/audio/BUILD
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Placeholder: load py_proto_library
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
licenses(["notice"])
@ -145,6 +146,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings",
"@com_google_audio_tools//audio/dsp/mfcc",
"@eigen_archive//:eigen3",
@ -163,8 +165,9 @@ cc_library(
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:logging",
"//mediapipe/util:time_series_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
"@com_google_audio_tools//audio/dsp:resampler",
"@com_google_audio_tools//audio/dsp:resampler_q",
@ -185,6 +188,7 @@ cc_library(
"//mediapipe/framework/port:core_proto",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_util",
"@com_google_absl//absl/log:absl_check",
],
alwayslink = 1,
)
@ -219,13 +223,12 @@ cc_library(
deps = [
":time_series_framer_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_audio_tools//audio/dsp:window_functions",
"@eigen_archive//:eigen3",
],
@ -296,6 +299,7 @@ cc_test(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_test_util",
"@com_google_absl//absl/log:absl_log",
"@com_google_audio_tools//audio/dsp:number_util",
"@eigen_archive//:eigen3",
],
@ -319,6 +323,21 @@ cc_test(
],
)
cc_binary(
name = "time_series_framer_calculator_benchmark",
srcs = ["time_series_framer_calculator_benchmark.cc"],
deps = [
":time_series_framer_calculator",
":time_series_framer_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:time_series_header_cc_proto",
"@com_google_absl//absl/log:absl_check",
"@com_google_benchmark//:benchmark",
],
)
cc_test(
name = "time_series_framer_calculator_test",
srcs = ["time_series_framer_calculator_test.cc"],
@ -333,6 +352,7 @@ cc_test(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_test_util",
"@com_google_absl//absl/log:absl_log",
"@com_google_audio_tools//audio/dsp:window_functions",
"@eigen_archive//:eigen3",
],

3
mediapipe/calculators/audio/mfcc_mel_calculators.cc
View File

@ -23,6 +23,7 @@
#include <vector>
#include "Eigen/Core"
#include "absl/log/absl_check.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/string_view.h"
#include "absl/strings/substitute.h"
@ -138,7 +139,7 @@ absl::Status FramewiseTransformCalculatorBase::Process(CalculatorContext* cc) {
TransformFrame(input_frame, &output_frame);
// Copy output from vector<float> to Eigen::Vector.
CHECK_EQ(output_frame.size(), num_output_channels_);
ABSL_CHECK_EQ(output_frame.size(), num_output_channels_);
Eigen::Map<const Eigen::MatrixXd> output_frame_map(&output_frame[0],
output_frame.size(), 1);
output->col(frame) = output_frame_map.cast<float>();

8
mediapipe/calculators/audio/rational_factor_resample_calculator.cc
View File

@ -16,6 +16,8 @@
#include "mediapipe/calculators/audio/rational_factor_resample_calculator.h"
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "audio/dsp/resampler_q.h"
using audio_dsp::Resampler;
@ -45,9 +47,9 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
if (matrix->cols() == 0) {
matrix->resize(matrix->rows(), vec.size());
} else {
CHECK_EQ(vec.size(), matrix->cols());
ABSL_CHECK_EQ(vec.size(), matrix->cols());
}
CHECK_LT(channel, matrix->rows());
ABSL_CHECK_LT(channel, matrix->rows());
matrix->row(channel) =
Eigen::Map<const Eigen::ArrayXf>(vec.data(), vec.size());
}
@ -77,7 +79,7 @@ absl::Status RationalFactorResampleCalculator::Open(CalculatorContext* cc) {
r = ResamplerFromOptions(source_sample_rate_, target_sample_rate_,
resample_options);
if (!r) {
LOG(ERROR) << "Failed to initialize resampler.";
ABSL_LOG(ERROR) << "Failed to initialize resampler.";
return absl::UnknownError("Failed to initialize resampler.");
}
}

1
mediapipe/calculators/audio/rational_factor_resample_calculator.h
View File

@ -27,7 +27,6 @@
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/time_series_header.pb.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/util/time_series_util.h"
namespace mediapipe {

51
mediapipe/calculators/audio/spectrogram_calculator.cc
View File

@ -210,6 +210,23 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
// Factor to convert ln(SQUARED_MAGNITUDE) to deciBels = 10.0/ln(10.0).
const float SpectrogramCalculator::kLnSquaredMagnitudeToDb = 4.342944819032518;
namespace {
std::unique_ptr<audio_dsp::WindowFunction> MakeWindowFun(
const SpectrogramCalculatorOptions::WindowType window_type) {
switch (window_type) {
// The cosine window and square root of Hann are equivalent.
case SpectrogramCalculatorOptions::COSINE:
case SpectrogramCalculatorOptions::SQRT_HANN:
return std::make_unique<audio_dsp::CosineWindow>();
case SpectrogramCalculatorOptions::HANN:
return std::make_unique<audio_dsp::HannWindow>();
case SpectrogramCalculatorOptions::HAMMING:
return std::make_unique<audio_dsp::HammingWindow>();
}
return nullptr;
}
} // namespace
absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
SpectrogramCalculatorOptions spectrogram_options =
cc->Options<SpectrogramCalculatorOptions>();
@ -266,28 +283,14 @@ absl::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
output_scale_ = spectrogram_options.output_scale();
std::vector<double> window;
switch (spectrogram_options.window_type()) {
case SpectrogramCalculatorOptions::COSINE:
audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
&window);
break;
case SpectrogramCalculatorOptions::HANN:
audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
&window);
break;
case SpectrogramCalculatorOptions::HAMMING:
audio_dsp::HammingWindow().GetPeriodicSamples(frame_duration_samples_,
&window);
break;
case SpectrogramCalculatorOptions::SQRT_HANN: {
audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
&window);
absl::c_transform(window, window.begin(),
[](double x) { return std::sqrt(x); });
break;
}
auto window_fun = MakeWindowFun(spectrogram_options.window_type());
if (window_fun == nullptr) {
return absl::Status(absl::StatusCode::kInvalidArgument,
absl::StrCat("Invalid window type ",
spectrogram_options.window_type()));
}
std::vector<double> window;
window_fun->GetPeriodicSamples(frame_duration_samples_, &window);
// Propagate settings down to the actual Spectrogram object.
spectrogram_generators_.clear();
@ -433,9 +436,9 @@ absl::Status SpectrogramCalculator::ProcessVectorToOutput(
absl::Status SpectrogramCalculator::ProcessVector(const Matrix& input_stream,
CalculatorContext* cc) {
switch (output_type_) {
// These blocks deliberately ignore clang-format to preserve the
// "silhouette" of the different cases.
// clang-format off
// These blocks deliberately ignore clang-format to preserve the
// "silhouette" of the different cases.
// clang-format off
case SpectrogramCalculatorOptions::COMPLEX: {
return ProcessVectorToOutput(
input_stream,

2
mediapipe/calculators/audio/spectrogram_calculator.proto
View File

@ -68,7 +68,7 @@ message SpectrogramCalculatorOptions {
HANN = 0;
HAMMING = 1;
COSINE = 2;
SQRT_HANN = 4;
SQRT_HANN = 4; // Alias of COSINE.
}
optional WindowType window_type = 6 [default = HANN];

11
mediapipe/calculators/audio/spectrogram_calculator_test.cc
View File

@ -22,6 +22,7 @@
#include <vector>
#include "Eigen/Core"
#include "absl/log/absl_log.h"
#include "audio/dsp/number_util.h"
#include "mediapipe/calculators/audio/spectrogram_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@ -882,11 +883,11 @@ void BM_ProcessDC(benchmark::State& state) {
const CalculatorRunner::StreamContents& output = runner.Outputs().Index(0);
const Matrix& output_matrix = output.packets[0].Get<Matrix>();
LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x"
<< output_matrix.cols();
LOG(INFO) << "First values=" << output_matrix(0, 0) << ", "
<< output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", "
<< output_matrix(3, 0);
ABSL_LOG(INFO) << "Output matrix=" << output_matrix.rows() << "x"
<< output_matrix.cols();
ABSL_LOG(INFO) << "First values=" << output_matrix(0, 0) << ", "
<< output_matrix(1, 0) << ", " << output_matrix(2, 0) << ", "
<< output_matrix(3, 0);
}
BENCHMARK(BM_ProcessDC);

3
mediapipe/calculators/audio/stabilized_log_calculator.cc
View File

@ -18,6 +18,7 @@
#include <memory>
#include <string>
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h"
@ -59,7 +60,7 @@ class StabilizedLogCalculator : public CalculatorBase {
output_scale_ = stabilized_log_calculator_options.output_scale();
check_nonnegativity_ =
stabilized_log_calculator_options.check_nonnegativity();
CHECK_GE(stabilizer_, 0.0)
ABSL_CHECK_GE(stabilizer_, 0.0)
<< "stabilizer must be >= 0.0, received a value of " << stabilizer_;
// If the input packets have a header, propagate the header to the output.

250
mediapipe/calculators/audio/time_series_framer_calculator.cc
View File

@ -15,19 +15,17 @@
// Defines TimeSeriesFramerCalculator.
#include <math.h>
#include <deque>
#include <memory>
#include <string>
#include <vector>
#include "Eigen/Core"
#include "absl/log/absl_check.h"
#include "audio/dsp/window_functions.h"
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/time_series_header.pb.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/timestamp.h"
#include "mediapipe/util/time_series_util.h"
namespace mediapipe {
@ -88,11 +86,6 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
absl::Status Close(CalculatorContext* cc) override;
private:
// Adds input data to the internal buffer.
void EnqueueInput(CalculatorContext* cc);
// Constructs and emits framed output packets.
void FrameOutput(CalculatorContext* cc);
Timestamp CurrentOutputTimestamp() {
if (use_local_timestamp_) {
return current_timestamp_;
@ -106,21 +99,13 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
Timestamp::kTimestampUnitsPerSecond);
}
// Returns the timestamp of a sample on a base, which is usually the time
// stamp of a packet.
Timestamp CurrentSampleTimestamp(const Timestamp& timestamp_base,
int64_t number_of_samples) {
return timestamp_base + round(number_of_samples / sample_rate_ *
Timestamp::kTimestampUnitsPerSecond);
}
// The number of input samples to advance after the current output frame is
// emitted.
int next_frame_step_samples() const {
// All numbers are in input samples.
const int64_t current_output_frame_start = static_cast<int64_t>(
round(cumulative_output_frames_ * average_frame_step_samples_));
CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
ABSL_CHECK_EQ(current_output_frame_start, cumulative_completed_samples_);
const int64_t next_output_frame_start = static_cast<int64_t>(
round((cumulative_output_frames_ + 1) * average_frame_step_samples_));
return next_output_frame_start - current_output_frame_start;
@ -142,61 +127,174 @@ class TimeSeriesFramerCalculator : public CalculatorBase {
Timestamp initial_input_timestamp_;
// The current timestamp is updated along with the incoming packets.
Timestamp current_timestamp_;
int num_channels_;
// Each entry in this deque consists of a single sample, i.e. a
// single column vector, and its timestamp.
std::deque<std::pair<Matrix, Timestamp>> sample_buffer_;
// Samples are buffered in a vector of sample blocks.
class SampleBlockBuffer {
public:
// Initializes the buffer.
void Init(double sample_rate, int num_channels) {
ts_units_per_sample_ = Timestamp::kTimestampUnitsPerSecond / sample_rate;
num_channels_ = num_channels;
num_samples_ = 0;
first_block_offset_ = 0;
}
// Number of channels, equal to the number of rows in each Matrix.
int num_channels() const { return num_channels_; }
// Total number of available samples over all blocks.
int num_samples() const { return num_samples_; }
// Pushes a new block of samples on the back of the buffer with `timestamp`
// being the input timestamp of the packet containing the Matrix.
void Push(const Matrix& samples, Timestamp timestamp);
// Copies `count` samples from the front of the buffer. If there are fewer
// samples than this, the result is zero padded to have `count` samples.
// The timestamp of the last copied sample is written to *last_timestamp.
// This output is used below to update `current_timestamp_`, which is only
// used when `use_local_timestamp` is true.
Matrix CopySamples(int count, Timestamp* last_timestamp) const;
// Drops `count` samples from the front of the buffer. If `count` exceeds
// `num_samples()`, the buffer is emptied. Returns how many samples were
// dropped.
int DropSamples(int count);
private:
struct Block {
// Matrix of num_channels rows by num_samples columns, a block of possibly
// multiple samples.
Matrix samples;
// Timestamp of the first sample in the Block. This comes from the input
// packet's timestamp that contains this Matrix.
Timestamp timestamp;
Block() : timestamp(Timestamp::Unstarted()) {}
Block(const Matrix& samples, Timestamp timestamp)
: samples(samples), timestamp(timestamp) {}
int num_samples() const { return samples.cols(); }
};
std::vector<Block> blocks_;
// Number of timestamp units per sample. Used to compute timestamps as
// nth sample timestamp = base_timestamp + round(ts_units_per_sample_ * n).
double ts_units_per_sample_;
// Number of rows in each Matrix.
int num_channels_;
// The total number of samples over all blocks, equal to
// (sum_i blocks_[i].num_samples()) - first_block_offset_.
int num_samples_;
// The number of samples in the first block that have been discarded. This
// way we can cheaply represent "partially discarding" a block.
int first_block_offset_;
} sample_buffer_;
bool use_window_;
Matrix window_;
Eigen::RowVectorXf window_;
bool use_local_timestamp_;
};
REGISTER_CALCULATOR(TimeSeriesFramerCalculator);
void TimeSeriesFramerCalculator::EnqueueInput(CalculatorContext* cc) {
const Matrix& input_frame = cc->Inputs().Index(0).Get<Matrix>();
for (int i = 0; i < input_frame.cols(); ++i) {
sample_buffer_.emplace_back(std::make_pair(
input_frame.col(i), CurrentSampleTimestamp(cc->InputTimestamp(), i)));
}
void TimeSeriesFramerCalculator::SampleBlockBuffer::Push(const Matrix& samples,
Timestamp timestamp) {
num_samples_ += samples.cols();
blocks_.emplace_back(samples, timestamp);
}
void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
while (sample_buffer_.size() >=
Matrix TimeSeriesFramerCalculator::SampleBlockBuffer::CopySamples(
int count, Timestamp* last_timestamp) const {
Matrix copied(num_channels_, count);
if (!blocks_.empty()) {
int num_copied = 0;
// First block has an offset for samples that have been discarded.
int offset = first_block_offset_;
int n;
Timestamp last_block_ts;
int last_sample_index;
for (auto it = blocks_.begin(); it != blocks_.end() && count > 0; ++it) {
n = std::min(it->num_samples() - offset, count);
// Copy `n` samples from the next block.
copied.middleCols(num_copied, n) = it->samples.middleCols(offset, n);
count -= n;
num_copied += n;
last_block_ts = it->timestamp;
last_sample_index = offset + n - 1;
offset = 0; // No samples have been discarded in subsequent blocks.
}
// Compute the timestamp of the last copied sample.
*last_timestamp =
last_block_ts + std::round(ts_units_per_sample_ * last_sample_index);
}
if (count > 0) {
copied.rightCols(count).setZero(); // Zero pad if needed.
}
return copied;
}
int TimeSeriesFramerCalculator::SampleBlockBuffer::DropSamples(int count) {
if (blocks_.empty()) {
return 0;
}
auto block_it = blocks_.begin();
if (first_block_offset_ + count < block_it->num_samples()) {
// `count` is less than the remaining samples in the first block.
first_block_offset_ += count;
num_samples_ -= count;
return count;
}
int num_samples_dropped = block_it->num_samples() - first_block_offset_;
count -= num_samples_dropped;
first_block_offset_ = 0;
for (++block_it; block_it != blocks_.end(); ++block_it) {
if (block_it->num_samples() > count) {
break;
}
num_samples_dropped += block_it->num_samples();
count -= block_it->num_samples();
}
blocks_.erase(blocks_.begin(), block_it); // Drop whole blocks.
if (!blocks_.empty()) {
first_block_offset_ = count; // Drop part of the next block.
num_samples_dropped += count;
}
num_samples_ -= num_samples_dropped;
return num_samples_dropped;
}
absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
if (initial_input_timestamp_ == Timestamp::Unstarted()) {
initial_input_timestamp_ = cc->InputTimestamp();
current_timestamp_ = initial_input_timestamp_;
}
// Add input data to the internal buffer.
sample_buffer_.Push(cc->Inputs().Index(0).Get<Matrix>(),
cc->InputTimestamp());
// Construct and emit framed output packets.
while (sample_buffer_.num_samples() >=
frame_duration_samples_ + samples_still_to_drop_) {
while (samples_still_to_drop_ > 0) {
sample_buffer_.pop_front();
--samples_still_to_drop_;
}
sample_buffer_.DropSamples(samples_still_to_drop_);
Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
&current_timestamp_);
const int frame_step_samples = next_frame_step_samples();
std::unique_ptr<Matrix> output_frame(
new Matrix(num_channels_, frame_duration_samples_));
for (int i = 0; i < std::min(frame_step_samples, frame_duration_samples_);
++i) {
output_frame->col(i) = sample_buffer_.front().first;
current_timestamp_ = sample_buffer_.front().second;
sample_buffer_.pop_front();
}
const int frame_overlap_samples =
frame_duration_samples_ - frame_step_samples;
if (frame_overlap_samples > 0) {
for (int i = 0; i < frame_overlap_samples; ++i) {
output_frame->col(i + frame_step_samples) = sample_buffer_[i].first;
current_timestamp_ = sample_buffer_[i].second;
}
} else {
samples_still_to_drop_ = -frame_overlap_samples;
}
samples_still_to_drop_ = frame_step_samples;
if (use_window_) {
*output_frame = (output_frame->array() * window_.array()).matrix();
// Apply the window to each row of output_frame.
output_frame.array().rowwise() *= window_.array();
}
cc->Outputs().Index(0).Add(output_frame.release(),
CurrentOutputTimestamp());
cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
.At(CurrentOutputTimestamp()));
++cumulative_output_frames_;
cumulative_completed_samples_ += frame_step_samples;
}
@ -206,35 +304,18 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
// fact to enable packet queueing optimizations.
cc->Outputs().Index(0).SetNextTimestampBound(CumulativeOutputTimestamp());
}
}
absl::Status TimeSeriesFramerCalculator::Process(CalculatorContext* cc) {
if (initial_input_timestamp_ == Timestamp::Unstarted()) {
initial_input_timestamp_ = cc->InputTimestamp();
current_timestamp_ = initial_input_timestamp_;
}
EnqueueInput(cc);
FrameOutput(cc);
return absl::OkStatus();
}
absl::Status TimeSeriesFramerCalculator::Close(CalculatorContext* cc) {
while (samples_still_to_drop_ > 0 && !sample_buffer_.empty()) {
sample_buffer_.pop_front();
--samples_still_to_drop_;
}
if (!sample_buffer_.empty() && pad_final_packet_) {
std::unique_ptr<Matrix> output_frame(new Matrix);
output_frame->setZero(num_channels_, frame_duration_samples_);
for (int i = 0; i < sample_buffer_.size(); ++i) {
output_frame->col(i) = sample_buffer_[i].first;
current_timestamp_ = sample_buffer_[i].second;
}
sample_buffer_.DropSamples(samples_still_to_drop_);
cc->Outputs().Index(0).Add(output_frame.release(),
CurrentOutputTimestamp());
if (sample_buffer_.num_samples() > 0 && pad_final_packet_) {
Matrix output_frame = sample_buffer_.CopySamples(frame_duration_samples_,
&current_timestamp_);
cc->Outputs().Index(0).AddPacket(MakePacket<Matrix>(std::move(output_frame))
.At(CurrentOutputTimestamp()));
}
return absl::OkStatus();
@ -258,7 +339,7 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
cc->Inputs().Index(0).Header(), &input_header));
sample_rate_ = input_header.sample_rate();
num_channels_ = input_header.num_channels();
sample_buffer_.Init(sample_rate_, input_header.num_channels());
frame_duration_samples_ = time_series_util::SecondsToSamples(
framer_options.frame_duration_seconds(), sample_rate_);
RET_CHECK_GT(frame_duration_samples_, 0)
@ -312,9 +393,8 @@ absl::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
}
if (use_window_) {
window_ = Matrix::Ones(num_channels_, 1) *
Eigen::Map<Eigen::MatrixXd>(window_vector.data(), 1,
frame_duration_samples_)
window_ = Eigen::Map<Eigen::RowVectorXd>(window_vector.data(),
frame_duration_samples_)
.cast<float>();
}
use_local_timestamp_ = framer_options.use_local_timestamp();

93
mediapipe/calculators/audio/time_series_framer_calculator_benchmark.cc Normal file
View File

@ -0,0 +1,93 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Benchmark for TimeSeriesFramerCalculator.
#include <memory>
#include <random>
#include <vector>
#include "absl/log/absl_check.h"
#include "benchmark/benchmark.h"
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/time_series_header.pb.h"
#include "mediapipe/framework/packet.h"
using ::mediapipe::Matrix;
void BM_TimeSeriesFramerCalculator(benchmark::State& state) {
constexpr float kSampleRate = 32000.0;
constexpr int kNumChannels = 2;
constexpr int kFrameDurationSeconds = 5.0;
std::mt19937 rng(0 /*seed*/);
// Input around a half second's worth of samples at a time.
std::uniform_int_distribution<int> input_size_dist(15000, 17000);
// Generate a pool of random blocks of samples up front.
std::vector<Matrix> sample_pool;
sample_pool.reserve(20);
for (int i = 0; i < 20; ++i) {
sample_pool.push_back(Matrix::Random(kNumChannels, input_size_dist(rng)));
}
std::uniform_int_distribution<int> pool_index_dist(0, sample_pool.size() - 1);
mediapipe::CalculatorGraphConfig config;
config.add_input_stream("input");
config.add_output_stream("output");
auto* node = config.add_node();
node->set_calculator("TimeSeriesFramerCalculator");
node->add_input_stream("input");
node->add_output_stream("output");
mediapipe::TimeSeriesFramerCalculatorOptions* options =
node->mutable_options()->MutableExtension(
mediapipe::TimeSeriesFramerCalculatorOptions::ext);
options->set_frame_duration_seconds(kFrameDurationSeconds);
for (auto _ : state) {
state.PauseTiming(); // Pause benchmark timing.
// Prepare input packets of random blocks of samples.
std::vector<mediapipe::Packet> input_packets;
input_packets.reserve(32);
float t = 0;
for (int i = 0; i < 32; ++i) {
auto samples =
std::make_unique<Matrix>(sample_pool[pool_index_dist(rng)]);
const int num_samples = samples->cols();
input_packets.push_back(mediapipe::Adopt(samples.release())
.At(mediapipe::Timestamp::FromSeconds(t)));
t += num_samples / kSampleRate;
}
// Initialize graph.
mediapipe::CalculatorGraph graph;
ABSL_CHECK_OK(graph.Initialize(config));
// Prepare input header.
auto header = std::make_unique<mediapipe::TimeSeriesHeader>();
header->set_sample_rate(kSampleRate);
header->set_num_channels(kNumChannels);
state.ResumeTiming(); // Resume benchmark timing.
ABSL_CHECK_OK(graph.StartRun({}, {{"input", Adopt(header.release())}}));
for (auto& packet : input_packets) {
ABSL_CHECK_OK(graph.AddPacketToInputStream("input", packet));
}
ABSL_CHECK(!graph.HasError());
ABSL_CHECK_OK(graph.CloseAllInputStreams());
ABSL_CHECK_OK(graph.WaitUntilIdle());
}
}
BENCHMARK(BM_TimeSeriesFramerCalculator);
BENCHMARK_MAIN();

12
mediapipe/calculators/audio/time_series_framer_calculator_test.cc
View File

@ -19,6 +19,7 @@
#include <vector>
#include "Eigen/Core"
#include "absl/log/absl_log.h"
#include "audio/dsp/window_functions.h"
#include "mediapipe/calculators/audio/time_series_framer_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@ -186,11 +187,12 @@ class TimeSeriesFramerCalculatorTest
const int num_unique_output_samples =
round((output().packets.size() - 1) * frame_step_samples) +
frame_duration_samples;
LOG(INFO) << "packets.size()=" << output().packets.size()
<< " frame_duration_samples=" << frame_duration_samples
<< " frame_step_samples=" << frame_step_samples
<< " num_input_samples_=" << num_input_samples_
<< " num_unique_output_samples=" << num_unique_output_samples;
ABSL_LOG(INFO) << "packets.size()=" << output().packets.size()
<< " frame_duration_samples=" << frame_duration_samples
<< " frame_step_samples=" << frame_step_samples
<< " num_input_samples_=" << num_input_samples_
<< " num_unique_output_samples="
<< num_unique_output_samples;
const int num_padding_samples =
num_unique_output_samples - num_input_samples_;
if (options_.pad_final_packet()) {

42
mediapipe/calculators/core/BUILD
View File

@ -117,6 +117,7 @@ mediapipe_proto_library(
"//mediapipe/framework:calculator_proto",
"//mediapipe/framework/formats:classification_proto",
"//mediapipe/framework/formats:landmark_proto",
"//mediapipe/framework/formats:matrix_data_proto",
"//mediapipe/framework/formats:time_series_header_proto",
],
)
@ -192,17 +193,19 @@ cc_library(
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_contract",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework:packet",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:gpu_buffer",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
],
alwayslink = 1,
)
@ -215,18 +218,18 @@ cc_library(
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_contract",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/util:render_data_cc_proto",
"@com_google_absl//absl/status",
"@org_tensorflow//tensorflow/lite:framework",
],
alwayslink = 1,
@ -287,6 +290,7 @@ cc_library(
"//mediapipe/framework/api2:node",
"//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:integral_types",
@ -295,8 +299,7 @@ cc_library(
"//mediapipe/util:render_data_cc_proto",
"@org_tensorflow//tensorflow/lite:framework",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//mediapipe:ios": [],
":ios_or_disable_gpu": [],
"//conditions:default": [
"@org_tensorflow//tensorflow/lite/delegates/gpu/gl:gl_buffer",
],
@ -378,17 +381,6 @@ cc_library(
alwayslink = 1,
)
cc_library(
name = "clip_detection_vector_size_calculator",
srcs = ["clip_detection_vector_size_calculator.cc"],
deps = [
":clip_vector_size_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:detection_cc_proto",
],
alwayslink = 1,
)
cc_test(
name = "clip_vector_size_calculator_test",
srcs = ["clip_vector_size_calculator_test.cc"],
@ -590,6 +582,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util",
"@com_google_absl//absl/log:absl_check",
],
alwayslink = 1,
)
@ -605,6 +598,7 @@ cc_test(
"//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings",
],
)
@ -637,6 +631,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
],
alwayslink = 1,
)
@ -784,10 +779,11 @@ cc_library(
"//mediapipe/framework/deps:random",
"//mediapipe/framework/formats:video_stream_header",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
],
alwayslink = 1,
@ -843,6 +839,7 @@ cc_test(
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/log:absl_check",
"@eigen_archive//:eigen3",
],
)
@ -904,6 +901,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:rect_cc_proto",
@ -1029,6 +1027,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
],
alwayslink = 1,
)
@ -1067,6 +1066,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log:absl_log",
],
)
@ -1113,6 +1113,7 @@ cc_library(
"//mediapipe/framework/api2:node",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
],
alwayslink = 1,
)
@ -1136,6 +1137,7 @@ cc_library(
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
@ -1164,6 +1166,7 @@ cc_library(
"//mediapipe/framework:collection_item_id",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:ret_check",
@ -1238,6 +1241,7 @@ cc_library(
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],

21
mediapipe/calculators/core/begin_loop_calculator.cc
View File

@ -17,10 +17,13 @@
#include <vector>
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/gpu/gpu_buffer.h"
namespace mediapipe {
@ -60,4 +63,22 @@ REGISTER_CALCULATOR(BeginLoopUint64tCalculator);
typedef BeginLoopCalculator<std::vector<Tensor>> BeginLoopTensorCalculator;
REGISTER_CALCULATOR(BeginLoopTensorCalculator);
// A calculator to process std::vector<mediapipe::ImageFrame>.
typedef BeginLoopCalculator<std::vector<ImageFrame>>
BeginLoopImageFrameCalculator;
REGISTER_CALCULATOR(BeginLoopImageFrameCalculator);
// A calculator to process std::vector<mediapipe::GpuBuffer>.
typedef BeginLoopCalculator<std::vector<GpuBuffer>>
BeginLoopGpuBufferCalculator;
REGISTER_CALCULATOR(BeginLoopGpuBufferCalculator);
// A calculator to process std::vector<mediapipe::Image>.
typedef BeginLoopCalculator<std::vector<Image>> BeginLoopImageCalculator;
REGISTER_CALCULATOR(BeginLoopImageCalculator);
// A calculator to process std::vector<float>.
typedef BeginLoopCalculator<std::vector<float>> BeginLoopFloatCalculator;
REGISTER_CALCULATOR(BeginLoopFloatCalculator);
} // namespace mediapipe

54
mediapipe/calculators/core/begin_loop_calculator.h
View File

@ -15,47 +15,57 @@
#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
#include "absl/status/status.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
namespace mediapipe {
// Calculator for implementing loops on iterable collections inside a MediaPipe
// graph.
// graph. Assume InputIterT is an iterable for type InputT, and OutputIterT is
// an iterable for type OutputT, e.g. vector<InputT> and vector<OutputT>.
// First, instantiate specializations in the loop calculators' implementations
// if missing:
// BeginLoopInputTCalculator = BeginLoopCalculator<InputIterT>
// EndLoopOutputTCalculator = EndLoopCalculator<OutputIterT>
// Then, the following graph transforms an item of type InputIterT to an
// OutputIterT by applying InputToOutputConverter to every element:
//
// It is designed to be used like:
//
// node {
// calculator: "BeginLoopWithIterableCalculator"
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
// node { # Type @timestamp
// calculator: "BeginLoopInputTCalculator"
// input_stream: "ITERABLE:input_iterable" # InputIterT @iterable_ts
// input_stream: "CLONE:extra_input" # ExtraT @extra_ts
// output_stream: "ITEM:input_iterator" # InputT @loop_internal_ts
// output_stream: "CLONE:cloned_extra_input" # ExtraT @loop_internal_ts
// output_stream: "BATCH_END:iterable_ts" # Timestamp @loop_internal_ts
// }
//
// node {
// calculator: "ElementToBlaConverterSubgraph"
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
// calculator: "InputToOutputConverter"
// input_stream: "INPUT:input_iterator" # InputT @loop_internal_ts
// input_stream: "EXTRA:cloned_extra_input" # ExtraT @loop_internal_ts
// output_stream: "OUTPUT:output_iterator" # OutputT @loop_internal_ts
// }
//
// node {
// calculator: "EndLoopWithOutputCalculator"
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
// output_stream: "ITERABLE:aggregated_result" # IterableU @ext_ts
// calculator: "EndLoopOutputTCalculator"
// input_stream: "ITEM:output_iterator" # OutputT @loop_internal_ts
// input_stream: "BATCH_END:iterable_ts" # Timestamp @loop_internal_ts
// output_stream: "ITERABLE:output_iterable" # OutputIterT @iterable_ts
// }
//
// The resulting 'output_iterable' has the same timestamp as 'input_iterable'.
// The output packets of this calculator are part of the loop body and have
// loop-internal timestamps that are unrelated to the input iterator timestamp.
//
// Input streams tagged with "CLONE" are cloned to the corresponding output
// streams at loop timestamps. This ensures that a MediaPipe graph or sub-graph
// can run multiple times, once per element in the "ITERABLE" for each pakcet
// clone of the packets in the "CLONE" input streams.
// streams at loop-internal timestamps. This ensures that a MediaPipe graph or
// sub-graph can run multiple times, once per element in the "ITERABLE" for each
// packet clone of the packets in the "CLONE" input streams. Think of CLONEd
// inputs as loop-wide constants.
template <typename IterableT>
class BeginLoopCalculator : public CalculatorBase {
using ItemT = typename IterableT::value_type;

8
mediapipe/calculators/core/concatenate_vector_calculator.cc
View File

@ -17,6 +17,7 @@
#include <vector>
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/integral_types.h"
@ -55,6 +56,10 @@ MEDIAPIPE_REGISTER_NODE(ConcatenateUInt64VectorCalculator);
typedef ConcatenateVectorCalculator<bool> ConcatenateBoolVectorCalculator;
MEDIAPIPE_REGISTER_NODE(ConcatenateBoolVectorCalculator);
typedef ConcatenateVectorCalculator<std::string>
ConcatenateStringVectorCalculator;
MEDIAPIPE_REGISTER_NODE(ConcatenateStringVectorCalculator);
// Example config:
// node {
// calculator: "ConcatenateTfLiteTensorVectorCalculator"
@ -100,4 +105,7 @@ typedef ConcatenateVectorCalculator<mediapipe::RenderData>
ConcatenateRenderDataVectorCalculator;
MEDIAPIPE_REGISTER_NODE(ConcatenateRenderDataVectorCalculator);
typedef ConcatenateVectorCalculator<mediapipe::Image>
ConcatenateImageVectorCalculator;
MEDIAPIPE_REGISTER_NODE(ConcatenateImageVectorCalculator);
} // namespace mediapipe

25
mediapipe/calculators/core/concatenate_vector_calculator_test.cc
View File

@ -30,13 +30,15 @@ namespace mediapipe {
typedef ConcatenateVectorCalculator<int> TestConcatenateIntVectorCalculator;
MEDIAPIPE_REGISTER_NODE(TestConcatenateIntVectorCalculator);
void AddInputVector(int index, const std::vector<int>& input, int64_t timestamp,
template <typename T>
void AddInputVector(int index, const std::vector<T>& input, int64_t timestamp,
CalculatorRunner* runner) {
runner->MutableInputs()->Index(index).packets.push_back(
MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
MakePacket<std::vector<T>>(input).At(Timestamp(timestamp)));
}
void AddInputVectors(const std::vector<std::vector<int>>& inputs,
template <typename T>
void AddInputVectors(const std::vector<std::vector<T>>& inputs,
int64_t timestamp, CalculatorRunner* runner) {
for (int i = 0; i < inputs.size(); ++i) {
AddInputVector(i, inputs[i], timestamp, runner);
@ -382,6 +384,23 @@ TEST(ConcatenateFloatVectorCalculatorTest, OneEmptyStreamNoOutput) {
EXPECT_EQ(0, outputs.size());
}
TEST(ConcatenateStringVectorCalculatorTest, OneTimestamp) {
CalculatorRunner runner("ConcatenateStringVectorCalculator",
/*options_string=*/"", /*num_inputs=*/3,
/*num_outputs=*/1, /*num_side_packets=*/0);
std::vector<std::vector<std::string>> inputs = {
{"a", "b"}, {"c"}, {"d", "e", "f"}};
AddInputVectors(inputs, /*timestamp=*/1, &runner);
MP_ASSERT_OK(runner.Run());
const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
EXPECT_EQ(1, outputs.size());
EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
std::vector<std::string> expected_vector = {"a", "b", "c", "d", "e", "f"};
EXPECT_EQ(expected_vector, outputs[0].Get<std::vector<std::string>>());
}
typedef ConcatenateVectorCalculator<std::unique_ptr<int>>
TestConcatenateUniqueIntPtrCalculator;
MEDIAPIPE_REGISTER_NODE(TestConcatenateUniqueIntPtrCalculator);

9
mediapipe/calculators/core/constant_side_packet_calculator.cc
View File

@ -19,6 +19,7 @@
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix_data.pb.h"
#include "mediapipe/framework/formats/time_series_header.pb.h"
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/integral_types.h"
@ -85,8 +86,12 @@ class ConstantSidePacketCalculator : public CalculatorBase {
packet.Set<LandmarkList>();
} else if (packet_options.has_double_value()) {
packet.Set<double>();
} else if (packet_options.has_matrix_data_value()) {
packet.Set<MatrixData>();
} else if (packet_options.has_time_series_header_value()) {
packet.Set<TimeSeriesHeader>();
} else if (packet_options.has_int64_value()) {
packet.Set<int64_t>();
} else {
return absl::InvalidArgumentError(
"None of supported values were specified in options.");
@ -121,9 +126,13 @@ class ConstantSidePacketCalculator : public CalculatorBase {
MakePacket<LandmarkList>(packet_options.landmark_list_value()));
} else if (packet_options.has_double_value()) {
packet.Set(MakePacket<double>(packet_options.double_value()));
} else if (packet_options.has_matrix_data_value()) {
packet.Set(MakePacket<MatrixData>(packet_options.matrix_data_value()));
} else if (packet_options.has_time_series_header_value()) {
packet.Set(MakePacket<TimeSeriesHeader>(
packet_options.time_series_header_value()));
} else if (packet_options.has_int64_value()) {
packet.Set(MakePacket<int64_t>(packet_options.int64_value()));
} else {
return absl::InvalidArgumentError(
"None of supported values were specified in options.");

7
mediapipe/calculators/core/constant_side_packet_calculator.proto
View File

@ -19,6 +19,7 @@ package mediapipe;
import "mediapipe/framework/calculator.proto";
import "mediapipe/framework/formats/classification.proto";
import "mediapipe/framework/formats/landmark.proto";
import "mediapipe/framework/formats/matrix_data.proto";
import "mediapipe/framework/formats/time_series_header.proto";
message ConstantSidePacketCalculatorOptions {
@ -29,14 +30,16 @@ message ConstantSidePacketCalculatorOptions {
message ConstantSidePacket {
oneof value {
int32 int_value = 1;
uint64 uint64_value = 5;
int64 int64_value = 11;
float float_value = 2;
double double_value = 9;
bool bool_value = 3;
string string_value = 4;
uint64 uint64_value = 5;
ClassificationList classification_list_value = 6;
LandmarkList landmark_list_value = 7;
double double_value = 9;
TimeSeriesHeader time_series_header_value = 10;
MatrixData matrix_data_value = 12;
}
}

2
mediapipe/calculators/core/constant_side_packet_calculator_test.cc
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cstdint>
#include <string>
#include "absl/strings/string_view.h"
@ -58,6 +59,7 @@ TEST(ConstantSidePacketCalculatorTest, EveryPossibleType) {
DoTestSingleSidePacket("{ float_value: 6.5f }", 6.5f);
DoTestSingleSidePacket("{ bool_value: true }", true);
DoTestSingleSidePacket<std::string>(R"({ string_value: "str" })", "str");
DoTestSingleSidePacket<int64_t>("{ int64_value: 63 }", 63);
}
TEST(ConstantSidePacketCalculatorTest, MultiplePackets) {

18
mediapipe/calculators/core/end_loop_calculator.cc
View File

@ -14,15 +14,19 @@
#include "mediapipe/calculators/core/end_loop_calculator.h"
#include <array>
#include <utility>
#include <vector>
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/util/render_data.pb.h"
#include "tensorflow/lite/interpreter.h"
@ -68,8 +72,22 @@ REGISTER_CALCULATOR(EndLoopMatrixCalculator);
typedef EndLoopCalculator<std::vector<Tensor>> EndLoopTensorCalculator;
REGISTER_CALCULATOR(EndLoopTensorCalculator);
typedef EndLoopCalculator<std::vector<ImageFrame>> EndLoopImageFrameCalculator;
REGISTER_CALCULATOR(EndLoopImageFrameCalculator);
typedef EndLoopCalculator<std::vector<GpuBuffer>> EndLoopGpuBufferCalculator;
REGISTER_CALCULATOR(EndLoopGpuBufferCalculator);
typedef EndLoopCalculator<std::vector<::mediapipe::Image>>
EndLoopImageCalculator;
REGISTER_CALCULATOR(EndLoopImageCalculator);
typedef EndLoopCalculator<std::vector<std::array<float, 16>>>
EndLoopAffineMatrixCalculator;
REGISTER_CALCULATOR(EndLoopAffineMatrixCalculator);
typedef EndLoopCalculator<std::vector<std::pair<int, int>>>
EndLoopImageSizeCalculator;
REGISTER_CALCULATOR(EndLoopImageSizeCalculator);
} // namespace mediapipe

28
mediapipe/calculators/core/end_loop_calculator.h
View File

@ -17,13 +17,11 @@
#include <type_traits>
#include "absl/status/status.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/collection_item_id.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
@ -33,27 +31,7 @@ namespace mediapipe {
// from the "BATCH_END" tagged input stream, it emits the aggregated results
// at the original timestamp contained in the "BATCH_END" input stream.
//
// It is designed to be used like:
//
// node {
// calculator: "BeginLoopWithIterableCalculator"
// input_stream: "ITERABLE:input_iterable" # IterableT @ext_ts
// output_stream: "ITEM:input_element" # ItemT @loop_internal_ts
// output_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
// }
//
// node {
// calculator: "ElementToBlaConverterSubgraph"
// input_stream: "ITEM:input_to_loop_body" # ItemT @loop_internal_ts
// output_stream: "BLA:output_of_loop_body" # ItemU @loop_internal_ts
// }
//
// node {
// calculator: "EndLoopWithOutputCalculator"
// input_stream: "ITEM:output_of_loop_body" # ItemU @loop_internal_ts
// input_stream: "BATCH_END:ext_ts" # Timestamp @loop_internal_ts
// output_stream: "ITERABLE:aggregated_result" # IterableU @ext_ts
// }
// See BeginLoopCalculator for a usage example.
template <typename IterableT>
class EndLoopCalculator : public CalculatorBase {
using ItemT = typename IterableT::value_type;
@ -79,7 +57,7 @@ class EndLoopCalculator : public CalculatorBase {
}
// Try to consume the item and move it into the collection. If the items
// are not consumable, then try to copy them instead. If the items are
// not copiable, then an error will be returned.
// not copyable, then an error will be returned.
auto item_ptr_or = cc->Inputs().Tag("ITEM").Value().Consume<ItemT>();
if (item_ptr_or.ok()) {
input_stream_collection_->push_back(std::move(*item_ptr_or.value()));

2
mediapipe/calculators/core/flow_limiter_calculator.cc
View File

@ -42,7 +42,7 @@ constexpr char kOptionsTag[] = "OPTIONS";
//
// Increasing `max_in_flight` to 2 or more can yield the better throughput
// when the graph exhibits a high degree of pipeline parallelism. Decreasing
// `max_in_flight` to 0 can yield a better average latency, but at the cost of
// `max_in_queue` to 0 can yield a better average latency, but at the cost of
// lower throughput (lower framerate) due to the time during which the graph
// is idle awaiting the next input frame.
//

31
mediapipe/calculators/core/gate_calculator.cc
View File

@ -26,19 +26,15 @@ constexpr char kStateChangeTag[] = "STATE_CHANGE";
constexpr char kDisallowTag[] = "DISALLOW";
constexpr char kAllowTag[] = "ALLOW";
enum GateState {
GATE_UNINITIALIZED,
GATE_ALLOW,
GATE_DISALLOW,
};
std::string ToString(GateState state) {
std::string ToString(GateCalculatorOptions::GateState state) {
switch (state) {
case GATE_UNINITIALIZED:
case GateCalculatorOptions::UNSPECIFIED:
return "UNSPECIFIED";
case GateCalculatorOptions::GATE_UNINITIALIZED:
return "UNINITIALIZED";
case GATE_ALLOW:
case GateCalculatorOptions::GATE_ALLOW:
return "ALLOW";
case GATE_DISALLOW:
case GateCalculatorOptions::GATE_DISALLOW:
return "DISALLOW";
}
DLOG(FATAL) << "Unknown GateState";
@ -153,10 +149,12 @@ class GateCalculator : public CalculatorBase {
cc->SetOffset(TimestampDiff(0));
num_data_streams_ = cc->Inputs().NumEntries("");
last_gate_state_ = GATE_UNINITIALIZED;
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
last_gate_state_ = options.initial_gate_state();
RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs()));
empty_packets_as_allow_ = options.empty_packets_as_allow();
if (!use_side_packet_for_allow_disallow_ &&
@ -184,10 +182,12 @@ class GateCalculator : public CalculatorBase {
allow = !cc->Inputs().Tag(kDisallowTag).Get<bool>();
}
}
const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
const GateCalculatorOptions::GateState new_gate_state =
allow ? GateCalculatorOptions::GATE_ALLOW
: GateCalculatorOptions::GATE_DISALLOW;
if (cc->Outputs().HasTag(kStateChangeTag)) {
if (last_gate_state_ != GATE_UNINITIALIZED &&
if (last_gate_state_ != GateCalculatorOptions::GATE_UNINITIALIZED &&
last_gate_state_ != new_gate_state) {
VLOG(2) << "State transition in " << cc->NodeName() << " @ "
<< cc->InputTimestamp().Value() << " from "
@ -223,7 +223,8 @@ class GateCalculator : public CalculatorBase {
}
private:
GateState last_gate_state_ = GATE_UNINITIALIZED;
GateCalculatorOptions::GateState last_gate_state_ =
GateCalculatorOptions::GATE_UNINITIALIZED;
int num_data_streams_;
bool empty_packets_as_allow_;
bool use_side_packet_for_allow_disallow_ = false;

9
mediapipe/calculators/core/gate_calculator.proto
View File

@ -31,4 +31,13 @@ message GateCalculatorOptions {
// Whether to allow or disallow the input streams to pass when no
// ALLOW/DISALLOW input or side input is specified.
optional bool allow = 2 [default = false];
enum GateState {
UNSPECIFIED = 0;
GATE_UNINITIALIZED = 1;
GATE_ALLOW = 2;
GATE_DISALLOW = 3;
}
optional GateState initial_gate_state = 3 [default = GATE_UNINITIALIZED];
}

35
mediapipe/calculators/core/gate_calculator_test.cc
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/log/absl_log.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/port/gtest.h"
@ -356,18 +357,18 @@ TEST_F(GateCalculatorTest, AllowWithStateChangeNoDataStreams) {
RunTimeStepWithoutDataStream(kTimestampValue2, "ALLOW", true);
constexpr int64_t kTimestampValue3 = 45;
RunTimeStepWithoutDataStream(kTimestampValue3, "ALLOW", false);
LOG(INFO) << "a";
ABSL_LOG(INFO) << "a";
const std::vector<Packet>& output =
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
LOG(INFO) << "s";
ABSL_LOG(INFO) << "s";
ASSERT_EQ(2, output.size());
LOG(INFO) << "d";
ABSL_LOG(INFO) << "d";
EXPECT_EQ(kTimestampValue1, output[0].Timestamp().Value());
EXPECT_EQ(kTimestampValue3, output[1].Timestamp().Value());
LOG(INFO) << "f";
ABSL_LOG(INFO) << "f";
EXPECT_EQ(true, output[0].Get<bool>()); // Allow.
EXPECT_EQ(false, output[1].Get<bool>()); // Disallow.
LOG(INFO) << "g";
ABSL_LOG(INFO) << "g";
}
TEST_F(GateCalculatorTest, DisallowWithStateChange) {
@ -458,5 +459,29 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
ASSERT_EQ(0, output.size());
}
// Must detect allow value for first timestamp as a state change when the
// initial state is set to GATE_DISALLOW.
TEST_F(GateCalculatorTest, StateChangeTriggeredWithInitialGateStateOption) {
SetRunner(R"(
calculator: "GateCalculator"
input_stream: "test_input"
input_stream: "ALLOW:allow"
output_stream: "test_output"
output_stream: "STATE_CHANGE:state_change"
options: {
[mediapipe.GateCalculatorOptions.ext] {
initial_gate_state: GATE_DISALLOW
}
}
)");
constexpr int64_t kTimestampValue0 = 42;
RunTimeStep(kTimestampValue0, "ALLOW", true);
const std::vector<Packet>& output =
runner()->Outputs().Get("STATE_CHANGE", 0).packets;
ASSERT_EQ(1, output.size());
}
} // namespace
} // namespace mediapipe

8
mediapipe/calculators/core/get_vector_item_calculator.cc
View File

@ -17,6 +17,7 @@
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
namespace mediapipe {
namespace api2 {
@ -37,5 +38,12 @@ using GetDetectionVectorItemCalculator =
GetVectorItemCalculator<mediapipe::Detection>;
REGISTER_CALCULATOR(GetDetectionVectorItemCalculator);
using GetNormalizedRectVectorItemCalculator =
GetVectorItemCalculator<NormalizedRect>;
REGISTER_CALCULATOR(GetNormalizedRectVectorItemCalculator);
using GetRectVectorItemCalculator = GetVectorItemCalculator<Rect>;
REGISTER_CALCULATOR(GetRectVectorItemCalculator);
} // namespace api2
} // namespace mediapipe

3
mediapipe/calculators/core/immediate_mux_calculator.cc
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/log/absl_log.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
@ -78,7 +79,7 @@ absl::Status ImmediateMuxCalculator::Process(CalculatorContext* cc) {
if (packet.Timestamp() >= cc->Outputs().Index(0).NextTimestampBound()) {
cc->Outputs().Index(0).AddPacket(packet);
} else {
LOG_FIRST_N(WARNING, 5)
ABSL_LOG_FIRST_N(WARNING, 5)
<< "Dropping a packet with timestamp " << packet.Timestamp();
}
if (cc->Outputs().NumEntries() >= 2) {

3
mediapipe/calculators/core/matrix_multiply_calculator_test.cc
View File

@ -16,6 +16,7 @@
#include <vector>
#include "Eigen/Core"
#include "absl/log/absl_check.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/matrix.h"
@ -209,7 +210,7 @@ TEST(MatrixMultiplyCalculatorTest, Multiply) {
MatrixFromTextProto(kSamplesText, &samples);
Matrix expected;
MatrixFromTextProto(kExpectedText, &expected);
CHECK_EQ(samples.cols(), expected.cols());
ABSL_CHECK_EQ(samples.cols(), expected.cols());
for (int i = 0; i < samples.cols(); ++i) {
// Take a column from samples and produce a packet with just that

2
mediapipe/calculators/core/matrix_to_vector_calculator_test.cc
View File

@ -35,7 +35,7 @@ class MatrixToVectorCalculatorTest
void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; }
void AppendInput(const std::vector<float>& column_major_data,
int64 timestamp) {
int64_t timestamp) {
ASSERT_EQ(num_input_samples_ * num_input_channels_,
column_major_data.size());
Eigen::Map<const Matrix> data_map(&column_major_data[0],

7
mediapipe/calculators/core/merge_calculator.cc
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/log/absl_log.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h"
@ -53,7 +54,7 @@ class MergeCalculator : public Node {
static absl::Status UpdateContract(CalculatorContract* cc) {
RET_CHECK_GT(kIn(cc).Count(), 0) << "Needs at least one input stream";
if (kIn(cc).Count() == 1) {
LOG(WARNING)
ABSL_LOG(WARNING)
<< "MergeCalculator expects multiple input streams to merge but is "
"receiving only one. Make sure the calculator is configured "
"correctly or consider removing this calculator to reduce "
@ -72,8 +73,8 @@ class MergeCalculator : public Node {
}
}
LOG(WARNING) << "Empty input packets at timestamp "
<< cc->InputTimestamp().Value();
ABSL_LOG(WARNING) << "Empty input packets at timestamp "
<< cc->InputTimestamp().Value();
return absl::OkStatus();
}

2
mediapipe/calculators/core/merge_to_vector_calculator.cc
View File

@ -1,4 +1,4 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
/* Copyright 2022 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

2
mediapipe/calculators/core/merge_to_vector_calculator.h
View File

@ -1,4 +1,4 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
/* Copyright 2022 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

55
mediapipe/calculators/core/packet_resampler_calculator.cc
View File

@ -16,6 +16,9 @@
#include <memory>
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
namespace {
// Reflect an integer against the lower and upper bound of an interval.
int64_t ReflectBetween(int64_t ts, int64_t ts_min, int64_t ts_max) {
@ -177,7 +180,7 @@ PacketResamplerCalculator::GetSamplingStrategy(
const PacketResamplerCalculatorOptions& options) {
if (options.reproducible_sampling()) {
if (!options.jitter_with_reflection()) {
LOG(WARNING)
ABSL_LOG(WARNING)
<< "reproducible_sampling enabled w/ jitter_with_reflection "
"disabled. "
<< "reproducible_sampling always uses jitter with reflection, "
@ -200,15 +203,15 @@ PacketResamplerCalculator::GetSamplingStrategy(
Timestamp PacketResamplerCalculator::PeriodIndexToTimestamp(
int64_t index) const {
CHECK_EQ(jitter_, 0.0);
CHECK_NE(first_timestamp_, Timestamp::Unset());
ABSL_CHECK_EQ(jitter_, 0.0);
ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset());
return first_timestamp_ + TimestampDiffFromSeconds(index / frame_rate_);
}
int64_t PacketResamplerCalculator::TimestampToPeriodIndex(
Timestamp timestamp) const {
CHECK_EQ(jitter_, 0.0);
CHECK_NE(first_timestamp_, Timestamp::Unset());
ABSL_CHECK_EQ(jitter_, 0.0);
ABSL_CHECK_NE(first_timestamp_, Timestamp::Unset());
return MathUtil::SafeRound<int64_t, double>(
(timestamp - first_timestamp_).Seconds() * frame_rate_);
}
@ -229,13 +232,15 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) {
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
"the actual value.";
ABSL_LOG(WARNING)
<< "VideoHeader::frame_rate holds the target value and not "
"the actual value.";
}
if (calculator_->flush_last_packet_) {
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
"ignored, because we are adding jitter.";
ABSL_LOG(WARNING)
<< "PacketResamplerCalculatorOptions.flush_last_packet is "
"ignored, because we are adding jitter.";
}
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
@ -254,7 +259,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Open(CalculatorContext* cc) {
}
absl::Status LegacyJitterWithReflectionStrategy::Close(CalculatorContext* cc) {
if (!packet_reservoir_->IsEmpty()) {
LOG(INFO) << "Emitting pack from reservoir.";
ABSL_LOG(INFO) << "Emitting pack from reservoir.";
calculator_->OutputWithinLimits(cc, packet_reservoir_->GetSample());
}
return absl::OkStatus();
@ -285,7 +290,7 @@ absl::Status LegacyJitterWithReflectionStrategy::Process(
if (calculator_->frame_time_usec_ <
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
LOG_FIRST_N(WARNING, 2)
ABSL_LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling.";
}
@ -340,8 +345,8 @@ void LegacyJitterWithReflectionStrategy::UpdateNextOutputTimestampWithJitter() {
next_output_timestamp_ = Timestamp(ReflectBetween(
next_output_timestamp_.Value(), next_output_timestamp_min_.Value(),
next_output_timestamp_max_.Value()));
CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
ABSL_CHECK_GE(next_output_timestamp_, next_output_timestamp_min_);
ABSL_CHECK_LT(next_output_timestamp_, next_output_timestamp_max_);
}
absl::Status ReproducibleJitterWithReflectionStrategy::Open(
@ -352,13 +357,15 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Open(
if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) {
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
"the actual value.";
ABSL_LOG(WARNING)
<< "VideoHeader::frame_rate holds the target value and not "
"the actual value.";
}
if (calculator_->flush_last_packet_) {
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
"ignored, because we are adding jitter.";
ABSL_LOG(WARNING)
<< "PacketResamplerCalculatorOptions.flush_last_packet is "
"ignored, because we are adding jitter.";
}
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
@ -411,7 +418,7 @@ absl::Status ReproducibleJitterWithReflectionStrategy::Process(
// Note, if the stream is upsampling, this could lead to the same packet
// being emitted twice. Upsampling and jitter doesn't make much sense
// but does technically work.
LOG_FIRST_N(WARNING, 2)
ABSL_LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling.";
}
@ -499,13 +506,15 @@ absl::Status JitterWithoutReflectionStrategy::Open(CalculatorContext* cc) {
if (resampler_options.output_header() !=
PacketResamplerCalculatorOptions::NONE) {
LOG(WARNING) << "VideoHeader::frame_rate holds the target value and not "
"the actual value.";
ABSL_LOG(WARNING)
<< "VideoHeader::frame_rate holds the target value and not "
"the actual value.";
}
if (calculator_->flush_last_packet_) {
LOG(WARNING) << "PacketResamplerCalculatorOptions.flush_last_packet is "
"ignored, because we are adding jitter.";
ABSL_LOG(WARNING)
<< "PacketResamplerCalculatorOptions.flush_last_packet is "
"ignored, because we are adding jitter.";
}
const auto& seed = cc->InputSidePackets().Tag(kSeedTag).Get<std::string>();
@ -555,7 +564,7 @@ absl::Status JitterWithoutReflectionStrategy::Process(CalculatorContext* cc) {
if (calculator_->frame_time_usec_ <
(cc->InputTimestamp() - calculator_->last_packet_.Timestamp()).Value()) {
LOG_FIRST_N(WARNING, 2)
ABSL_LOG_FIRST_N(WARNING, 2)
<< "Adding jitter is not very useful when upsampling.";
}

1
mediapipe/calculators/core/packet_resampler_calculator.h
View File

@ -13,7 +13,6 @@
#include "mediapipe/framework/deps/random_base.h"
#include "mediapipe/framework/formats/video_stream_header.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"

20
mediapipe/calculators/core/packet_resampler_calculator_test.cc
View File

@ -51,9 +51,9 @@ class SimpleRunner : public CalculatorRunner {
virtual ~SimpleRunner() {}
void SetInput(const std::vector<int64>& timestamp_list) {
void SetInput(const std::vector<int64_t>& timestamp_list) {
MutableInputs()->Index(0).packets.clear();
for (const int64 ts : timestamp_list) {
for (const int64_t ts : timestamp_list) {
MutableInputs()->Index(0).packets.push_back(
Adopt(new std::string(absl::StrCat("Frame #", ts)))
.At(Timestamp(ts)));
@ -72,8 +72,8 @@ class SimpleRunner : public CalculatorRunner {
}
void CheckOutputTimestamps(
const std::vector<int64>& expected_frames,
const std::vector<int64>& expected_timestamps) const {
const std::vector<int64_t>& expected_frames,
const std::vector<int64_t>& expected_timestamps) const {
EXPECT_EQ(expected_frames.size(), Outputs().Index(0).packets.size());
EXPECT_EQ(expected_timestamps.size(), Outputs().Index(0).packets.size());
int count = 0;
@ -112,7 +112,7 @@ MATCHER_P2(PacketAtTimestamp, payload, timestamp,
*result_listener << "at incorrect timestamp = " << arg.Timestamp().Value();
return false;
}
int64 actual_payload = arg.template Get<int64>();
int64_t actual_payload = arg.template Get<int64_t>();
if (actual_payload != payload) {
*result_listener << "with incorrect payload = " << actual_payload;
return false;
@ -137,18 +137,18 @@ class ReproducibleJitterWithReflectionStrategyForTesting
//
// An EXPECT will fail if sequence is less than the number requested during
// processing.
static std::vector<uint64> random_sequence;
static std::vector<uint64_t> random_sequence;
protected:
virtual uint64 GetNextRandom(uint64 n) {
virtual uint64_t GetNextRandom(uint64_t n) {
EXPECT_LT(sequence_index_, random_sequence.size());
return random_sequence[sequence_index_++] % n;
}
private:
int32 sequence_index_ = 0;
int32_t sequence_index_ = 0;
};
std::vector<uint64>
std::vector<uint64_t>
ReproducibleJitterWithReflectionStrategyForTesting::random_sequence;
// PacketResamplerCalculator child class which injects a specified stream
@ -469,7 +469,7 @@ TEST(PacketResamplerCalculatorTest, SetVideoHeader) {
}
)pb"));
for (const int64 ts : {0, 5000, 10010, 15001, 19990}) {
for (const int64_t ts : {0, 5000, 10010, 15001, 19990}) {
runner.MutableInputs()->Tag(kDataTag).packets.push_back(
Adopt(new std::string(absl::StrCat("Frame #", ts))).At(Timestamp(ts)));
}

20
mediapipe/calculators/core/packet_thinner_calculator.cc
View File

@ -17,6 +17,7 @@
#include <cmath> // for ceil
#include <memory>
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h"
@ -160,8 +161,8 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
thinner_type_ = options.thinner_type();
// This check enables us to assume only two thinner types exist in Process()
CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
ABSL_CHECK(thinner_type_ == PacketThinnerCalculatorOptions::ASYNC ||
thinner_type_ == PacketThinnerCalculatorOptions::SYNC)
<< "Unsupported thinner type.";
if (thinner_type_ == PacketThinnerCalculatorOptions::ASYNC) {
@ -177,7 +178,8 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
} else {
period_ = TimestampDiff(options.period());
}
CHECK_LT(TimestampDiff(0), period_) << "Specified period must be positive.";
ABSL_CHECK_LT(TimestampDiff(0), period_)
<< "Specified period must be positive.";
if (options.has_start_time()) {
start_time_ = Timestamp(options.start_time());
@ -189,7 +191,7 @@ absl::Status PacketThinnerCalculator::Open(CalculatorContext* cc) {
end_time_ =
options.has_end_time() ? Timestamp(options.end_time()) : Timestamp::Max();
CHECK_LT(start_time_, end_time_)
ABSL_CHECK_LT(start_time_, end_time_)
<< "Invalid PacketThinner: start_time must be earlier than end_time";
sync_output_timestamps_ = options.sync_output_timestamps();
@ -232,7 +234,7 @@ absl::Status PacketThinnerCalculator::Close(CalculatorContext* cc) {
// Emit any saved packets before quitting.
if (!saved_packet_.IsEmpty()) {
// Only sync thinner should have saved packets.
CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
ABSL_CHECK_EQ(PacketThinnerCalculatorOptions::SYNC, thinner_type_);
if (sync_output_timestamps_) {
cc->Outputs().Index(0).AddPacket(
saved_packet_.At(NearestSyncTimestamp(saved_packet_.Timestamp())));
@ -269,7 +271,7 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess(
const Timestamp saved_sync = NearestSyncTimestamp(saved);
const Timestamp now = cc->InputTimestamp();
const Timestamp now_sync = NearestSyncTimestamp(now);
CHECK_LE(saved_sync, now_sync);
ABSL_CHECK_LE(saved_sync, now_sync);
if (saved_sync == now_sync) {
// Saved Packet is in same interval as current packet.
// Replace saved packet with current if it is at least as
@ -295,7 +297,7 @@ absl::Status PacketThinnerCalculator::SyncThinnerProcess(
}
Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
CHECK_NE(start_time_, Timestamp::Unset())
ABSL_CHECK_NE(start_time_, Timestamp::Unset())
<< "Method only valid for sync thinner calculator.";
// Computation is done using int64 arithmetic. No easy way to avoid
@ -303,12 +305,12 @@ Timestamp PacketThinnerCalculator::NearestSyncTimestamp(Timestamp now) const {
const int64_t now64 = now.Value();
const int64_t start64 = start_time_.Value();
const int64_t period64 = period_.Value();
CHECK_LE(0, period64);
ABSL_CHECK_LE(0, period64);
// Round now64 to its closest interval (units of period64).
int64_t sync64 =
(now64 - start64 + period64 / 2) / period64 * period64 + start64;
CHECK_LE(abs(now64 - sync64), period64 / 2)
ABSL_CHECK_LE(abs(now64 - sync64), period64 / 2)
<< "start64: " << start64 << "; now64: " << now64
<< "; sync64: " << sync64;

3
mediapipe/calculators/core/packet_thinner_calculator_test.cc
View File

@ -16,6 +16,7 @@
#include <string>
#include <vector>
#include "absl/log/absl_check.h"
#include "absl/strings/str_cat.h"
#include "mediapipe/calculators/core/packet_thinner_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@ -70,7 +71,7 @@ class SimpleRunner : public CalculatorRunner {
}
double GetFrameRate() const {
CHECK(!Outputs().Index(0).header.IsEmpty());
ABSL_CHECK(!Outputs().Index(0).header.IsEmpty());
return Outputs().Index(0).header.Get<VideoHeader>().frame_rate;
}
};

5
mediapipe/calculators/core/previous_loopback_calculator.cc
View File

@ -123,7 +123,10 @@ class PreviousLoopbackCalculator : public Node {
// However, LOOP packet is empty.
kPrevLoop(cc).SetNextTimestampBound(main_spec.timestamp + 1);
} else {
kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
// Avoids sending leftovers to a stream that's already closed.
if (!kPrevLoop(cc).IsClosed()) {
kPrevLoop(cc).Send(loop_candidate.At(main_spec.timestamp));
}
}
loop_packets_.pop_front();
main_packet_specs_.pop_front();

6
mediapipe/calculators/core/previous_loopback_calculator_test.cc
View File

@ -43,8 +43,8 @@ constexpr char kDisallowTag[] = "DISALLOW";
// Returns the timestamp values for a vector of Packets.
// TODO: puth this kind of test util in a common place.
std::vector<int64> TimestampValues(const std::vector<Packet>& packets) {
std::vector<int64> result;
std::vector<int64_t> TimestampValues(const std::vector<Packet>& packets) {
std::vector<int64_t> result;
for (const Packet& packet : packets) {
result.push_back(packet.Timestamp().Value());
}
@ -371,7 +371,7 @@ TEST(PreviousLoopbackCalculator, EmptyLoopForever) {
for (int main_ts = 0; main_ts < 50; ++main_ts) {
send_packet("in", main_ts);
MP_EXPECT_OK(graph_.WaitUntilIdle());
std::vector<int64> ts_values = TimestampValues(outputs);
std::vector<int64_t> ts_values = TimestampValues(outputs);
EXPECT_EQ(ts_values.size(), main_ts + 1);
for (int j = 0; j < main_ts + 1; ++j) {
EXPECT_EQ(ts_values[j], j);

3
mediapipe/calculators/core/sequence_shift_calculator.cc
View File

@ -14,6 +14,7 @@
#include <deque>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/core/sequence_shift_calculator.pb.h"
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
@ -101,7 +102,7 @@ void SequenceShiftCalculator::ProcessPositiveOffset(CalculatorContext* cc) {
kOut(cc).Send(packet_cache_.front().At(cc->InputTimestamp()));
packet_cache_.pop_front();
} else if (emit_empty_packets_before_first_packet_) {
LOG(FATAL) << "Not supported yet";
ABSL_LOG(FATAL) << "Not supported yet";
}
// Store current packet for later output.
packet_cache_.push_back(kIn(cc).packet());

6
mediapipe/calculators/core/side_packet_to_stream_calculator.cc
View File

@ -121,7 +121,7 @@ absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
if (cc->Outputs().HasTag(kTagAtTimestamp)) {
RET_CHECK_EQ(num_entries + 1, cc->InputSidePackets().NumEntries())
<< "For AT_TIMESTAMP tag, 2 input side packets are required.";
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Set<int64>();
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Set<int64_t>();
} else {
RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries())
<< "Same number of input side packets and output streams is required.";
@ -178,8 +178,8 @@ absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
}
} else if (cc->Outputs().HasTag(kTagAtTimestamp)) {
int64 timestamp =
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Get<int64>();
int64_t timestamp =
cc->InputSidePackets().Tag(kTagSideInputTimestamp).Get<int64_t>();
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
cc->Outputs()
.Get(output_tag_, i)

9
mediapipe/calculators/core/split_vector_calculator.cc
View File

@ -18,6 +18,7 @@
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/rect.pb.h"
@ -86,4 +87,12 @@ REGISTER_CALCULATOR(SplitUint64tVectorCalculator);
typedef SplitVectorCalculator<float, false> SplitFloatVectorCalculator;
REGISTER_CALCULATOR(SplitFloatVectorCalculator);
typedef SplitVectorCalculator<mediapipe::Image, false>
SplitImageVectorCalculator;
REGISTER_CALCULATOR(SplitImageVectorCalculator);
typedef SplitVectorCalculator<std::array<float, 16>, false>
SplitAffineMatrixVectorCalculator;
REGISTER_CALCULATOR(SplitAffineMatrixVectorCalculator);
} // namespace mediapipe

22
mediapipe/calculators/core/stream_to_side_packet_calculator.cc
View File

@ -12,11 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/api2/node.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/timestamp.h"
namespace mediapipe {
namespace api2 {
// A calculator that takes a packet of an input stream and converts it to an
// output side packet. This calculator only works under the assumption that the
@ -28,21 +30,21 @@ namespace mediapipe {
// input_stream: "stream"
// output_side_packet: "side_packet"
// }
class StreamToSidePacketCalculator : public mediapipe::CalculatorBase {
class StreamToSidePacketCalculator : public Node {
public:
static absl::Status GetContract(mediapipe::CalculatorContract* cc) {
cc->Inputs().Index(0).SetAny();
cc->OutputSidePackets().Index(0).SetAny();
return absl::OkStatus();
}
static constexpr Input<AnyType>::Optional kIn{""};
static constexpr SideOutput<SameType<kIn>> kOut{""};
MEDIAPIPE_NODE_CONTRACT(kIn, kOut);
absl::Status Process(mediapipe::CalculatorContext* cc) override {
mediapipe::Packet& packet = cc->Inputs().Index(0).Value();
cc->OutputSidePackets().Index(0).Set(
packet.At(mediapipe::Timestamp::Unset()));
kOut(cc).Set(
kIn(cc).packet().As<AnyType>().At(mediapipe::Timestamp::Unset()));
return absl::OkStatus();
}
};
REGISTER_CALCULATOR(StreamToSidePacketCalculator);
MEDIAPIPE_REGISTER_NODE(StreamToSidePacketCalculator);
} // namespace api2
} // namespace mediapipe

54
mediapipe/calculators/image/BUILD
View File

@ -97,6 +97,7 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:source_location",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
],
alwayslink = 1,
)
@ -125,6 +126,7 @@ cc_library(
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
],
alwayslink = 1,
)
@ -135,7 +137,6 @@ cc_library(
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status",
],
@ -152,11 +153,11 @@ cc_library(
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
"@com_google_absl//absl/log:absl_log",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
@ -203,6 +204,7 @@ cc_library(
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings",
] + select({
"//mediapipe/gpu:disable_gpu": [],
@ -262,9 +264,12 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/gpu:scale_mode_cc_proto",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_base_hdr",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_quad_renderer",
"//mediapipe/gpu:gl_simple_shaders",
@ -274,6 +279,36 @@ cc_library(
alwayslink = 1,
)
cc_test(
name = "image_transformation_calculator_test",
srcs = ["image_transformation_calculator_test.cc"],
data = ["//mediapipe/calculators/image/testdata:test_images"],
tags = [
"desktop_only_test",
],
deps = [
":image_transformation_calculator",
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:gtest",
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
"//third_party:opencv",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest_main",
],
)
cc_library(
name = "image_cropping_calculator",
srcs = ["image_cropping_calculator.cc"],
@ -301,6 +336,7 @@ cc_library(
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_log",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
@ -317,6 +353,7 @@ cc_library(
cc_test(
name = "image_cropping_calculator_test",
srcs = ["image_cropping_calculator_test.cc"],
tags = ["not_run:arm"],
deps = [
":image_cropping_calculator",
":image_cropping_calculator_cc_proto",
@ -396,6 +433,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings",
],
)
@ -420,6 +458,8 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/util:image_frame_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
"@libyuv",
],
@ -625,9 +665,9 @@ cc_library(
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:vector",
"@com_google_absl//absl/log:absl_log",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
@ -650,6 +690,7 @@ cc_library(
cc_test(
name = "segmentation_smoothing_calculator_test",
srcs = ["segmentation_smoothing_calculator_test.cc"],
tags = ["not_run:arm"],
deps = [
":image_clone_calculator",
":image_clone_calculator_cc_proto",
@ -664,6 +705,7 @@ cc_test(
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log:absl_log",
],
)
@ -686,6 +728,7 @@ cc_library(
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/gpu:shader_util",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
@ -771,7 +814,10 @@ cc_test(
"//mediapipe/calculators/tensor:testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero_interp_cubic.png",
"//mediapipe/calculators/tensor:testdata/image_to_tensor/noop_except_range.png",
],
tags = ["desktop_only_test"],
tags = [
"desktop_only_test",
"not_run:arm",
],
deps = [
":affine_transformation",
":image_transformation_calculator",

7
mediapipe/calculators/image/affine_transformation_runner_gl.cc
View File

@ -20,6 +20,7 @@
#include "Eigen/Core"
#include "Eigen/Geometry"
#include "Eigen/LU"
#include "absl/log/absl_log.h"
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
@ -53,6 +54,10 @@ bool IsMatrixVerticalFlipNeeded(GpuOrigin::Mode gpu_origin) {
#endif // __APPLE__
case GpuOrigin::TOP_LEFT:
return false;
default:
ABSL_LOG(ERROR) << "Incorrect GpuOrigin: "
<< static_cast<int>(gpu_origin);
return true;
}
}
@ -384,6 +389,8 @@ class GlTextureWarpAffineRunner
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, 0);
glFlush();
return absl::OkStatus();
}

7
mediapipe/calculators/image/bilateral_filter_calculator.cc
View File

@ -15,6 +15,7 @@
#include <memory>
#include <string>
#include "absl/log/absl_check.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/image/bilateral_filter_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@ -112,7 +113,7 @@ class BilateralFilterCalculator : public CalculatorBase {
REGISTER_CALCULATOR(BilateralFilterCalculator);
absl::Status BilateralFilterCalculator::GetContract(CalculatorContract* cc) {
CHECK_GE(cc->Inputs().NumEntries(), 1);
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
if (cc->Inputs().HasTag(kInputFrameTag) &&
cc->Inputs().HasTag(kInputFrameTagGpu)) {
@ -183,8 +184,8 @@ absl::Status BilateralFilterCalculator::Open(CalculatorContext* cc) {
sigma_color_ = options_.sigma_color();
sigma_space_ = options_.sigma_space();
CHECK_GE(sigma_color_, 0.0);
CHECK_GE(sigma_space_, 0.0);
ABSL_CHECK_GE(sigma_color_, 0.0);
ABSL_CHECK_GE(sigma_space_, 0.0);
if (!use_gpu_) sigma_color_ *= 255.0;
if (use_gpu_) {

5
mediapipe/calculators/image/color_convert_calculator.cc
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/log/absl_check.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
@ -25,8 +26,8 @@
namespace mediapipe {
namespace {
void SetColorChannel(int channel, uint8 value, cv::Mat* mat) {
CHECK(mat->depth() == CV_8U);
CHECK(channel < mat->channels());
ABSL_CHECK(mat->depth() == CV_8U);
ABSL_CHECK(channel < mat->channels());
const int step = mat->channels();
for (int r = 0; r < mat->rows; ++r) {
uint8* row_ptr = mat->ptr<uint8>(r);

6
mediapipe/calculators/image/image_cropping_calculator.cc
View File

@ -16,6 +16,7 @@
#include <cmath>
#include "absl/log/absl_log.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/formats/rect.pb.h"
@ -202,8 +203,9 @@ absl::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
switch (options.border_mode()) {
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU "
<< "implementation and will fall back into BORDER_REPLICATE";
ABSL_LOG(WARNING)
<< "BORDER_ZERO mode is not supported by GPU "
<< "implementation and will fall back into BORDER_REPLICATE";
break;
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
break;

107
mediapipe/calculators/image/image_transformation_calculator.cc
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/status/status.h"
#include "mediapipe/calculators/image/image_transformation_calculator.pb.h"
#include "mediapipe/calculators/image/rotation_mode.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@ -27,6 +28,7 @@
#include "mediapipe/gpu/scale_mode.pb.h"
#if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_quad_renderer.h"
#include "mediapipe/gpu/gl_simple_shaders.h"
@ -60,42 +62,42 @@ constexpr char kVideoPrestreamTag[] = "VIDEO_PRESTREAM";
int RotationModeToDegrees(mediapipe::RotationMode_Mode rotation) {
switch (rotation) {
case mediapipe::RotationMode_Mode_UNKNOWN:
case mediapipe::RotationMode_Mode_ROTATION_0:
case mediapipe::RotationMode::UNKNOWN:
case mediapipe::RotationMode::ROTATION_0:
return 0;
case mediapipe::RotationMode_Mode_ROTATION_90:
case mediapipe::RotationMode::ROTATION_90:
return 90;
case mediapipe::RotationMode_Mode_ROTATION_180:
case mediapipe::RotationMode::ROTATION_180:
return 180;
case mediapipe::RotationMode_Mode_ROTATION_270:
case mediapipe::RotationMode::ROTATION_270:
return 270;
}
}
mediapipe::RotationMode_Mode DegreesToRotationMode(int degrees) {
switch (degrees) {
case 0:
return mediapipe::RotationMode_Mode_ROTATION_0;
return mediapipe::RotationMode::ROTATION_0;
case 90:
return mediapipe::RotationMode_Mode_ROTATION_90;
return mediapipe::RotationMode::ROTATION_90;
case 180:
return mediapipe::RotationMode_Mode_ROTATION_180;
return mediapipe::RotationMode::ROTATION_180;
case 270:
return mediapipe::RotationMode_Mode_ROTATION_270;
return mediapipe::RotationMode::ROTATION_270;
default:
return mediapipe::RotationMode_Mode_UNKNOWN;
return mediapipe::RotationMode::UNKNOWN;
}
}
mediapipe::ScaleMode_Mode ParseScaleMode(
mediapipe::ScaleMode_Mode scale_mode,
mediapipe::ScaleMode_Mode default_mode) {
switch (scale_mode) {
case mediapipe::ScaleMode_Mode_DEFAULT:
case mediapipe::ScaleMode::DEFAULT:
return default_mode;
case mediapipe::ScaleMode_Mode_STRETCH:
case mediapipe::ScaleMode::STRETCH:
return scale_mode;
case mediapipe::ScaleMode_Mode_FIT:
case mediapipe::ScaleMode::FIT:
return scale_mode;
case mediapipe::ScaleMode_Mode_FILL_AND_CROP:
case mediapipe::ScaleMode::FILL_AND_CROP:
return scale_mode;
default:
return default_mode;
@ -208,6 +210,8 @@ class ImageTransformationCalculator : public CalculatorBase {
bool use_gpu_ = false;
cv::Scalar padding_color_;
ImageTransformationCalculatorOptions::InterpolationMode interpolation_mode_;
#if !MEDIAPIPE_DISABLE_GPU
GlCalculatorHelper gpu_helper_;
std::unique_ptr<QuadRenderer> rgb_renderer_;
@ -343,6 +347,11 @@ absl::Status ImageTransformationCalculator::Open(CalculatorContext* cc) {
options_.padding_color().green(),
options_.padding_color().blue());
interpolation_mode_ = options_.interpolation_mode();
if (options_.interpolation_mode() ==
ImageTransformationCalculatorOptions::DEFAULT) {
interpolation_mode_ = ImageTransformationCalculatorOptions::LINEAR;
}
if (use_gpu_) {
#if !MEDIAPIPE_DISABLE_GPU
// Let the helper access the GL context information.
@ -457,26 +466,48 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
ComputeOutputDimensions(input_width, input_height, &output_width,
&output_height);
int opencv_interpolation_mode = cv::INTER_LINEAR;
if (output_width_ > 0 && output_height_ > 0) {
cv::Mat scaled_mat;
if (scale_mode_ == mediapipe::ScaleMode_Mode_STRETCH) {
int scale_flag =
input_mat.cols > output_width_ && input_mat.rows > output_height_
? cv::INTER_AREA
: cv::INTER_LINEAR;
if (scale_mode_ == mediapipe::ScaleMode::STRETCH) {
if (interpolation_mode_ == ImageTransformationCalculatorOptions::LINEAR) {
// Use INTER_AREA for downscaling if interpolation mode is set to
// LINEAR.
if (input_mat.cols > output_width_ && input_mat.rows > output_height_) {
opencv_interpolation_mode = cv::INTER_AREA;
} else {
opencv_interpolation_mode = cv::INTER_LINEAR;
}
} else {
opencv_interpolation_mode = cv::INTER_NEAREST;
}
cv::resize(input_mat, scaled_mat, cv::Size(output_width_, output_height_),
0, 0, scale_flag);
0, 0, opencv_interpolation_mode);
} else {
const float scale =
std::min(static_cast<float>(output_width_) / input_width,
static_cast<float>(output_height_) / input_height);
const int target_width = std::round(input_width * scale);
const int target_height = std::round(input_height * scale);
int scale_flag = scale < 1.0f ? cv::INTER_AREA : cv::INTER_LINEAR;
if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
if (interpolation_mode_ == ImageTransformationCalculatorOptions::LINEAR) {
// Use INTER_AREA for downscaling if interpolation mode is set to
// LINEAR.
if (scale < 1.0f) {
opencv_interpolation_mode = cv::INTER_AREA;
} else {
opencv_interpolation_mode = cv::INTER_LINEAR;
}
} else {
opencv_interpolation_mode = cv::INTER_NEAREST;
}
if (scale_mode_ == mediapipe::ScaleMode::FIT) {
cv::Mat intermediate_mat;
cv::resize(input_mat, intermediate_mat,
cv::Size(target_width, target_height), 0, 0, scale_flag);
cv::Size(target_width, target_height), 0, 0,
opencv_interpolation_mode);
const int top = (output_height_ - target_height) / 2;
const int bottom = output_height_ - target_height - top;
const int left = (output_width_ - target_width) / 2;
@ -488,7 +519,7 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
padding_color_);
} else {
cv::resize(input_mat, scaled_mat, cv::Size(target_width, target_height),
0, 0, scale_flag);
0, 0, opencv_interpolation_mode);
output_width = target_width;
output_height = target_height;
}
@ -514,17 +545,17 @@ absl::Status ImageTransformationCalculator::RenderCpu(CalculatorContext* cc) {
cv::warpAffine(input_mat, rotated_mat, rotation_mat, rotated_size);
} else {
switch (rotation_) {
case mediapipe::RotationMode_Mode_UNKNOWN:
case mediapipe::RotationMode_Mode_ROTATION_0:
case mediapipe::RotationMode::UNKNOWN:
case mediapipe::RotationMode::ROTATION_0:
rotated_mat = input_mat;
break;
case mediapipe::RotationMode_Mode_ROTATION_90:
case mediapipe::RotationMode::ROTATION_90:
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_COUNTERCLOCKWISE);
break;
case mediapipe::RotationMode_Mode_ROTATION_180:
case mediapipe::RotationMode::ROTATION_180:
cv::rotate(input_mat, rotated_mat, cv::ROTATE_180);
break;
case mediapipe::RotationMode_Mode_ROTATION_270:
case mediapipe::RotationMode::ROTATION_270:
cv::rotate(input_mat, rotated_mat, cv::ROTATE_90_CLOCKWISE);
break;
}
@ -561,7 +592,7 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
ComputeOutputDimensions(input_width, input_height, &output_width,
&output_height);
if (scale_mode_ == mediapipe::ScaleMode_Mode_FILL_AND_CROP) {
if (scale_mode_ == mediapipe::ScaleMode::FILL_AND_CROP) {
const float scale =
std::min(static_cast<float>(output_width_) / input_width,
static_cast<float>(output_height_) / input_height);
@ -628,6 +659,12 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
glActiveTexture(GL_TEXTURE1);
glBindTexture(src1.target(), src1.name());
if (interpolation_mode_ == ImageTransformationCalculatorOptions::NEAREST) {
// TODO: revert texture params.
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
}
MP_RETURN_IF_ERROR(renderer->GlRender(
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
rotation, flip_horizontally_, flip_vertically_,
@ -652,8 +689,8 @@ void ImageTransformationCalculator::ComputeOutputDimensions(
if (output_width_ > 0 && output_height_ > 0) {
*output_width = output_width_;
*output_height = output_height_;
} else if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 ||
rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) {
} else if (rotation_ == mediapipe::RotationMode::ROTATION_90 ||
rotation_ == mediapipe::RotationMode::ROTATION_270) {
*output_width = input_height;
*output_height = input_width;
} else {
@ -666,9 +703,9 @@ void ImageTransformationCalculator::ComputeOutputLetterboxPadding(
int input_width, int input_height, int output_width, int output_height,
std::array<float, 4>* padding) {
padding->fill(0.f);
if (scale_mode_ == mediapipe::ScaleMode_Mode_FIT) {
if (rotation_ == mediapipe::RotationMode_Mode_ROTATION_90 ||
rotation_ == mediapipe::RotationMode_Mode_ROTATION_270) {
if (scale_mode_ == mediapipe::ScaleMode::FIT) {
if (rotation_ == mediapipe::RotationMode::ROTATION_90 ||
rotation_ == mediapipe::RotationMode::ROTATION_270) {
std::swap(input_width, input_height);
}
const float input_aspect_ratio =

11
mediapipe/calculators/image/image_transformation_calculator.proto
View File

@ -54,4 +54,15 @@ message ImageTransformationCalculatorOptions {
// The color for the padding. This option is only used when the scale mode is
// FIT. Default is black. This is for CPU only.
optional Color padding_color = 8;
// Interpolation method to use. Note that on CPU when LINEAR is specified,
// INTER_LINEAR is used for upscaling and INTER_AREA is used for downscaling.
enum InterpolationMode {
DEFAULT = 0;
LINEAR = 1;
NEAREST = 2;
}
// Mode DEFAULT will use LINEAR interpolation.
optional InterpolationMode interpolation_mode = 9;
}

315
mediapipe/calculators/image/image_transformation_calculator_test.cc Normal file
View File

@ -0,0 +1,315 @@
#include <string>
#include <utility>
#include <vector>
#include "absl/container/flat_hash_set.h"
#include "absl/flags/flag.h"
#include "absl/strings/substitute.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "testing/base/public/gmock.h"
#include "testing/base/public/googletest.h"
#include "third_party/OpenCV/core.hpp" // IWYU pragma: keep
#include "third_party/OpenCV/core/mat.hpp"
namespace mediapipe {
namespace {
absl::flat_hash_set<int> computeUniqueValues(const cv::Mat& mat) {
// Compute the unique values in cv::Mat
absl::flat_hash_set<int> unique_values;
for (int i = 0; i < mat.rows; i++) {
for (int j = 0; j < mat.cols; j++) {
unique_values.insert(mat.at<unsigned char>(i, j));
}
}
return unique_values;
}
TEST(ImageTransformationCalculatorTest, NearestNeighborResizing) {
cv::Mat input_mat;
cv::cvtColor(cv::imread(file::JoinPath("./",
"/mediapipe/calculators/"
"image/testdata/binary_mask.png")),
input_mat, cv::COLOR_BGR2GRAY);
Packet input_image_packet = MakePacket<ImageFrame>(
ImageFormat::GRAY8, input_mat.size().width, input_mat.size().height);
input_mat.copyTo(formats::MatView(&(input_image_packet.Get<ImageFrame>())));
std::vector<std::pair<int, int>> output_dims{
{256, 333}, {512, 512}, {1024, 1024}};
for (auto& output_dim : output_dims) {
Packet input_output_dim_packet =
MakePacket<std::pair<int, int>>(output_dim);
std::vector<std::string> scale_modes{"FIT", "STRETCH"};
for (const auto& scale_mode : scale_modes) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
absl::Substitute(R"(
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE:input_image"
input_stream: "OUTPUT_DIMENSIONS:image_size"
output_stream: "IMAGE:output_image"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
scale_mode: $0
interpolation_mode: NEAREST
}
})",
scale_mode));
CalculatorRunner runner(node_config);
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
input_image_packet.At(Timestamp(0)));
runner.MutableInputs()
->Tag("OUTPUT_DIMENSIONS")
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
MP_ASSERT_OK(runner.Run());
const auto& outputs = runner.Outputs();
ASSERT_EQ(outputs.NumEntries(), 1);
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
ASSERT_EQ(packets.size(), 1);
const auto& result = packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, result.Width());
ASSERT_EQ(output_dim.second, result.Height());
auto unique_input_values = computeUniqueValues(input_mat);
auto unique_output_values =
computeUniqueValues(formats::MatView(&result));
EXPECT_THAT(unique_input_values,
::testing::ContainerEq(unique_output_values));
}
}
}
TEST(ImageTransformationCalculatorTest,
NearestNeighborResizingWorksForFloatInput) {
cv::Mat input_mat;
cv::cvtColor(cv::imread(file::JoinPath("./",
"/mediapipe/calculators/"
"image/testdata/binary_mask.png")),
input_mat, cv::COLOR_BGR2GRAY);
Packet input_image_packet = MakePacket<ImageFrame>(
ImageFormat::VEC32F1, input_mat.size().width, input_mat.size().height);
cv::Mat packet_mat_view =
formats::MatView(&(input_image_packet.Get<ImageFrame>()));
input_mat.convertTo(packet_mat_view, CV_32FC1, 1 / 255.f);
std::vector<std::pair<int, int>> output_dims{
{256, 333}, {512, 512}, {1024, 1024}};
for (auto& output_dim : output_dims) {
Packet input_output_dim_packet =
MakePacket<std::pair<int, int>>(output_dim);
std::vector<std::string> scale_modes{"FIT", "STRETCH"};
for (const auto& scale_mode : scale_modes) {
CalculatorGraphConfig::Node node_config =
ParseTextProtoOrDie<CalculatorGraphConfig::Node>(
absl::Substitute(R"(
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE:input_image"
input_stream: "OUTPUT_DIMENSIONS:image_size"
output_stream: "IMAGE:output_image"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
scale_mode: $0
interpolation_mode: NEAREST
}
})",
scale_mode));
CalculatorRunner runner(node_config);
runner.MutableInputs()->Tag("IMAGE").packets.push_back(
input_image_packet.At(Timestamp(0)));
runner.MutableInputs()
->Tag("OUTPUT_DIMENSIONS")
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
MP_ASSERT_OK(runner.Run());
const auto& outputs = runner.Outputs();
ASSERT_EQ(outputs.NumEntries(), 1);
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
ASSERT_EQ(packets.size(), 1);
const auto& result = packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, result.Width());
ASSERT_EQ(output_dim.second, result.Height());
auto unique_input_values = computeUniqueValues(packet_mat_view);
auto unique_output_values =
computeUniqueValues(formats::MatView(&result));
EXPECT_THAT(unique_input_values,
::testing::ContainerEq(unique_output_values));
}
}
}
TEST(ImageTransformationCalculatorTest, NearestNeighborResizingGpu) {
cv::Mat input_mat;
cv::cvtColor(cv::imread(file::JoinPath("./",
"/mediapipe/calculators/"
"image/testdata/binary_mask.png")),
input_mat, cv::COLOR_BGR2RGBA);
std::vector<std::pair<int, int>> output_dims{
{256, 333}, {512, 512}, {1024, 1024}};
for (auto& output_dim : output_dims) {
std::vector<std::string> scale_modes{"FIT"}; //, "STRETCH"};
for (const auto& scale_mode : scale_modes) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
absl::Substitute(R"(
input_stream: "input_image"
input_stream: "image_size"
output_stream: "output_image"
node {
calculator: "ImageFrameToGpuBufferCalculator"
input_stream: "input_image"
output_stream: "input_image_gpu"
}
node {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:input_image_gpu"
input_stream: "OUTPUT_DIMENSIONS:image_size"
output_stream: "IMAGE_GPU:output_image_gpu"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
scale_mode: $0
interpolation_mode: NEAREST
}
}
}
node {
calculator: "GpuBufferToImageFrameCalculator"
input_stream: "output_image_gpu"
output_stream: "output_image"
})",
scale_mode));
ImageFrame input_image(ImageFormat::SRGBA, input_mat.size().width,
input_mat.size().height);
input_mat.copyTo(formats::MatView(&input_image));
std::vector<Packet> output_image_packets;
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
CalculatorGraph graph(graph_config);
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image",
MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0))));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"image_size",
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_THAT(output_image_packets, testing::SizeIs(1));
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, output_image.Width());
ASSERT_EQ(output_dim.second, output_image.Height());
auto unique_input_values = computeUniqueValues(input_mat);
auto unique_output_values =
computeUniqueValues(formats::MatView(&output_image));
EXPECT_THAT(unique_input_values,
::testing::ContainerEq(unique_output_values));
}
}
}
TEST(ImageTransformationCalculatorTest,
NearestNeighborResizingWorksForFloatTexture) {
cv::Mat input_mat;
cv::cvtColor(cv::imread(file::JoinPath("./",
"/mediapipe/calculators/"
"image/testdata/binary_mask.png")),
input_mat, cv::COLOR_BGR2GRAY);
Packet input_image_packet = MakePacket<ImageFrame>(
ImageFormat::VEC32F1, input_mat.size().width, input_mat.size().height);
cv::Mat packet_mat_view =
formats::MatView(&(input_image_packet.Get<ImageFrame>()));
input_mat.convertTo(packet_mat_view, CV_32FC1, 1 / 255.f);
std::vector<std::pair<int, int>> output_dims{
{256, 333}, {512, 512}, {1024, 1024}};
for (auto& output_dim : output_dims) {
std::vector<std::string> scale_modes{"FIT"}; //, "STRETCH"};
for (const auto& scale_mode : scale_modes) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
absl::Substitute(R"(
input_stream: "input_image"
input_stream: "image_size"
output_stream: "output_image"
node {
calculator: "ImageFrameToGpuBufferCalculator"
input_stream: "input_image"
output_stream: "input_image_gpu"
}
node {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:input_image_gpu"
input_stream: "OUTPUT_DIMENSIONS:image_size"
output_stream: "IMAGE_GPU:output_image_gpu"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
scale_mode: $0
interpolation_mode: NEAREST
}
}
}
node {
calculator: "GpuBufferToImageFrameCalculator"
input_stream: "output_image_gpu"
output_stream: "output_image"
})",
scale_mode));
std::vector<Packet> output_image_packets;
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
CalculatorGraph graph(graph_config);
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image", input_image_packet.At(Timestamp(0))));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"image_size",
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_THAT(output_image_packets, testing::SizeIs(1));
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, output_image.Width());
ASSERT_EQ(output_dim.second, output_image.Height());
auto unique_input_values = computeUniqueValues(packet_mat_view);
auto unique_output_values =
computeUniqueValues(formats::MatView(&output_image));
EXPECT_THAT(unique_input_values,
::testing::ContainerEq(unique_output_values));
}
}
}
} // namespace
} // namespace mediapipe

3
mediapipe/calculators/image/opencv_image_encoder_calculator.cc
View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/image/opencv_image_encoder_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
@ -61,7 +62,7 @@ absl::Status OpenCvImageEncoderCalculator::Open(CalculatorContext* cc) {
absl::Status OpenCvImageEncoderCalculator::Process(CalculatorContext* cc) {
const ImageFrame& image_frame = cc->Inputs().Index(0).Get<ImageFrame>();
CHECK_EQ(1, image_frame.ByteDepth());
ABSL_CHECK_EQ(1, image_frame.ByteDepth());
std::unique_ptr<OpenCvImageEncoderCalculatorResults> encoded_result =
absl::make_unique<OpenCvImageEncoderCalculatorResults>();

15
mediapipe/calculators/image/scale_image_calculator.cc
View File

@ -18,6 +18,8 @@
#include <memory>
#include <string>
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/substitute.h"
#include "libyuv/scale.h"
@ -293,7 +295,7 @@ absl::Status ScaleImageCalculator::InitializeFrameInfo(CalculatorContext* cc) {
header->width = output_width_;
header->height = output_height_;
header->format = output_format_;
LOG(INFO) << "OUTPUTTING HEADER on stream";
ABSL_LOG(INFO) << "OUTPUTTING HEADER on stream";
cc->Outputs()
.Tag("VIDEO_HEADER")
.Add(header.release(), Timestamp::PreStream());
@ -393,10 +395,11 @@ absl::Status ScaleImageCalculator::Open(CalculatorContext* cc) {
.SetHeader(Adopt(output_header.release()));
has_header_ = true;
} else {
LOG(WARNING) << "Stream had a VideoHeader which didn't have sufficient "
"information. "
"Dropping VideoHeader and trying to deduce needed "
"information.";
ABSL_LOG(WARNING)
<< "Stream had a VideoHeader which didn't have sufficient "
"information. "
"Dropping VideoHeader and trying to deduce needed "
"information.";
input_width_ = 0;
input_height_ = 0;
if (!options_.has_input_format()) {
@ -507,7 +510,7 @@ absl::Status ScaleImageCalculator::ValidateImageFrame(
absl::Status ScaleImageCalculator::ValidateYUVImage(CalculatorContext* cc,
const YUVImage& yuv_image) {
CHECK_EQ(input_format_, ImageFormat::YCBCR420P);
ABSL_CHECK_EQ(input_format_, ImageFormat::YCBCR420P);
if (!has_header_) {
if (input_width_ != yuv_image.width() ||
input_height_ != yuv_image.height()) {

17
mediapipe/calculators/image/scale_image_utils.cc
View File

@ -18,6 +18,7 @@
#include <string>
#include "absl/log/absl_check.h"
#include "absl/strings/str_split.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
@ -40,10 +41,10 @@ absl::Status FindCropDimensions(int input_width, int input_height, //
const std::string& max_aspect_ratio, //
int* crop_width, int* crop_height, //
int* col_start, int* row_start) {
CHECK(crop_width);
CHECK(crop_height);
CHECK(col_start);
CHECK(row_start);
ABSL_CHECK(crop_width);
ABSL_CHECK(crop_height);
ABSL_CHECK(col_start);
ABSL_CHECK(row_start);
double min_aspect_ratio_q = 0.0;
double max_aspect_ratio_q = 0.0;
@ -83,8 +84,8 @@ absl::Status FindCropDimensions(int input_width, int input_height, //
}
}
CHECK_LE(*crop_width, input_width);
CHECK_LE(*crop_height, input_height);
ABSL_CHECK_LE(*crop_width, input_width);
ABSL_CHECK_LE(*crop_height, input_height);
return absl::OkStatus();
}
@ -96,8 +97,8 @@ absl::Status FindOutputDimensions(int input_width, //
bool preserve_aspect_ratio, //
int scale_to_multiple_of, //
int* output_width, int* output_height) {
CHECK(output_width);
CHECK(output_height);
ABSL_CHECK(output_width);
ABSL_CHECK(output_height);
if (target_max_area > 0 && input_width * input_height > target_max_area) {
preserve_aspect_ratio = true;

6
mediapipe/calculators/image/segmentation_smoothing_calculator.cc
View File

@ -15,13 +15,13 @@
#include <algorithm>
#include <memory>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/vector.h"
@ -110,7 +110,7 @@ REGISTER_CALCULATOR(SegmentationSmoothingCalculator);
absl::Status SegmentationSmoothingCalculator::GetContract(
CalculatorContract* cc) {
CHECK_GE(cc->Inputs().NumEntries(), 1);
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
cc->Inputs().Tag(kCurrentMaskTag).Set<Image>();
cc->Inputs().Tag(kPreviousMaskTag).Set<Image>();
@ -273,7 +273,7 @@ absl::Status SegmentationSmoothingCalculator::RenderGpu(CalculatorContext* cc) {
const auto& previous_frame = cc->Inputs().Tag(kPreviousMaskTag).Get<Image>();
if (previous_frame.format() != current_frame.format()) {
LOG(ERROR) << "Warning: mixing input format types. ";
ABSL_LOG(ERROR) << "Warning: mixing input format types. ";
}
auto previous_texture = gpu_helper_.CreateSourceTexture(previous_frame);

3
mediapipe/calculators/image/segmentation_smoothing_calculator_test.cc
View File

@ -14,6 +14,7 @@
#include <memory>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/image/segmentation_smoothing_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
@ -169,7 +170,7 @@ void RunTest(bool use_gpu, float mix_ratio, cv::Mat& test_result) {
}
}
} else {
LOG(ERROR) << "invalid ratio";
ABSL_LOG(ERROR) << "invalid ratio";
}
}

8
mediapipe/calculators/image/set_alpha_calculator.cc
View File

@ -14,13 +14,13 @@
#include <memory>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/image/set_alpha_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/status.h"
@ -142,7 +142,7 @@ class SetAlphaCalculator : public CalculatorBase {
REGISTER_CALCULATOR(SetAlphaCalculator);
absl::Status SetAlphaCalculator::GetContract(CalculatorContract* cc) {
CHECK_GE(cc->Inputs().NumEntries(), 1);
RET_CHECK_GE(cc->Inputs().NumEntries(), 1);
bool use_gpu = false;
@ -268,7 +268,7 @@ absl::Status SetAlphaCalculator::RenderCpu(CalculatorContext* cc) {
const auto& input_frame = cc->Inputs().Tag(kInputFrameTag).Get<ImageFrame>();
const cv::Mat input_mat = formats::MatView(&input_frame);
if (!(input_mat.type() == CV_8UC3 || input_mat.type() == CV_8UC4)) {
LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported";
ABSL_LOG(ERROR) << "Only 3 or 4 channel 8-bit input image supported";
}
// Setup destination image
@ -328,7 +328,7 @@ absl::Status SetAlphaCalculator::RenderGpu(CalculatorContext* cc) {
cc->Inputs().Tag(kInputFrameTagGpu).Get<mediapipe::GpuBuffer>();
if (!(input_frame.format() == mediapipe::GpuBufferFormat::kBGRA32 ||
input_frame.format() == mediapipe::GpuBufferFormat::kRGB24)) {
LOG(ERROR) << "Only RGB or RGBA input image supported";
ABSL_LOG(ERROR) << "Only RGB or RGBA input image supported";
}
auto input_texture = gpu_helper_.CreateSourceTexture(input_frame);

1
mediapipe/calculators/image/testdata/BUILD vendored
View File

@ -18,6 +18,7 @@ licenses(["notice"])
filegroup(
name = "test_images",
srcs = [
"binary_mask.png",
"dino.jpg",
"dino_quality_50.jpg",
"dino_quality_80.jpg",

BIN
mediapipe/calculators/image/testdata/binary_mask.png vendored Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 771 B

2
mediapipe/calculators/image/yuv_to_image_calculator.cc
View File

@ -38,7 +38,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
buf[0] = (fourcc >> 24) & 0xff;
buf[1] = (fourcc >> 16) & 0xff;
buf[2] = (fourcc >> 8) & 0xff;
buf[3] = (fourcc)&0xff;
buf[3] = (fourcc) & 0xff;
buf[4] = 0;
return std::string(buf);
}

2
mediapipe/calculators/internal/BUILD
View File

@ -31,12 +31,14 @@ mediapipe_proto_library(
cc_library(
name = "callback_packet_calculator",
srcs = ["callback_packet_calculator.cc"],
hdrs = ["callback_packet_calculator.h"],
visibility = ["//mediapipe/framework:__subpackages__"],
deps = [
":callback_packet_calculator_cc_proto",
"//mediapipe/framework:calculator_base",
"//mediapipe/framework:calculator_registry",
"//mediapipe/framework:output_side_packet",
"@com_google_absl//absl/status",
],
alwayslink = 1,
)

97
mediapipe/calculators/internal/callback_packet_calculator.cc
View File

@ -11,10 +11,12 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/internal/callback_packet_calculator.h"
#include <functional>
#include <string>
#include "absl/status/status.h"
#include "mediapipe/calculators/internal/callback_packet_calculator.pb.h" // NOLINT
#include "mediapipe/framework/calculator_base.h"
#include "mediapipe/framework/calculator_registry.h"
@ -39,64 +41,55 @@ void DumpPostStreamPacket(Packet* post_stream_packet, const Packet& packet) {
*post_stream_packet = packet;
}
}
} // namespace
// Creates a callback which takes a packet and stores it either in a
// vector of packets or stores only the packet at PostStream timestamp.
// The kind of callback is controlled by an option. The callback is
// a std::function and is directly usable by CallbackCalculator.
// Since the options for the packet generator include a serialized pointer
// value, the resulting callback is only valid on the original machine
// while that pointer is still alive.
class CallbackPacketCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc) {
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
switch (options.type()) {
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
cc->OutputSidePackets()
.Index(0)
.Set<std::function<void(const Packet&)>>();
break;
default:
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Invalid type of callback to produce.";
}
return absl::OkStatus();
}
absl::Status Open(CalculatorContext* cc) override {
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
void* ptr;
if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) {
absl::Status CallbackPacketCalculator::GetContract(CalculatorContract* cc) {
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
switch (options.type()) {
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
cc->OutputSidePackets()
.Index(0)
.Set<std::function<void(const Packet&)>>();
break;
default:
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Stored pointer value in options is invalid.";
}
switch (options.type()) {
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
cc->OutputSidePackets().Index(0).Set(
MakePacket<std::function<void(const Packet&)>>(std::bind(
&DumpToVector, reinterpret_cast<std::vector<Packet>*>(ptr),
std::placeholders::_1)));
break;
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
cc->OutputSidePackets().Index(0).Set(
MakePacket<std::function<void(const Packet&)>>(
std::bind(&DumpPostStreamPacket, reinterpret_cast<Packet*>(ptr),
std::placeholders::_1)));
break;
default:
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Invalid type to dump into.";
}
return absl::OkStatus();
<< "Invalid type of callback to produce.";
}
return absl::OkStatus();
}
absl::Status Process(CalculatorContext* cc) override {
return absl::OkStatus();
absl::Status CallbackPacketCalculator::Open(CalculatorContext* cc) {
const auto& options = cc->Options<CallbackPacketCalculatorOptions>();
void* ptr;
if (sscanf(options.pointer().c_str(), "%p", &ptr) != 1) {
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Stored pointer value in options is invalid.";
}
};
switch (options.type()) {
case CallbackPacketCalculatorOptions::VECTOR_PACKET:
cc->OutputSidePackets().Index(0).Set(
MakePacket<std::function<void(const Packet&)>>(std::bind(
&DumpToVector, reinterpret_cast<std::vector<Packet>*>(ptr),
std::placeholders::_1)));
break;
case CallbackPacketCalculatorOptions::POST_STREAM_PACKET:
cc->OutputSidePackets().Index(0).Set(
MakePacket<std::function<void(const Packet&)>>(
std::bind(&DumpPostStreamPacket, reinterpret_cast<Packet*>(ptr),
std::placeholders::_1)));
break;
default:
return mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
<< "Invalid type to dump into.";
}
return absl::OkStatus();
}
absl::Status CallbackPacketCalculator::Process(CalculatorContext* cc) {
return absl::OkStatus();
}
REGISTER_CALCULATOR(CallbackPacketCalculator);

39
mediapipe/calculators/internal/callback_packet_calculator.h Normal file
View File

@ -0,0 +1,39 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
#define MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_
#include "absl/status/status.h"
#include "mediapipe/framework/calculator_base.h"
namespace mediapipe {
// Creates a callback which takes a packet and stores it either in a
// vector of packets or stores only the packet at PostStream timestamp.
// The kind of callback is controlled by an option. The callback is
// a std::function and is directly usable by CallbackCalculator.
// Since the options for the packet generator include a serialized pointer
// value, the resulting callback is only valid on the original machine
// while that pointer is still alive.
class CallbackPacketCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_INTERNAL_CALLBACK_PACKET_CALCULATOR_H_

30
mediapipe/calculators/tensor/BUILD
View File

@ -87,6 +87,7 @@ cc_library(
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/util:time_series_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
@ -181,6 +182,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/api2:node",
"//mediapipe/framework/formats:tensor",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/status",
],
alwayslink = 1,
@ -198,6 +200,7 @@ cc_test(
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log:absl_check",
"@org_tensorflow//tensorflow/lite/c:common",
],
)
@ -228,7 +231,6 @@ cc_library(
"//mediapipe/tasks/metadata:metadata_schema_cc",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
],
alwayslink = 1,
@ -280,7 +282,6 @@ cc_library(
"//mediapipe/tasks/cc/text/tokenizers:tokenizer_utils",
"//mediapipe/tasks/metadata:metadata_schema_cc",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
],
alwayslink = 1,
)
@ -394,7 +395,7 @@ mediapipe_proto_library(
# If you want to have precise control of which implementations to include (e.g. for strict binary
# size concerns), depend on those implementations directly, and do not depend on
# :inference_calculator.
# In all cases, use "InferenceCalulator" in your graphs.
# In all cases, use "InferenceCalculator" in your graphs.
cc_library_with_tflite(
name = "inference_calculator_interface",
srcs = ["inference_calculator.cc"],
@ -447,6 +448,7 @@ cc_library(
"//mediapipe/framework/deps:file_path",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/util/tflite:tflite_gpu_runner",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
@ -476,6 +478,7 @@ cc_library(
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/objc:mediapipe_framework_ios",
"//mediapipe/util/tflite:config",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings:str_format",
"@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
@ -622,6 +625,7 @@ mediapipe_proto_library(
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/gpu:gpu_origin_proto",
],
)
@ -651,10 +655,21 @@ cc_library(
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_buffer_format",
"//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/util:resource_util",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings:str_format",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": ["tensor_converter_calculator_gpu_deps"],
}) + select({
"//mediapipe:apple": [
"//third_party/apple_frameworks:MetalKit",
],
"//conditions:default": [],
}),
alwayslink = 1,
)
@ -696,6 +711,7 @@ cc_test(
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/memory",
@ -734,6 +750,8 @@ cc_library(
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/formats/object_detection:anchor_cc_proto",
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span",
] + selects.with_or({
@ -790,6 +808,7 @@ cc_library(
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/log:absl_check",
],
alwayslink = 1,
)
@ -982,6 +1001,8 @@ cc_library(
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_origin_cc_proto",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [":image_to_tensor_calculator_gpu_deps"],
@ -1052,6 +1073,7 @@ cc_test(
"testdata/image_to_tensor/medium_sub_rect_with_rotation_border_zero.png",
"testdata/image_to_tensor/noop_except_range.png",
],
tags = ["not_run:arm"],
deps = [
":image_to_tensor_calculator",
":image_to_tensor_converter",
@ -1073,6 +1095,7 @@ cc_test(
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/util:image_test_utils",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
@ -1200,6 +1223,7 @@ cc_library(
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
],
}),

34
mediapipe/calculators/tensor/audio_to_tensor_calculator.cc
View File

@ -20,6 +20,7 @@
#include <utility>
#include <vector>
#include "absl/log/absl_check.h"
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
@ -282,18 +283,23 @@ absl::Status AudioToTensorCalculator::Open(CalculatorContext* cc) {
if (options.has_volume_gain_db()) {
gain_ = pow(10, options.volume_gain_db() / 20.0);
}
RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
!kAudioIn(cc).Header().IsEmpty())
<< "Must either specify the time series header of the \"AUDIO\" stream "
"or have the \"SAMPLE_RATE\" stream connected.";
if (!kAudioIn(cc).Header().IsEmpty()) {
mediapipe::TimeSeriesHeader input_header;
MP_RETURN_IF_ERROR(mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
kAudioIn(cc).Header(), &input_header));
if (stream_mode_) {
MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
} else {
source_sample_rate_ = input_header.sample_rate();
if (options.has_source_sample_rate()) {
source_sample_rate_ = options.source_sample_rate();
} else {
RET_CHECK(kAudioSampleRateIn(cc).IsConnected() ^
!kAudioIn(cc).Header().IsEmpty())
<< "Must either specify the time series header of the \"AUDIO\" stream "
"or have the \"SAMPLE_RATE\" stream connected.";
if (!kAudioIn(cc).Header().IsEmpty()) {
mediapipe::TimeSeriesHeader input_header;
MP_RETURN_IF_ERROR(
mediapipe::time_series_util::FillTimeSeriesHeaderIfValid(
kAudioIn(cc).Header(), &input_header));
if (stream_mode_) {
MP_RETURN_IF_ERROR(SetupStreamingResampler(input_header.sample_rate()));
} else {
source_sample_rate_ = input_header.sample_rate();
}
}
}
AppendZerosToSampleBuffer(padding_samples_before_);
@ -343,7 +349,7 @@ absl::Status AudioToTensorCalculator::Process(CalculatorContext* cc) {
return absl::InvalidArgumentError(
"The audio data should be stored in column-major.");
}
CHECK(channels_match || mono_output);
ABSL_CHECK(channels_match || mono_output);
const Matrix& input = channels_match ? input_frame
// Mono mixdown.
: input_frame.colwise().mean();
@ -452,7 +458,7 @@ absl::Status AudioToTensorCalculator::SetupStreamingResampler(
}
void AudioToTensorCalculator::AppendZerosToSampleBuffer(int num_samples) {
CHECK_GE(num_samples, 0); // Ensured by `UpdateContract`.
ABSL_CHECK_GE(num_samples, 0); // Ensured by `UpdateContract`.
if (num_samples == 0) {
return;
}

3
mediapipe/calculators/tensor/audio_to_tensor_calculator.proto
View File

@ -85,4 +85,7 @@ message AudioToTensorCalculatorOptions {
// The volume gain, measured in dB.
// Scale the input audio amplitude by 10^(volume_gain_db/20).
optional double volume_gain_db = 12;
// The source number of samples per second (hertz) of the input audio buffers.
optional double source_sample_rate = 13;
}

4
mediapipe/calculators/tensor/bert_preprocessor_calculator.cc
View File

@ -22,7 +22,6 @@
#include "absl/container/flat_hash_set.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/ascii.h"
#include "absl/strings/string_view.h"
#include "absl/strings/substitute.h"
@ -244,7 +243,8 @@ std::vector<Tensor> BertPreprocessorCalculator::GenerateInputTensors(
input_tensors.reserve(kNumInputTensorsForBert);
for (int i = 0; i < kNumInputTensorsForBert; ++i) {
input_tensors.push_back(
{Tensor::ElementType::kInt32, Tensor::Shape({tensor_size})});
{Tensor::ElementType::kInt32,
Tensor::Shape({1, tensor_size}, has_dynamic_input_tensors_)});
}
std::memcpy(input_tensors[input_ids_tensor_index_]
.GetCpuWriteView()

2
mediapipe/calculators/tensor/bert_preprocessor_calculator.proto
View File

@ -1,4 +1,4 @@
/* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
/* Copyright 2022 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

3
mediapipe/calculators/tensor/feedback_tensors_calculator_test.cc
View File

@ -18,6 +18,7 @@
#include <utility>
#include <vector>
#include "absl/log/absl_check.h"
#include "mediapipe/calculators/tensor/feedback_tensors_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@ -65,7 +66,7 @@ template <typename T>
Tensor MakeTensor(std::initializer_list<int> shape,
std::initializer_list<T> values) {
Tensor tensor(TensorElementType<T>::value, shape);
CHECK_EQ(values.size(), tensor.shape().num_elements())
ABSL_CHECK_EQ(values.size(), tensor.shape().num_elements())
<< "The size of `values` is incompatible with `shape`";
absl::c_copy(values, tensor.GetCpuWriteView().buffer<T>());
return tensor;

7
mediapipe/calculators/tensor/image_to_tensor_calculator.cc
View File

@ -16,6 +16,7 @@
#include <memory>
#include <vector>
#include "absl/log/absl_log.h"
#include "mediapipe/calculators/tensor/image_to_tensor_calculator.pb.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
#include "mediapipe/calculators/tensor/image_to_tensor_utils.h"
@ -284,9 +285,9 @@ class ImageToTensorCalculator : public Node {
cc, GetBorderMode(options_.border_mode()),
GetOutputTensorType(/*uses_gpu=*/false, params_)));
#else
LOG(FATAL) << "Cannot create image to tensor CPU converter since "
"MEDIAPIPE_DISABLE_OPENCV is defined and "
"MEDIAPIPE_ENABLE_HALIDE is not defined.";
ABSL_LOG(FATAL) << "Cannot create image to tensor CPU converter since "
"MEDIAPIPE_DISABLE_OPENCV is defined and "
"MEDIAPIPE_ENABLE_HALIDE is not defined.";
#endif // !MEDIAPIPE_DISABLE_HALIDE
}
}

3
mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc
View File

@ -18,6 +18,7 @@
#include <vector>
#include "absl/flags/flag.h"
#include "absl/log/absl_check.h"
#include "absl/memory/memory.h"
#include "absl/strings/str_format.h"
#include "absl/strings/substitute.h"
@ -205,7 +206,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
} else if (image_channels == 1) {
return ImageFormat::GRAY8;
}
CHECK(false) << "Unsupported input image channles: " << image_channels;
ABSL_CHECK(false) << "Unsupported input image channles: " << image_channels;
}
Packet MakeImageFramePacket(cv::Mat input) {

3
mediapipe/calculators/tensor/image_to_tensor_converter_frame_buffer.cc
View File

@ -95,7 +95,8 @@ absl::Status FrameBufferProcessor::Convert(const mediapipe::Image& input,
static_cast<int>(range_max) == 255);
}
auto input_frame = input.GetGpuBuffer().GetReadView<FrameBuffer>();
auto input_frame =
input.GetGpuBuffer(/*upload_to_gpu=*/false).GetReadView<FrameBuffer>();
const auto& output_shape = output_tensor.shape();
MP_RETURN_IF_ERROR(ValidateTensorShape(output_shape));
FrameBuffer::Dimension output_dimension{/*width=*/output_shape.dims[2],

3
mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.cc
View File

@ -22,6 +22,7 @@
#include <memory>
#include <vector>
#include "absl/log/absl_log.h"
#include "absl/strings/str_cat.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter.h"
#include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_utils.h"
@ -259,7 +260,7 @@ class GlProcessor : public ImageToTensorConverter {
// error. So in that case, we'll grab the transpose of our original matrix
// and send that instead.
const auto gl_context = mediapipe::GlContext::GetCurrent();
LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread.";
ABSL_LOG_IF(FATAL, !gl_context) << "GlContext is not bound to the thread.";
if (gl_context->GetGlVersion() == mediapipe::GlVersion::kGLES2) {
GetTransposedRotatedSubRectToRectTransformMatrix(
sub_rect, texture.width(), texture.height(), flip_horizontaly,

Some files were not shown because too many files have changed in this diff Show More