Project import generated by Copybara.

GitOrigin-RevId: 50714fe28298d7b707eff7304547d89d6ec34a54
This commit is contained in:
MediaPipe Team 2019-11-21 12:01:51 -08:00 committed by mgyong
parent 9437483827
commit 48bcbb115f
60 changed files with 1662 additions and 111 deletions

View File

@ -10,11 +10,13 @@
## ML Solutions in MediaPipe
* [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md)
* [Multi-hand Tracking](mediapipe/docs/multi_hand_tracking_mobile_gpu.md)
* [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md)
* [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md)
* [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
![object_detection](mediapipe/docs/images/mobile/object_detection_android_gpu_small.gif)
@ -23,7 +25,7 @@
Follow these [instructions](mediapipe/docs/install.md).
## Getting started
See mobile and desktop [examples](mediapipe/docs/examples.md).
See mobile, desktop and Google Coral [examples](mediapipe/docs/examples.md).
## Documentation
[MediaPipe Read-the-Docs](https://mediapipe.readthedocs.io/) or [docs.mediapipe.dev](https://docs.mediapipe.dev)
@ -41,6 +43,7 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
* [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
## Events
* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)

View File

@ -149,11 +149,10 @@ new_local_repository(
http_archive(
name = "android_opencv",
sha256 = "056b849842e4fa8751d09edbb64530cfa7a63c84ccd232d0ace330e27ba55d0b",
build_file = "@//third_party:opencv_android.BUILD",
strip_prefix = "OpenCV-android-sdk",
type = "zip",
url = "https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip",
url = "https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip",
)
# After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will
@ -184,13 +183,18 @@ maven_install(
artifacts = [
"androidx.annotation:annotation:aar:1.1.0",
"androidx.appcompat:appcompat:aar:1.1.0-rc01",
"androidx.camera:camera-core:aar:1.0.0-alpha06",
"androidx.camera:camera-camera2:aar:1.0.0-alpha06",
"androidx.constraintlayout:constraintlayout:aar:1.1.3",
"androidx.core:core:aar:1.1.0-rc03",
"androidx.legacy:legacy-support-v4:aar:1.0.0",
"androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
"com.google.android.material:material:aar:1.0.0-rc01",
],
repositories = ["https://dl.google.com/dl/android/maven2"],
repositories = [
"https://dl.google.com/dl/android/maven2",
"https://repo1.maven.org/maven2",
],
)
maven_server(

View File

@ -36,7 +36,8 @@ message ScaleImageCalculatorOptions {
// If ratio is positive, crop the image to this minimum and maximum
// aspect ratio (preserving the center of the frame). This is done
// before scaling.
// before scaling. The string must contain "/", so to disable cropping,
// set both to "0/1".
// For example, for a min_aspect_ratio of "9/16" and max of "16/9" the
// following cropping will occur:
// 1920x1080 (which is 16:9) is not cropped

View File

@ -85,7 +85,7 @@ class TFRecordReaderCalculator : public CalculatorBase {
tensorflow::io::RecordReader reader(file.get(),
tensorflow::io::RecordReaderOptions());
tensorflow::uint64 offset = 0;
std::string example_str;
tensorflow::tstring example_str;
const int target_idx =
cc->InputSidePackets().HasTag(kRecordIndex)
? cc->InputSidePackets().Tag(kRecordIndex).Get<int>()
@ -98,7 +98,7 @@ class TFRecordReaderCalculator : public CalculatorBase {
if (current_idx == target_idx) {
if (cc->OutputSidePackets().HasTag(kExampleTag)) {
tensorflow::Example tf_example;
tf_example.ParseFromString(example_str);
tf_example.ParseFromArray(example_str.data(), example_str.size());
cc->OutputSidePackets()
.Tag(kExampleTag)
.Set(MakePacket<tensorflow::Example>(std::move(tf_example)));

View File

@ -64,6 +64,28 @@ typedef id<MTLBuffer> GpuTensor;
size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; } // NOLINT
} // namespace
#if defined(MEDIAPIPE_EDGE_TPU)
#include "edgetpu.h"
// Creates and returns an Edge TPU interpreter to run the given edgetpu model.
std::unique_ptr<tflite::Interpreter> BuildEdgeTpuInterpreter(
const tflite::FlatBufferModel& model,
tflite::ops::builtin::BuiltinOpResolver* resolver,
edgetpu::EdgeTpuContext* edgetpu_context) {
resolver->AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp());
std::unique_ptr<tflite::Interpreter> interpreter;
if (tflite::InterpreterBuilder(model, *resolver)(&interpreter) != kTfLiteOk) {
std::cerr << "Failed to build edge TPU interpreter." << std::endl;
}
interpreter->SetExternalContext(kTfLiteEdgeTpuContext, edgetpu_context);
interpreter->SetNumThreads(1);
if (interpreter->AllocateTensors() != kTfLiteOk) {
std::cerr << "Failed to allocate edge TPU tensors." << std::endl;
}
return interpreter;
}
#endif // MEDIAPIPE_EDGE_TPU
// TfLiteInferenceCalculator File Layout:
// * Header
// * Core
@ -162,6 +184,11 @@ class TfLiteInferenceCalculator : public CalculatorBase {
TFLBufferConvert* converter_from_BPHWC4_ = nil;
#endif
#if defined(MEDIAPIPE_EDGE_TPU)
std::shared_ptr<edgetpu::EdgeTpuContext> edgetpu_context_ =
edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice();
#endif
std::string model_path_ = "";
bool gpu_inference_ = false;
bool gpu_input_ = false;
@ -425,6 +452,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
#endif
delegate_ = nullptr;
}
#if defined(MEDIAPIPE_EDGE_TPU)
edgetpu_context_.reset();
#endif
return ::mediapipe::OkStatus();
}
@ -458,16 +488,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
model_ = tflite::FlatBufferModel::BuildFromFile(model_path_.c_str());
RET_CHECK(model_);
tflite::ops::builtin::BuiltinOpResolver op_resolver;
if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
const auto& op_resolver =
cc->InputSidePackets()
.Tag("CUSTOM_OP_RESOLVER")
.Get<tflite::ops::builtin::BuiltinOpResolver>();
tflite::InterpreterBuilder(*model_, op_resolver)(&interpreter_);
} else {
const tflite::ops::builtin::BuiltinOpResolver op_resolver;
tflite::InterpreterBuilder(*model_, op_resolver)(&interpreter_);
op_resolver = cc->InputSidePackets()
.Tag("CUSTOM_OP_RESOLVER")
.Get<tflite::ops::builtin::BuiltinOpResolver>();
}
#if defined(MEDIAPIPE_EDGE_TPU)
interpreter_ =
BuildEdgeTpuInterpreter(*model_, &op_resolver, edgetpu_context_.get());
#else
tflite::InterpreterBuilder(*model_, op_resolver)(&interpreter_);
#endif // MEDIAPIPE_EDGE_TPU
RET_CHECK(interpreter_);

View File

@ -93,6 +93,7 @@ REGISTER_CALCULATOR(LabelsToRenderDataCalculator);
}
::mediapipe::Status LabelsToRenderDataCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
options_ = cc->Options<LabelsToRenderDataCalculatorOptions>();
num_colors_ = options_.color_size();
label_height_px_ = std::ceil(options_.font_height_px() * kFontHeightScale);

View File

@ -92,7 +92,7 @@ project.
MediaPipe depends on OpenCV, you will need to copy the precompiled OpenCV so
files into app/src/main/jniLibs. You can download the official OpenCV
Android SDK from
[here](https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip)
[here](https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip)
and run:
```bash

View File

@ -157,3 +157,20 @@ how to use MediaPipe with a TFLite model for hair segmentation on desktop using
GPU with live video from a webcam.
* [Desktop GPU](./hair_segmentation_desktop.md)
## Google Coral (machine learning acceleration with Google EdgeTPU)
Below are code samples on how to run MediaPipe on Google Coral Dev Board.
### Object Detection on Coral
[Object Detection on Coral with Webcam](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
shows how to run quantized object detection TFlite model accelerated with
EdgeTPU on
[Google Coral Dev Board](https://coral.withgoogle.com/products/dev-board).
### Face Detection on Coral
[Face Detection on Coral with Webcam](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
shows how to use quantized face detection TFlite model accelerated with EdgeTPU
on [Google Coral Dev Board](https://coral.withgoogle.com/products/dev-board).

View File

@ -629,7 +629,7 @@ to load both dependencies:
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java4");
System.loadLibrary("opencv_java3");
}
```

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.3 MiB

View File

@ -156,7 +156,7 @@ node {
output_stream: "multi_hand_rects"
node_options: {
[type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
min_similarity_threshold: 0.1
min_similarity_threshold: 0.5
}
}
}

View File

@ -219,7 +219,7 @@ node {
output_stream: "multi_hand_rects"
node_options: {
[type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
min_similarity_threshold: 0.1
min_similarity_threshold: 0.5
}
}
}

View File

@ -47,7 +47,7 @@ public class MainActivity extends AppCompatActivity {
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java4");
System.loadLibrary("opencv_java3");
}
// {@link SurfaceTexture} where the camera-preview frames can be accessed.

View File

@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java4");
System.loadLibrary("opencv_java3");
}
// {@link SurfaceTexture} where the camera-preview frames can be accessed.

View File

@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java4");
System.loadLibrary("opencv_java3");
}
// {@link SurfaceTexture} where the camera-preview frames can be accessed.

View File

@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java4");
System.loadLibrary("opencv_java3");
}
// {@link SurfaceTexture} where the camera-preview frames can be accessed.

View File

@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java4");
System.loadLibrary("opencv_java3");
}
// {@link SurfaceTexture} where the camera-preview frames can be accessed.

View File

@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java4");
System.loadLibrary("opencv_java3");
}
// {@link SurfaceTexture} where the camera-preview frames can be accessed.

View File

@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java4");
System.loadLibrary("opencv_java3");
}
// {@link SurfaceTexture} where the camera-preview frames can be accessed.

View File

@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java4");
System.loadLibrary("opencv_java3");
}
// {@link SurfaceTexture} where the camera-preview frames can be accessed.

View File

@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
static {
// Load all native libraries needed by the app.
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java4");
System.loadLibrary("opencv_java3");
}
// {@link SurfaceTexture} where the camera-preview frames can be accessed.

View File

@ -0,0 +1,56 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
package(default_visibility = [
"//visibility:public",
])
# Graph Runner
cc_library(
name = "demo_run_graph_main",
srcs = ["demo_run_graph_main.cc"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:opencv_highgui",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:opencv_video",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
],
)
# Demos
cc_binary(
name = "object_detection_cpu",
deps = [
"//mediapipe/examples/coral:demo_run_graph_main",
"//mediapipe/graphs/object_detection:desktop_tflite_calculators",
],
)
cc_binary(
name = "face_detection_cpu",
deps = [
"//mediapipe/examples/coral:demo_run_graph_main",
"//mediapipe/graphs/face_detection:desktop_tflite_calculators",
],
)

View File

@ -0,0 +1,87 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#==== ! Prerequisite ! ====
# $ sh mediapipe/examples/coral/setup.sh
#====
# for opencv 3.2 default
FROM ubuntu:18.04
MAINTAINER <mediapipe@google.com>
WORKDIR /mediapipe
ENV DEBIAN_FRONTEND=noninteractive
# Install MediaPipe & Coral deps
COPY update_sources.sh /
RUN /update_sources.sh
RUN dpkg --add-architecture armhf
RUN dpkg --add-architecture arm64
RUN apt-get update && apt-get install -y \
build-essential \
crossbuild-essential-arm64 \
libusb-1.0-0-dev \
libusb-1.0-0-dev:arm64 \
zlib1g-dev \
zlib1g-dev:arm64 \
pkg-config \
zip \
unzip \
curl \
wget \
git \
python \
python-pip \
python3-pip \
vim-common \
ca-certificates \
emacs \
software-properties-common && \
add-apt-repository -y ppa:openjdk-r/ppa && \
apt-get update && apt-get install -y openjdk-8-jdk
RUN pip install --upgrade setuptools
RUN pip install future
RUN pip3 install six
COPY . /mediapipe/
# Install bazel
ARG BAZEL_VERSION=0.29.1
RUN mkdir /bazel && \
wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
wget --no-check-certificate -O /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
chmod +x /bazel/installer.sh && \
/bazel/installer.sh && \
rm -f /bazel/installer.sh
# OpenCV (3.2 default in 18.04)
RUN apt-get update && apt-get install -y libopencv-dev
# Opencv libs copied from coral device into opencv32_arm64_libs
RUN cp opencv32_arm64_libs/* /usr/lib/aarch64-linux-gnu/.
# Edge tpu header and lib
RUN git clone https://github.com/google-coral/edgetpu.git /edgetpu
RUN cp /edgetpu/libedgetpu/direct/aarch64/libedgetpu.so.1.0 /usr/lib/aarch64-linux-gnu/libedgetpu.so
# See mediapipe/examples/coral/README.md to finish setup

View File

@ -0,0 +1,137 @@
# Coral Dev Board Setup (experimental)
**Dislaimer**: Running MediaPipe on Coral is experimental, and this process may not be exact and is subject to change. These instructions have only been tested on the coral dev board with OS version _mendel day_, and may vary for different devices and workstations.
This file describes how to prepare a Google Coral Dev Board and setup a linux Docker container for building MediaPipe applications that run on Edge TPU.
## Before creating the Docker
* (on host machine) run _setup.sh_ from MediaPipe root directory
sh mediapipe/examples/coral/setup.sh
* Setup the coral device via [here](https://coral.withgoogle.com/docs/dev-board/get-started/), and ensure the _mdt_ command works
* (on coral device) prepare MediaPipe
cd ~
sudo apt-get install git
git clone https://github.com/google/mediapipe.git
mkdir mediapipe/bazel-bin
* (on coral device) install opencv 3.2
sudo apt-get update && apt-get install -y libopencv-dev
* (on coral device) find all opencv libs
find /usr/lib/aarch64-linux-gnu/ -name 'libopencv*so'
* (on host machine) copy core opencv libs from coral device to a local folder inside MediaPipe checkout:
# in root level mediapipe folder #
mdt pull /usr/lib/aarch64-linux-gnu/libopencv_core.so opencv32_arm64_libs
mdt pull /usr/lib/aarch64-linux-gnu/libopencv_calib3d.so opencv32_arm64_libs
mdt pull /usr/lib/aarch64-linux-gnu/libopencv_features2d.so opencv32_arm64_libs
mdt pull /usr/lib/aarch64-linux-gnu/libopencv_highgui.so opencv32_arm64_libs
mdt pull /usr/lib/aarch64-linux-gnu/libopencv_imgcodecs.so opencv32_arm64_libs
mdt pull /usr/lib/aarch64-linux-gnu/libopencv_imgproc.so opencv32_arm64_libs
mdt pull /usr/lib/aarch64-linux-gnu/libopencv_video.so opencv32_arm64_libs
mdt pull /usr/lib/aarch64-linux-gnu/libopencv_videoio.so opencv32_arm64_libs
* (on host machine) Create and start the docker environment
# from mediapipe root level directory #
docker build -t coral .
docker run -it --name coral coral:latest
## Inside the Docker environment
* Update library paths in /mediapipe/third_party/opencv_linux.BUILD
(replace 'x86_64-linux-gnu' with 'aarch64-linux-gnu')
"lib/aarch64-linux-gnu/libopencv_core.so",
"lib/aarch64-linux-gnu/libopencv_calib3d.so",
"lib/aarch64-linux-gnu/libopencv_features2d.so",
"lib/aarch64-linux-gnu/libopencv_highgui.so",
"lib/aarch64-linux-gnu/libopencv_imgcodecs.so",
"lib/aarch64-linux-gnu/libopencv_imgproc.so",
"lib/aarch64-linux-gnu/libopencv_video.so",
"lib/aarch64-linux-gnu/libopencv_videoio.so",
* Attempt to build hello world (to download external deps)
bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world:hello_world
* Edit /mediapipe/bazel-mediapipe/external/com_github_glog_glog/src/signalhandler.cc
on line 78, replace
return (void*)context->PC_FROM_UCONTEXT;
with
return NULL;
* Edit /edgetpu/libedgetpu/BUILD
to add this build target
cc_library(
name = "lib",
srcs = [
"libedgetpu.so",
],
visibility = ["//visibility:public"],
)
* Edit *tflite_inference_calculator.cc* BUILD rules:
sed -i 's/\":tflite_inference_calculator_cc_proto\",/\":tflite_inference_calculator_cc_proto\",\n\t\"@edgetpu\/\/:header\",\n\t\"@libedgetpu\/\/:lib\",/g' mediapipe/calculators/tflite/BUILD
The above command should add
"@edgetpu//:header",
"@libedgetpu//:lib",
to the _deps_ of tflite_inference_calculator.cc
#### Now try cross-compiling for device
* Object detection demo
bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:object_detection_cpu
Copy object_detection_cpu binary to the MediaPipe checkout on the coral device
# outside docker env, open new terminal on host machine #
docker ps
docker cp <container-id>:/mediapipe/bazel-bin/mediapipe/examples/coral/object_detection_cpu /tmp/.
mdt push /tmp/object_detection_cpu /home/mendel/mediapipe/bazel-bin/.
* Face detection demo
bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:face_detection_cpu
Copy face_detection_cpu binary to the MediaPipe checkout on the coral device
# outside docker env, open new terminal on host machine #
docker ps
docker cp <container-id>:/mediapipe/bazel-bin/mediapipe/examples/coral/face_detection_cpu /tmp/.
mdt push /tmp/face_detection_cpu /home/mendel/mediapipe/bazel-bin/.
## On the coral device (with display)
# Object detection
cd ~/mediapipe
chmod +x bazel-bin/object_detection_cpu
export GLOG_logtostderr=1
bazel-bin/object_detection_cpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt
# Face detection
cd ~/mediapipe
chmod +x bazel-bin/face_detection_cpu
export GLOG_logtostderr=1
bazel-bin/face_detection_cpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt

View File

@ -0,0 +1,313 @@
workspace(name = "mediapipe")
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
skylib_version = "0.8.0"
http_archive(
name = "bazel_skylib",
type = "tar.gz",
url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format (skylib_version, skylib_version),
sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
)
load("@bazel_skylib//lib:versions.bzl", "versions")
versions.check(minimum_bazel_version = "0.24.1")
# ABSL cpp library.
http_archive(
name = "com_google_absl",
# Head commit on 2019-04-12.
# TODO: Switch to the latest absl version when the problem gets
# fixed.
urls = [
"https://github.com/abseil/abseil-cpp/archive/a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a.tar.gz",
],
sha256 = "d437920d1434c766d22e85773b899c77c672b8b4865d5dc2cd61a29fdff3cf03",
strip_prefix = "abseil-cpp-a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a",
)
http_archive(
name = "rules_cc",
strip_prefix = "rules_cc-master",
urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
)
# GoogleTest/GoogleMock framework. Used by most unit-tests.
http_archive(
name = "com_google_googletest",
urls = ["https://github.com/google/googletest/archive/master.zip"],
strip_prefix = "googletest-master",
)
# Google Benchmark library.
http_archive(
name = "com_google_benchmark",
urls = ["https://github.com/google/benchmark/archive/master.zip"],
strip_prefix = "benchmark-master",
build_file = "@//third_party:benchmark.BUILD",
)
# gflags needed by glog
http_archive(
name = "com_github_gflags_gflags",
sha256 = "6e16c8bc91b1310a44f3965e616383dbda48f83e8c1eaa2370a215057b00cabe",
strip_prefix = "gflags-77592648e3f3be87d6c7123eb81cbad75f9aef5a",
urls = [
"https://mirror.bazel.build/github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
"https://github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
],
)
# glog
http_archive(
name = "com_github_glog_glog",
url = "https://github.com/google/glog/archive/v0.3.5.zip",
sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
strip_prefix = "glog-0.3.5",
build_file = "@//third_party:glog.BUILD",
patches = [
"@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff"
],
patch_args = [
"-p1",
],
)
# libyuv
http_archive(
name = "libyuv",
urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/refs/heads/master.tar.gz"],
build_file = "@//third_party:libyuv.BUILD",
)
http_archive(
name = "com_google_protobuf_javalite",
sha256 = "79d102c61e2a479a0b7e5fc167bcfaa4832a0c6aad4a75fa7da0480564931bcc",
strip_prefix = "protobuf-384989534b2246d413dbcd750744faab2607b516",
urls = ["https://github.com/google/protobuf/archive/384989534b2246d413dbcd750744faab2607b516.zip"],
)
http_archive(
name = "com_google_audio_tools",
strip_prefix = "multichannel-audio-tools-master",
urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
)
# Needed by TensorFlow
http_archive(
name = "io_bazel_rules_closure",
sha256 = "e0a111000aeed2051f29fcc7a3f83be3ad8c6c93c186e64beb1ad313f0c7f9f9",
strip_prefix = "rules_closure-cf1e44edb908e9616030cc83d085989b8e6cd6df",
urls = [
"http://mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz",
"https://github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz", # 2019-04-04
],
)
# 2019-11-12
_TENSORFLOW_GIT_COMMIT = "a5f9bcd64453ff3d1f64cb4da4786db3d2da7f82"
_TENSORFLOW_SHA256= "f2b6f2ab2ffe63e86eccd3ce4bea6b7197383d726638dfeeebcdc1e7de73f075"
http_archive(
name = "org_tensorflow",
urls = [
"https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
"https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
],
strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
sha256 = _TENSORFLOW_SHA256,
)
load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
tf_workspace(tf_repo_name = "org_tensorflow")
# Please run
# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
# libopencv-imgproc-dev libopencv-video-dev
new_local_repository(
name = "linux_opencv",
build_file = "@//third_party:opencv_linux.BUILD",
path = "/usr",
)
new_local_repository(
name = "linux_ffmpeg",
build_file = "@//third_party:ffmpeg_linux.BUILD",
path = "/usr"
)
# Please run $ brew install opencv@3
new_local_repository(
name = "macos_opencv",
build_file = "@//third_party:opencv_macos.BUILD",
path = "/usr",
)
new_local_repository(
name = "macos_ffmpeg",
build_file = "@//third_party:ffmpeg_macos.BUILD",
path = "/usr",
)
http_archive(
name = "android_opencv",
sha256 = "056b849842e4fa8751d09edbb64530cfa7a63c84ccd232d0ace330e27ba55d0b",
build_file = "@//third_party:opencv_android.BUILD",
strip_prefix = "OpenCV-android-sdk",
type = "zip",
url = "https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip",
)
# After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will
# trigger duplicate symbol errors in the linking stage of building a mediapipe ios app.
# To get a higher version of OpenCV for iOS, opencv2.framework needs to be built from source with
# '-DBUILD_PROTOBUF=OFF -DBUILD_opencv_dnn=OFF'.
http_archive(
name = "ios_opencv",
sha256 = "7dd536d06f59e6e1156b546bd581523d8df92ce83440002885ec5abc06558de2",
build_file = "@//third_party:opencv_ios.BUILD",
type = "zip",
url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
)
RULES_JVM_EXTERNAL_TAG = "2.2"
RULES_JVM_EXTERNAL_SHA = "f1203ce04e232ab6fdd81897cf0ff76f2c04c0741424d192f28e65ae752ce2d6"
http_archive(
name = "rules_jvm_external",
strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
sha256 = RULES_JVM_EXTERNAL_SHA,
url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
)
load("@rules_jvm_external//:defs.bzl", "maven_install")
maven_install(
artifacts = [
"androidx.annotation:annotation:aar:1.1.0",
"androidx.appcompat:appcompat:aar:1.1.0-rc01",
"androidx.constraintlayout:constraintlayout:aar:1.1.3",
"androidx.core:core:aar:1.1.0-rc03",
"androidx.legacy:legacy-support-v4:aar:1.0.0",
"androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
"com.google.android.material:material:aar:1.0.0-rc01",
],
repositories = ["https://dl.google.com/dl/android/maven2"],
)
maven_server(
name = "google_server",
url = "https://dl.google.com/dl/android/maven2",
)
maven_jar(
name = "androidx_lifecycle",
artifact = "androidx.lifecycle:lifecycle-common:2.0.0",
sha1 = "e070ffae07452331bc5684734fce6831d531785c",
server = "google_server",
)
maven_jar(
name = "androidx_concurrent_futures",
artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
server = "google_server",
)
maven_jar(
name = "com_google_guava_android",
artifact = "com.google.guava:guava:27.0.1-android",
sha1 = "b7e1c37f66ef193796ccd7ea6e80c2b05426182d",
)
maven_jar(
name = "com_google_common_flogger",
artifact = "com.google.flogger:flogger:0.3.1",
sha1 = "585030fe1ec709760cbef997a459729fb965df0e",
)
maven_jar(
name = "com_google_common_flogger_system_backend",
artifact = "com.google.flogger:flogger-system-backend:0.3.1",
sha1 = "287b569d76abcd82f9de87fe41829fbc7ebd8ac9",
)
maven_jar(
name = "com_google_code_findbugs",
artifact = "com.google.code.findbugs:jsr305:3.0.2",
sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d",
)
# You may run setup_android.sh to install Android SDK and NDK.
android_ndk_repository(
name = "androidndk",
)
android_sdk_repository(
name = "androidsdk",
)
# iOS basic build deps.
http_archive(
name = "build_bazel_rules_apple",
sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e",
strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3",
url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz",
)
load(
"@build_bazel_rules_apple//apple:repositories.bzl",
"apple_rules_dependencies",
)
apple_rules_dependencies()
load(
"@build_bazel_rules_swift//swift:repositories.bzl",
"swift_rules_dependencies",
)
swift_rules_dependencies()
load(
"@build_bazel_apple_support//lib:repositories.bzl",
"apple_support_dependencies",
)
apple_support_dependencies()
# More iOS deps.
http_archive(
name = "google_toolbox_for_mac",
url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip",
sha256 = "e3ac053813c989a88703556df4dc4466e424e30d32108433ed6beaec76ba4fdc",
strip_prefix = "google-toolbox-for-mac-2.2.1",
build_file = "@//third_party:google_toolbox_for_mac.BUILD",
)
# Coral
#COMMIT=$(git ls-remote https://github.com/google-coral/crosstool master | awk '{print $1}')
#SHA256=$(curl -L "https://github.com/google-coral/crosstool/archive/${COMMIT}.tar.gz" | sha256sum | awk '{print $1}')
http_archive(
name = "coral_crosstool",
sha256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb",
strip_prefix = "crosstool-9e00d5be43bf001f883b5700f5d04882fea00229",
urls = [
"https://github.com/google-coral/crosstool/archive/9e00d5be43bf001f883b5700f5d04882fea00229.tar.gz",
],
)
load("@coral_crosstool//:configure.bzl", "cc_crosstool")
cc_crosstool(name = "crosstool")
# EdgeTPU
new_local_repository(
name = "edgetpu",
path = "/edgetpu/libedgetpu",
build_file = "/edgetpu/libedgetpu/BUILD"
)
new_local_repository(
name = "libedgetpu",
path = "/usr/lib/aarch64-linux-gnu",
build_file = "/edgetpu/libedgetpu/BUILD"
)

View File

@ -0,0 +1,151 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// An example of sending OpenCV webcam frames into a MediaPipe graph.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/opencv_highgui_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/opencv_video_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
constexpr char kInputStream[] = "input_video";
constexpr char kOutputStream[] = "output_video";
constexpr char kWindowName[] = "MediaPipe";
DEFINE_string(
calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
DEFINE_string(input_video_path, "",
"Full path of video to load. "
"If not provided, attempt to use a webcam.");
DEFINE_string(output_video_path, "",
"Full path of where to save result (.mp4 only). "
"If not provided, show result in a window.");
::mediapipe::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
LOG(INFO) << "Get calculator graph config contents: "
<< calculator_graph_config_contents;
mediapipe::CalculatorGraphConfig config =
mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
calculator_graph_config_contents);
LOG(INFO) << "Initialize the calculator graph.";
mediapipe::CalculatorGraph graph;
MP_RETURN_IF_ERROR(graph.Initialize(config));
LOG(INFO) << "Initialize the camera or load the video.";
cv::VideoCapture capture;
const bool load_video = !FLAGS_input_video_path.empty();
if (load_video) {
capture.open(FLAGS_input_video_path);
} else {
capture.open(0);
}
RET_CHECK(capture.isOpened());
cv::VideoWriter writer;
const bool save_video = !FLAGS_output_video_path.empty();
if (save_video) {
LOG(INFO) << "Prepare video writer.";
cv::Mat test_frame;
capture.read(test_frame); // Consume first frame.
capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0); // Rewind to beginning.
writer.open(FLAGS_output_video_path,
mediapipe::fourcc('a', 'v', 'c', '1'), // .mp4
capture.get(cv::CAP_PROP_FPS), test_frame.size());
RET_CHECK(writer.isOpened());
} else {
cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
capture.set(cv::CAP_PROP_FRAME_WIDTH, 640);
capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480);
capture.set(cv::CAP_PROP_AUTOFOCUS, 0);
capture.set(cv::CAP_PROP_FOCUS, 1);
capture.set(cv::CAP_PROP_FPS, 30);
}
LOG(INFO) << "Start running the calculator graph.";
ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller,
graph.AddOutputStreamPoller(kOutputStream));
MP_RETURN_IF_ERROR(graph.StartRun({}));
LOG(INFO) << "Start grabbing and processing frames.";
size_t frame_timestamp = 0;
bool grab_frames = true;
while (grab_frames) {
// Capture opencv camera or video frame.
cv::Mat camera_frame_raw;
capture >> camera_frame_raw;
if (camera_frame_raw.empty()) break; // End of video.
cv::Mat camera_frame;
cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB);
if (!load_video) {
cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1);
}
// Wrap Mat into an ImageFrame.
auto input_frame = absl::make_unique<mediapipe::ImageFrame>(
mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows,
mediapipe::ImageFrame::kDefaultAlignmentBoundary);
cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get());
camera_frame.copyTo(input_frame_mat);
// Send image packet into the graph.
MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
kInputStream, mediapipe::Adopt(input_frame.release())
.At(mediapipe::Timestamp(frame_timestamp++))));
// Get the graph result packet, or stop if that fails.
mediapipe::Packet packet;
if (!poller.Next(&packet)) break;
auto& output_frame = packet.Get<mediapipe::ImageFrame>();
// Convert back to opencv for display or saving.
cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame);
cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
if (save_video) {
writer.write(output_frame_mat);
} else {
cv::imshow(kWindowName, output_frame_mat);
// Press any key to exit.
const int pressed_key = cv::waitKey(5);
if (pressed_key >= 0 && pressed_key != 255) grab_frames = false;
}
}
LOG(INFO) << "Shutting down.";
if (writer.isOpened()) writer.release();
MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream));
return graph.WaitUntilDone();
}
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
::mediapipe::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
} else {
LOG(INFO) << "Success!";
}
return 0;
}

View File

@ -0,0 +1,189 @@
# MediaPipe graph that performs face detection with TensorFlow Lite on CPU.
# Used in the examples in
# mediapipe/examples/coral:face_detection_cpu.
# Images on GPU coming into and out of the graph.
input_stream: "input_video"
output_stream: "output_video"
# Throttles the images flowing downstream for flow control. It passes through
# the very first incoming image unaltered, and waits for
# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish
# generating the corresponding detections before it passes through another
# image. All images that come in while waiting are dropped, limiting the number
# of in-flight images between this calculator and
# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between
# from queuing up incoming images and data excessively, which leads to increased
# latency and memory usage, unwanted in real-time mobile applications. It also
# eliminates unnecessarily computation, e.g., a transformed image produced by
# ImageTransformationCalculator may get dropped downstream if the subsequent
# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy
# processing previous inputs.
node {
calculator: "FlowLimiterCalculator"
input_stream: "input_video"
input_stream: "FINISHED:detections"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_input_video"
}
# Transforms the input image on CPU to a 128x128 image. To scale the input
# image, the scale_mode option is set to FIT to preserve the aspect ratio,
# resulting in potential letterboxing in the transformed image.
node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE:throttled_input_video"
output_stream: "IMAGE:transformed_input_video_cpu"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext] {
output_width: 128
output_height: 128
scale_mode: FIT
}
}
}
# Converts the transformed input image on CPU into an image tensor stored as a
# TfLiteTensor.
node {
calculator: "TfLiteConverterCalculator"
input_stream: "IMAGE:transformed_input_video_cpu"
output_stream: "TENSORS:image_tensor"
options: {
[mediapipe.TfLiteConverterCalculatorOptions.ext] {
use_quantized_tensors: true
}
}
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "TfLiteInferenceCalculator"
input_stream: "TENSORS:image_tensor"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
model_path: "mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite"
}
}
}
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 4
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 128
input_size_width: 128
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 8
strides: 16
strides: 16
strides: 16
aspect_ratios: 1.0
fixed_anchor_size: true
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TfLiteTensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:detections"
options: {
[mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 896
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 128.0
y_scale: 128.0
h_scale: 128.0
w_scale: 128.0
min_score_thresh: 0.75
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
return_empty_detections: true
}
}
}
# Maps detection label IDs to the corresponding label text ("Face"). The label
# map is provided in the label_map_path option.
node {
calculator: "DetectionLabelIdToTextCalculator"
input_stream: "filtered_detections"
output_stream: "labeled_detections"
options: {
[mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
}
}
}
# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
# letterboxed image (after image transformation with the FIT scale mode) to the
# corresponding locations on the same image with the letterbox removed (the
# input image to the graph before image transformation).
node {
calculator: "DetectionLetterboxRemovalCalculator"
input_stream: "DETECTIONS:labeled_detections"
input_stream: "LETTERBOX_PADDING:letterbox_padding"
output_stream: "DETECTIONS:output_detections"
}
# Converts the detections to drawing primitives for annotation overlay.
node {
calculator: "DetectionsToRenderDataCalculator"
input_stream: "DETECTIONS:output_detections"
output_stream: "RENDER_DATA:render_data"
options: {
[mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
thickness: 4.0
color { r: 255 g: 0 b: 0 }
}
}
}
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
}

View File

@ -0,0 +1,179 @@
# MediaPipe graph that performs object detection with TensorFlow Lite on CPU.
# Used in the examples in
# mediapipie/examples/coral:object_detection_cpu.
# Images on CPU coming into and out of the graph.
input_stream: "input_video"
output_stream: "output_video"
# Throttles the images flowing downstream for flow control. It passes through
# the very first incoming image unaltered, and waits for
# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish
# generating the corresponding detections before it passes through another
# image. All images that come in while waiting are dropped, limiting the number
# of in-flight images between this calculator and
# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between
# from queuing up incoming images and data excessively, which leads to increased
# latency and memory usage, unwanted in real-time mobile applications. It also
# eliminates unnecessarily computation, e.g., a transformed image produced by
# ImageTransformationCalculator may get dropped downstream if the subsequent
# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy
# processing previous inputs.
node {
calculator: "FlowLimiterCalculator"
input_stream: "input_video"
input_stream: "FINISHED:detections"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_input_video"
}
# Transforms the input image on CPU to a 320x320 image. To scale the image, by
# default it uses the STRETCH scale mode that maps the entire input image to the
# entire transformed image. As a result, image aspect ratio may be changed and
# objects in the image may be deformed (stretched or squeezed), but the object
# detection model used in this graph is agnostic to that deformation.
node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE:throttled_input_video"
output_stream: "IMAGE:transformed_input_video"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext] {
output_width: 300
output_height: 300
}
}
}
# Converts the transformed input image on CPU into an image tensor stored as a
# TfLiteTensor.
node {
calculator: "TfLiteConverterCalculator"
input_stream: "IMAGE:transformed_input_video"
output_stream: "TENSORS:image_tensor"
options: {
[mediapipe.TfLiteConverterCalculatorOptions.ext] {
use_quantized_tensors: true
}
}
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "TfLiteInferenceCalculator"
input_stream: "TENSORS:image_tensor"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
model_path: "mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite"
}
}
}
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 6
min_scale: 0.2
max_scale: 0.95
input_size_height: 300
input_size_width: 300
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 16
strides: 32
strides: 64
strides: 128
strides: 256
strides: 512
aspect_ratios: 1.0
aspect_ratios: 2.0
aspect_ratios: 0.5
aspect_ratios: 3.0
aspect_ratios: 0.3333
reduce_boxes_in_lowest_layer: true
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TfLiteTensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:detections"
options: {
[mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
num_classes: 91
num_boxes: 2034
num_coords: 4
ignore_classes: 0
sigmoid_score: true
apply_exponential_on_box_size: true
x_scale: 10.0
y_scale: 10.0
h_scale: 5.0
w_scale: 5.0
min_score_thresh: 0.6
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.4
max_num_detections: 3
overlap_type: INTERSECTION_OVER_UNION
return_empty_detections: true
}
}
}
# Maps detection label IDs to the corresponding label text. The label map is
# provided in the label_map_path option.
node {
calculator: "DetectionLabelIdToTextCalculator"
input_stream: "filtered_detections"
output_stream: "output_detections"
options: {
[mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
label_map_path: "mediapipe/examples/coral/models/object_detection_labelmap.txt"
}
}
}
# Converts the detections to drawing primitives for annotation overlay.
node {
calculator: "DetectionsToRenderDataCalculator"
input_stream: "DETECTIONS:output_detections"
output_stream: "RENDER_DATA:render_data"
options: {
[mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
thickness: 4.0
color { r: 255 g: 0 b: 0 }
}
}
}
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
}

View File

@ -0,0 +1,90 @@
person
bicycle
car
motorcycle
airplane
bus
train
truck
boat
traffic light
fire hydrant
???
stop sign
parking meter
bench
bird
cat
dog
horse
sheep
cow
elephant
bear
zebra
giraffe
???
backpack
umbrella
???
???
handbag
tie
suitcase
frisbee
skis
snowboard
sports ball
kite
baseball bat
baseball glove
skateboard
surfboard
tennis racket
bottle
???
wine glass
cup
fork
knife
spoon
bowl
banana
apple
sandwich
orange
broccoli
carrot
hot dog
pizza
donut
cake
chair
couch
potted plant
bed
???
dining table
???
???
toilet
???
tv
laptop
mouse
remote
keyboard
cell phone
microwave
oven
toaster
sink
refrigerator
???
book
clock
vase
scissors
teddy bear
hair drier
toothbrush

View File

@ -0,0 +1,21 @@
#!/bin/sh
set -e
set -v
echo 'Please run this from root level mediapipe directory! \n Ex:'
echo ' sh mediapipe/examples/coral/setup.sh '
sleep 3
mkdir opencv32_arm64_libs
cp mediapipe/examples/coral/update_sources.sh update_sources.sh
chmod +x update_sources.sh
mv Dockerfile Dockerfile.orig
cp mediapipe/examples/coral/Dockerfile Dockerfile
cp WORKSPACE WORKSPACE.orig
cp mediapipe/examples/coral/WORKSPACE WORKSPACE

View File

@ -0,0 +1,11 @@
#!/bin/bash
# To run in the Coral Docker environment.
. /etc/os-release
sed -i "s/deb\ /deb \[arch=amd64\]\ /g" /etc/apt/sources.list
echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME} main universe" >> /etc/apt/sources.list
echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-updates main universe" >> /etc/apt/sources.list
echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-security main universe" >> /etc/apt/sources.list

View File

@ -9,7 +9,9 @@ bazel build -c opt mediapipe/examples/desktop/hello_world:hello_world
and then run it using:
```
bazel-bin/mediapipe/examples/desktop/hello_world/hello_world --logtostderr
export GLOG_logtostderr=1
bazel-bin/mediapipe/examples/desktop/hello_world/hello_world
```
**TFlite Object Detection**
@ -23,10 +25,11 @@ bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_
and run it using:
```
export GLOG_logtostderr=1
bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \
--calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt \
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file \
--alsologtostderr
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
```
**TensorFlow Object Detection**
@ -34,6 +37,8 @@ bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \
To build the object detection demo using a TensorFlow model on desktop, use:
```
export GLOG_logtostderr=1
bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_tensorflow \
--define MEDIAPIPE_DISABLE_GPU=1
```
@ -41,8 +46,68 @@ bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_
and run it using:
```
export GLOG_logtostderr=1
bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tensorflow \
--calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt \
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
--alsologtostderr
```
**TFlite Hand Detection**
To build the hand detection demo using a TFLite model on desktop, use:
```
bazel build -c opt mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1
```
and run it using:
```
export GLOG_logtostderr=1
bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_tflite \
--calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_detection_desktop.pbtxt \
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
```
**TFlite Hand Tracking**
To build the hand tracking demo using a TFLite model on desktop, use:
```
bazel build -c opt mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1
```
and run it using:
```
export GLOG_logtostderr=1
bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_tflite \
--calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_desktop.pbtxt \
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
```
**TFlite Multi-Hand Tracking**
To build the multi-hand tracking demo using a TFLite model on desktop, use:
```
bazel build -c opt mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1
```
and run it using:
```
export GLOG_logtostderr=1
bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_tflite \
--calculator_graph_config_file=mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt \
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
```
To change the number of hands to `x` in this application, change:
1. `min_size:x` in `CollectionHasMinSizeCalculatorOptions` in `mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt`.
2. `max_vec_size:x` in `ClipVectorSizeCalculatorOptions` in `mediapipe/examples/dekstop/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt`.

View File

@ -93,15 +93,15 @@ FILEPATTERN = "kinetics_700_%s_25fps_rgb_flow"
SPLITS = {
"train": {
"shards": 1000,
"examples": 541632
"examples": 541490
},
"validate": {
"shards": 100,
"examples": 34727
"examples": 34715
},
"test": {
"shards": 100,
"examples": 69347
"examples": 69321
},
"custom": {
"csv": None, # Add a CSV for your own data here.

View File

@ -1524,7 +1524,9 @@ cc_test(
"//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
"//mediapipe/framework/stream_handler:immediate_input_stream_handler",
"//mediapipe/framework/stream_handler:mux_input_stream_handler",
"//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
"//mediapipe/framework/tool:sink",
"@com_google_absl//absl/strings",
],
)

View File

@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/strings/str_replace.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/canonical_errors.h"
@ -642,6 +643,36 @@ REGISTER_CALCULATOR(OffsetBoundCalculator);
// A Calculator that produces a packet for each call to Process.
class BoundToPacketCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
cc->Inputs().Index(i).SetAny();
}
for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
cc->Outputs().Index(i).Set<Timestamp>();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) final {
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
Timestamp t = cc->Inputs().Index(i).Value().Timestamp();
cc->Outputs().Index(i).AddPacket(
mediapipe::MakePacket<Timestamp>(t).At(cc->InputTimestamp()));
}
return ::mediapipe::OkStatus();
}
};
REGISTER_CALCULATOR(BoundToPacketCalculator);
// A Calculator that produces packets at timestamps beyond the input timestamp.
class FuturePacketCalculator : public CalculatorBase {
static constexpr int64 kOutputFutureMicros = 3;
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<int>();
@ -654,11 +685,14 @@ class BoundToPacketCalculator : public CalculatorBase {
}
::mediapipe::Status Process(CalculatorContext* cc) final {
cc->Outputs().Index(0).AddPacket(Adopt(new int(33)));
const Packet& packet = cc->Inputs().Index(0).Value();
Timestamp timestamp =
Timestamp(packet.Timestamp().Value() + kOutputFutureMicros);
cc->Outputs().Index(0).AddPacket(packet.At(timestamp));
return ::mediapipe::OkStatus();
}
};
REGISTER_CALCULATOR(BoundToPacketCalculator);
REGISTER_CALCULATOR(FuturePacketCalculator);
// Verifies that SetOffset still propagates when Process is called and
// produces no output packets.
@ -964,5 +998,111 @@ TEST(CalculatorGraphBoundsTest, LastPacketCheck) {
MP_ASSERT_OK(graph.WaitUntilDone());
}
// Shows that bounds are indicated for input streams without input packets.
void TestBoundsForEmptyInputs(std::string input_stream_handler) {
// FuturePacketCalculator and OffsetBoundCalculator produce future ts bounds.
// BoundToPacketCalculator reports all of its bounds, including empty inputs.
std::string config_str = R"(
input_stream: 'input'
node {
calculator: 'FuturePacketCalculator'
input_stream: 'input'
output_stream: 'futures'
}
node {
calculator: 'OffsetBoundCalculator'
input_stream: 'futures'
output_stream: 'bounds'
}
node {
calculator: 'BoundToPacketCalculator'
input_stream: 'input'
input_stream: 'bounds'
output_stream: 'input_ts'
output_stream: 'bounds_ts'
input_stream_handler { $input_stream_handler }
}
)";
absl::StrReplaceAll({{"$input_stream_handler", input_stream_handler}},
&config_str);
CalculatorGraphConfig config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(config_str);
CalculatorGraph graph;
std::vector<Packet> input_ts_packets;
std::vector<Packet> bounds_ts_packets;
MP_ASSERT_OK(graph.Initialize(config));
MP_ASSERT_OK(graph.ObserveOutputStream("input_ts", [&](const Packet& p) {
input_ts_packets.push_back(p);
return ::mediapipe::OkStatus();
}));
MP_ASSERT_OK(graph.ObserveOutputStream("bounds_ts", [&](const Packet& p) {
bounds_ts_packets.push_back(p);
return ::mediapipe::OkStatus();
}));
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(graph.WaitUntilIdle());
// Add four packets into the graph, with timedtamps 0, 10, 20, 30.
constexpr int kNumInputs = 4;
for (int i = 0; i < kNumInputs; ++i) {
Packet p = MakePacket<int>(33).At(Timestamp(i * 10));
MP_ASSERT_OK(graph.AddPacketToInputStream("input", p));
MP_ASSERT_OK(graph.WaitUntilIdle());
}
// Packets arrive. The input packet timestamps are: 0, 10, 20, 30.
// The corresponding empty packet timestamps are: 3, 13, 23, 33.
MP_ASSERT_OK(graph.WaitUntilIdle());
EXPECT_EQ(input_ts_packets.size(), 4);
EXPECT_EQ(bounds_ts_packets.size(), 4);
// The timestamp bounds from OffsetBoundCalculator are: 3, 13, 23, 33.
// Because the process call waits for the input packets and not for
// the empty packets, the first empty packet timestamp can be
// either Timestamp::Unstarted() or Timestamp(3).
std::vector<Timestamp> expected = {Timestamp::Unstarted(), Timestamp(3),
Timestamp(13), Timestamp(23),
Timestamp(33)};
for (int i = 0; i < bounds_ts_packets.size(); ++i) {
Timestamp ts = bounds_ts_packets[i].Get<Timestamp>();
EXPECT_GE(ts, expected[i]);
EXPECT_LE(ts, expected[i + 1]);
}
// Shutdown the graph.
MP_ASSERT_OK(graph.CloseAllPacketSources());
MP_ASSERT_OK(graph.WaitUntilDone());
}
// Shows that bounds are indicated for input streams without input packets.
TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_Immediate) {
TestBoundsForEmptyInputs(R"(
input_stream_handler: "ImmediateInputStreamHandler")");
}
// Shows that bounds are indicated for input streams without input packets.
TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_Default) {
TestBoundsForEmptyInputs(R"(
input_stream_handler: "DefaultInputStreamHandler")");
}
// Shows that bounds are indicated for input streams without input packets.
TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_SyncSet) {
TestBoundsForEmptyInputs(R"(
input_stream_handler: "SyncSetInputStreamHandler")");
}
// Shows that bounds are indicated for input streams without input packets.
TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_SyncSets) {
TestBoundsForEmptyInputs(R"(
input_stream_handler: "SyncSetInputStreamHandler"
options {
[mediapipe.SyncSetInputStreamHandlerOptions.ext] {
sync_set { tag_index: ":0" }
}
}
)");
}
} // namespace
} // namespace mediapipe

View File

@ -27,72 +27,67 @@ package(
proto_library(
name = "detection_proto",
srcs = ["detection.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework/formats:location_data_proto"],
)
proto_library(
name = "classification_proto",
srcs = ["classification.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
)
proto_library(
name = "image_format_proto",
srcs = ["image_format.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
)
proto_library(
name = "matrix_data_proto",
srcs = ["matrix_data.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
)
proto_library(
name = "location_data_proto",
srcs = ["location_data.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework/formats/annotation:rasterization_proto"],
)
proto_library(
name = "time_series_header_proto",
srcs = ["time_series_header.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
)
mediapipe_cc_proto_library(
name = "detection_cc_proto",
srcs = ["detection.proto"],
cc_deps = [":location_data_cc_proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
deps = [":detection_proto"],
)
mediapipe_cc_proto_library(
name = "classification_cc_proto",
srcs = ["classification.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
deps = [":classification_proto"],
)
mediapipe_cc_proto_library(
name = "image_format_cc_proto",
srcs = ["image_format.proto"],
visibility = [
"//mediapipe:__subpackages__",
"//mediapipe/java/com/google/mediapipe/framework:__subpackages__",
],
visibility = ["//visibility:public"],
deps = [":image_format_proto"],
)
mediapipe_cc_proto_library(
name = "matrix_data_cc_proto",
srcs = ["matrix_data.proto"],
visibility = [
"//mediapipe:__subpackages__",
"//mediapipe/java/com/google/mediapipe/framework:__subpackages__",
],
visibility = ["//visibility:public"],
deps = [":matrix_data_proto"],
)
@ -100,17 +95,14 @@ mediapipe_cc_proto_library(
name = "location_data_cc_proto",
srcs = ["location_data.proto"],
cc_deps = ["//mediapipe/framework/formats/annotation:rasterization_cc_proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
deps = [":location_data_proto"],
)
mediapipe_cc_proto_library(
name = "time_series_header_cc_proto",
srcs = ["time_series_header.proto"],
visibility = [
"//mediapipe:__subpackages__",
"//mediapipe/java/com/google/mediapipe/framework:__subpackages__",
],
visibility = ["//visibility:public"],
deps = [":time_series_header_proto"],
)
@ -249,14 +241,14 @@ cc_test(
proto_library(
name = "rect_proto",
srcs = ["rect.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
)
mediapipe_cc_proto_library(
name = "rect_cc_proto",
srcs = ["rect.proto"],
visibility = [
"//mediapipe:__subpackages__",
"//visibility:public",
],
deps = [":rect_proto"],
)
@ -264,12 +256,12 @@ mediapipe_cc_proto_library(
proto_library(
name = "landmark_proto",
srcs = ["landmark.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
)
mediapipe_cc_proto_library(
name = "landmark_cc_proto",
srcs = ["landmark.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
deps = [":landmark_proto"],
)

View File

@ -25,26 +25,27 @@ package(default_visibility = ["//visibility:private"])
proto_library(
name = "locus_proto",
srcs = ["locus.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework/formats/annotation:rasterization_proto"],
)
proto_library(
name = "rasterization_proto",
srcs = ["rasterization.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
)
mediapipe_cc_proto_library(
name = "locus_cc_proto",
srcs = ["locus.proto"],
cc_deps = [":rasterization_cc_proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
deps = [":locus_proto"],
)
mediapipe_cc_proto_library(
name = "rasterization_cc_proto",
srcs = ["rasterization.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
deps = [":rasterization_proto"],
)

View File

@ -25,6 +25,11 @@ message Landmark {
optional float z = 3;
}
// Group of Landmark protos.
message LandmarkList {
repeated Landmark landmark = 1;
}
// A normalized version of above Landmark proto. All coordiates should be within
// [0, 1].
message NormalizedLandmark {

View File

@ -27,12 +27,13 @@ package(default_visibility = ["//visibility:private"])
proto_library(
name = "optical_flow_field_data_proto",
srcs = ["optical_flow_field_data.proto"],
visibility = ["//visibility:public"],
)
mediapipe_cc_proto_library(
name = "optical_flow_field_data_cc_proto",
srcs = ["optical_flow_field_data.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
deps = [":optical_flow_field_data_proto"],
)
@ -41,7 +42,7 @@ cc_library(
srcs = ["optical_flow_field.cc"],
hdrs = ["optical_flow_field.h"],
visibility = [
"//mediapipe:__subpackages__",
"//visibility:public",
],
deps = [
"//mediapipe/framework:type_map",

View File

@ -24,12 +24,12 @@ package(default_visibility = ["//visibility:private"])
proto_library(
name = "anchor_proto",
srcs = ["anchor.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
)
mediapipe_cc_proto_library(
name = "anchor_cc_proto",
srcs = ["anchor.proto"],
visibility = ["//mediapipe:__subpackages__"],
visibility = ["//visibility:public"],
deps = [":anchor_proto"],
)

View File

@ -229,6 +229,11 @@ Timestamp InputStreamManager::MinTimestampOrBound(bool* is_empty) const {
if (is_empty) {
*is_empty = queue_.empty();
}
return MinTimestampOrBoundHelper();
}
Timestamp InputStreamManager::MinTimestampOrBoundHelper() const
EXCLUSIVE_LOCKS_REQUIRED(stream_mutex_) {
return queue_.empty() ? next_timestamp_bound_ : queue_.front().Timestamp();
}
@ -271,7 +276,9 @@ Packet InputStreamManager::PopPacketAtTimestamp(Timestamp timestamp,
}
// Clear value_ if it doesn't have exactly the right timestamp.
if (current_timestamp != timestamp) {
packet = Packet();
// The timestamp bound reported when no packet is sent.
Timestamp bound = MinTimestampOrBoundHelper();
packet = Packet().At(bound.PreviousAllowedInStream());
++(*num_packets_dropped);
}

View File

@ -189,6 +189,9 @@ class InputStreamManager {
// Returns true if the next timestamp bound reaches Timestamp::Done().
bool IsDone() const EXCLUSIVE_LOCKS_REQUIRED(stream_mutex_);
// Returns the smallest timestamp at which this stream might see an input.
Timestamp MinTimestampOrBoundHelper() const;
mutable absl::Mutex stream_mutex_;
std::deque<Packet> queue_ GUARDED_BY(stream_mutex_);
// The number of packets added to queue_. Used to verify a packet at

View File

@ -115,9 +115,10 @@ void ImmediateInputStreamHandler::FillInputSet(Timestamp input_timestamp,
AddPacketToShard(&input_set->Get(id), std::move(current_packet),
stream_is_done);
} else {
bool empty = false;
bool is_done = stream->MinTimestampOrBound(&empty) == Timestamp::Done();
AddPacketToShard(&input_set->Get(id), Packet(), is_done);
Timestamp bound = stream->MinTimestampOrBound(nullptr);
AddPacketToShard(&input_set->Get(id),
Packet().At(bound.PreviousAllowedInStream()),
bound == Timestamp::Done());
}
}
}

View File

@ -56,9 +56,16 @@ class SyncSetInputStreamHandler : public InputStreamHandler {
NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override;
// Only invoked when associated GetNodeReadiness() returned kReadyForProcess.
// Populates packets for the ready sync-set, and populates timestamp bounds
// for all sync-sets.
void FillInputSet(Timestamp input_timestamp,
InputStreamShardSet* input_set) override;
// Populates timestamp bounds for streams outside the ready sync-set.
void FillInputBounds(Timestamp input_timestamp,
InputStreamShardSet* input_set)
EXCLUSIVE_LOCKS_REQUIRED(mutex_);
private:
absl::Mutex mutex_;
// The ids of each set of inputs.
@ -181,6 +188,22 @@ NodeReadiness SyncSetInputStreamHandler::GetNodeReadiness(
return NodeReadiness::kNotReady;
}
void SyncSetInputStreamHandler::FillInputBounds(
Timestamp input_timestamp, InputStreamShardSet* input_set) {
for (int i = 0; i < sync_sets_.size(); ++i) {
if (i != ready_sync_set_index_) {
// Set the input streams for the not-ready sync sets.
for (CollectionItemId id : sync_sets_[i]) {
const auto stream = input_stream_managers_.Get(id);
Timestamp bound = stream->MinTimestampOrBound(nullptr);
AddPacketToShard(&input_set->Get(id),
Packet().At(bound.PreviousAllowedInStream()),
bound == Timestamp::Done());
}
}
}
}
void SyncSetInputStreamHandler::FillInputSet(Timestamp input_timestamp,
InputStreamShardSet* input_set) {
// Assume that all current packets are already cleared.
@ -202,6 +225,7 @@ void SyncSetInputStreamHandler::FillInputSet(Timestamp input_timestamp,
AddPacketToShard(&input_set->Get(id), std::move(current_packet),
stream_is_done);
}
FillInputBounds(input_timestamp, input_set);
ready_sync_set_index_ = -1;
ready_timestamp_ = Timestamp::Done();
}

View File

@ -123,13 +123,23 @@ std::string TimestampDiff::DebugString() const {
Timestamp Timestamp::NextAllowedInStream() const {
CHECK(IsAllowedInStream()) << "Timestamp is: " << DebugString();
if (IsRangeValue() && *this != Max()) {
return *this + 1;
} else {
// Returning this value effectively means no futher timestamps may
// occur (however, the stream is not yet closed).
if (*this >= Max() || *this == PreStream()) {
// Indicates that no further timestamps may occur.
return OneOverPostStream();
} else if (*this < Min()) {
return Min();
}
return *this + 1;
}
Timestamp Timestamp::PreviousAllowedInStream() const {
if (*this <= Min() || *this == PostStream()) {
// Indicates that no previous timestamps may occur.
return Unstarted();
} else if (*this > Max()) {
return Max();
}
return *this - 1;
}
std::ostream& operator<<(std::ostream& os, Timestamp arg) {

View File

@ -182,12 +182,15 @@ class Timestamp {
Timestamp operator++(int);
Timestamp operator--(int);
// Returns the next timestamp at which a Packet may arrive in a stream, given
// that the current Packet is at *this timestamp. CHECKs that
// this->IsAllowedInStream()==true. Returns Timestamp::OneOverPostStream() if
// no Packets may follow one with the given timestamp.
// Returns the next timestamp in the range [Min .. Max], or
// OneOverPostStream() if no Packets may follow one with this timestamp.
// CHECKs that this->IsAllowedInStream().
Timestamp NextAllowedInStream() const;
// Returns the previous timestamp in the range [Min .. Max], or
// Unstarted() if no Packets may preceed one with this timestamp.
Timestamp PreviousAllowedInStream() const;
private:
TimestampBaseType timestamp_;
};

View File

@ -132,7 +132,7 @@ def _metal_library_impl(ctx):
),
)
# This ridiculous circumlocution is needed because new_objc_provider rejects
# This circumlocution is needed because new_objc_provider rejects
# an empty depset, with the error:
# "Value for key header must be a set of File, instead found set of unknown."
# It also rejects an explicit "None".

View File

@ -94,7 +94,7 @@ node {
output_stream: "multi_hand_rects"
node_options: {
[type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
min_similarity_threshold: 0.1
min_similarity_threshold: 0.5
}
}
}

View File

@ -85,7 +85,7 @@ node {
output_stream: "multi_hand_rects"
node_options: {
[type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
min_similarity_threshold: 0.1
min_similarity_threshold: 0.5
}
}
}

View File

@ -105,7 +105,7 @@ node {
output_stream: "multi_hand_rects"
node_options: {
[type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
min_similarity_threshold: 0.1
min_similarity_threshold: 0.5
}
}
}

View File

@ -104,7 +104,7 @@ node {
y_scale: 256.0
h_scale: 256.0
w_scale: 256.0
min_score_thresh: 0.5
min_score_thresh: 0.7
}
}
}
@ -117,7 +117,6 @@ node {
node_options: {
[type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
min_suppression_threshold: 0.3
min_score_threshold: 0.5
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
return_empty_detections: true

View File

@ -340,9 +340,7 @@ AudioPacketProcessor::AudioPacketProcessor(const AudioStreamOptions& options)
DCHECK(absl::little_endian::IsLittleEndian());
}
mediapipe::Status AudioPacketProcessor::Open(int id,
AVStream* stream) {
mediapipe::Status AudioPacketProcessor::Open(int id, AVStream* stream) {
id_ = id;
avcodec_ = avcodec_find_decoder(stream->codecpar->codec_id);
if (!avcodec_) {

View File

@ -274,10 +274,11 @@ float TimestampsToRate(int64 first_timestamp, int64 second_timestamp) {
// overwriting with modified values.
if (!GetUnmodifiedBBoxTimestampSize(prefix, *sequence)) {
for (int i = 0; i < num_frames; ++i) {
if (bbox_index_if_annotated[i] >= 0 &&
GetBBoxIsAnnotatedAt(prefix, *sequence, i)) {
AddUnmodifiedBBoxTimestamp(
prefix, box_timestamps[bbox_index_if_annotated[i]], sequence);
const int bbox_index = bbox_index_if_annotated[i];
if (bbox_index >= 0 &&
GetBBoxIsAnnotatedAt(prefix, *sequence, bbox_index)) {
AddUnmodifiedBBoxTimestamp(prefix, box_timestamps[bbox_index],
sequence);
}
}
}

View File

@ -738,10 +738,11 @@ TEST(MediaSequenceTest,
AddImageTimestamp(10, &sequence);
AddImageTimestamp(20, &sequence);
AddImageTimestamp(30, &sequence);
AddImageTimestamp(40, &sequence);
AddBBoxTimestamp(9, &sequence);
AddBBoxTimestamp(21, &sequence);
AddBBoxTimestamp(22, &sequence); // Will be dropped in the output.
AddBBoxTimestamp(11, &sequence);
AddBBoxTimestamp(12, &sequence); // Will be dropped in the output.
AddBBoxTimestamp(39, &sequence);
std::vector<std::vector<Location>> bboxes = {
{Location::CreateRelativeBBoxLocation(0.1, 0.2, 0.7, 0.7)},
@ -753,32 +754,35 @@ TEST(MediaSequenceTest,
MP_ASSERT_OK(ReconcileMetadata(true, false, &sequence));
ASSERT_EQ(GetBBoxTimestampSize(sequence), 3);
ASSERT_EQ(GetBBoxTimestampSize(sequence), 4);
ASSERT_EQ(GetBBoxTimestampAt(sequence, 0), 10);
ASSERT_EQ(GetBBoxTimestampAt(sequence, 1), 20);
ASSERT_EQ(GetBBoxTimestampAt(sequence, 2), 30);
ASSERT_EQ(GetBBoxTimestampAt(sequence, 3), 40);
ASSERT_EQ(GetBBoxIsAnnotatedSize(sequence), 3);
ASSERT_EQ(GetBBoxIsAnnotatedSize(sequence), 4);
ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 0), true);
ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 1), true);
ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 1), false);
ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 2), false);
ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 3), true);
// Unmodified timestamp is only stored for is_annotated == true.
ASSERT_EQ(GetUnmodifiedBBoxTimestampSize(sequence), 2);
ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 9);
ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 21);
ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 11);
ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 39);
// A second reconciliation should not corrupt unmodified bbox timestamps.
MP_ASSERT_OK(ReconcileMetadata(true, false, &sequence));
ASSERT_EQ(GetBBoxTimestampSize(sequence), 3);
ASSERT_EQ(GetBBoxTimestampSize(sequence), 4);
ASSERT_EQ(GetBBoxTimestampAt(sequence, 0), 10);
ASSERT_EQ(GetBBoxTimestampAt(sequence, 1), 20);
ASSERT_EQ(GetBBoxTimestampAt(sequence, 2), 30);
ASSERT_EQ(GetBBoxTimestampAt(sequence, 3), 40);
ASSERT_EQ(GetUnmodifiedBBoxTimestampSize(sequence), 2);
ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 9);
ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 21);
ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 11);
ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 39);
}
TEST(MediaSequenceTest, ReconcileMetadataBoxAnnotationsFillsMissing) {

13
third_party/BUILD vendored
View File

@ -118,13 +118,16 @@ android_library(
],
)
# TODO: Get the AARs from Google's Maven Repository.
aar_import(
android_library(
name = "camerax_core",
aar = "camera-core-1.0.0-alpha01.aar",
exports = [
"@maven//:androidx_camera_camera_core",
],
)
aar_import(
android_library(
name = "camera2",
aar = "camera-camera2-1.0.0-alpha01.aar",
exports = [
"@maven//:androidx_camera_camera_camera2",
],
)

Binary file not shown.

Binary file not shown.

View File

@ -5,7 +5,7 @@ licenses(["notice"]) # BSD license
exports_files(["LICENSE"])
OPENCV_LIBRARY_NAME = "libopencv_java4.so"
OPENCV_LIBRARY_NAME = "libopencv_java3.so"
OPENCVANDROIDSDK_NATIVELIBS_PATH = "sdk/native/libs/"