Project import generated by Copybara.

GitOrigin-RevId: 50714fe28298d7b707eff7304547d89d6ec34a54

README.md

@@ -10,11 +10,13 @@
 ## ML Solutions in MediaPipe
 
 * [Hand Tracking](mediapipe/docs/hand_tracking_mobile_gpu.md)
+* [Multi-hand Tracking](mediapipe/docs/multi_hand_tracking_mobile_gpu.md)
 * [Face Detection](mediapipe/docs/face_detection_mobile_gpu.md)
 * [Hair Segmentation](mediapipe/docs/hair_segmentation_mobile_gpu.md)
 * [Object Detection](mediapipe/docs/object_detection_mobile_gpu.md)
 
 ![hand_tracking](mediapipe/docs/images/mobile/hand_tracking_3d_android_gpu_small.gif)
+![multi-hand_tracking](mediapipe/docs/images/mobile/multi_hand_tracking_android_gpu_small.gif)
 ![face_detection](mediapipe/docs/images/mobile/face_detection_android_gpu_small.gif)
 ![hair_segmentation](mediapipe/docs/images/mobile/hair_segmentation_android_gpu_small.gif)
 ![object_detection](mediapipe/docs/images/mobile/object_detection_android_gpu_small.gif)
@@ -23,7 +25,7 @@
 Follow these [instructions](mediapipe/docs/install.md).
 
 ## Getting started
-See mobile and desktop [examples](mediapipe/docs/examples.md).
+See mobile, desktop and Google Coral [examples](mediapipe/docs/examples.md).
 
 ## Documentation
 [MediaPipe Read-the-Docs](https://mediapipe.readthedocs.io/) or [docs.mediapipe.dev](https://docs.mediapipe.dev)
@@ -41,6 +43,7 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
 * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
 
 ## Events
+* [AI Nextcon 2020, 12-16 Feb 2020, Seattle](http://aisea20.xnextcon.com/)
 * [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
 * [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
 * [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)

WORKSPACE

@@ -149,11 +149,10 @@ new_local_repository(
 
 http_archive(
     name = "android_opencv",
-    sha256 = "056b849842e4fa8751d09edbb64530cfa7a63c84ccd232d0ace330e27ba55d0b",
     build_file = "@//third_party:opencv_android.BUILD",
     strip_prefix = "OpenCV-android-sdk",
     type = "zip",
-    url = "https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip",
+    url = "https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip",
 )
 
 # After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will
@@ -184,13 +183,18 @@ maven_install(
     artifacts = [
         "androidx.annotation:annotation:aar:1.1.0",
         "androidx.appcompat:appcompat:aar:1.1.0-rc01",
+        "androidx.camera:camera-core:aar:1.0.0-alpha06",
+        "androidx.camera:camera-camera2:aar:1.0.0-alpha06",
         "androidx.constraintlayout:constraintlayout:aar:1.1.3",
         "androidx.core:core:aar:1.1.0-rc03",
         "androidx.legacy:legacy-support-v4:aar:1.0.0",
         "androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
         "com.google.android.material:material:aar:1.0.0-rc01",
     ],
-    repositories = ["https://dl.google.com/dl/android/maven2"],
+   repositories = [
+        "https://dl.google.com/dl/android/maven2",
+        "https://repo1.maven.org/maven2",
+    ],
 )
 
 maven_server(

mediapipe/calculators/image/scale_image_calculator.proto

@@ -36,7 +36,8 @@ message ScaleImageCalculatorOptions {
 
   // If ratio is positive, crop the image to this minimum and maximum
   // aspect ratio (preserving the center of the frame).  This is done
-  // before scaling.
+  // before scaling.  The string must contain "/", so to disable cropping,
+  // set both to "0/1".
   //   For example, for a min_aspect_ratio of "9/16" and max of "16/9" the
   //   following cropping will occur:
   //       1920x1080 (which is 16:9) is not cropped

mediapipe/calculators/tensorflow/tfrecord_reader_calculator.cc

@@ -85,7 +85,7 @@ class TFRecordReaderCalculator : public CalculatorBase {
   tensorflow::io::RecordReader reader(file.get(),
                                       tensorflow::io::RecordReaderOptions());
   tensorflow::uint64 offset = 0;
-  std::string example_str;
+  tensorflow::tstring example_str;
   const int target_idx =
       cc->InputSidePackets().HasTag(kRecordIndex)
           ? cc->InputSidePackets().Tag(kRecordIndex).Get<int>()
@@ -98,7 +98,7 @@ class TFRecordReaderCalculator : public CalculatorBase {
     if (current_idx == target_idx) {
       if (cc->OutputSidePackets().HasTag(kExampleTag)) {
         tensorflow::Example tf_example;
-        tf_example.ParseFromString(example_str);
+        tf_example.ParseFromArray(example_str.data(), example_str.size());
         cc->OutputSidePackets()
             .Tag(kExampleTag)
             .Set(MakePacket<tensorflow::Example>(std::move(tf_example)));

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -64,6 +64,28 @@ typedef id<MTLBuffer> GpuTensor;
 size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; }  // NOLINT
 }  // namespace
 
+#if defined(MEDIAPIPE_EDGE_TPU)
+#include "edgetpu.h"
+
+// Creates and returns an Edge TPU interpreter to run the given edgetpu model.
+std::unique_ptr<tflite::Interpreter> BuildEdgeTpuInterpreter(
+    const tflite::FlatBufferModel& model,
+    tflite::ops::builtin::BuiltinOpResolver* resolver,
+    edgetpu::EdgeTpuContext* edgetpu_context) {
+  resolver->AddCustom(edgetpu::kCustomOp, edgetpu::RegisterCustomOp());
+  std::unique_ptr<tflite::Interpreter> interpreter;
+  if (tflite::InterpreterBuilder(model, *resolver)(&interpreter) != kTfLiteOk) {
+    std::cerr << "Failed to build edge TPU interpreter." << std::endl;
+  }
+  interpreter->SetExternalContext(kTfLiteEdgeTpuContext, edgetpu_context);
+  interpreter->SetNumThreads(1);
+  if (interpreter->AllocateTensors() != kTfLiteOk) {
+    std::cerr << "Failed to allocate edge TPU tensors." << std::endl;
+  }
+  return interpreter;
+}
+#endif  // MEDIAPIPE_EDGE_TPU
+
 // TfLiteInferenceCalculator File Layout:
 //  * Header
 //  * Core
@@ -162,6 +184,11 @@ class TfLiteInferenceCalculator : public CalculatorBase {
   TFLBufferConvert* converter_from_BPHWC4_ = nil;
 #endif
 
+#if defined(MEDIAPIPE_EDGE_TPU)
+  std::shared_ptr<edgetpu::EdgeTpuContext> edgetpu_context_ =
+      edgetpu::EdgeTpuManager::GetSingleton()->OpenDevice();
+#endif
+
   std::string model_path_ = "";
   bool gpu_inference_ = false;
   bool gpu_input_ = false;
@@ -425,6 +452,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 #endif
     delegate_ = nullptr;
   }
+#if defined(MEDIAPIPE_EDGE_TPU)
+  edgetpu_context_.reset();
+#endif
   return ::mediapipe::OkStatus();
 }
 
@@ -458,16 +488,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   model_ = tflite::FlatBufferModel::BuildFromFile(model_path_.c_str());
   RET_CHECK(model_);
 
+  tflite::ops::builtin::BuiltinOpResolver op_resolver;
   if (cc->InputSidePackets().HasTag("CUSTOM_OP_RESOLVER")) {
-    const auto& op_resolver =
+    op_resolver = cc->InputSidePackets()
-        cc->InputSidePackets()
                       .Tag("CUSTOM_OP_RESOLVER")
                       .Get<tflite::ops::builtin::BuiltinOpResolver>();
-    tflite::InterpreterBuilder(*model_, op_resolver)(&interpreter_);
-  } else {
-    const tflite::ops::builtin::BuiltinOpResolver op_resolver;
-    tflite::InterpreterBuilder(*model_, op_resolver)(&interpreter_);
   }
+#if defined(MEDIAPIPE_EDGE_TPU)
+  interpreter_ =
+      BuildEdgeTpuInterpreter(*model_, &op_resolver, edgetpu_context_.get());
+#else
+  tflite::InterpreterBuilder(*model_, op_resolver)(&interpreter_);
+#endif  // MEDIAPIPE_EDGE_TPU
 
   RET_CHECK(interpreter_);
 

mediapipe/calculators/util/labels_to_render_data_calculator.cc

@@ -93,6 +93,7 @@ REGISTER_CALCULATOR(LabelsToRenderDataCalculator);
 }
 
 ::mediapipe::Status LabelsToRenderDataCalculator::Open(CalculatorContext* cc) {
+  cc->SetOffset(TimestampDiff(0));
   options_ = cc->Options<LabelsToRenderDataCalculatorOptions>();
   num_colors_ = options_.color_size();
   label_height_px_ = std::ceil(options_.font_height_px() * kFontHeightScale);

mediapipe/docs/android_archive_library.md

@@ -92,7 +92,7 @@ project.
     MediaPipe depends on OpenCV, you will need to copy the precompiled OpenCV so
     files into app/src/main/jniLibs. You can download the official OpenCV
     Android SDK from
-    [here](https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip)
+    [here](https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip)
     and run:
 
     ```bash

mediapipe/docs/examples.md

@@ -157,3 +157,20 @@ how to use MediaPipe with a TFLite model for hair segmentation on desktop using
 GPU with live video from a webcam.
 
 *   [Desktop GPU](./hair_segmentation_desktop.md)
+
+## Google Coral (machine learning acceleration with Google EdgeTPU)
+
+Below are code samples on how to run MediaPipe on Google Coral Dev Board.
+
+### Object Detection on Coral
+
+[Object Detection on Coral with Webcam](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
+shows how to run quantized object detection TFlite model accelerated with
+EdgeTPU on
+[Google Coral Dev Board](https://coral.withgoogle.com/products/dev-board).
+
+### Face Detection on Coral
+
+[Face Detection on Coral with Webcam](https://github.com/google/mediapipe/tree/master/mediapipe/examples/coral/README.md)
+shows how to use quantized face detection TFlite model accelerated with EdgeTPU
+on [Google Coral Dev Board](https://coral.withgoogle.com/products/dev-board).

mediapipe/docs/hello_world_android.md

@@ -629,7 +629,7 @@ to load both dependencies:
 static {
   // Load all native libraries needed by the app.
   System.loadLibrary("mediapipe_jni");
-  System.loadLibrary("opencv_java4");
+  System.loadLibrary("opencv_java3");
 }
 ```
 

mediapipe/docs/multi_hand_tracking_desktop.md

@@ -156,7 +156,7 @@ node {
   output_stream: "multi_hand_rects"
   node_options: {
     [type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
-      min_similarity_threshold: 0.1
+      min_similarity_threshold: 0.5
     }
   }
 }

mediapipe/docs/multi_hand_tracking_mobile_gpu.md

@@ -219,7 +219,7 @@ node {
   output_stream: "multi_hand_rects"
   node_options: {
     [type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
-      min_similarity_threshold: 0.1
+      min_similarity_threshold: 0.5
     }
   }
 }

mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/MainActivity.java

@@ -47,7 +47,7 @@ public class MainActivity extends AppCompatActivity {
   static {
     // Load all native libraries needed by the app.
     System.loadLibrary("mediapipe_jni");
-    System.loadLibrary("opencv_java4");
+    System.loadLibrary("opencv_java3");
   }
 
   // {@link SurfaceTexture} where the camera-preview frames can be accessed.

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/MainActivity.java

@@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
   static {
     // Load all native libraries needed by the app.
     System.loadLibrary("mediapipe_jni");
-    System.loadLibrary("opencv_java4");
+    System.loadLibrary("opencv_java3");
   }
 
   // {@link SurfaceTexture} where the camera-preview frames can be accessed.

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/MainActivity.java

@@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
   static {
     // Load all native libraries needed by the app.
     System.loadLibrary("mediapipe_jni");
-    System.loadLibrary("opencv_java4");
+    System.loadLibrary("opencv_java3");
   }
 
   // {@link SurfaceTexture} where the camera-preview frames can be accessed.

mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/MainActivity.java

@@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
   static {
     // Load all native libraries needed by the app.
     System.loadLibrary("mediapipe_jni");
-    System.loadLibrary("opencv_java4");
+    System.loadLibrary("opencv_java3");
   }
 
   // {@link SurfaceTexture} where the camera-preview frames can be accessed.

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/MainActivity.java

@@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
   static {
     // Load all native libraries needed by the app.
     System.loadLibrary("mediapipe_jni");
-    System.loadLibrary("opencv_java4");
+    System.loadLibrary("opencv_java3");
   }
 
   // {@link SurfaceTexture} where the camera-preview frames can be accessed.

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java

@@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
   static {
     // Load all native libraries needed by the app.
     System.loadLibrary("mediapipe_jni");
-    System.loadLibrary("opencv_java4");
+    System.loadLibrary("opencv_java3");
   }
 
   // {@link SurfaceTexture} where the camera-preview frames can be accessed.

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java

@@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
   static {
     // Load all native libraries needed by the app.
     System.loadLibrary("mediapipe_jni");
-    System.loadLibrary("opencv_java4");
+    System.loadLibrary("opencv_java3");
   }
 
   // {@link SurfaceTexture} where the camera-preview frames can be accessed.

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/MainActivity.java

@@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
   static {
     // Load all native libraries needed by the app.
     System.loadLibrary("mediapipe_jni");
-    System.loadLibrary("opencv_java4");
+    System.loadLibrary("opencv_java3");
   }
 
   // {@link SurfaceTexture} where the camera-preview frames can be accessed.

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/MainActivity.java

@@ -48,7 +48,7 @@ public class MainActivity extends AppCompatActivity {
   static {
     // Load all native libraries needed by the app.
     System.loadLibrary("mediapipe_jni");
-    System.loadLibrary("opencv_java4");
+    System.loadLibrary("opencv_java3");
   }
 
   // {@link SurfaceTexture} where the camera-preview frames can be accessed.

mediapipe/examples/coral/BUILD

@@ -0,0 +1,56 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = [
+    "//visibility:public",
+])
+
+# Graph Runner
+
+cc_library(
+    name = "demo_run_graph_main",
+    srcs = ["demo_run_graph_main.cc"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/port:commandlineflags",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:opencv_highgui",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:opencv_video",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+    ],
+)
+
+# Demos
+
+cc_binary(
+    name = "object_detection_cpu",
+    deps = [
+        "//mediapipe/examples/coral:demo_run_graph_main",
+        "//mediapipe/graphs/object_detection:desktop_tflite_calculators",
+    ],
+)
+
+cc_binary(
+    name = "face_detection_cpu",
+    deps = [
+        "//mediapipe/examples/coral:demo_run_graph_main",
+        "//mediapipe/graphs/face_detection:desktop_tflite_calculators",
+    ],
+)

mediapipe/examples/coral/Dockerfile

@@ -0,0 +1,87 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#==== ! Prerequisite ! ====
+# $ sh mediapipe/examples/coral/setup.sh
+#====
+
+# for opencv 3.2 default
+FROM ubuntu:18.04
+
+MAINTAINER <mediapipe@google.com>
+
+WORKDIR /mediapipe
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install MediaPipe & Coral deps
+
+COPY update_sources.sh /
+RUN /update_sources.sh
+
+RUN dpkg --add-architecture armhf
+RUN dpkg --add-architecture arm64
+RUN apt-get update && apt-get install -y \
+  build-essential \
+  crossbuild-essential-arm64 \
+  libusb-1.0-0-dev \
+  libusb-1.0-0-dev:arm64 \
+  zlib1g-dev \
+  zlib1g-dev:arm64 \
+  pkg-config \
+  zip \
+  unzip \
+  curl \
+  wget \
+  git \
+  python \
+  python-pip \
+  python3-pip \
+  vim-common \
+  ca-certificates \
+  emacs \
+  software-properties-common && \
+  add-apt-repository -y ppa:openjdk-r/ppa && \
+  apt-get update && apt-get install -y openjdk-8-jdk
+
+RUN pip install --upgrade setuptools
+RUN pip install future
+RUN pip3 install six
+
+COPY . /mediapipe/
+
+# Install bazel
+
+ARG BAZEL_VERSION=0.29.1
+RUN mkdir /bazel && \
+    wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
+    wget --no-check-certificate -O  /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \
+    chmod +x /bazel/installer.sh && \
+    /bazel/installer.sh  && \
+    rm -f /bazel/installer.sh
+
+# OpenCV (3.2 default in 18.04)
+
+RUN apt-get update && apt-get install -y libopencv-dev
+
+# Opencv libs copied from coral device into opencv32_arm64_libs
+
+RUN cp opencv32_arm64_libs/* /usr/lib/aarch64-linux-gnu/.
+
+# Edge tpu header and lib
+
+RUN git clone https://github.com/google-coral/edgetpu.git /edgetpu
+RUN cp /edgetpu/libedgetpu/direct/aarch64/libedgetpu.so.1.0 /usr/lib/aarch64-linux-gnu/libedgetpu.so
+
+# See mediapipe/examples/coral/README.md to finish setup

mediapipe/examples/coral/README.md

@@ -0,0 +1,137 @@
+# Coral Dev Board Setup (experimental)
+
+**Dislaimer**: Running MediaPipe on Coral is experimental, and this process may not be exact and is subject to change. These instructions have only been tested on the coral dev board with OS version _mendel day_, and may vary for different devices and workstations.
+
+This file describes how to prepare a Google Coral Dev Board and setup a linux Docker container for building MediaPipe applications that run on Edge TPU.
+
+## Before creating the Docker
+
+* (on host machine) run _setup.sh_ from MediaPipe root directory
+
+        sh mediapipe/examples/coral/setup.sh
+
+* Setup the coral device via [here](https://coral.withgoogle.com/docs/dev-board/get-started/), and ensure the _mdt_ command works
+
+* (on coral device) prepare MediaPipe
+
+        cd ~
+        sudo apt-get install git
+        git clone https://github.com/google/mediapipe.git
+        mkdir mediapipe/bazel-bin
+
+* (on coral device) install opencv 3.2
+
+        sudo apt-get update && apt-get install -y libopencv-dev
+
+* (on coral device) find all opencv libs
+
+        find /usr/lib/aarch64-linux-gnu/ -name 'libopencv*so'
+
+* (on host machine) copy core opencv libs from coral device to a local folder inside MediaPipe checkout:
+
+        # in root level mediapipe folder #
+        mdt pull /usr/lib/aarch64-linux-gnu/libopencv_core.so opencv32_arm64_libs
+        mdt pull /usr/lib/aarch64-linux-gnu/libopencv_calib3d.so opencv32_arm64_libs
+        mdt pull /usr/lib/aarch64-linux-gnu/libopencv_features2d.so opencv32_arm64_libs
+        mdt pull /usr/lib/aarch64-linux-gnu/libopencv_highgui.so opencv32_arm64_libs
+        mdt pull /usr/lib/aarch64-linux-gnu/libopencv_imgcodecs.so opencv32_arm64_libs
+        mdt pull /usr/lib/aarch64-linux-gnu/libopencv_imgproc.so opencv32_arm64_libs
+        mdt pull /usr/lib/aarch64-linux-gnu/libopencv_video.so opencv32_arm64_libs
+        mdt pull /usr/lib/aarch64-linux-gnu/libopencv_videoio.so opencv32_arm64_libs
+
+* (on host machine) Create and start the docker environment
+
+        # from mediapipe root level directory #
+        docker build -t coral .
+        docker run -it --name coral coral:latest
+
+## Inside the Docker environment
+
+* Update library paths in /mediapipe/third_party/opencv_linux.BUILD
+
+  (replace 'x86_64-linux-gnu' with 'aarch64-linux-gnu')
+
+        "lib/aarch64-linux-gnu/libopencv_core.so",
+        "lib/aarch64-linux-gnu/libopencv_calib3d.so",
+        "lib/aarch64-linux-gnu/libopencv_features2d.so",
+        "lib/aarch64-linux-gnu/libopencv_highgui.so",
+        "lib/aarch64-linux-gnu/libopencv_imgcodecs.so",
+        "lib/aarch64-linux-gnu/libopencv_imgproc.so",
+        "lib/aarch64-linux-gnu/libopencv_video.so",
+        "lib/aarch64-linux-gnu/libopencv_videoio.so",
+
+* Attempt to build hello world (to download external deps)
+
+        bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/hello_world:hello_world
+
+* Edit  /mediapipe/bazel-mediapipe/external/com_github_glog_glog/src/signalhandler.cc
+
+      on line 78, replace
+
+        return (void*)context->PC_FROM_UCONTEXT;
+
+      with
+
+        return NULL;
+
+* Edit  /edgetpu/libedgetpu/BUILD
+
+      to add this build target
+
+         cc_library(
+           name = "lib",
+           srcs = [
+               "libedgetpu.so",
+           ],
+           visibility = ["//visibility:public"],
+         )
+
+* Edit *tflite_inference_calculator.cc*  BUILD rules:
+
+        sed -i 's/\":tflite_inference_calculator_cc_proto\",/\":tflite_inference_calculator_cc_proto\",\n\t\"@edgetpu\/\/:header\",\n\t\"@libedgetpu\/\/:lib\",/g' mediapipe/calculators/tflite/BUILD
+
+      The above command should add
+
+        "@edgetpu//:header",
+        "@libedgetpu//:lib",
+
+      to the _deps_ of tflite_inference_calculator.cc
+
+#### Now try cross-compiling for device
+
+* Object detection demo
+
+        bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:object_detection_cpu
+
+ Copy object_detection_cpu binary to the MediaPipe checkout on the coral device
+
+        # outside docker env, open new terminal on host machine #
+        docker ps
+        docker cp <container-id>:/mediapipe/bazel-bin/mediapipe/examples/coral/object_detection_cpu /tmp/.
+        mdt push /tmp/object_detection_cpu /home/mendel/mediapipe/bazel-bin/.
+
+* Face detection demo
+
+        bazel build -c opt --crosstool_top=@crosstool//:toolchains --compiler=gcc --cpu=aarch64 --define MEDIAPIPE_DISABLE_GPU=1 --copt -DMEDIAPIPE_EDGE_TPU --copt=-flax-vector-conversions mediapipe/examples/coral:face_detection_cpu
+
+ Copy face_detection_cpu binary to the MediaPipe checkout on the coral device
+
+        # outside docker env, open new terminal on host machine #
+        docker ps
+        docker cp <container-id>:/mediapipe/bazel-bin/mediapipe/examples/coral/face_detection_cpu /tmp/.
+        mdt push /tmp/face_detection_cpu /home/mendel/mediapipe/bazel-bin/.
+
+## On the coral device (with display)
+
+     # Object detection
+     cd ~/mediapipe
+     chmod +x bazel-bin/object_detection_cpu
+     export GLOG_logtostderr=1
+     bazel-bin/object_detection_cpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt
+
+     # Face detection
+     cd ~/mediapipe
+     chmod +x bazel-bin/face_detection_cpu
+     export GLOG_logtostderr=1
+     bazel-bin/face_detection_cpu --calculator_graph_config_file=mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt
+

mediapipe/examples/coral/WORKSPACE

@@ -0,0 +1,313 @@
+workspace(name = "mediapipe")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+skylib_version = "0.8.0"
+http_archive(
+    name = "bazel_skylib",
+    type = "tar.gz",
+    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format (skylib_version, skylib_version),
+    sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
+)
+load("@bazel_skylib//lib:versions.bzl", "versions")
+versions.check(minimum_bazel_version = "0.24.1")
+
+# ABSL cpp library.
+http_archive(
+    name = "com_google_absl",
+    # Head commit on 2019-04-12.
+    # TODO: Switch to the latest absl version when the problem gets
+    # fixed.
+    urls = [
+        "https://github.com/abseil/abseil-cpp/archive/a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a.tar.gz",
+    ],
+    sha256 = "d437920d1434c766d22e85773b899c77c672b8b4865d5dc2cd61a29fdff3cf03",
+    strip_prefix = "abseil-cpp-a02f62f456f2c4a7ecf2be3104fe0c6e16fbad9a",
+)
+
+http_archive(
+    name = "rules_cc",
+    strip_prefix = "rules_cc-master",
+    urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
+)
+
+# GoogleTest/GoogleMock framework. Used by most unit-tests.
+http_archive(
+     name = "com_google_googletest",
+     urls = ["https://github.com/google/googletest/archive/master.zip"],
+     strip_prefix = "googletest-master",
+)
+
+# Google Benchmark library.
+http_archive(
+    name = "com_google_benchmark",
+    urls = ["https://github.com/google/benchmark/archive/master.zip"],
+    strip_prefix = "benchmark-master",
+    build_file = "@//third_party:benchmark.BUILD",
+)
+
+# gflags needed by glog
+http_archive(
+    name = "com_github_gflags_gflags",
+    sha256 = "6e16c8bc91b1310a44f3965e616383dbda48f83e8c1eaa2370a215057b00cabe",
+    strip_prefix = "gflags-77592648e3f3be87d6c7123eb81cbad75f9aef5a",
+    urls = [
+        "https://mirror.bazel.build/github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
+        "https://github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
+    ],
+)
+
+# glog
+http_archive(
+    name = "com_github_glog_glog",
+    url = "https://github.com/google/glog/archive/v0.3.5.zip",
+    sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
+    strip_prefix = "glog-0.3.5",
+    build_file = "@//third_party:glog.BUILD",
+    patches = [
+        "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
+)
+
+# libyuv
+http_archive(
+    name = "libyuv",
+    urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/refs/heads/master.tar.gz"],
+    build_file = "@//third_party:libyuv.BUILD",
+)
+
+http_archive(
+    name = "com_google_protobuf_javalite",
+    sha256 = "79d102c61e2a479a0b7e5fc167bcfaa4832a0c6aad4a75fa7da0480564931bcc",
+    strip_prefix = "protobuf-384989534b2246d413dbcd750744faab2607b516",
+    urls = ["https://github.com/google/protobuf/archive/384989534b2246d413dbcd750744faab2607b516.zip"],
+)
+
+http_archive(
+    name = "com_google_audio_tools",
+    strip_prefix = "multichannel-audio-tools-master",
+    urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
+)
+
+# Needed by TensorFlow
+http_archive(
+    name = "io_bazel_rules_closure",
+    sha256 = "e0a111000aeed2051f29fcc7a3f83be3ad8c6c93c186e64beb1ad313f0c7f9f9",
+    strip_prefix = "rules_closure-cf1e44edb908e9616030cc83d085989b8e6cd6df",
+    urls = [
+        "http://mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz",
+        "https://github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz",  # 2019-04-04
+    ],
+)
+
+# 2019-11-12
+_TENSORFLOW_GIT_COMMIT = "a5f9bcd64453ff3d1f64cb4da4786db3d2da7f82"
+_TENSORFLOW_SHA256= "f2b6f2ab2ffe63e86eccd3ce4bea6b7197383d726638dfeeebcdc1e7de73f075"
+http_archive(
+    name = "org_tensorflow",
+    urls = [
+      "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
+      "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
+    ],
+    strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
+    sha256 = _TENSORFLOW_SHA256,
+)
+
+load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
+tf_workspace(tf_repo_name = "org_tensorflow")
+
+# Please run
+# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
+#                        libopencv-imgproc-dev libopencv-video-dev
+new_local_repository(
+    name = "linux_opencv",
+    build_file = "@//third_party:opencv_linux.BUILD",
+    path = "/usr",
+)
+
+new_local_repository(
+    name = "linux_ffmpeg",
+    build_file = "@//third_party:ffmpeg_linux.BUILD",
+    path = "/usr"
+)
+
+# Please run $ brew install opencv@3
+new_local_repository(
+    name = "macos_opencv",
+    build_file = "@//third_party:opencv_macos.BUILD",
+    path = "/usr",
+)
+
+new_local_repository(
+    name = "macos_ffmpeg",
+    build_file = "@//third_party:ffmpeg_macos.BUILD",
+    path = "/usr",
+)
+
+http_archive(
+    name = "android_opencv",
+    sha256 = "056b849842e4fa8751d09edbb64530cfa7a63c84ccd232d0ace330e27ba55d0b",
+    build_file = "@//third_party:opencv_android.BUILD",
+    strip_prefix = "OpenCV-android-sdk",
+    type = "zip",
+    url = "https://github.com/opencv/opencv/releases/download/4.1.0/opencv-4.1.0-android-sdk.zip",
+)
+
+# After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will
+# trigger duplicate symbol errors in the linking stage of building a mediapipe ios app.
+# To get a higher version of OpenCV for iOS, opencv2.framework needs to be built from source with
+# '-DBUILD_PROTOBUF=OFF -DBUILD_opencv_dnn=OFF'.
+http_archive(
+    name = "ios_opencv",
+    sha256 = "7dd536d06f59e6e1156b546bd581523d8df92ce83440002885ec5abc06558de2",
+    build_file = "@//third_party:opencv_ios.BUILD",
+    type = "zip",
+    url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
+)
+
+RULES_JVM_EXTERNAL_TAG = "2.2"
+RULES_JVM_EXTERNAL_SHA = "f1203ce04e232ab6fdd81897cf0ff76f2c04c0741424d192f28e65ae752ce2d6"
+
+http_archive(
+    name = "rules_jvm_external",
+    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
+    sha256 = RULES_JVM_EXTERNAL_SHA,
+    url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
+)
+
+load("@rules_jvm_external//:defs.bzl", "maven_install")
+
+maven_install(
+    artifacts = [
+        "androidx.annotation:annotation:aar:1.1.0",
+        "androidx.appcompat:appcompat:aar:1.1.0-rc01",
+        "androidx.constraintlayout:constraintlayout:aar:1.1.3",
+        "androidx.core:core:aar:1.1.0-rc03",
+        "androidx.legacy:legacy-support-v4:aar:1.0.0",
+        "androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
+        "com.google.android.material:material:aar:1.0.0-rc01",
+    ],
+    repositories = ["https://dl.google.com/dl/android/maven2"],
+)
+
+maven_server(
+    name = "google_server",
+    url = "https://dl.google.com/dl/android/maven2",
+)
+
+maven_jar(
+    name = "androidx_lifecycle",
+    artifact = "androidx.lifecycle:lifecycle-common:2.0.0",
+    sha1 = "e070ffae07452331bc5684734fce6831d531785c",
+    server = "google_server",
+)
+
+maven_jar(
+     name = "androidx_concurrent_futures",
+     artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
+     sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
+     server = "google_server",
+)
+
+maven_jar(
+    name = "com_google_guava_android",
+    artifact = "com.google.guava:guava:27.0.1-android",
+    sha1 = "b7e1c37f66ef193796ccd7ea6e80c2b05426182d",
+)
+
+maven_jar(
+    name = "com_google_common_flogger",
+    artifact = "com.google.flogger:flogger:0.3.1",
+    sha1 = "585030fe1ec709760cbef997a459729fb965df0e",
+)
+
+maven_jar(
+    name = "com_google_common_flogger_system_backend",
+    artifact = "com.google.flogger:flogger-system-backend:0.3.1",
+    sha1 = "287b569d76abcd82f9de87fe41829fbc7ebd8ac9",
+)
+
+maven_jar(
+    name = "com_google_code_findbugs",
+    artifact = "com.google.code.findbugs:jsr305:3.0.2",
+    sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d",
+)
+
+# You may run setup_android.sh to install Android SDK and NDK.
+android_ndk_repository(
+    name = "androidndk",
+)
+
+android_sdk_repository(
+    name = "androidsdk",
+)
+
+# iOS basic build deps.
+
+http_archive(
+    name = "build_bazel_rules_apple",
+    sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e",
+    strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3",
+    url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz",
+)
+
+load(
+    "@build_bazel_rules_apple//apple:repositories.bzl",
+    "apple_rules_dependencies",
+)
+
+apple_rules_dependencies()
+
+load(
+    "@build_bazel_rules_swift//swift:repositories.bzl",
+    "swift_rules_dependencies",
+)
+
+swift_rules_dependencies()
+
+load(
+    "@build_bazel_apple_support//lib:repositories.bzl",
+    "apple_support_dependencies",
+)
+
+apple_support_dependencies()
+
+# More iOS deps.
+
+http_archive(
+    name = "google_toolbox_for_mac",
+    url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip",
+    sha256 = "e3ac053813c989a88703556df4dc4466e424e30d32108433ed6beaec76ba4fdc",
+    strip_prefix = "google-toolbox-for-mac-2.2.1",
+    build_file = "@//third_party:google_toolbox_for_mac.BUILD",
+)
+
+
+# Coral
+#COMMIT=$(git ls-remote https://github.com/google-coral/crosstool master | awk '{print $1}')
+#SHA256=$(curl -L "https://github.com/google-coral/crosstool/archive/${COMMIT}.tar.gz" | sha256sum | awk '{print $1}')
+http_archive(
+    name = "coral_crosstool",
+    sha256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb",
+    strip_prefix = "crosstool-9e00d5be43bf001f883b5700f5d04882fea00229",
+    urls = [
+        "https://github.com/google-coral/crosstool/archive/9e00d5be43bf001f883b5700f5d04882fea00229.tar.gz",
+    ],
+)
+load("@coral_crosstool//:configure.bzl", "cc_crosstool")
+cc_crosstool(name = "crosstool")
+
+# EdgeTPU
+new_local_repository(
+    name = "edgetpu",
+    path = "/edgetpu/libedgetpu",
+    build_file = "/edgetpu/libedgetpu/BUILD"
+)
+new_local_repository(
+    name = "libedgetpu",
+    path = "/usr/lib/aarch64-linux-gnu",
+    build_file = "/edgetpu/libedgetpu/BUILD"
+)

mediapipe/examples/coral/demo_run_graph_main.cc

@@ -0,0 +1,151 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// An example of sending OpenCV webcam frames into a MediaPipe graph.
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/port/commandlineflags.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/opencv_highgui_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/opencv_video_inc.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+
+constexpr char kInputStream[] = "input_video";
+constexpr char kOutputStream[] = "output_video";
+constexpr char kWindowName[] = "MediaPipe";
+
+DEFINE_string(
+    calculator_graph_config_file, "",
+    "Name of file containing text format CalculatorGraphConfig proto.");
+DEFINE_string(input_video_path, "",
+              "Full path of video to load. "
+              "If not provided, attempt to use a webcam.");
+DEFINE_string(output_video_path, "",
+              "Full path of where to save result (.mp4 only). "
+              "If not provided, show result in a window.");
+
+::mediapipe::Status RunMPPGraph() {
+  std::string calculator_graph_config_contents;
+  MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
+      FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
+  LOG(INFO) << "Get calculator graph config contents: "
+            << calculator_graph_config_contents;
+  mediapipe::CalculatorGraphConfig config =
+      mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
+          calculator_graph_config_contents);
+
+  LOG(INFO) << "Initialize the calculator graph.";
+  mediapipe::CalculatorGraph graph;
+  MP_RETURN_IF_ERROR(graph.Initialize(config));
+
+  LOG(INFO) << "Initialize the camera or load the video.";
+  cv::VideoCapture capture;
+  const bool load_video = !FLAGS_input_video_path.empty();
+  if (load_video) {
+    capture.open(FLAGS_input_video_path);
+  } else {
+    capture.open(0);
+  }
+  RET_CHECK(capture.isOpened());
+
+  cv::VideoWriter writer;
+  const bool save_video = !FLAGS_output_video_path.empty();
+  if (save_video) {
+    LOG(INFO) << "Prepare video writer.";
+    cv::Mat test_frame;
+    capture.read(test_frame);                    // Consume first frame.
+    capture.set(cv::CAP_PROP_POS_AVI_RATIO, 0);  // Rewind to beginning.
+    writer.open(FLAGS_output_video_path,
+                mediapipe::fourcc('a', 'v', 'c', '1'),  // .mp4
+                capture.get(cv::CAP_PROP_FPS), test_frame.size());
+    RET_CHECK(writer.isOpened());
+  } else {
+    cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1);
+    capture.set(cv::CAP_PROP_FRAME_WIDTH, 640);
+    capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480);
+    capture.set(cv::CAP_PROP_AUTOFOCUS, 0);
+    capture.set(cv::CAP_PROP_FOCUS, 1);
+    capture.set(cv::CAP_PROP_FPS, 30);
+  }
+
+  LOG(INFO) << "Start running the calculator graph.";
+  ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller poller,
+                   graph.AddOutputStreamPoller(kOutputStream));
+  MP_RETURN_IF_ERROR(graph.StartRun({}));
+
+  LOG(INFO) << "Start grabbing and processing frames.";
+  size_t frame_timestamp = 0;
+  bool grab_frames = true;
+  while (grab_frames) {
+    // Capture opencv camera or video frame.
+    cv::Mat camera_frame_raw;
+    capture >> camera_frame_raw;
+    if (camera_frame_raw.empty()) break;  // End of video.
+    cv::Mat camera_frame;
+    cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB);
+    if (!load_video) {
+      cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1);
+    }
+
+    // Wrap Mat into an ImageFrame.
+    auto input_frame = absl::make_unique<mediapipe::ImageFrame>(
+        mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows,
+        mediapipe::ImageFrame::kDefaultAlignmentBoundary);
+    cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get());
+    camera_frame.copyTo(input_frame_mat);
+
+    // Send image packet into the graph.
+    MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
+        kInputStream, mediapipe::Adopt(input_frame.release())
+                          .At(mediapipe::Timestamp(frame_timestamp++))));
+
+    // Get the graph result packet, or stop if that fails.
+    mediapipe::Packet packet;
+    if (!poller.Next(&packet)) break;
+    auto& output_frame = packet.Get<mediapipe::ImageFrame>();
+
+    // Convert back to opencv for display or saving.
+    cv::Mat output_frame_mat = mediapipe::formats::MatView(&output_frame);
+    cv::cvtColor(output_frame_mat, output_frame_mat, cv::COLOR_RGB2BGR);
+    if (save_video) {
+      writer.write(output_frame_mat);
+    } else {
+      cv::imshow(kWindowName, output_frame_mat);
+      // Press any key to exit.
+      const int pressed_key = cv::waitKey(5);
+      if (pressed_key >= 0 && pressed_key != 255) grab_frames = false;
+    }
+  }
+
+  LOG(INFO) << "Shutting down.";
+  if (writer.isOpened()) writer.release();
+  MP_RETURN_IF_ERROR(graph.CloseInputStream(kInputStream));
+  return graph.WaitUntilDone();
+}
+
+int main(int argc, char** argv) {
+  google::InitGoogleLogging(argv[0]);
+  gflags::ParseCommandLineFlags(&argc, &argv, true);
+  ::mediapipe::Status run_status = RunMPPGraph();
+  if (!run_status.ok()) {
+    LOG(ERROR) << "Failed to run the graph: " << run_status.message();
+  } else {
+    LOG(INFO) << "Success!";
+  }
+  return 0;
+}

mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt

@@ -0,0 +1,189 @@
+# MediaPipe graph that performs face detection with TensorFlow Lite on CPU.
+# Used in the examples in
+# mediapipe/examples/coral:face_detection_cpu.
+
+# Images on GPU coming into and out of the graph.
+input_stream: "input_video"
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for
+# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish
+# generating the corresponding detections before it passes through another
+# image. All images that come in while waiting are dropped, limiting the number
+# of in-flight images between this calculator and
+# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between
+# from queuing up incoming images and data excessively, which leads to increased
+# latency and memory usage, unwanted in real-time mobile applications. It also
+# eliminates unnecessarily computation, e.g., a transformed image produced by
+# ImageTransformationCalculator may get dropped downstream if the subsequent
+# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy
+# processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:detections"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Transforms the input image on CPU to a 128x128 image. To scale the input
+# image, the scale_mode option is set to FIT to preserve the aspect ratio,
+# resulting in potential letterboxing in the transformed image.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:throttled_input_video"
+  output_stream: "IMAGE:transformed_input_video_cpu"
+  output_stream: "LETTERBOX_PADDING:letterbox_padding"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 128
+      output_height: 128
+      scale_mode: FIT
+    }
+  }
+}
+
+# Converts the transformed input image on CPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE:transformed_input_video_cpu"
+  output_stream: "TENSORS:image_tensor"
+  options: {
+    [mediapipe.TfLiteConverterCalculatorOptions.ext] {
+      use_quantized_tensors: true
+    }
+  }
+}
+
+# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:image_tensor"
+  output_stream: "TENSORS:detection_tensors"
+  options: {
+    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/examples/coral/models/face-detector-quantized_edgetpu.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  options: {
+    [mediapipe.SsdAnchorsCalculatorOptions.ext] {
+      num_layers: 4
+      min_scale: 0.1484375
+      max_scale: 0.75
+      input_size_height: 128
+      input_size_width: 128
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 8
+      strides: 16
+      strides: 16
+      strides: 16
+      aspect_ratios: 1.0
+      fixed_anchor_size: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  options: {
+    [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
+      num_classes: 1
+      num_boxes: 896
+      num_coords: 16
+      box_coord_offset: 0
+      keypoint_coord_offset: 4
+      num_keypoints: 6
+      num_values_per_keypoint: 2
+      sigmoid_score: true
+      score_clipping_thresh: 100.0
+      reverse_output_order: true
+      x_scale: 128.0
+      y_scale: 128.0
+      h_scale: 128.0
+      w_scale: 128.0
+      min_score_thresh: 0.75
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  options: {
+    [mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
+      min_suppression_threshold: 0.3
+      overlap_type: INTERSECTION_OVER_UNION
+      algorithm: WEIGHTED
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text ("Face"). The label
+# map is provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "labeled_detections"
+  options: {
+    [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
+      label_map_path: "mediapipe/models/face_detection_front_labelmap.txt"
+    }
+  }
+}
+
+# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
+# letterboxed image (after image transformation with the FIT scale mode) to the
+# corresponding locations on the same image with the letterbox removed (the
+# input image to the graph before image transformation).
+node {
+  calculator: "DetectionLetterboxRemovalCalculator"
+  input_stream: "DETECTIONS:labeled_detections"
+  input_stream: "LETTERBOX_PADDING:letterbox_padding"
+  output_stream: "DETECTIONS:output_detections"
+}
+
+# Converts the detections to drawing primitives for annotation overlay.
+node {
+  calculator: "DetectionsToRenderDataCalculator"
+  input_stream: "DETECTIONS:output_detections"
+  output_stream: "RENDER_DATA:render_data"
+  options: {
+    [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
+      thickness: 4.0
+      color { r: 255 g: 0 b: 0 }
+    }
+  }
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "INPUT_FRAME:throttled_input_video"
+  input_stream: "render_data"
+  output_stream: "OUTPUT_FRAME:output_video"
+}
+

mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt

@@ -0,0 +1,179 @@
+# MediaPipe graph that performs object detection with TensorFlow Lite on CPU.
+# Used in the examples in
+# mediapipie/examples/coral:object_detection_cpu.
+
+# Images on CPU coming into and out of the graph.
+input_stream: "input_video"
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for
+# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish
+# generating the corresponding detections before it passes through another
+# image. All images that come in while waiting are dropped, limiting the number
+# of in-flight images between this calculator and
+# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between
+# from queuing up incoming images and data excessively, which leads to increased
+# latency and memory usage, unwanted in real-time mobile applications. It also
+# eliminates unnecessarily computation, e.g., a transformed image produced by
+# ImageTransformationCalculator may get dropped downstream if the subsequent
+# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy
+# processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:detections"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Transforms the input image on CPU to a 320x320 image. To scale the image, by
+# default it uses the STRETCH scale mode that maps the entire input image to the
+# entire transformed image. As a result, image aspect ratio may be changed and
+# objects in the image may be deformed (stretched or squeezed), but the object
+# detection model used in this graph is agnostic to that deformation.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE:throttled_input_video"
+  output_stream: "IMAGE:transformed_input_video"
+  options: {
+    [mediapipe.ImageTransformationCalculatorOptions.ext] {
+      output_width: 300
+      output_height: 300
+    }
+  }
+}
+
+# Converts the transformed input image on CPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE:transformed_input_video"
+  output_stream: "TENSORS:image_tensor"
+  options: {
+    [mediapipe.TfLiteConverterCalculatorOptions.ext] {
+      use_quantized_tensors: true
+    }
+  }
+}
+
+# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS:image_tensor"
+  output_stream: "TENSORS:detection_tensors"
+  options: {
+    [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
+      model_path: "mediapipe/examples/coral/models/object-detector-quantized_edgetpu.tflite"
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  options: {
+    [mediapipe.SsdAnchorsCalculatorOptions.ext] {
+      num_layers: 6
+      min_scale: 0.2
+      max_scale: 0.95
+      input_size_height: 300
+      input_size_width: 300
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 16
+      strides: 32
+      strides: 64
+      strides: 128
+      strides: 256
+      strides: 512
+      aspect_ratios: 1.0
+      aspect_ratios: 2.0
+      aspect_ratios: 0.5
+      aspect_ratios: 3.0
+      aspect_ratios: 0.3333
+      reduce_boxes_in_lowest_layer: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  options: {
+    [mediapipe.TfLiteTensorsToDetectionsCalculatorOptions.ext] {
+      num_classes: 91
+      num_boxes: 2034
+      num_coords: 4
+      ignore_classes: 0
+      sigmoid_score: true
+      apply_exponential_on_box_size: true
+      x_scale: 10.0
+      y_scale: 10.0
+      h_scale: 5.0
+      w_scale: 5.0
+      min_score_thresh: 0.6
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  options: {
+    [mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
+      min_suppression_threshold: 0.4
+      max_num_detections: 3
+      overlap_type: INTERSECTION_OVER_UNION
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text. The label map is
+# provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "output_detections"
+  options: {
+    [mediapipe.DetectionLabelIdToTextCalculatorOptions.ext] {
+      label_map_path: "mediapipe/examples/coral/models/object_detection_labelmap.txt"
+    }
+  }
+}
+
+# Converts the detections to drawing primitives for annotation overlay.
+node {
+  calculator: "DetectionsToRenderDataCalculator"
+  input_stream: "DETECTIONS:output_detections"
+  output_stream: "RENDER_DATA:render_data"
+  options: {
+    [mediapipe.DetectionsToRenderDataCalculatorOptions.ext] {
+      thickness: 4.0
+      color { r: 255 g: 0 b: 0 }
+    }
+  }
+}
+
+# Draws annotations and overlays them on top of the input images.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "INPUT_FRAME:throttled_input_video"
+  input_stream: "render_data"
+  output_stream: "OUTPUT_FRAME:output_video"
+}

mediapipe/examples/coral/models/object_detection_labelmap.txt

@@ -0,0 +1,90 @@
+person
+bicycle
+car
+motorcycle
+airplane
+bus
+train
+truck
+boat
+traffic light
+fire hydrant
+???
+stop sign
+parking meter
+bench
+bird
+cat
+dog
+horse
+sheep
+cow
+elephant
+bear
+zebra
+giraffe
+???
+backpack
+umbrella
+???
+???
+handbag
+tie
+suitcase
+frisbee
+skis
+snowboard
+sports ball
+kite
+baseball bat
+baseball glove
+skateboard
+surfboard
+tennis racket
+bottle
+???
+wine glass
+cup
+fork
+knife
+spoon
+bowl
+banana
+apple
+sandwich
+orange
+broccoli
+carrot
+hot dog
+pizza
+donut
+cake
+chair
+couch
+potted plant
+bed
+???
+dining table
+???
+???
+toilet
+???
+tv
+laptop
+mouse
+remote
+keyboard
+cell phone
+microwave
+oven
+toaster
+sink
+refrigerator
+???
+book
+clock
+vase
+scissors
+teddy bear
+hair drier
+toothbrush

mediapipe/examples/coral/setup.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+set -e
+set -v
+
+echo 'Please run this from root level mediapipe directory! \n Ex:'
+echo '  sh mediapipe/examples/coral/setup.sh  '
+
+sleep 3
+
+mkdir opencv32_arm64_libs
+
+cp mediapipe/examples/coral/update_sources.sh update_sources.sh
+chmod +x update_sources.sh
+
+mv Dockerfile Dockerfile.orig
+cp mediapipe/examples/coral/Dockerfile Dockerfile
+
+cp WORKSPACE WORKSPACE.orig
+cp mediapipe/examples/coral/WORKSPACE WORKSPACE
+

mediapipe/examples/coral/update_sources.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# To run in the Coral Docker environment.
+
+. /etc/os-release
+
+sed -i "s/deb\ /deb \[arch=amd64\]\ /g" /etc/apt/sources.list
+
+echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME} main universe" >> /etc/apt/sources.list
+echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-updates main universe" >> /etc/apt/sources.list
+echo "deb [arch=arm64,armhf] http://ports.ubuntu.com/ubuntu-ports ${UBUNTU_CODENAME}-security main universe" >> /etc/apt/sources.list

mediapipe/examples/desktop/README.md

@@ -9,7 +9,9 @@ bazel build -c opt mediapipe/examples/desktop/hello_world:hello_world
 and then run it using:
 
 ```
-bazel-bin/mediapipe/examples/desktop/hello_world/hello_world --logtostderr
+export GLOG_logtostderr=1
+
+bazel-bin/mediapipe/examples/desktop/hello_world/hello_world
 ```
 
 **TFlite Object Detection**
@@ -23,10 +25,11 @@ bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_
 and run it using:
 
 ```
+export GLOG_logtostderr=1
+
 bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \
   --calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt \
-  --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file \
+  --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
-  --alsologtostderr
 ```
 
 **TensorFlow Object Detection**
@@ -34,6 +37,8 @@ bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tflite \
 To build the object detection demo using a TensorFlow model on desktop, use:
 
 ```
+export GLOG_logtostderr=1
+
 bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_tensorflow \
   --define MEDIAPIPE_DISABLE_GPU=1
 ```
@@ -41,8 +46,68 @@ bazel build -c opt mediapipe/examples/desktop/object_detection:object_detection_
 and run it using:
 
 ```
+export GLOG_logtostderr=1
+
 bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tensorflow  \
   --calculator_graph_config_file=mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt  \
   --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
-  --alsologtostderr
 ```
+
+**TFlite Hand Detection**
+
+To build the hand detection demo using a TFLite model on desktop, use:
+
+```
+bazel build -c opt mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1
+```
+
+and run it using:
+
+```
+export GLOG_logtostderr=1
+
+bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_tflite \
+  --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_detection_desktop.pbtxt \
+  --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
+```
+
+**TFlite Hand Tracking**
+
+To build the hand tracking demo using a TFLite model on desktop, use:
+
+```
+bazel build -c opt mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1
+```
+
+and run it using:
+
+```
+export GLOG_logtostderr=1
+
+bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_tflite \
+  --calculator_graph_config_file=mediapipe/graphs/hand_tracking/hand_tracking_desktop.pbtxt \
+  --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
+```
+
+**TFlite Multi-Hand Tracking**
+
+To build the multi-hand tracking demo using a TFLite model on desktop, use:
+
+```
+bazel build -c opt mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_tflite --define MEDIAPIPE_DISABLE_GPU=1
+```
+
+and run it using:
+
+```
+export GLOG_logtostderr=1
+
+bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_tflite \
+  --calculator_graph_config_file=mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt \
+  --input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
+```
+
+To change the number of hands to `x` in this application, change:
+
+1. `min_size:x` in `CollectionHasMinSizeCalculatorOptions` in `mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt`.
+2. `max_vec_size:x` in `ClipVectorSizeCalculatorOptions` in `mediapipe/examples/dekstop/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt`.

mediapipe/examples/desktop/media_sequence/kinetics_dataset.py

@@ -93,15 +93,15 @@ FILEPATTERN = "kinetics_700_%s_25fps_rgb_flow"
 SPLITS = {
     "train": {
         "shards": 1000,
-        "examples": 541632
+        "examples": 541490
     },
     "validate": {
         "shards": 100,
-        "examples": 34727
+        "examples": 34715
     },
     "test": {
         "shards": 100,
-        "examples": 69347
+        "examples": 69321
     },
     "custom": {
         "csv": None,  # Add a CSV for your own data here.

mediapipe/framework/BUILD

@@ -1524,7 +1524,9 @@ cc_test(
         "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
         "//mediapipe/framework/stream_handler:immediate_input_stream_handler",
         "//mediapipe/framework/stream_handler:mux_input_stream_handler",
+        "//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
         "//mediapipe/framework/tool:sink",
+        "@com_google_absl//absl/strings",
     ],
 )
 

mediapipe/framework/calculator_graph_bounds_test.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "absl/strings/str_replace.h"
 #include "mediapipe/framework/calculator_context.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/port/canonical_errors.h"
@@ -642,6 +643,36 @@ REGISTER_CALCULATOR(OffsetBoundCalculator);
 
 // A Calculator that produces a packet for each call to Process.
 class BoundToPacketCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    for (int i = 0; i < cc->Inputs().NumEntries(); ++i) {
+      cc->Inputs().Index(i).SetAny();
+    }
+    for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
+      cc->Outputs().Index(i).Set<Timestamp>();
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) final {
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) final {
+    for (int i = 0; i < cc->Outputs().NumEntries(); ++i) {
+      Timestamp t = cc->Inputs().Index(i).Value().Timestamp();
+      cc->Outputs().Index(i).AddPacket(
+          mediapipe::MakePacket<Timestamp>(t).At(cc->InputTimestamp()));
+    }
+    return ::mediapipe::OkStatus();
+  }
+};
+REGISTER_CALCULATOR(BoundToPacketCalculator);
+
+// A Calculator that produces packets at timestamps beyond the input timestamp.
+class FuturePacketCalculator : public CalculatorBase {
+  static constexpr int64 kOutputFutureMicros = 3;
+
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
     cc->Inputs().Index(0).Set<int>();
@@ -654,11 +685,14 @@ class BoundToPacketCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Process(CalculatorContext* cc) final {
-    cc->Outputs().Index(0).AddPacket(Adopt(new int(33)));
+    const Packet& packet = cc->Inputs().Index(0).Value();
+    Timestamp timestamp =
+        Timestamp(packet.Timestamp().Value() + kOutputFutureMicros);
+    cc->Outputs().Index(0).AddPacket(packet.At(timestamp));
     return ::mediapipe::OkStatus();
   }
 };
-REGISTER_CALCULATOR(BoundToPacketCalculator);
+REGISTER_CALCULATOR(FuturePacketCalculator);
 
 // Verifies that SetOffset still propagates when Process is called and
 // produces no output packets.
@@ -964,5 +998,111 @@ TEST(CalculatorGraphBoundsTest, LastPacketCheck) {
   MP_ASSERT_OK(graph.WaitUntilDone());
 }
 
+// Shows that bounds are indicated for input streams without input packets.
+void TestBoundsForEmptyInputs(std::string input_stream_handler) {
+  // FuturePacketCalculator and OffsetBoundCalculator produce future ts bounds.
+  // BoundToPacketCalculator reports all of its bounds, including empty inputs.
+  std::string config_str = R"(
+            input_stream: 'input'
+            node {
+              calculator: 'FuturePacketCalculator'
+              input_stream: 'input'
+              output_stream: 'futures'
+            }
+            node {
+              calculator: 'OffsetBoundCalculator'
+              input_stream: 'futures'
+              output_stream: 'bounds'
+            }
+            node {
+              calculator: 'BoundToPacketCalculator'
+              input_stream: 'input'
+              input_stream: 'bounds'
+              output_stream: 'input_ts'
+              output_stream: 'bounds_ts'
+              input_stream_handler { $input_stream_handler }
+            }
+          )";
+  absl::StrReplaceAll({{"$input_stream_handler", input_stream_handler}},
+                      &config_str);
+  CalculatorGraphConfig config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(config_str);
+  CalculatorGraph graph;
+  std::vector<Packet> input_ts_packets;
+  std::vector<Packet> bounds_ts_packets;
+  MP_ASSERT_OK(graph.Initialize(config));
+  MP_ASSERT_OK(graph.ObserveOutputStream("input_ts", [&](const Packet& p) {
+    input_ts_packets.push_back(p);
+    return ::mediapipe::OkStatus();
+  }));
+  MP_ASSERT_OK(graph.ObserveOutputStream("bounds_ts", [&](const Packet& p) {
+    bounds_ts_packets.push_back(p);
+    return ::mediapipe::OkStatus();
+  }));
+  MP_ASSERT_OK(graph.StartRun({}));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // Add four packets into the graph, with timedtamps 0, 10, 20, 30.
+  constexpr int kNumInputs = 4;
+  for (int i = 0; i < kNumInputs; ++i) {
+    Packet p = MakePacket<int>(33).At(Timestamp(i * 10));
+    MP_ASSERT_OK(graph.AddPacketToInputStream("input", p));
+    MP_ASSERT_OK(graph.WaitUntilIdle());
+  }
+
+  // Packets arrive.  The input packet timestamps are: 0, 10, 20, 30.
+  // The corresponding empty packet timestamps are: 3, 13, 23, 33.
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+  EXPECT_EQ(input_ts_packets.size(), 4);
+  EXPECT_EQ(bounds_ts_packets.size(), 4);
+
+  // The timestamp bounds from OffsetBoundCalculator are: 3, 13, 23, 33.
+  // Because the process call waits for the input packets and not for
+  // the empty packets, the first empty packet timestamp can be
+  // either Timestamp::Unstarted() or Timestamp(3).
+  std::vector<Timestamp> expected = {Timestamp::Unstarted(), Timestamp(3),
+                                     Timestamp(13), Timestamp(23),
+                                     Timestamp(33)};
+  for (int i = 0; i < bounds_ts_packets.size(); ++i) {
+    Timestamp ts = bounds_ts_packets[i].Get<Timestamp>();
+    EXPECT_GE(ts, expected[i]);
+    EXPECT_LE(ts, expected[i + 1]);
+  }
+
+  // Shutdown the graph.
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+// Shows that bounds are indicated for input streams without input packets.
+TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_Immediate) {
+  TestBoundsForEmptyInputs(R"(
+      input_stream_handler: "ImmediateInputStreamHandler")");
+}
+
+// Shows that bounds are indicated for input streams without input packets.
+TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_Default) {
+  TestBoundsForEmptyInputs(R"(
+      input_stream_handler: "DefaultInputStreamHandler")");
+}
+
+// Shows that bounds are indicated for input streams without input packets.
+TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_SyncSet) {
+  TestBoundsForEmptyInputs(R"(
+     input_stream_handler: "SyncSetInputStreamHandler")");
+}
+
+// Shows that bounds are indicated for input streams without input packets.
+TEST(CalculatorGraphBoundsTest, BoundsForEmptyInputs_SyncSets) {
+  TestBoundsForEmptyInputs(R"(
+      input_stream_handler: "SyncSetInputStreamHandler"
+      options {
+        [mediapipe.SyncSetInputStreamHandlerOptions.ext] {
+          sync_set { tag_index: ":0" }
+        }
+      }
+    )");
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/framework/formats/BUILD

@@ -27,72 +27,67 @@ package(
 proto_library(
     name = "detection_proto",
     srcs = ["detection.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = ["//mediapipe/framework/formats:location_data_proto"],
 )
 
 proto_library(
     name = "classification_proto",
     srcs = ["classification.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
 )
 
 proto_library(
     name = "image_format_proto",
     srcs = ["image_format.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
 )
 
 proto_library(
     name = "matrix_data_proto",
     srcs = ["matrix_data.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
 )
 
 proto_library(
     name = "location_data_proto",
     srcs = ["location_data.proto"],
+    visibility = ["//visibility:public"],
     deps = ["//mediapipe/framework/formats/annotation:rasterization_proto"],
 )
 
 proto_library(
     name = "time_series_header_proto",
     srcs = ["time_series_header.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
 )
 
 mediapipe_cc_proto_library(
     name = "detection_cc_proto",
     srcs = ["detection.proto"],
     cc_deps = [":location_data_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":detection_proto"],
 )
 
 mediapipe_cc_proto_library(
     name = "classification_cc_proto",
     srcs = ["classification.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":classification_proto"],
 )
 
 mediapipe_cc_proto_library(
     name = "image_format_cc_proto",
     srcs = ["image_format.proto"],
-    visibility = [
+    visibility = ["//visibility:public"],
-        "//mediapipe:__subpackages__",
-        "//mediapipe/java/com/google/mediapipe/framework:__subpackages__",
-    ],
     deps = [":image_format_proto"],
 )
 
 mediapipe_cc_proto_library(
     name = "matrix_data_cc_proto",
     srcs = ["matrix_data.proto"],
-    visibility = [
+    visibility = ["//visibility:public"],
-        "//mediapipe:__subpackages__",
-        "//mediapipe/java/com/google/mediapipe/framework:__subpackages__",
-    ],
     deps = [":matrix_data_proto"],
 )
 
@@ -100,17 +95,14 @@ mediapipe_cc_proto_library(
     name = "location_data_cc_proto",
     srcs = ["location_data.proto"],
     cc_deps = ["//mediapipe/framework/formats/annotation:rasterization_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":location_data_proto"],
 )
 
 mediapipe_cc_proto_library(
     name = "time_series_header_cc_proto",
     srcs = ["time_series_header.proto"],
-    visibility = [
+    visibility = ["//visibility:public"],
-        "//mediapipe:__subpackages__",
-        "//mediapipe/java/com/google/mediapipe/framework:__subpackages__",
-    ],
     deps = [":time_series_header_proto"],
 )
 
@@ -249,14 +241,14 @@ cc_test(
 proto_library(
     name = "rect_proto",
     srcs = ["rect.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
 )
 
 mediapipe_cc_proto_library(
     name = "rect_cc_proto",
     srcs = ["rect.proto"],
     visibility = [
-        "//mediapipe:__subpackages__",
+        "//visibility:public",
     ],
     deps = [":rect_proto"],
 )
@@ -264,12 +256,12 @@ mediapipe_cc_proto_library(
 proto_library(
     name = "landmark_proto",
     srcs = ["landmark.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
 )
 
 mediapipe_cc_proto_library(
     name = "landmark_cc_proto",
     srcs = ["landmark.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":landmark_proto"],
 )

mediapipe/framework/formats/annotation/BUILD

@@ -25,26 +25,27 @@ package(default_visibility = ["//visibility:private"])
 proto_library(
     name = "locus_proto",
     srcs = ["locus.proto"],
+    visibility = ["//visibility:public"],
     deps = ["//mediapipe/framework/formats/annotation:rasterization_proto"],
 )
 
 proto_library(
     name = "rasterization_proto",
     srcs = ["rasterization.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
 )
 
 mediapipe_cc_proto_library(
     name = "locus_cc_proto",
     srcs = ["locus.proto"],
     cc_deps = [":rasterization_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":locus_proto"],
 )
 
 mediapipe_cc_proto_library(
     name = "rasterization_cc_proto",
     srcs = ["rasterization.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":rasterization_proto"],
 )

mediapipe/framework/formats/landmark.proto

@@ -25,6 +25,11 @@ message Landmark {
   optional float z = 3;
 }
 
+// Group of Landmark protos.
+message LandmarkList {
+  repeated Landmark landmark = 1;
+}
+
 // A normalized version of above Landmark proto. All coordiates should be within
 // [0, 1].
 message NormalizedLandmark {

mediapipe/framework/formats/motion/BUILD

@@ -27,12 +27,13 @@ package(default_visibility = ["//visibility:private"])
 proto_library(
     name = "optical_flow_field_data_proto",
     srcs = ["optical_flow_field_data.proto"],
+    visibility = ["//visibility:public"],
 )
 
 mediapipe_cc_proto_library(
     name = "optical_flow_field_data_cc_proto",
     srcs = ["optical_flow_field_data.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":optical_flow_field_data_proto"],
 )
 
@@ -41,7 +42,7 @@ cc_library(
     srcs = ["optical_flow_field.cc"],
     hdrs = ["optical_flow_field.h"],
     visibility = [
-        "//mediapipe:__subpackages__",
+        "//visibility:public",
     ],
     deps = [
         "//mediapipe/framework:type_map",

mediapipe/framework/formats/object_detection/BUILD

@@ -24,12 +24,12 @@ package(default_visibility = ["//visibility:private"])
 proto_library(
     name = "anchor_proto",
     srcs = ["anchor.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
 )
 
 mediapipe_cc_proto_library(
     name = "anchor_cc_proto",
     srcs = ["anchor.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":anchor_proto"],
 )

mediapipe/framework/input_stream_manager.cc

@@ -229,6 +229,11 @@ Timestamp InputStreamManager::MinTimestampOrBound(bool* is_empty) const {
   if (is_empty) {
     *is_empty = queue_.empty();
   }
+  return MinTimestampOrBoundHelper();
+}
+
+Timestamp InputStreamManager::MinTimestampOrBoundHelper() const
+    EXCLUSIVE_LOCKS_REQUIRED(stream_mutex_) {
   return queue_.empty() ? next_timestamp_bound_ : queue_.front().Timestamp();
 }
 
@@ -271,7 +276,9 @@ Packet InputStreamManager::PopPacketAtTimestamp(Timestamp timestamp,
     }
     // Clear value_ if it doesn't have exactly the right timestamp.
     if (current_timestamp != timestamp) {
-      packet = Packet();
+      // The timestamp bound reported when no packet is sent.
+      Timestamp bound = MinTimestampOrBoundHelper();
+      packet = Packet().At(bound.PreviousAllowedInStream());
       ++(*num_packets_dropped);
     }
 

mediapipe/framework/input_stream_manager.h

@@ -189,6 +189,9 @@ class InputStreamManager {
   // Returns true if the next timestamp bound reaches Timestamp::Done().
   bool IsDone() const EXCLUSIVE_LOCKS_REQUIRED(stream_mutex_);
 
+  // Returns the smallest timestamp at which this stream might see an input.
+  Timestamp MinTimestampOrBoundHelper() const;
+
   mutable absl::Mutex stream_mutex_;
   std::deque<Packet> queue_ GUARDED_BY(stream_mutex_);
   // The number of packets added to queue_.  Used to verify a packet at

mediapipe/framework/stream_handler/immediate_input_stream_handler.cc

@@ -115,9 +115,10 @@ void ImmediateInputStreamHandler::FillInputSet(Timestamp input_timestamp,
       AddPacketToShard(&input_set->Get(id), std::move(current_packet),
                        stream_is_done);
     } else {
-      bool empty = false;
+      Timestamp bound = stream->MinTimestampOrBound(nullptr);
-      bool is_done = stream->MinTimestampOrBound(&empty) == Timestamp::Done();
+      AddPacketToShard(&input_set->Get(id),
-      AddPacketToShard(&input_set->Get(id), Packet(), is_done);
+                       Packet().At(bound.PreviousAllowedInStream()),
+                       bound == Timestamp::Done());
     }
   }
 }

mediapipe/framework/stream_handler/sync_set_input_stream_handler.cc

@@ -56,9 +56,16 @@ class SyncSetInputStreamHandler : public InputStreamHandler {
   NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override;
 
   // Only invoked when associated GetNodeReadiness() returned kReadyForProcess.
+  // Populates packets for the ready sync-set, and populates timestamp bounds
+  // for all sync-sets.
   void FillInputSet(Timestamp input_timestamp,
                     InputStreamShardSet* input_set) override;
 
+  // Populates timestamp bounds for streams outside the ready sync-set.
+  void FillInputBounds(Timestamp input_timestamp,
+                       InputStreamShardSet* input_set)
+      EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+
  private:
   absl::Mutex mutex_;
   // The ids of each set of inputs.
@@ -181,6 +188,22 @@ NodeReadiness SyncSetInputStreamHandler::GetNodeReadiness(
   return NodeReadiness::kNotReady;
 }
 
+void SyncSetInputStreamHandler::FillInputBounds(
+    Timestamp input_timestamp, InputStreamShardSet* input_set) {
+  for (int i = 0; i < sync_sets_.size(); ++i) {
+    if (i != ready_sync_set_index_) {
+      // Set the input streams for the not-ready sync sets.
+      for (CollectionItemId id : sync_sets_[i]) {
+        const auto stream = input_stream_managers_.Get(id);
+        Timestamp bound = stream->MinTimestampOrBound(nullptr);
+        AddPacketToShard(&input_set->Get(id),
+                         Packet().At(bound.PreviousAllowedInStream()),
+                         bound == Timestamp::Done());
+      }
+    }
+  }
+}
+
 void SyncSetInputStreamHandler::FillInputSet(Timestamp input_timestamp,
                                              InputStreamShardSet* input_set) {
   // Assume that all current packets are already cleared.
@@ -202,6 +225,7 @@ void SyncSetInputStreamHandler::FillInputSet(Timestamp input_timestamp,
     AddPacketToShard(&input_set->Get(id), std::move(current_packet),
                      stream_is_done);
   }
+  FillInputBounds(input_timestamp, input_set);
   ready_sync_set_index_ = -1;
   ready_timestamp_ = Timestamp::Done();
 }

mediapipe/framework/timestamp.cc

@@ -123,13 +123,23 @@ std::string TimestampDiff::DebugString() const {
 
 Timestamp Timestamp::NextAllowedInStream() const {
   CHECK(IsAllowedInStream()) << "Timestamp is: " << DebugString();
-  if (IsRangeValue() && *this != Max()) {
+  if (*this >= Max() || *this == PreStream()) {
-    return *this + 1;
+    // Indicates that no further timestamps may occur.
-  } else {
-    // Returning this value effectively means no futher timestamps may
-    // occur (however, the stream is not yet closed).
     return OneOverPostStream();
+  } else if (*this < Min()) {
+    return Min();
   }
+  return *this + 1;
+}
+
+Timestamp Timestamp::PreviousAllowedInStream() const {
+  if (*this <= Min() || *this == PostStream()) {
+    // Indicates that no previous timestamps may occur.
+    return Unstarted();
+  } else if (*this > Max()) {
+    return Max();
+  }
+  return *this - 1;
 }
 
 std::ostream& operator<<(std::ostream& os, Timestamp arg) {

mediapipe/framework/timestamp.h

@@ -182,12 +182,15 @@ class Timestamp {
   Timestamp operator++(int);
   Timestamp operator--(int);
 
-  // Returns the next timestamp at which a Packet may arrive in a stream, given
+  // Returns the next timestamp in the range [Min .. Max], or
-  // that the current Packet is at *this timestamp. CHECKs that
+  // OneOverPostStream() if no Packets may follow one with this timestamp.
-  // this->IsAllowedInStream()==true. Returns Timestamp::OneOverPostStream() if
+  // CHECKs that this->IsAllowedInStream().
-  // no Packets may follow one with the given timestamp.
   Timestamp NextAllowedInStream() const;
 
+  // Returns the previous timestamp in the range [Min .. Max], or
+  // Unstarted() if no Packets may preceed one with this timestamp.
+  Timestamp PreviousAllowedInStream() const;
+
  private:
   TimestampBaseType timestamp_;
 };

mediapipe/gpu/metal.bzl

@@ -132,7 +132,7 @@ def _metal_library_impl(ctx):
         ),
     )
 
-    # This ridiculous circumlocution is needed because new_objc_provider rejects
+    # This circumlocution is needed because new_objc_provider rejects
     # an empty depset, with the error:
     # "Value for key header must be a set of File, instead found set of unknown."
     # It also rejects an explicit "None".

mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt

@@ -94,7 +94,7 @@ node {
   output_stream: "multi_hand_rects"
   node_options: {
     [type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
-      min_similarity_threshold: 0.1
+      min_similarity_threshold: 0.5
     }
   }
 }

mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt

@@ -85,7 +85,7 @@ node {
   output_stream: "multi_hand_rects"
   node_options: {
     [type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
-      min_similarity_threshold: 0.1
+      min_similarity_threshold: 0.5
     }
   }
 }

mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt

@@ -105,7 +105,7 @@ node {
   output_stream: "multi_hand_rects"
   node_options: {
     [type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
-      min_similarity_threshold: 0.1
+      min_similarity_threshold: 0.5
     }
   }
 }

mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt

@@ -104,7 +104,7 @@ node {
       y_scale: 256.0
       h_scale: 256.0
       w_scale: 256.0
-      min_score_thresh: 0.5
+      min_score_thresh: 0.7
     }
   }
 }
@@ -117,7 +117,6 @@ node {
   node_options: {
     [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
       min_suppression_threshold: 0.3
-      min_score_threshold: 0.5
       overlap_type: INTERSECTION_OVER_UNION
       algorithm: WEIGHTED
       return_empty_detections: true

mediapipe/util/audio_decoder.cc

@@ -340,9 +340,7 @@ AudioPacketProcessor::AudioPacketProcessor(const AudioStreamOptions& options)
   DCHECK(absl::little_endian::IsLittleEndian());
 }
 
-mediapipe::Status AudioPacketProcessor::Open(int id,
+mediapipe::Status AudioPacketProcessor::Open(int id, AVStream* stream) {
-
-                                             AVStream* stream) {
   id_ = id;
   avcodec_ = avcodec_find_decoder(stream->codecpar->codec_id);
   if (!avcodec_) {

mediapipe/util/sequence/media_sequence.cc

@@ -274,10 +274,11 @@ float TimestampsToRate(int64 first_timestamp, int64 second_timestamp) {
     // overwriting with modified values.
     if (!GetUnmodifiedBBoxTimestampSize(prefix, *sequence)) {
       for (int i = 0; i < num_frames; ++i) {
-        if (bbox_index_if_annotated[i] >= 0 &&
+        const int bbox_index = bbox_index_if_annotated[i];
-            GetBBoxIsAnnotatedAt(prefix, *sequence, i)) {
+        if (bbox_index >= 0 &&
-          AddUnmodifiedBBoxTimestamp(
+            GetBBoxIsAnnotatedAt(prefix, *sequence, bbox_index)) {
-              prefix, box_timestamps[bbox_index_if_annotated[i]], sequence);
+          AddUnmodifiedBBoxTimestamp(prefix, box_timestamps[bbox_index],
+                                     sequence);
         }
       }
     }

mediapipe/util/sequence/media_sequence_test.cc

@@ -738,10 +738,11 @@ TEST(MediaSequenceTest,
   AddImageTimestamp(10, &sequence);
   AddImageTimestamp(20, &sequence);
   AddImageTimestamp(30, &sequence);
+  AddImageTimestamp(40, &sequence);
 
-  AddBBoxTimestamp(9, &sequence);
+  AddBBoxTimestamp(11, &sequence);
-  AddBBoxTimestamp(21, &sequence);
+  AddBBoxTimestamp(12, &sequence);  // Will be dropped in the output.
-  AddBBoxTimestamp(22, &sequence);  // Will be dropped in the output.
+  AddBBoxTimestamp(39, &sequence);
 
   std::vector<std::vector<Location>> bboxes = {
       {Location::CreateRelativeBBoxLocation(0.1, 0.2, 0.7, 0.7)},
@@ -753,32 +754,35 @@ TEST(MediaSequenceTest,
 
   MP_ASSERT_OK(ReconcileMetadata(true, false, &sequence));
 
-  ASSERT_EQ(GetBBoxTimestampSize(sequence), 3);
+  ASSERT_EQ(GetBBoxTimestampSize(sequence), 4);
   ASSERT_EQ(GetBBoxTimestampAt(sequence, 0), 10);
   ASSERT_EQ(GetBBoxTimestampAt(sequence, 1), 20);
   ASSERT_EQ(GetBBoxTimestampAt(sequence, 2), 30);
+  ASSERT_EQ(GetBBoxTimestampAt(sequence, 3), 40);
 
-  ASSERT_EQ(GetBBoxIsAnnotatedSize(sequence), 3);
+  ASSERT_EQ(GetBBoxIsAnnotatedSize(sequence), 4);
   ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 0), true);
-  ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 1), true);
+  ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 1), false);
   ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 2), false);
+  ASSERT_EQ(GetBBoxIsAnnotatedAt(sequence, 3), true);
 
   // Unmodified timestamp is only stored for is_annotated == true.
   ASSERT_EQ(GetUnmodifiedBBoxTimestampSize(sequence), 2);
-  ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 9);
+  ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 11);
-  ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 21);
+  ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 39);
 
   // A second reconciliation should not corrupt unmodified bbox timestamps.
   MP_ASSERT_OK(ReconcileMetadata(true, false, &sequence));
 
-  ASSERT_EQ(GetBBoxTimestampSize(sequence), 3);
+  ASSERT_EQ(GetBBoxTimestampSize(sequence), 4);
   ASSERT_EQ(GetBBoxTimestampAt(sequence, 0), 10);
   ASSERT_EQ(GetBBoxTimestampAt(sequence, 1), 20);
   ASSERT_EQ(GetBBoxTimestampAt(sequence, 2), 30);
+  ASSERT_EQ(GetBBoxTimestampAt(sequence, 3), 40);
 
   ASSERT_EQ(GetUnmodifiedBBoxTimestampSize(sequence), 2);
-  ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 9);
+  ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 0), 11);
-  ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 21);
+  ASSERT_EQ(GetUnmodifiedBBoxTimestampAt(sequence, 1), 39);
 }
 
 TEST(MediaSequenceTest, ReconcileMetadataBoxAnnotationsFillsMissing) {

third_party/BUILD

@@ -118,13 +118,16 @@ android_library(
     ],
 )
 
-# TODO: Get the AARs from Google's Maven Repository.
+android_library(
-aar_import(
     name = "camerax_core",
-    aar = "camera-core-1.0.0-alpha01.aar",
+    exports = [
+        "@maven//:androidx_camera_camera_core",
+    ],
 )
 
-aar_import(
+android_library(
     name = "camera2",
-    aar = "camera-camera2-1.0.0-alpha01.aar",
+    exports = [
+        "@maven//:androidx_camera_camera_camera2",
+    ],
 )

third_party/opencv_android.BUILD

@@ -5,7 +5,7 @@ licenses(["notice"])  # BSD license
 
 exports_files(["LICENSE"])
 
-OPENCV_LIBRARY_NAME = "libopencv_java4.so"
+OPENCV_LIBRARY_NAME = "libopencv_java3.so"
 
 OPENCVANDROIDSDK_NATIVELIBS_PATH = "sdk/native/libs/"