Project import generated by Copybara.

GitOrigin-RevId: 53a42bf7ad836321123cb7b6c80b0f2e13fbf83e

.bazelrc

@@ -12,9 +12,9 @@ build --copt='-Wno-comment'
 build --copt='-Wno-return-type'
 build --copt='-Wno-unused-local-typedefs'
 build --copt='-Wno-ignored-attributes'
-# Temporarily set the incompatibility flag for Bazel 0.27.0 and above
+
-build --incompatible_disable_deprecated_attr_params=false
+# Tensorflow needs remote repo
-build --incompatible_depset_is_not_iterable=false
+build --experimental_repo_remote_exec
 
 # Sets the default Apple platform to macOS.
 build --apple_platform_type=macos
@@ -23,6 +23,10 @@ build --apple_platform_type=macos
 build --apple_generate_dsym
 
 # Android configs.
+# Note: the documentation tells us to use @androidndk//:default_crosstool, but
+# the automatic configuration transition uses //external:android/crosstool.
+# Using it here works and spares us from having two different config_settings
+# for Android.
 build:android --crosstool_top=//external:android/crosstool
 build:android --host_crosstool_top=@bazel_tools//tools/cpp:toolchain
 build:android --linkopt=-landroid

Dockerfile

@@ -48,7 +48,7 @@ RUN pip install future
 RUN pip3 install six
 
 # Install bazel
-ARG BAZEL_VERSION=1.1.0
+ARG BAZEL_VERSION=2.0.0
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/b\
 azel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \

WORKSPACE

@@ -2,16 +2,15 @@ workspace(name = "mediapipe")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-skylib_version = "0.8.0"
+skylib_version = "0.9.0"
 http_archive(
     name = "bazel_skylib",
     type = "tar.gz",
-    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format (skylib_version, skylib_version),
+    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel_skylib-{}.tar.gz".format (skylib_version, skylib_version),
-    sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
+    sha256 = "1dde365491125a3db70731e25658dfdd3bc5dbdfd11b840b3e987ecf043c7ca0",
 )
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "1.0.0",
+versions.check(minimum_bazel_version = "2.0.0")
-               maximum_bazel_version = "1.2.1")
 
 
 # ABSL cpp library lts_2020_02_25
@@ -93,11 +92,13 @@ http_archive(
     build_file = "@//third_party:libyuv.BUILD",
 )
 
+# Note: protobuf-javalite is no longer released as a separate download, it's included in the main Java download.
+# ...but the Java download is currently broken, so we use the "source" download.
 http_archive(
     name = "com_google_protobuf_javalite",
-    sha256 = "79d102c61e2a479a0b7e5fc167bcfaa4832a0c6aad4a75fa7da0480564931bcc",
+    sha256 = "a79d19dcdf9139fa4b81206e318e33d245c4c9da1ffed21c87288ed4380426f9",
-    strip_prefix = "protobuf-384989534b2246d413dbcd750744faab2607b516",
+    strip_prefix = "protobuf-3.11.4",
-    urls = ["https://github.com/google/protobuf/archive/384989534b2246d413dbcd750744faab2607b516.zip"],
+    urls = ["https://github.com/protocolbuffers/protobuf/archive/v3.11.4.tar.gz"],
 )
 
 http_archive(
@@ -117,23 +118,18 @@ http_archive(
     ],
 )
 
-# 2020-02-12
+# 2020-04-01
-# The last commit before TensorFlow switched to Bazel 2.0
+_TENSORFLOW_GIT_COMMIT = "805e47cea96c7e8c6fccf494d40a2392dc99fdd8"
-_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328"
+_TENSORFLOW_SHA256= "9ee3ae604c2e1345ac60345becee6d659364721513f9cb8652eb2e7138320ca5"
-_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13"
 http_archive(
     name = "org_tensorflow",
     urls = [
       "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
       "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
     ],
-    # A compatibility patch
     patches = [
-        "@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff",
+        "@//third_party:org_tensorflow_compatibility_fixes.diff",
-        # Updates for XNNPACK: https://github.com/tensorflow/tensorflow/commit/cfc31e324c8de6b52f752a39cb161d99d853ca99
+        "@//third_party:org_tensorflow_protobuf_updates.diff",
-        "@//third_party:org_tensorflow_cfc31e324c8de6b52f752a39cb161d99d853ca99.diff",
-        # CpuInfo's build rule fixes.
-        "@//third_party:org_tensorflow_9696366bcadab23a25c773b3ed405bac8ded4d0d.diff",
     ],
     patch_args = [
         "-p1",
@@ -158,10 +154,6 @@ http_archive(
     sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
 )
 
-# Please run
-# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
-#                        libopencv-calib3d-dev libopencv-features2d-dev \
-#                        libopencv-imgproc-dev libopencv-video-dev
 new_local_repository(
     name = "linux_opencv",
     build_file = "@//third_party:opencv_linux.BUILD",
@@ -174,7 +166,6 @@ new_local_repository(
     path = "/usr"
 )
 
-# Please run $ brew install opencv@3
 new_local_repository(
     name = "macos_opencv",
     build_file = "@//third_party:opencv_macos.BUILD",
@@ -207,79 +198,6 @@ http_archive(
     url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
 )
 
-RULES_JVM_EXTERNAL_TAG = "2.2"
-RULES_JVM_EXTERNAL_SHA = "f1203ce04e232ab6fdd81897cf0ff76f2c04c0741424d192f28e65ae752ce2d6"
-
-http_archive(
-    name = "rules_jvm_external",
-    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
-    sha256 = RULES_JVM_EXTERNAL_SHA,
-    url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
-)
-
-load("@rules_jvm_external//:defs.bzl", "maven_install")
-
-maven_install(
-    artifacts = [
-        "androidx.annotation:annotation:aar:1.1.0",
-        "androidx.appcompat:appcompat:aar:1.1.0-rc01",
-        "androidx.camera:camera-core:aar:1.0.0-alpha06",
-        "androidx.camera:camera-camera2:aar:1.0.0-alpha06",
-        "androidx.constraintlayout:constraintlayout:aar:1.1.3",
-        "androidx.core:core:aar:1.1.0-rc03",
-        "androidx.legacy:legacy-support-v4:aar:1.0.0",
-        "androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
-        "com.google.android.material:material:aar:1.0.0-rc01",
-    ],
-    repositories = [
-        "https://dl.google.com/dl/android/maven2",
-        "https://repo1.maven.org/maven2",
-    ],
-)
-
-maven_server(
-    name = "google_server",
-    url = "https://dl.google.com/dl/android/maven2",
-)
-
-maven_jar(
-    name = "androidx_lifecycle",
-    artifact = "androidx.lifecycle:lifecycle-common:2.0.0",
-    sha1 = "e070ffae07452331bc5684734fce6831d531785c",
-    server = "google_server",
-)
-
-maven_jar(
-    name = "androidx_concurrent_futures",
-    artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
-    sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
-    server = "google_server",
-)
-
-maven_jar(
-    name = "com_google_guava_android",
-    artifact = "com.google.guava:guava:27.0.1-android",
-    sha1 = "b7e1c37f66ef193796ccd7ea6e80c2b05426182d",
-)
-
-maven_jar(
-    name = "com_google_common_flogger",
-    artifact = "com.google.flogger:flogger:0.3.1",
-    sha1 = "585030fe1ec709760cbef997a459729fb965df0e",
-)
-
-maven_jar(
-    name = "com_google_common_flogger_system_backend",
-    artifact = "com.google.flogger:flogger-system-backend:0.3.1",
-    sha1 = "287b569d76abcd82f9de87fe41829fbc7ebd8ac9",
-)
-
-maven_jar(
-    name = "com_google_code_findbugs",
-    artifact = "com.google.code.findbugs:jsr305:3.0.2",
-    sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d",
-)
-
 # You may run setup_android.sh to install Android SDK and NDK.
 android_ndk_repository(
     name = "androidndk",
@@ -293,9 +211,15 @@ android_sdk_repository(
 
 http_archive(
     name = "build_bazel_rules_apple",
-    sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e",
+    sha256 = "7a7afdd4869bb201c9352eed2daf37294d42b093579b70423490c1b4d4f6ce42",
-    strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3",
+    url = "https://github.com/bazelbuild/rules_apple/releases/download/0.19.0/rules_apple.0.19.0.tar.gz",
-    url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz",
+    patches = [
+        # Bypass checking ios unit test runner when building MP ios applications.
+        "@//third_party:build_bazel_rules_apple_bypass_test_runner_check.diff"
+    ],
+    patch_args = [
+        "-p1",
+    ],
 )
 
 load(
@@ -329,3 +253,49 @@ http_archive(
     build_file = "@//third_party:google_toolbox_for_mac.BUILD",
 )
 
+# Maven dependencies.
+
+RULES_JVM_EXTERNAL_TAG = "3.2"
+RULES_JVM_EXTERNAL_SHA = "82262ff4223c5fda6fb7ff8bd63db8131b51b413d26eb49e3131037e79e324af"
+
+http_archive(
+    name = "rules_jvm_external",
+    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
+    sha256 = RULES_JVM_EXTERNAL_SHA,
+    url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
+)
+
+load("@rules_jvm_external//:defs.bzl", "maven_install")
+
+# Important: there can only be one maven_install rule. Add new maven deps here.
+maven_install(
+    name = "maven",
+    artifacts = [
+        "junit:junit:4.12",
+        "androidx.test.espresso:espresso-core:3.1.1",
+        "org.hamcrest:hamcrest-library:1.3",
+        "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
+        "androidx.lifecycle:lifecycle-common:2.0.0",
+        "androidx.annotation:annotation:aar:1.1.0",
+        "androidx.appcompat:appcompat:aar:1.1.0-rc01",
+        "androidx.camera:camera-core:aar:1.0.0-alpha06",
+        "androidx.camera:camera-camera2:aar:1.0.0-alpha06",
+        "androidx.constraintlayout:constraintlayout:aar:1.1.3",
+        "androidx.core:core:aar:1.1.0-rc03",
+        "androidx.legacy:legacy-support-v4:aar:1.0.0",
+        "androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
+        "com.google.android.material:material:aar:1.0.0-rc01",
+        "com.google.code.findbugs:jsr305:3.0.2",
+        "com.google.flogger:flogger-system-backend:0.3.1",
+        "com.google.flogger:flogger:0.3.1",
+        "com.google.guava:guava:27.0.1-android",
+    ],
+    repositories = [
+        "https://jcenter.bintray.com",
+        "https://maven.google.com",
+        "https://dl.google.com/dl/android/maven2",
+        "https://repo1.maven.org/maven2",
+    ],
+    fetch_sources = True,
+    version_conflict_policy = "pinned",
+)

mediapipe/BUILD

@@ -14,6 +14,9 @@
 
 licenses(["notice"])  # Apache 2.0
 
+# Note: yes, these need to use "//external:android/crosstool", not
+# @androidndk//:default_crosstool.
+
 config_setting(
     name = "android",
     values = {"crosstool_top": "//external:android/crosstool"},

mediapipe/calculators/core/BUILD

@@ -230,6 +230,7 @@ cc_library(
         "//mediapipe/framework:packet",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:ret_check",
@@ -257,6 +258,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:render_data_cc_proto",
+        "@org_tensorflow//tensorflow/lite:framework",
     ],
     alwayslink = 1,
 )
@@ -779,6 +781,7 @@ cc_library(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:landmark_cc_proto",
         "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/util:resource_util",

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -18,6 +18,7 @@
 
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 
 namespace mediapipe {
@@ -37,4 +38,8 @@ typedef BeginLoopCalculator<std::vector<::mediapipe::Detection>>
     BeginLoopDetectionCalculator;
 REGISTER_CALCULATOR(BeginLoopDetectionCalculator);
 
+// A calculator to process std::vector<Matrix>.
+typedef BeginLoopCalculator<std::vector<Matrix>> BeginLoopMatrixCalculator;
+REGISTER_CALCULATOR(BeginLoopMatrixCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/end_loop_calculator.cc

@@ -20,6 +20,7 @@
 #include "mediapipe/framework/formats/landmark.pb.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "mediapipe/util/render_data.pb.h"
+#include "tensorflow/lite/interpreter.h"
 
 namespace mediapipe {
 
@@ -42,4 +43,7 @@ typedef EndLoopCalculator<std::vector<::mediapipe::ClassificationList>>
     EndLoopClassificationListCalculator;
 REGISTER_CALCULATOR(EndLoopClassificationListCalculator);
 
+typedef EndLoopCalculator<std::vector<TfLiteTensor>> EndLoopTensorCalculator;
+REGISTER_CALCULATOR(EndLoopTensorCalculator);
+
 }  // namespace mediapipe

mediapipe/calculators/core/split_vector_calculator.cc

@@ -18,6 +18,7 @@
 
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/matrix.h"
 #include "mediapipe/framework/formats/rect.pb.h"
 #include "tensorflow/lite/interpreter.h"
 
@@ -57,6 +58,9 @@ typedef SplitVectorCalculator<::mediapipe::NormalizedRect, false>
     SplitNormalizedRectVectorCalculator;
 REGISTER_CALCULATOR(SplitNormalizedRectVectorCalculator);
 
+typedef SplitVectorCalculator<Matrix, false> SplitMatrixVectorCalculator;
+REGISTER_CALCULATOR(SplitMatrixVectorCalculator);
+
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
 typedef SplitVectorCalculator<::tflite::gpu::gl::GlBuffer, true>
     MovableSplitGlBufferVectorCalculator;

mediapipe/calculators/image/BUILD

@@ -86,6 +86,14 @@ mediapipe_cc_proto_library(
     deps = [":opencv_image_encoder_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "opencv_encoded_image_to_image_frame_calculator_cc_proto",
+    srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":opencv_encoded_image_to_image_frame_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "mask_overlay_calculator_cc_proto",
     srcs = ["mask_overlay_calculator.proto"],
@@ -172,6 +180,7 @@ cc_library(
     srcs = ["opencv_encoded_image_to_image_frame_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        ":opencv_encoded_image_to_image_frame_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame_opencv",
         "//mediapipe/framework/port:opencv_imgcodecs",
@@ -557,6 +566,27 @@ proto_library(
     deps = ["//mediapipe/framework:calculator_proto"],
 )
 
+proto_library(
+    name = "opencv_encoded_image_to_image_frame_calculator_proto",
+    srcs = ["opencv_encoded_image_to_image_frame_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+proto_library(
+    name = "feature_detector_calculator_proto",
+    srcs = ["feature_detector_calculator.proto"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "feature_detector_calculator_cc_proto",
+    srcs = ["feature_detector_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":feature_detector_calculator_proto"],
+)
+
 cc_library(
     name = "mask_overlay_calculator",
     srcs = ["mask_overlay_calculator.cc"],
@@ -572,3 +602,30 @@ cc_library(
     ],
     alwayslink = 1,
 )
+
+cc_library(
+    name = "feature_detector_calculator",
+    srcs = ["feature_detector_calculator.cc"],
+    visibility = ["//mediapipe:__subpackages__"],
+    deps = [
+        ":feature_detector_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:video_stream_header",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_features2d",
+        "//mediapipe/framework/port:opencv_imgproc",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/port:threadpool",
+        "//mediapipe/framework/tool:options_util",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/synchronization",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ],
+    alwayslink = 1,
+)

mediapipe/calculators/image/feature_detector_calculator.cc

@@ -0,0 +1,210 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "absl/synchronization/blocking_counter.h"
+#include "mediapipe/calculators/image/feature_detector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_features2d_inc.h"
+#include "mediapipe/framework/port/opencv_imgproc_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/threadpool.h"
+#include "mediapipe/framework/tool/options_util.h"
+#include "tensorflow/lite/interpreter.h"
+
+namespace mediapipe {
+
+const char kOptionsTag[] = "OPTIONS";
+const int kPatchSize = 32;
+const int kNumThreads = 16;
+
+// A calculator to apply local feature detection.
+// Input stream:
+//   IMAGE: Input image frame of type ImageFrame from video stream.
+// Output streams:
+//   FEATURES: The detected keypoints from input image as vector<cv::KeyPoint>.
+//   PATCHES:  Optional output the extracted patches as vector<cv::Mat>
+class FeatureDetectorCalculator : public CalculatorBase {
+ public:
+  ~FeatureDetectorCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  FeatureDetectorCalculatorOptions options_;
+  cv::Ptr<cv::Feature2D> feature_detector_;
+  std::unique_ptr<::mediapipe::ThreadPool> pool_;
+
+  // Create image pyramid based on input image.
+  void ComputeImagePyramid(const cv::Mat& input_image,
+                           std::vector<cv::Mat>* image_pyramid);
+
+  // Extract the patch for single feature with image pyramid.
+  cv::Mat ExtractPatch(const cv::KeyPoint& feature,
+                       const std::vector<cv::Mat>& image_pyramid);
+};
+
+REGISTER_CALCULATOR(FeatureDetectorCalculator);
+
+::mediapipe::Status FeatureDetectorCalculator::GetContract(
+    CalculatorContract* cc) {
+  if (cc->Inputs().HasTag("IMAGE")) {
+    cc->Inputs().Tag("IMAGE").Set<ImageFrame>();
+  }
+  if (cc->Outputs().HasTag("FEATURES")) {
+    cc->Outputs().Tag("FEATURES").Set<std::vector<cv::KeyPoint>>();
+  }
+  if (cc->Outputs().HasTag("LANDMARKS")) {
+    cc->Outputs().Tag("LANDMARKS").Set<NormalizedLandmarkList>();
+  }
+  if (cc->Outputs().HasTag("PATCHES")) {
+    cc->Outputs().Tag("PATCHES").Set<std::vector<TfLiteTensor>>();
+  }
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FeatureDetectorCalculator::Open(CalculatorContext* cc) {
+  options_ =
+      tool::RetrieveOptions(cc->Options(), cc->InputSidePackets(), kOptionsTag)
+          .GetExtension(FeatureDetectorCalculatorOptions::ext);
+  feature_detector_ = cv::ORB::create(
+      options_.max_features(), options_.scale_factor(),
+      options_.pyramid_level(), kPatchSize - 1, 0, 2, cv::ORB::FAST_SCORE);
+  pool_ = absl::make_unique<::mediapipe::ThreadPool>("ThreadPool", kNumThreads);
+  pool_->StartWorkers();
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status FeatureDetectorCalculator::Process(CalculatorContext* cc) {
+  const Timestamp& timestamp = cc->InputTimestamp();
+  if (timestamp == Timestamp::PreStream()) {
+    // Indicator packet.
+    return ::mediapipe::OkStatus();
+  }
+  InputStream* input_frame = &(cc->Inputs().Tag("IMAGE"));
+  cv::Mat input_view = formats::MatView(&input_frame->Get<ImageFrame>());
+  cv::Mat grayscale_view;
+  cv::cvtColor(input_view, grayscale_view, cv::COLOR_RGB2GRAY);
+
+  std::vector<cv::KeyPoint> keypoints;
+  feature_detector_->detect(grayscale_view, keypoints);
+  if (keypoints.size() > options_.max_features()) {
+    keypoints.resize(options_.max_features());
+  }
+
+  if (cc->Outputs().HasTag("FEATURES")) {
+    auto features_ptr = absl::make_unique<std::vector<cv::KeyPoint>>(keypoints);
+    cc->Outputs().Tag("FEATURES").Add(features_ptr.release(), timestamp);
+  }
+
+  if (cc->Outputs().HasTag("LANDMARKS")) {
+    auto landmarks_ptr = absl::make_unique<NormalizedLandmarkList>();
+    for (int j = 0; j < keypoints.size(); ++j) {
+      auto feature_landmark = landmarks_ptr->add_landmark();
+      feature_landmark->set_x(keypoints[j].pt.x / grayscale_view.cols);
+      feature_landmark->set_y(keypoints[j].pt.y / grayscale_view.rows);
+    }
+    cc->Outputs().Tag("LANDMARKS").Add(landmarks_ptr.release(), timestamp);
+  }
+
+  if (cc->Outputs().HasTag("PATCHES")) {
+    std::vector<cv::Mat> image_pyramid;
+    ComputeImagePyramid(grayscale_view, &image_pyramid);
+    std::vector<cv::Mat> patch_mat;
+    patch_mat.resize(keypoints.size());
+    absl::BlockingCounter counter(keypoints.size());
+    for (int i = 0; i < keypoints.size(); i++) {
+      pool_->Schedule(
+          [this, &image_pyramid, &keypoints, &patch_mat, i, &counter] {
+            patch_mat[i] = ExtractPatch(keypoints[i], image_pyramid);
+            counter.DecrementCount();
+          });
+    }
+    counter.Wait();
+    const int batch_size = options_.max_features();
+    auto patches = absl::make_unique<std::vector<TfLiteTensor>>();
+    TfLiteTensor tensor;
+    tensor.type = kTfLiteFloat32;
+    tensor.dims = TfLiteIntArrayCreate(4);
+    tensor.dims->data[0] = batch_size;
+    tensor.dims->data[1] = kPatchSize;
+    tensor.dims->data[2] = kPatchSize;
+    tensor.dims->data[3] = 1;
+    int num_bytes = batch_size * kPatchSize * kPatchSize * sizeof(float);
+    tensor.data.data = malloc(num_bytes);
+    tensor.bytes = num_bytes;
+    tensor.allocation_type = kTfLiteArenaRw;
+    float* tensor_buffer = tensor.data.f;
+    for (int i = 0; i < keypoints.size(); i++) {
+      for (int j = 0; j < patch_mat[i].rows; ++j) {
+        for (int k = 0; k < patch_mat[i].cols; ++k) {
+          *tensor_buffer++ = patch_mat[i].at<uchar>(j, k) / 128.0f - 1.0f;
+        }
+      }
+    }
+    for (int i = keypoints.size() * kPatchSize * kPatchSize; i < num_bytes / 4;
+         i++) {
+      *tensor_buffer++ = 0;
+    }
+
+    patches->emplace_back(tensor);
+    cc->Outputs().Tag("PATCHES").Add(patches.release(), timestamp);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+void FeatureDetectorCalculator::ComputeImagePyramid(
+    const cv::Mat& input_image, std::vector<cv::Mat>* image_pyramid) {
+  cv::Mat tmp_image = input_image;
+  cv::Mat src_image = input_image;
+  for (int i = 0; i < options_.pyramid_level(); ++i) {
+    image_pyramid->push_back(src_image);
+    cv::resize(src_image, tmp_image, cv::Size(), 1.0f / options_.scale_factor(),
+               1.0f / options_.scale_factor());
+    src_image = tmp_image;
+  }
+}
+
+cv::Mat FeatureDetectorCalculator::ExtractPatch(
+    const cv::KeyPoint& feature, const std::vector<cv::Mat>& image_pyramid) {
+  cv::Mat img = image_pyramid[feature.octave];
+  float scale_factor = 1 / pow(options_.scale_factor(), feature.octave);
+  cv::Point2f center =
+      cv::Point2f(feature.pt.x * scale_factor, feature.pt.y * scale_factor);
+  cv::Mat rot = cv::getRotationMatrix2D(center, feature.angle, 1.0);
+  rot.at<double>(0, 2) += kPatchSize / 2 - center.x;
+  rot.at<double>(1, 2) += kPatchSize / 2 - center.y;
+  cv::Mat cropped_img;
+  // perform the affine transformation
+  cv::warpAffine(img, cropped_img, rot, cv::Size(kPatchSize, kPatchSize),
+                 cv::INTER_LINEAR);
+  return cropped_img;
+}
+
+}  // namespace mediapipe

mediapipe/calculators/image/feature_detector_calculator.proto

@@ -0,0 +1,24 @@
+// Options for FeatureDetectorCalculator
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message FeatureDetectorCalculatorOptions {
+  extend CalculatorOptions {
+    optional FeatureDetectorCalculatorOptions ext = 278741680;
+  }
+
+  // Set to true if output patches, otherwise only output cv::KeyPoint
+  optional bool output_patch = 1;
+
+  // The max number of detected features.
+  optional int32 max_features = 2 [default = 200];
+
+  // The number of pyramid levels.
+  optional int32 pyramid_level = 3 [default = 4];
+
+  // Pyramid decimation ratio.
+  optional float scale_factor = 4 [default = 1.2];
+}

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -219,8 +219,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
   const auto& input_img = cc->Inputs().Tag(kImageTag).Get<ImageFrame>();
   cv::Mat input_mat = formats::MatView(&input_img);
 
-  auto [target_width, target_height, rect_center_x, rect_center_y, rotation] =
+  RectSpec specs = GetCropSpecs(cc, input_img.Width(), input_img.Height());
-      GetCropSpecs(cc, input_img.Width(), input_img.Height());
+  int target_width = specs.width, target_height = specs.height,
+      rect_center_x = specs.center_x, rect_center_y = specs.center_y;
+  float rotation = specs.rotation;
 
   // Get border mode and value for OpenCV.
   int border_mode;
@@ -403,8 +405,10 @@ void ImageCroppingCalculator::GetOutputDimensions(CalculatorContext* cc,
                                                   int src_width, int src_height,
                                                   int* dst_width,
                                                   int* dst_height) {
-  auto [crop_width, crop_height, x_center, y_center, rotation] =
+  RectSpec specs = GetCropSpecs(cc, src_width, src_height);
-      GetCropSpecs(cc, src_width, src_height);
+  int crop_width = specs.width, crop_height = specs.height,
+      x_center = specs.center_x, y_center = specs.center_y;
+  float rotation = specs.rotation;
 
   const float half_width = crop_width / 2.0f;
   const float half_height = crop_height / 2.0f;

mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.cc

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include "mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/image_frame_opencv.h"
 #include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
@@ -34,7 +35,11 @@ namespace mediapipe {
 class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
   ::mediapipe::Status Process(CalculatorContext* cc) override;
+
+ private:
+  mediapipe::OpenCvEncodedImageToImageFrameCalculatorOptions options_;
 };
 
 ::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::GetContract(
@@ -44,13 +49,29 @@ class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Open(
+    CalculatorContext* cc) {
+  options_ =
+      cc->Options<mediapipe::OpenCvEncodedImageToImageFrameCalculatorOptions>();
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status OpenCvEncodedImageToImageFrameCalculator::Process(
     CalculatorContext* cc) {
   const std::string& contents = cc->Inputs().Index(0).Get<std::string>();
   const std::vector<char> contents_vector(contents.begin(), contents.end());
-  cv::Mat decoded_mat =
+  cv::Mat decoded_mat;
-      cv::imdecode(contents_vector, -1 /* return the loaded image as-is */);
+  if (options_.apply_orientation_from_exif_data()) {
-
+    // We want to respect the orientation from the EXIF data, which
+    // IMREAD_UNCHANGED ignores, but otherwise we want to be as permissive as
+    // possible with our reading flags. Therefore, we use IMREAD_ANYCOLOR and
+    // IMREAD_ANYDEPTH.
+    decoded_mat = cv::imdecode(contents_vector,
+                               cv::IMREAD_ANYCOLOR | cv::IMREAD_ANYDEPTH);
+  } else {
+    // Return the loaded image as-is
+    decoded_mat = cv::imdecode(contents_vector, cv::IMREAD_UNCHANGED);
+  }
   ImageFormat::Format image_format = ImageFormat::UNKNOWN;
   cv::Mat output_mat;
   switch (decoded_mat.channels()) {
@@ -70,7 +91,8 @@ class OpenCvEncodedImageToImageFrameCalculator : public CalculatorBase {
              << "Unsupported number of channels: " << decoded_mat.channels();
   }
   std::unique_ptr<ImageFrame> output_frame = absl::make_unique<ImageFrame>(
-      image_format, decoded_mat.size().width, decoded_mat.size().height);
+      image_format, decoded_mat.size().width, decoded_mat.size().height,
+      ImageFrame::kGlDefaultAlignmentBoundary);
   output_mat.copyTo(formats::MatView(output_frame.get()));
   cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
   return ::mediapipe::OkStatus();

mediapipe/calculators/image/opencv_encoded_image_to_image_frame_calculator.proto

@@ -0,0 +1,30 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message OpenCvEncodedImageToImageFrameCalculatorOptions {
+  extend CalculatorOptions {
+    optional OpenCvEncodedImageToImageFrameCalculatorOptions ext = 303447308;
+  }
+
+  // If set, we will attempt to automatically apply the orientation specified by
+  // the image's EXIF data when loading the image. Otherwise, the image data
+  // will be loaded as-is.
+  optional bool apply_orientation_from_exif_data = 1 [default = false];
+}

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc

@@ -51,10 +51,11 @@ static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH";
 #endif
 }
 
-// If options.convert_signature_to_tags() will convert letters to uppercase
+// If options.convert_signature_to_tags() is set, will convert letters to
-// and replace /'s with _'s. If set, this enables the standard SavedModel
+// uppercase and replace /'s and -'s with _'s. This enables the standard
-// classification, regression, and prediction signatures to be used as
+// SavedModel classification, regression, and prediction signatures to be used
-// uppercase INPUTS and OUTPUTS tags for streams.
+// as uppercase INPUTS and OUTPUTS tags for streams and supports other common
+// patterns.
 const std::string MaybeConvertSignatureToTag(
     const std::string& name,
     const TensorFlowSessionFromSavedModelCalculatorOptions& options) {
@@ -64,6 +65,7 @@ const std::string MaybeConvertSignatureToTag(
     std::transform(name.begin(), name.end(), output.begin(),
                    [](unsigned char c) { return std::toupper(c); });
     output = absl::StrReplaceAll(output, {{"/", "_"}});
+    output = absl::StrReplaceAll(output, {{"-", "_"}});
     return output;
   } else {
     return name;

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.proto

@@ -32,8 +32,8 @@ message TensorFlowSessionFromSavedModelCalculatorOptions {
   // The name of the generic signature to load into the mapping from tags to
   // tensor names.
   optional string signature_name = 2 [default = "serving_default"];
-  // Whether to convert the signature keys to uppercase and switch /'s to
+  // Whether to convert the signature keys to uppercase as well as switch /'s
-  // _'s, which enables standard signatures to be used as Tags.
+  // and -'s to _'s, which enables common signatures to be used as Tags.
   optional bool convert_signature_to_tags = 3 [default = true];
   // If true, saved_model_path can have multiple exported models in
   // subdirectories saved_model_path/%08d and the alphabetically last (i.e.,

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc

@@ -53,10 +53,11 @@ static constexpr char kStringSavedModelPath[] = "STRING_SAVED_MODEL_PATH";
 #endif
 }
 
-// If options.convert_signature_to_tags() will convert letters to uppercase
+// If options.convert_signature_to_tags() is set, will convert letters to
-// and replace /'s with _'s. If set, this enables the standard SavedModel
+// uppercase and replace /'s and -'s with _'s. This enables the standard
-// classification, regression, and prediction signatures to be used as
+// SavedModel classification, regression, and prediction signatures to be used
-// uppercase INPUTS and OUTPUTS tags for streams.
+// as uppercase INPUTS and OUTPUTS tags for streams and supports other common
+// patterns.
 const std::string MaybeConvertSignatureToTag(
     const std::string& name,
     const TensorFlowSessionFromSavedModelGeneratorOptions& options) {
@@ -66,6 +67,7 @@ const std::string MaybeConvertSignatureToTag(
     std::transform(name.begin(), name.end(), output.begin(),
                    [](unsigned char c) { return std::toupper(c); });
     output = absl::StrReplaceAll(output, {{"/", "_"}});
+    output = absl::StrReplaceAll(output, {{"-", "_"}});
     return output;
   } else {
     return name;

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.proto

@@ -32,8 +32,8 @@ message TensorFlowSessionFromSavedModelGeneratorOptions {
   // The name of the generic signature to load into the mapping from tags to
   // tensor names.
   optional string signature_name = 2 [default = "serving_default"];
-  // Whether to convert the signature keys to uppercase and switch /'s to
+  // Whether to convert the signature keys to uppercase as well as switch /'s
-  // _'s, which enables standard signatures to be used as Tags.
+  // and -'s to _'s, which enables common signatures to be used as Tags.
   optional bool convert_signature_to_tags = 3 [default = true];
   // If true, saved_model_path can have multiple exported models in
   // subdirectories saved_model_path/%08d and the alphabetically last (i.e.,

mediapipe/calculators/tflite/BUILD

@@ -451,7 +451,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -673,7 +673,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     const auto& input_indices = interpreter_->inputs();
     gpu_data_in_.resize(input_indices.size());
     for (int i = 0; i < input_indices.size(); ++i) {
-      const TfLiteTensor* tensor = interpreter_->tensor(input_indices[0]);
+      const TfLiteTensor* tensor = interpreter_->tensor(input_indices[i]);
       gpu_data_in_[i] = absl::make_unique<GPUData>();
       gpu_data_in_[i]->elements = 1;
       for (int d = 0; d < tensor->dims->size; ++d) {

mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc

@@ -145,7 +145,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
           ? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>()
           : options_.flip_horizontally();
 
-  flip_horizontally_ =
+  flip_vertically_ =
       cc->InputSidePackets().HasTag("FLIP_VERTICALLY")
           ? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>()
           : options_.flip_vertically();

mediapipe/calculators/tflite/util.h

@@ -19,7 +19,7 @@
   do {                                                     \
     const auto status = (call);                            \
     if (ABSL_PREDICT_FALSE(!status.ok()))                  \
-      return ::mediapipe::InternalError(status.error_message()); \
+      return ::mediapipe::InternalError(status.message()); \
   } while (0);
 
 #endif  // MEDIAPIPE_CALCULATORS_TFLITE_UTIL_H_

mediapipe/calculators/util/BUILD

@@ -321,7 +321,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [
@@ -349,7 +349,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [
@@ -926,7 +926,7 @@ cc_library(
         "//mediapipe:android": [
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/util/android/file/base",
         ],
         "//mediapipe:macos": [
@@ -971,9 +971,9 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework:calculator_framework",
-        "//mediapipe/framework/port:file_helpers",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "//mediapipe/util:resource_util",
     ],
     alwayslink = 1,
 )

mediapipe/calculators/util/annotation_overlay_calculator.cc

@@ -55,7 +55,7 @@ size_t RoundUp(size_t n, size_t m) { return ((n + m - 1) / m) * m; }  // NOLINT
 // When using GPU, this color will become transparent when the calculator
 // merges the annotation overlay with the image frame. As a result, drawing in
 // this color is not supported and it should be set to something unlikely used.
-constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
+constexpr uchar kAnnotationBackgroundColor = 2;  // Grayscale value.
 }  // namespace
 
 // A calculator for rendering data on images.
@@ -491,11 +491,9 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     if (format != mediapipe::ImageFormat::SRGBA &&
         format != mediapipe::ImageFormat::SRGB)
       RET_CHECK_FAIL() << "Unsupported GPU input format: " << format;
-
+    image_mat = absl::make_unique<cv::Mat>(height_, width_, CV_8UC3);
-    image_mat = absl::make_unique<cv::Mat>(
+    memset(image_mat->data, kAnnotationBackgroundColor,
-        height_, width_, CV_8UC3,
+           height_ * width_ * image_mat->elemSize());
-        cv::Scalar(kAnnotationBackgroundColor[0], kAnnotationBackgroundColor[1],
-                   kAnnotationBackgroundColor[2]));
   } else {
     image_mat = absl::make_unique<cv::Mat>(
         options_.canvas_height_px(), options_.canvas_width_px(), CV_8UC3,
@@ -617,9 +615,9 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
   glUniform1i(glGetUniformLocation(program_, "input_frame"), 1);
   glUniform1i(glGetUniformLocation(program_, "overlay"), 2);
   glUniform3f(glGetUniformLocation(program_, "transparent_color"),
-              kAnnotationBackgroundColor[0] / 255.0,
+              kAnnotationBackgroundColor / 255.0,
-              kAnnotationBackgroundColor[1] / 255.0,
+              kAnnotationBackgroundColor / 255.0,
-              kAnnotationBackgroundColor[2] / 255.0);
+              kAnnotationBackgroundColor / 255.0);
 
   // Init texture for opencv rendered frame.
   const auto& input_frame =

mediapipe/calculators/util/labels_to_render_data_calculator.cc

@@ -128,18 +128,21 @@ REGISTER_CALCULATOR(LabelsToRenderDataCalculator);
   } else {
     const std::vector<std::string>& label_vector =
         cc->Inputs().Tag("LABELS").Get<std::vector<std::string>>();
-    std::vector<float> score_vector;
-    if (cc->Inputs().HasTag("SCORES")) {
-      score_vector = cc->Inputs().Tag("SCORES").Get<std::vector<float>>();
-    }
-    CHECK_EQ(label_vector.size(), score_vector.size());
     labels.resize(label_vector.size());
-    scores.resize(label_vector.size());
     for (int i = 0; i < label_vector.size(); ++i) {
       labels[i] = label_vector[i];
+    }
+
+    if (cc->Inputs().HasTag("SCORES")) {
+      std::vector<float> score_vector =
+          cc->Inputs().Tag("SCORES").Get<std::vector<float>>();
+      CHECK_EQ(label_vector.size(), score_vector.size());
+      scores.resize(label_vector.size());
+      for (int i = 0; i < label_vector.size(); ++i) {
         scores[i] = score_vector[i];
       }
     }
+  }
 
   RenderData render_data;
   int num_label = std::min((int)labels.size(), options_.max_num_labels());

mediapipe/calculators/util/local_file_contents_calculator.cc

@@ -16,34 +16,80 @@
 #include <string>
 
 #include "mediapipe/framework/calculator_framework.h"
-#include "mediapipe/framework/port/file_helpers.h"
 #include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/resource_util.h"
 
 namespace mediapipe {
+
+namespace {
+
+constexpr char kFilePathTag[] = "FILE_PATH";
+constexpr char kContentsTag[] = "CONTENTS";
+
+}  // namespace
+
 // The calculator takes the path to the local file as an input side packet and
 // outputs the contents of that file.
 //
+// NOTE: file loading can be batched by providing multiple input/output side
+// packets.
+//
 // Example config:
 // node {
 //   calculator: "LocalFileContentsCalculator"
 //   input_side_packet: "FILE_PATH:file_path"
 //   output_side_packet: "CONTENTS:contents"
 // }
+//
+// node {
+//   calculator: "LocalFileContentsCalculator"
+//   input_side_packet: "FILE_PATH:0:file_path1"
+//   input_side_packet: "FILE_PATH:1:file_path2"
+//   ...
+//   output_side_packet: "CONTENTS:0:contents1"
+//   output_side_packet: "CONTENTS:1:contents2"
+//   ...
+// }
 class LocalFileContentsCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
-    cc->InputSidePackets().Tag("FILE_PATH").Set<std::string>();
+    RET_CHECK(cc->InputSidePackets().HasTag(kFilePathTag))
-    cc->OutputSidePackets().Tag("CONTENTS").Set<std::string>();
+        << "Missing PATH input side packet(s)";
+    RET_CHECK(cc->OutputSidePackets().HasTag(kContentsTag))
+        << "Missing CONTENTS output side packet(s)";
+
+    RET_CHECK_EQ(cc->InputSidePackets().NumEntries(kFilePathTag),
+                 cc->OutputSidePackets().NumEntries(kContentsTag))
+        << "Same number of input streams and output streams is required.";
+
+    for (CollectionItemId id = cc->InputSidePackets().BeginId(kFilePathTag);
+         id != cc->InputSidePackets().EndId(kFilePathTag); ++id) {
+      cc->InputSidePackets().Get(id).Set<std::string>();
+    }
+
+    for (CollectionItemId id = cc->OutputSidePackets().BeginId(kContentsTag);
+         id != cc->OutputSidePackets().EndId(kContentsTag); ++id) {
+      cc->OutputSidePackets().Get(id).Set<std::string>();
+    }
+
     return ::mediapipe::OkStatus();
   }
 
   ::mediapipe::Status Open(CalculatorContext* cc) override {
+    CollectionItemId input_id = cc->InputSidePackets().BeginId(kFilePathTag);
+    CollectionItemId output_id = cc->OutputSidePackets().BeginId(kContentsTag);
+    // Number of inputs and outpus is the same according to the contract.
+    for (; input_id != cc->InputSidePackets().EndId(kFilePathTag);
+         ++input_id, ++output_id) {
+      std::string file_path =
+          cc->InputSidePackets().Get(input_id).Get<std::string>();
+      ASSIGN_OR_RETURN(file_path, PathToResourceAsFile(file_path));
+
       std::string contents;
-    MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
+      MP_RETURN_IF_ERROR(GetResourceContents(file_path, &contents));
-        cc->InputSidePackets().Tag("FILE_PATH").Get<std::string>(), &contents));
+      cc->OutputSidePackets().Get(output_id).Set(
-    cc->OutputSidePackets()
+          MakePacket<std::string>(std::move(contents)));
-        .Tag("CONTENTS")
+    }
-        .Set(MakePacket<std::string>(std::move(contents)));
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/util/timed_box_list_to_render_data_calculator.cc

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <algorithm>
+
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
@@ -76,14 +78,16 @@ void AddTimedBoxProtoToRenderData(
     RenderAnnotation::Text* text = label_annotation->mutable_text();
     text->set_display_text(box_proto.label());
     text->set_normalized(true);
-    constexpr float text_left_start = 0.3f;
+    constexpr float text_left_start = 0.2f;
     text->set_left((1.0f - text_left_start) * box_proto.left() +
                    text_left_start * box_proto.right());
     constexpr float text_baseline = 0.6f;
     text->set_baseline(text_baseline * box_proto.bottom() +
                        (1.0f - text_baseline) * box_proto.top());
-    constexpr float text_height = 0.2f;
+    constexpr float text_height = 0.1f;
-    text->set_font_height((box_proto.bottom() - box_proto.top()) * text_height);
+    text->set_font_height(std::min(box_proto.bottom() - box_proto.top(),
+                                   box_proto.right() - box_proto.left()) *
+                          text_height);
   }
 }
 

mediapipe/calculators/video/BUILD

@@ -65,6 +65,26 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "tracked_detection_manager_calculator_proto",
+    srcs = ["tracked_detection_manager_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:tracked_detection_manager_config_proto",
+    ],
+)
+
+proto_library(
+    name = "box_detector_calculator_proto",
+    srcs = ["box_detector_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+        "//mediapipe/util/tracking:box_detector_proto",
+    ],
+)
+
 proto_library(
     name = "video_pre_stream_calculator_proto",
     srcs = ["video_pre_stream_calculator.proto"],
@@ -107,6 +127,28 @@ mediapipe_cc_proto_library(
     deps = [":box_tracker_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "tracked_detection_manager_calculator_cc_proto",
+    srcs = ["tracked_detection_manager_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:tracked_detection_manager_config_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":tracked_detection_manager_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "box_detector_calculator_cc_proto",
+    srcs = ["box_detector_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/util/tracking:box_detector_cc_proto",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [":box_detector_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "video_pre_stream_calculator_cc_proto",
     srcs = ["video_pre_stream_calculator.proto"],
@@ -279,11 +321,54 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "box_detector_calculator",
+    srcs = ["box_detector_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":box_detector_calculator_cc_proto",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/formats:image_frame_opencv",
+        "//mediapipe/framework/formats:video_stream_header",  # fixdeps: keep -- required for exobazel build.
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:opencv_core",
+        "//mediapipe/framework/port:opencv_features2d",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:resource_util",
+        "//mediapipe/util/tracking",
+        "//mediapipe/util/tracking:box_detector",
+        "//mediapipe/util/tracking:box_tracker",
+        "//mediapipe/util/tracking:box_tracker_cc_proto",
+        "//mediapipe/util/tracking:flow_packager_cc_proto",
+        "//mediapipe/util/tracking:tracking_visualization_utilities",
+    ] + select({
+        "//mediapipe:android": [
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:ios": [
+            "//mediapipe/util/android/file/base",
+        ],
+        "//mediapipe:macos": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
+        "//conditions:default": [
+            "//mediapipe/framework/port:file_helpers",
+        ],
+    }),
+    alwayslink = 1,
+)
+
 cc_library(
     name = "tracked_detection_manager_calculator",
     srcs = ["tracked_detection_manager_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
+        ":tracked_detection_manager_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:detection_cc_proto",
         "//mediapipe/framework/formats:location_data_cc_proto",

mediapipe/calculators/video/box_detector_calculator.cc

@@ -0,0 +1,393 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#include <memory>
+#include <unordered_set>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/numbers.h"
+#include "mediapipe/calculators/video/box_detector_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/image_frame.h"
+#include "mediapipe/framework/formats/image_frame_opencv.h"
+#include "mediapipe/framework/formats/video_stream_header.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/logging.h"
+#include "mediapipe/framework/port/opencv_core_inc.h"
+#include "mediapipe/framework/port/opencv_features2d_inc.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/util/resource_util.h"
+#include "mediapipe/util/tracking/box_detector.h"
+#include "mediapipe/util/tracking/box_tracker.h"
+#include "mediapipe/util/tracking/box_tracker.pb.h"
+#include "mediapipe/util/tracking/flow_packager.pb.h"
+#include "mediapipe/util/tracking/tracking.h"
+#include "mediapipe/util/tracking/tracking_visualization_utilities.h"
+
+#if defined(MEDIAPIPE_MOBILE)
+#include "mediapipe/util/android/file/base/file.h"
+#include "mediapipe/util/android/file/base/helpers.h"
+#else
+#include "mediapipe/framework/port/file_helpers.h"
+#endif
+
+namespace mediapipe {
+
+// A calculator to detect reappeared box positions from single frame.
+//
+// Input stream:
+//   TRACKING: Input tracking data (proto TrackingData) containing features and
+//             descriptors.
+//   VIDEO:    Optional input video stream tracked boxes are rendered over
+//             (Required if VIZ is specified).
+//   FEATURES: Input feature points (std::vector<cv::KeyPoint>) in the original
+//             pixel space.
+//   DESCRIPTORS: Input feature descriptors (std::vector<float>). Actual feature
+//             dimension needs to be specified in detector_options.
+//   IMAGE_SIZE: Input image dimension.
+//   TRACKED_BOXES : input box tracking result (proto TimedBoxProtoList) from
+//             BoxTrackerCalculator.
+//   ADD_INDEX: Optional std::string containing binary format proto of type
+//             BoxDetectorIndex. Used for adding target index to the detector
+//             search index during runtime.
+//   CANCEL_OBJECT_ID: Optional id of box to be removed. This is recommended
+//             to be used with SyncSetInputStreamHandler.
+//   REACQ_SWITCH: Optional bool for swithcing on and off reacquisition
+//             functionality. User should initialize a graph with box detector
+//             calculator and be able to switch it on and off in runtime.
+//
+// Output streams:
+//   VIZ:   Optional output video stream with rendered box positions
+//          (requires VIDEO to be present)
+//   BOXES: Optional output stream of type TimedBoxProtoList for each lost box.
+//
+// Imput side packets:
+//   INDEX_PROTO_STRING: Optional std::string containing binary format proto of
+//   type
+//                       BoxDetectorIndex. Used for initializing box_detector
+//                       with predefined template images.
+//   FRAME_ALIGNMENT:    Optional integer to indicate alignment_boundary for
+//                       outputing ImageFrame in "VIZ" stream.
+//                       Set to ImageFrame::kDefaultAlignmentBoundary for
+//                       offline pipeline to be compatible with FFmpeg.
+//                       Set to ImageFrame::kGlDefaultAlignmentBoundary for Apps
+//                       to be compatible with GL renderer.
+//   OUTPUT_INDEX_FILENAME: File path to the output index file.
+
+class BoxDetectorCalculator : public CalculatorBase {
+ public:
+  ~BoxDetectorCalculator() override = default;
+
+  static ::mediapipe::Status GetContract(CalculatorContract* cc);
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
+  ::mediapipe::Status Process(CalculatorContext* cc) override;
+  ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  BoxDetectorCalculatorOptions options_;
+  std::unique_ptr<BoxDetectorInterface> box_detector_;
+  bool detector_switch_ = true;
+  uint32 frame_alignment_ = ImageFrame::kDefaultAlignmentBoundary;
+  bool write_index_ = false;
+  int box_id_ = 0;
+};
+
+REGISTER_CALCULATOR(BoxDetectorCalculator);
+
+::mediapipe::Status BoxDetectorCalculator::GetContract(CalculatorContract* cc) {
+  if (cc->Inputs().HasTag("TRACKING")) {
+    cc->Inputs().Tag("TRACKING").Set<TrackingData>();
+  }
+
+  if (cc->Inputs().HasTag("TRACKED_BOXES")) {
+    cc->Inputs().Tag("TRACKED_BOXES").Set<TimedBoxProtoList>();
+  }
+
+  if (cc->Inputs().HasTag("VIDEO")) {
+    cc->Inputs().Tag("VIDEO").Set<ImageFrame>();
+  }
+
+  if (cc->Inputs().HasTag("FEATURES")) {
+    RET_CHECK(cc->Inputs().HasTag("DESCRIPTORS"))
+        << "FEATURES and DESCRIPTORS need to be specified together.";
+    cc->Inputs().Tag("FEATURES").Set<std::vector<cv::KeyPoint>>();
+  }
+
+  if (cc->Inputs().HasTag("DESCRIPTORS")) {
+    RET_CHECK(cc->Inputs().HasTag("FEATURES"))
+        << "FEATURES and DESCRIPTORS need to be specified together.";
+    cc->Inputs().Tag("DESCRIPTORS").Set<std::vector<float>>();
+  }
+
+  if (cc->Inputs().HasTag("IMAGE_SIZE")) {
+    cc->Inputs().Tag("IMAGE_SIZE").Set<std::pair<int, int>>();
+  }
+
+  if (cc->Inputs().HasTag("ADD_INDEX")) {
+    cc->Inputs().Tag("ADD_INDEX").Set<std::string>();
+  }
+
+  if (cc->Inputs().HasTag("CANCEL_OBJECT_ID")) {
+    cc->Inputs().Tag("CANCEL_OBJECT_ID").Set<int>();
+  }
+
+  if (cc->Inputs().HasTag("REACQ_SWITCH")) {
+    cc->Inputs().Tag("REACQ_SWITCH").Set<bool>();
+  }
+
+  if (cc->Outputs().HasTag("BOXES")) {
+    cc->Outputs().Tag("BOXES").Set<TimedBoxProtoList>();
+  }
+
+  if (cc->Outputs().HasTag("VIZ")) {
+    RET_CHECK(cc->Inputs().HasTag("VIDEO"))
+        << "Output stream VIZ requires VIDEO to be present.";
+    cc->Outputs().Tag("VIZ").Set<ImageFrame>();
+  }
+
+  if (cc->InputSidePackets().HasTag("INDEX_PROTO_STRING")) {
+    cc->InputSidePackets().Tag("INDEX_PROTO_STRING").Set<std::string>();
+  }
+
+  if (cc->InputSidePackets().HasTag("OUTPUT_INDEX_FILENAME")) {
+    cc->InputSidePackets().Tag("OUTPUT_INDEX_FILENAME").Set<std::string>();
+  }
+
+  if (cc->InputSidePackets().HasTag("FRAME_ALIGNMENT")) {
+    cc->InputSidePackets().Tag("FRAME_ALIGNMENT").Set<int>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BoxDetectorCalculator::Open(CalculatorContext* cc) {
+  options_ = cc->Options<BoxDetectorCalculatorOptions>();
+  box_detector_ = BoxDetectorInterface::Create(options_.detector_options());
+
+  if (cc->InputSidePackets().HasTag("INDEX_PROTO_STRING")) {
+    BoxDetectorIndex predefined_index;
+    if (!predefined_index.ParseFromString(cc->InputSidePackets()
+                                              .Tag("INDEX_PROTO_STRING")
+                                              .Get<std::string>())) {
+      LOG(FATAL) << "failed to parse BoxDetectorIndex from INDEX_PROTO_STRING";
+    }
+    box_detector_->AddBoxDetectorIndex(predefined_index);
+  }
+
+  for (const auto& filename : options_.index_proto_filename()) {
+    std::string string_path;
+    ASSIGN_OR_RETURN(string_path, PathToResourceAsFile(filename));
+    std::string index_string;
+    MP_RETURN_IF_ERROR(file::GetContents(string_path, &index_string));
+    BoxDetectorIndex predefined_index;
+    if (!predefined_index.ParseFromString(index_string)) {
+      LOG(FATAL)
+          << "failed to parse BoxDetectorIndex from index_proto_filename";
+    }
+    box_detector_->AddBoxDetectorIndex(predefined_index);
+  }
+
+  if (cc->InputSidePackets().HasTag("OUTPUT_INDEX_FILENAME")) {
+    write_index_ = true;
+  }
+
+  if (cc->InputSidePackets().HasTag("FRAME_ALIGNMENT")) {
+    frame_alignment_ = cc->InputSidePackets().Tag("FRAME_ALIGNMENT").Get<int>();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BoxDetectorCalculator::Process(CalculatorContext* cc) {
+  const Timestamp timestamp = cc->InputTimestamp();
+  const int64 timestamp_msec = timestamp.Value() / 1000;
+
+  InputStream* cancel_object_id_stream =
+      cc->Inputs().HasTag("CANCEL_OBJECT_ID")
+          ? &(cc->Inputs().Tag("CANCEL_OBJECT_ID"))
+          : nullptr;
+  if (cancel_object_id_stream && !cancel_object_id_stream->IsEmpty()) {
+    const int cancel_object_id = cancel_object_id_stream->Get<int>();
+    box_detector_->CancelBoxDetection(cancel_object_id);
+  }
+
+  InputStream* add_index_stream = cc->Inputs().HasTag("ADD_INDEX")
+                                      ? &(cc->Inputs().Tag("ADD_INDEX"))
+                                      : nullptr;
+  if (add_index_stream && !add_index_stream->IsEmpty()) {
+    BoxDetectorIndex predefined_index;
+    if (!predefined_index.ParseFromString(
+            add_index_stream->Get<std::string>())) {
+      LOG(FATAL) << "failed to parse BoxDetectorIndex from ADD_INDEX";
+    }
+    box_detector_->AddBoxDetectorIndex(predefined_index);
+  }
+
+  InputStream* reacq_switch_stream = cc->Inputs().HasTag("REACQ_SWITCH")
+                                         ? &(cc->Inputs().Tag("REACQ_SWITCH"))
+                                         : nullptr;
+  if (reacq_switch_stream && !reacq_switch_stream->IsEmpty()) {
+    detector_switch_ = reacq_switch_stream->Get<bool>();
+  }
+
+  if (!detector_switch_) {
+    return ::mediapipe::OkStatus();
+  }
+
+  InputStream* track_stream = cc->Inputs().HasTag("TRACKING")
+                                  ? &(cc->Inputs().Tag("TRACKING"))
+                                  : nullptr;
+  InputStream* video_stream =
+      cc->Inputs().HasTag("VIDEO") ? &(cc->Inputs().Tag("VIDEO")) : nullptr;
+  InputStream* feature_stream = cc->Inputs().HasTag("FEATURES")
+                                    ? &(cc->Inputs().Tag("FEATURES"))
+                                    : nullptr;
+  InputStream* descriptor_stream = cc->Inputs().HasTag("DESCRIPTORS")
+                                       ? &(cc->Inputs().Tag("DESCRIPTORS"))
+                                       : nullptr;
+
+  CHECK(track_stream != nullptr || video_stream != nullptr ||
+        (feature_stream != nullptr && descriptor_stream != nullptr))
+      << "One and only one of {tracking_data, input image frame, "
+         "feature/descriptor} need to be valid.";
+
+  InputStream* tracked_boxes_stream = cc->Inputs().HasTag("TRACKED_BOXES")
+                                          ? &(cc->Inputs().Tag("TRACKED_BOXES"))
+                                          : nullptr;
+  std::unique_ptr<TimedBoxProtoList> detected_boxes(new TimedBoxProtoList());
+
+  if (track_stream != nullptr) {
+    // Detect from tracking data
+    if (track_stream->IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    const TrackingData& tracking_data = track_stream->Get<TrackingData>();
+
+    CHECK(tracked_boxes_stream != nullptr) << "tracked_boxes needed.";
+
+    const TimedBoxProtoList tracked_boxes =
+        tracked_boxes_stream->Get<TimedBoxProtoList>();
+
+    box_detector_->DetectAndAddBox(tracking_data, tracked_boxes, timestamp_msec,
+                                   detected_boxes.get());
+  } else if (video_stream != nullptr) {
+    // Detect from input frame
+    if (video_stream->IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    TimedBoxProtoList tracked_boxes;
+    if (tracked_boxes_stream != nullptr && !tracked_boxes_stream->IsEmpty()) {
+      tracked_boxes = tracked_boxes_stream->Get<TimedBoxProtoList>();
+    }
+
+    // Just directly pass along the image frame data as-is for detection; we
+    // don't need to worry about conforming to a specific alignment here.
+    const cv::Mat input_view =
+        formats::MatView(&video_stream->Get<ImageFrame>());
+    box_detector_->DetectAndAddBox(input_view, tracked_boxes, timestamp_msec,
+                                   detected_boxes.get());
+  } else {
+    if (feature_stream->IsEmpty() || descriptor_stream->IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    const auto& image_size =
+        cc->Inputs().Tag("IMAGE_SIZE").Get<std::pair<int, int>>();
+    float inv_scale = 1.0f / std::max(image_size.first, image_size.second);
+
+    TimedBoxProtoList tracked_boxes;
+    if (tracked_boxes_stream != nullptr && !tracked_boxes_stream->IsEmpty()) {
+      tracked_boxes = tracked_boxes_stream->Get<TimedBoxProtoList>();
+    } else if (write_index_) {
+      auto* box_ptr = tracked_boxes.add_box();
+      box_ptr->set_id(box_id_);
+      box_ptr->set_reacquisition(true);
+      box_ptr->set_aspect_ratio((float)image_size.first /
+                                (float)image_size.second);
+
+      box_ptr->mutable_quad()->add_vertices(0);
+      box_ptr->mutable_quad()->add_vertices(0);
+
+      box_ptr->mutable_quad()->add_vertices(0);
+      box_ptr->mutable_quad()->add_vertices(1);
+
+      box_ptr->mutable_quad()->add_vertices(1);
+      box_ptr->mutable_quad()->add_vertices(1);
+
+      box_ptr->mutable_quad()->add_vertices(1);
+      box_ptr->mutable_quad()->add_vertices(0);
+
+      ++box_id_;
+    }
+
+    const auto& features = feature_stream->Get<std::vector<cv::KeyPoint>>();
+    const int feature_size = features.size();
+    std::vector<Vector2_f> features_vec(feature_size);
+
+    const auto& descriptors = descriptor_stream->Get<std::vector<float>>();
+    const int dims = options_.detector_options().descriptor_dims();
+    CHECK_GE(descriptors.size(), feature_size * dims);
+    cv::Mat descriptors_mat(feature_size, dims, CV_32F);
+    for (int j = 0; j < feature_size; ++j) {
+      features_vec[j].Set(features[j].pt.x * inv_scale,
+                          features[j].pt.y * inv_scale);
+      for (int i = 0; i < dims; ++i) {
+        descriptors_mat.at<float>(j, i) = descriptors[j * dims + i];
+      }
+    }
+
+    box_detector_->DetectAndAddBoxFromFeatures(
+        features_vec, descriptors_mat, tracked_boxes, timestamp_msec,
+        image_size.first * inv_scale, image_size.second * inv_scale,
+        detected_boxes.get());
+  }
+
+  if (cc->Outputs().HasTag("VIZ")) {
+    cv::Mat viz_view;
+    std::unique_ptr<ImageFrame> viz_frame;
+    if (video_stream != nullptr && !video_stream->IsEmpty()) {
+      viz_frame = absl::make_unique<ImageFrame>();
+      viz_frame->CopyFrom(video_stream->Get<ImageFrame>(), frame_alignment_);
+      viz_view = formats::MatView(viz_frame.get());
+    }
+    for (const auto& box : detected_boxes->box()) {
+      RenderBox(box, &viz_view);
+    }
+    cc->Outputs().Tag("VIZ").Add(viz_frame.release(), timestamp);
+  }
+
+  if (cc->Outputs().HasTag("BOXES")) {
+    cc->Outputs().Tag("BOXES").Add(detected_boxes.release(), timestamp);
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
+::mediapipe::Status BoxDetectorCalculator::Close(CalculatorContext* cc) {
+  if (write_index_) {
+    BoxDetectorIndex index = box_detector_->ObtainBoxDetectorIndex();
+    MEDIAPIPE_CHECK_OK(mediapipe::file::SetContents(
+        cc->InputSidePackets().Tag("OUTPUT_INDEX_FILENAME").Get<std::string>(),
+        index.SerializeAsString()));
+  }
+  return ::mediapipe::OkStatus();
+}
+
+}  // namespace mediapipe

mediapipe/calculators/video/box_detector_calculator.proto

@@ -0,0 +1,31 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/box_detector.proto";
+
+message BoxDetectorCalculatorOptions {
+  extend CalculatorOptions {
+    optional BoxDetectorCalculatorOptions ext = 289746530;
+  }
+
+  optional BoxDetectorOptions detector_options = 1;
+
+  // File path to the template index files.
+  repeated string index_proto_filename = 2;
+}

mediapipe/calculators/video/tracked_detection_manager_calculator.cc

@@ -18,6 +18,7 @@
 #include <vector>
 
 #include "absl/container/node_hash_map.h"
+#include "mediapipe/calculators/video/tracked_detection_manager_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/formats/detection.pb.h"
 #include "mediapipe/framework/formats/location_data.pb.h"
@@ -139,6 +140,7 @@ Detection GetAxisAlignedDetectionFromTrackedDetection(
 class TrackedDetectionManagerCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
 
   ::mediapipe::Status Process(CalculatorContext* cc) override;
 
@@ -184,6 +186,15 @@ REGISTER_CALCULATOR(TrackedDetectionManagerCalculator);
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status TrackedDetectionManagerCalculator::Open(
+    CalculatorContext* cc) {
+  mediapipe::TrackedDetectionManagerCalculatorOptions options =
+      cc->Options<mediapipe::TrackedDetectionManagerCalculatorOptions>();
+  tracked_detection_manager_.SetConfig(
+      options.tracked_detection_manager_options());
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status TrackedDetectionManagerCalculator::Process(
     CalculatorContext* cc) {
   if (cc->Inputs().HasTag("TRACKING_BOXES")) {

mediapipe/calculators/video/tracked_detection_manager_calculator.proto

@@ -0,0 +1,28 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+import "mediapipe/util/tracking/tracked_detection_manager_config.proto";
+
+message TrackedDetectionManagerCalculatorOptions {
+  extend CalculatorOptions {
+    optional TrackedDetectionManagerCalculatorOptions ext = 301970230;
+  }
+
+  optional TrackedDetectionManagerConfig tracked_detection_manager_options = 1;
+}

mediapipe/docs/face_detection_desktop.md

@@ -51,7 +51,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
 ```bash
 # Video from webcam running on desktop GPU
 # This works only for linux currently
-$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
     mediapipe/examples/desktop/face_detection:face_detection_gpu
 
 # It should print:

mediapipe/docs/gpu.md

@@ -190,10 +190,10 @@ within the MediaPipe framework:
 bazel build --define MEDIAPIPE_DISABLE_GPU=1  <my-target>
 
 # to enable full GPU support (OpenGL ES 3.1+ & Metal)
-bazel build --copt -DMESA_EGL_NO_X11_HEADERS  <my-target>
+bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 <my-target>
 
 # to enable only OpenGL ES 3.0 and below (no GLES 3.1+ features)
-bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DMEDIAPIPE_DISABLE_GL_COMPUTE  <my-target>
+bazel build --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 --copt -DMEDIAPIPE_DISABLE_GL_COMPUTE  <my-target>
 ```
 
 Note *MEDIAPIPE_DISABLE_GL_COMPUTE* is automatically defined on all Apple

mediapipe/docs/hair_segmentation_desktop.md

@@ -26,7 +26,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
 ```bash
 # Video from webcam running on desktop GPU
 # This works only for linux currently
-$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
     mediapipe/examples/desktop/hair_segmentation:hair_segmentation_gpu
 
 # It should print:

mediapipe/docs/hand_tracking_desktop.md

@@ -48,7 +48,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
 ```bash
 # Video from webcam running on desktop GPU
 # This works only for linux currently
-$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
     mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu
 
 # It should print:

mediapipe/docs/install.md

@@ -40,12 +40,11 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (version between 1.0.0 and 1.2.1).
+2.  Install Bazel.
 
     Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
-    to install Bazel manually. Note that MediaPipe doesn't support Bazel 2.0.0+
+    to install Bazel 2.0 or higher.
-    yet.
 
 3.  Install OpenCV and FFmpeg.
 
@@ -111,7 +110,7 @@ To build and run iOS apps:
     # To compile with GPU support, replace
     --define MEDIAPIPE_DISABLE_GPU=1
     # with
-    --copt -DMESA_EGL_NO_X11_HEADERS
+    --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11
     # when building GPU examples.
     ```
 
@@ -125,7 +124,7 @@ To build and run iOS apps:
         mediapipe/examples/desktop/hello_world:hello_world
 
     # If you are running on Linux desktop with GPU support enabled (via mesa drivers)
-    $ bazel run --copt -DMESA_EGL_NO_X11_HEADERS \
+    $ bazel run --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
         mediapipe/examples/desktop/hello_world:hello_world
 
     # Should print:
@@ -152,12 +151,11 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (version between 1.0.0 and 1.2.1).
+2.  Install Bazel.
 
     Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
-    to install Bazel manually. Note that MediaPipe doesn't support Bazel 2.0.0+
+    to install Bazel 2.0 or higher.
-    yet.
 
 3.  Install OpenCV.
 
@@ -241,23 +239,18 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-3.  Install Bazel (version between 1.0.0 and 1.1.0).
+3.  Install Bazel.
 
-    Option 1. Use package manager tool to install Bazel 1.1.0
+    Option 1. Use package manager tool to install Bazel
 
     ```bash
-    # If Bazel 1.1.0+ was installed.
+    $ brew install bazel
-    $ brew uninstall bazel
-    # Install Bazel 1.1.0
-    $ brew install https://raw.githubusercontent.com/bazelbuild/homebrew-tap/f8a0fa981bcb1784a0d0823e14867b844e94fb3d/Formula/bazel.rb
-    $ brew link bazel
     # Run 'bazel version' to check version of bazel
     ```
 
     Option 2. Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
-    to install any version of Bazel manually. Note that MediaPipe doesn't
+    to install Bazel 2.0 or higher.
-    support Bazel 1.1.0+ on macOS yet.
 
 4.  Install OpenCV and FFmpeg.
 
@@ -391,18 +384,18 @@ cameras. Alternatively, you use a video file as input.
     username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
     ```
 
-5.  Install Bazel (version between 1.0.0 and 1.2.1).
+5.  Install Bazel.
 
     ```bash
     username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
-    https://storage.googleapis.com/bazel/1.0.0/release/bazel-1.0.0-installer-linux-x86_64.sh && \
+    https://storage.googleapis.com/bazel/2.0.0/release/bazel-2.0.0-installer-linux-x86_64.sh && \
-    sudo mkdir -p /usr/local/bazel/1.0.0 && \
+    sudo mkdir -p /usr/local/bazel/2.0.0 && \
-    chmod 755 bazel-1.0.0-installer-linux-x86_64.sh && \
+    chmod 755 bazel-2.0.0-installer-linux-x86_64.sh && \
-    sudo ./bazel-1.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/1.0.0 && \
+    sudo ./bazel-2.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/2.0.0 && \
-    source /usr/local/bazel/1.0.0/lib/bazel/bin/bazel-complete.bash
+    source /usr/local/bazel/2.0.0/lib/bazel/bin/bazel-complete.bash
 
-    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/1.0.0/lib/bazel/bin/bazel version && \
+    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/2.0.0/lib/bazel/bin/bazel version && \
-    alias bazel='/usr/local/bazel/1.0.0/lib/bazel/bin/bazel'
+    alias bazel='/usr/local/bazel/2.0.0/lib/bazel/bin/bazel'
     ```
 
 6.  Checkout MediaPipe repository.

mediapipe/docs/media_sequence.md

@@ -89,11 +89,11 @@ process new data sets, in the documentation of
        dataset = d.as_dataset('test')
        # implement additional processing and batching here
        dataset_output = dataset.make_one_shot_iterator().get_next()
-       images = dataset_output=['images']
+       images = dataset_output['images']
-       labels = dataset_output=['labels']
+       labels = dataset_output['labels']
 
        with tf.Session() as sess:
-         images_, labels_ = sess.run(images, labels)
+         images_, labels_ = sess.run([images, labels])
        print('The shape of images_ is %s' % str(images_.shape))
        print('The shape of labels_ is %s' % str(labels_.shape))
     ```

mediapipe/docs/mediapipe_ios_setup.md

@@ -76,6 +76,10 @@ in the next section.
 
 2.  Open `mediapipe/Mediapipe.tulsiproj` using the Tulsi app.
 
+    Important: If Tulsi displays an error saying "Bazel could not be found",
+    press the "Bazel..." button in the Packages tab and select the `bazel`
+    executable in your homebrew `/bin/` directory.
+
 3.  Select the MediaPipe config in the Configs tab, then press the Generate
     button below. You will be asked for a location to save the Xcode project.
     Once the project is generated, it will be opened in Xcode.

mediapipe/docs/multi_hand_tracking_desktop.md

@@ -48,7 +48,7 @@ To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
 ```bash
 # Video from webcam running on desktop GPU
 # This works only for linux currently
-$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS --copt -DEGL_NO_X11 \
     mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_gpu
 
 # It should print:

mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/BUILD

@@ -60,8 +60,8 @@ android_library(
         "//third_party:androidx_appcompat",
         "//third_party:androidx_constraint_layout",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/BUILD

@@ -64,10 +64,10 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/BUILD

@@ -64,10 +64,10 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/BUILD

@@ -63,10 +63,10 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/BUILD

@@ -64,10 +64,10 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/BUILD

@@ -85,10 +85,10 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD

@@ -85,10 +85,10 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetection3d/BUILD

@@ -97,10 +97,10 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/BUILD

@@ -64,10 +64,10 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/BUILD

@@ -65,10 +65,10 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/android/src/java/com/google/mediapipe/apps/objecttrackinggpu/BUILD

@@ -64,9 +64,9 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
         "//third_party:opencv",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/examples/coral/Dockerfile

@@ -35,10 +35,8 @@ RUN dpkg --add-architecture arm64
 RUN apt-get update && apt-get install -y \
   build-essential \
   crossbuild-essential-arm64 \
-  libusb-1.0-0-dev \
   libusb-1.0-0-dev:arm64 \
-  zlib1g-dev \
+  zlibc:arm64 \
-  zlib1g-dev:arm64 \
   pkg-config \
   zip \
   unzip \
@@ -62,8 +60,8 @@ RUN pip3 install six
 COPY . /mediapipe/
 
 # Install bazel
-
+# Please match the current MediaPipe Bazel requirements according to docs.
-ARG BAZEL_VERSION=1.1.0
+ARG BAZEL_VERSION=2.0.0
 RUN mkdir /bazel && \
     wget --no-check-certificate -O /bazel/installer.sh "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
     wget --no-check-certificate -O  /bazel/LICENSE.txt "https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE" && \

mediapipe/examples/coral/WORKSPACE

@@ -1,356 +0,0 @@
-workspace(name = "mediapipe")
-
-load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
-
-skylib_version = "0.8.0"
-http_archive(
-    name = "bazel_skylib",
-    type = "tar.gz",
-    url = "https://github.com/bazelbuild/bazel-skylib/releases/download/{}/bazel-skylib.{}.tar.gz".format (skylib_version, skylib_version),
-    sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
-)
-load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "1.0.0",
-               maximum_bazel_version = "1.2.1")
-
-
-# ABSL cpp library lts_2020_02_25
-http_archive(
-    name = "com_google_absl",
-    urls = [
-        "https://github.com/abseil/abseil-cpp/archive/20200225.tar.gz",
-    ],
-    # Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
-    patches = [
-        "@//third_party:com_google_absl_f863b622fe13612433fdf43f76547d5edda0c93001.diff"
-    ],
-    patch_args = [
-        "-p1",
-    ],
-    strip_prefix = "abseil-cpp-20200225",
-    sha256 = "728a813291bdec2aa46eab8356ace9f75ac2ed9dfe2df5ab603c4e6c09f1c353"
-)
-
-http_archive(
-    name = "rules_cc",
-    strip_prefix = "rules_cc-master",
-    urls = ["https://github.com/bazelbuild/rules_cc/archive/master.zip"],
-)
-
-# GoogleTest/GoogleMock framework. Used by most unit-tests.
-http_archive(
-     name = "com_google_googletest",
-     urls = ["https://github.com/google/googletest/archive/master.zip"],
-     strip_prefix = "googletest-master",
-)
-
-# Google Benchmark library.
-http_archive(
-    name = "com_google_benchmark",
-    urls = ["https://github.com/google/benchmark/archive/master.zip"],
-    strip_prefix = "benchmark-master",
-    build_file = "@//third_party:benchmark.BUILD",
-)
-
-# gflags needed by glog
-http_archive(
-    name = "com_github_gflags_gflags",
-    sha256 = "6e16c8bc91b1310a44f3965e616383dbda48f83e8c1eaa2370a215057b00cabe",
-    strip_prefix = "gflags-77592648e3f3be87d6c7123eb81cbad75f9aef5a",
-    urls = [
-        "https://mirror.bazel.build/github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
-        "https://github.com/gflags/gflags/archive/77592648e3f3be87d6c7123eb81cbad75f9aef5a.tar.gz",
-    ],
-)
-
-# glog
-http_archive(
-    name = "com_github_glog_glog",
-    url = "https://github.com/google/glog/archive/v0.3.5.zip",
-    sha256 = "267103f8a1e9578978aa1dc256001e6529ef593e5aea38193d31c2872ee025e8",
-    strip_prefix = "glog-0.3.5",
-    build_file = "@//third_party:glog.BUILD",
-    patches = [
-        "@//third_party:com_github_glog_glog_9779e5ea6ef59562b030248947f787d1256132ae.diff"
-    ],
-    patch_args = [
-        "-p1",
-    ],
-)
-
-# easyexif
-http_archive(
-    name = "easyexif",
-    url = "https://github.com/mayanklahiri/easyexif/archive/master.zip",
-    strip_prefix = "easyexif-master",
-    build_file = "@//third_party:easyexif.BUILD",
-)
-
-# libyuv
-http_archive(
-    name = "libyuv",
-    urls = ["https://chromium.googlesource.com/libyuv/libyuv/+archive/refs/heads/master.tar.gz"],
-    build_file = "@//third_party:libyuv.BUILD",
-)
-
-http_archive(
-    name = "com_google_protobuf_javalite",
-    sha256 = "79d102c61e2a479a0b7e5fc167bcfaa4832a0c6aad4a75fa7da0480564931bcc",
-    strip_prefix = "protobuf-384989534b2246d413dbcd750744faab2607b516",
-    urls = ["https://github.com/google/protobuf/archive/384989534b2246d413dbcd750744faab2607b516.zip"],
-)
-
-http_archive(
-    name = "com_google_audio_tools",
-    strip_prefix = "multichannel-audio-tools-master",
-    urls = ["https://github.com/google/multichannel-audio-tools/archive/master.zip"],
-)
-
-# Needed by TensorFlow
-http_archive(
-    name = "io_bazel_rules_closure",
-    sha256 = "e0a111000aeed2051f29fcc7a3f83be3ad8c6c93c186e64beb1ad313f0c7f9f9",
-    strip_prefix = "rules_closure-cf1e44edb908e9616030cc83d085989b8e6cd6df",
-    urls = [
-        "http://mirror.tensorflow.org/github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz",
-        "https://github.com/bazelbuild/rules_closure/archive/cf1e44edb908e9616030cc83d085989b8e6cd6df.tar.gz",  # 2019-04-04
-    ],
-)
-
-# 2020-02-12
-# The last commit before TensorFlow switched to Bazel 2.0
-_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328"
-_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13"
-http_archive(
-    name = "org_tensorflow",
-    urls = [
-      "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
-      "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
-    ],
-    # A compatibility patch
-    patches = [
-        "@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff"
-    ],
-    patch_args = [
-        "-p1",
-    ],
-    strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
-    sha256 = _TENSORFLOW_SHA256,
-)
-
-load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
-tf_workspace(tf_repo_name = "org_tensorflow")
-
-http_archive(
-    name = "ceres_solver",
-    url = "https://github.com/ceres-solver/ceres-solver/archive/1.14.0.zip",
-    patches = [
-        "@//third_party:ceres_solver_9bf9588988236279e1262f75d7f4d85711dfa172.diff"
-    ],
-    patch_args = [
-        "-p1",
-    ],
-    strip_prefix = "ceres-solver-1.14.0",
-    sha256 = "5ba6d0db4e784621fda44a50c58bb23b0892684692f0c623e2063f9c19f192f1"
-)
-
-# Please run
-# $ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
-#                        libopencv-calib3d-dev libopencv-features2d-dev \
-#                        libopencv-imgproc-dev libopencv-video-dev
-new_local_repository(
-    name = "linux_opencv",
-    build_file = "@//third_party:opencv_linux.BUILD",
-    path = "/usr",
-)
-
-new_local_repository(
-    name = "linux_ffmpeg",
-    build_file = "@//third_party:ffmpeg_linux.BUILD",
-    path = "/usr"
-)
-
-# Please run $ brew install opencv@3
-new_local_repository(
-    name = "macos_opencv",
-    build_file = "@//third_party:opencv_macos.BUILD",
-    path = "/usr",
-)
-
-new_local_repository(
-    name = "macos_ffmpeg",
-    build_file = "@//third_party:ffmpeg_macos.BUILD",
-    path = "/usr",
-)
-
-http_archive(
-    name = "android_opencv",
-    build_file = "@//third_party:opencv_android.BUILD",
-    strip_prefix = "OpenCV-android-sdk",
-    type = "zip",
-    url = "https://github.com/opencv/opencv/releases/download/3.4.3/opencv-3.4.3-android-sdk.zip",
-)
-
-# After OpenCV 3.2.0, the pre-compiled opencv2.framework has google protobuf symbols, which will
-# trigger duplicate symbol errors in the linking stage of building a mediapipe ios app.
-# To get a higher version of OpenCV for iOS, opencv2.framework needs to be built from source with
-# '-DBUILD_PROTOBUF=OFF -DBUILD_opencv_dnn=OFF'.
-http_archive(
-    name = "ios_opencv",
-    sha256 = "7dd536d06f59e6e1156b546bd581523d8df92ce83440002885ec5abc06558de2",
-    build_file = "@//third_party:opencv_ios.BUILD",
-    type = "zip",
-    url = "https://github.com/opencv/opencv/releases/download/3.2.0/opencv-3.2.0-ios-framework.zip",
-)
-
-RULES_JVM_EXTERNAL_TAG = "2.2"
-RULES_JVM_EXTERNAL_SHA = "f1203ce04e232ab6fdd81897cf0ff76f2c04c0741424d192f28e65ae752ce2d6"
-
-http_archive(
-    name = "rules_jvm_external",
-    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
-    sha256 = RULES_JVM_EXTERNAL_SHA,
-    url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
-)
-
-load("@rules_jvm_external//:defs.bzl", "maven_install")
-
-maven_install(
-    artifacts = [
-        "androidx.annotation:annotation:aar:1.1.0",
-        "androidx.appcompat:appcompat:aar:1.1.0-rc01",
-        "androidx.camera:camera-core:aar:1.0.0-alpha06",
-        "androidx.camera:camera-camera2:aar:1.0.0-alpha06",
-        "androidx.constraintlayout:constraintlayout:aar:1.1.3",
-        "androidx.core:core:aar:1.1.0-rc03",
-        "androidx.legacy:legacy-support-v4:aar:1.0.0",
-        "androidx.recyclerview:recyclerview:aar:1.1.0-beta02",
-        "com.google.android.material:material:aar:1.0.0-rc01",
-    ],
-    repositories = [
-        "https://dl.google.com/dl/android/maven2",
-        "https://repo1.maven.org/maven2",
-    ],
-)
-
-maven_server(
-    name = "google_server",
-    url = "https://dl.google.com/dl/android/maven2",
-)
-
-maven_jar(
-    name = "androidx_lifecycle",
-    artifact = "androidx.lifecycle:lifecycle-common:2.0.0",
-    sha1 = "e070ffae07452331bc5684734fce6831d531785c",
-    server = "google_server",
-)
-
-maven_jar(
-    name = "androidx_concurrent_futures",
-    artifact = "androidx.concurrent:concurrent-futures:1.0.0-alpha03",
-    sha1 = "b528df95c7e2fefa2210c0c742bf3e491c1818ae",
-    server = "google_server",
-)
-
-maven_jar(
-    name = "com_google_guava_android",
-    artifact = "com.google.guava:guava:27.0.1-android",
-    sha1 = "b7e1c37f66ef193796ccd7ea6e80c2b05426182d",
-)
-
-maven_jar(
-    name = "com_google_common_flogger",
-    artifact = "com.google.flogger:flogger:0.3.1",
-    sha1 = "585030fe1ec709760cbef997a459729fb965df0e",
-)
-
-maven_jar(
-    name = "com_google_common_flogger_system_backend",
-    artifact = "com.google.flogger:flogger-system-backend:0.3.1",
-    sha1 = "287b569d76abcd82f9de87fe41829fbc7ebd8ac9",
-)
-
-maven_jar(
-    name = "com_google_code_findbugs",
-    artifact = "com.google.code.findbugs:jsr305:3.0.2",
-    sha1 = "25ea2e8b0c338a877313bd4672d3fe056ea78f0d",
-)
-
-# You may run setup_android.sh to install Android SDK and NDK.
-android_ndk_repository(
-    name = "androidndk",
-)
-
-android_sdk_repository(
-    name = "androidsdk",
-)
-
-# iOS basic build deps.
-
-http_archive(
-    name = "build_bazel_rules_apple",
-    sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e",
-    strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3",
-    url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz",
-)
-
-load(
-    "@build_bazel_rules_apple//apple:repositories.bzl",
-    "apple_rules_dependencies",
-)
-
-apple_rules_dependencies()
-
-load(
-    "@build_bazel_rules_swift//swift:repositories.bzl",
-    "swift_rules_dependencies",
-)
-
-swift_rules_dependencies()
-
-load(
-    "@build_bazel_apple_support//lib:repositories.bzl",
-    "apple_support_dependencies",
-)
-
-apple_support_dependencies()
-
-# More iOS deps.
-
-http_archive(
-    name = "google_toolbox_for_mac",
-    url = "https://github.com/google/google-toolbox-for-mac/archive/v2.2.1.zip",
-    sha256 = "e3ac053813c989a88703556df4dc4466e424e30d32108433ed6beaec76ba4fdc",
-    strip_prefix = "google-toolbox-for-mac-2.2.1",
-    build_file = "@//third_party:google_toolbox_for_mac.BUILD",
-)
-
-### Coral ###
-
-#COMMIT=$(git ls-remote https://github.com/google-coral/crosstool master | awk '{print $1}')
-#SHA256=$(curl -L "https://github.com/google-coral/crosstool/archive/${COMMIT}.tar.gz" | sha256sum | awk '{print $1}')
-# Oct 2019
-#COMMIT=9e00d5be43bf001f883b5700f5d04882fea00229
-#SHA256=cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb
-http_archive(
-    name = "coral_crosstool",
-    sha256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb",
-    strip_prefix = "crosstool-9e00d5be43bf001f883b5700f5d04882fea00229",
-    urls = [
-        "https://github.com/google-coral/crosstool/archive/9e00d5be43bf001f883b5700f5d04882fea00229.tar.gz",
-    ],
-)
-load("@coral_crosstool//:configure.bzl", "cc_crosstool")
-cc_crosstool(name = "crosstool")
-
-# EdgeTPU
-new_local_repository(
-    name = "edgetpu",
-    path = "/edgetpu/libedgetpu",
-    build_file = "/edgetpu/libedgetpu/BUILD"
-)
-new_local_repository(
-    name = "libedgetpu",
-    path = "/usr/lib/aarch64-linux-gnu",
-    build_file = "/edgetpu/libedgetpu/BUILD"
-)

mediapipe/examples/coral/WORKSPACE.coral

@@ -0,0 +1,30 @@
+
+### Coral additions to MediaPipe WORKSPACE ###
+
+#COMMIT=$(git ls-remote https://github.com/google-coral/crosstool master | awk '{print $1}')
+#SHA256=$(curl -L "https://github.com/google-coral/crosstool/archive/${COMMIT}.tar.gz" | sha256sum | awk '{print $1}')
+# Oct 2019
+#COMMIT=9e00d5be43bf001f883b5700f5d04882fea00229
+#SHA256=cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb
+http_archive(
+    name = "coral_crosstool",
+    sha256 = "cb31b1417ccdcf7dd9fca5ec63e1571672372c30427730255997a547569d2feb",
+    strip_prefix = "crosstool-9e00d5be43bf001f883b5700f5d04882fea00229",
+    urls = [
+        "https://github.com/google-coral/crosstool/archive/9e00d5be43bf001f883b5700f5d04882fea00229.tar.gz",
+    ],
+)
+load("@coral_crosstool//:configure.bzl", "cc_crosstool")
+cc_crosstool(name = "crosstool")
+
+# EdgeTPU
+new_local_repository(
+    name = "edgetpu",
+    path = "/edgetpu/libedgetpu",
+    build_file = "/edgetpu/libedgetpu/BUILD"
+)
+new_local_repository(
+    name = "libedgetpu",
+    path = "/usr/lib/aarch64-linux-gnu",
+    build_file = "/edgetpu/libedgetpu/BUILD"
+)

mediapipe/examples/coral/setup.sh

@@ -10,12 +10,25 @@ sleep 3
 
 mkdir -p opencv32_arm64_libs
 
+# prepare docker aux script
 cp mediapipe/examples/coral/update_sources.sh update_sources.sh
 chmod +x update_sources.sh
 
+# backup non-coral Dockerfile
 mv Dockerfile Dockerfile.orig
 cp mediapipe/examples/coral/Dockerfile Dockerfile
 
+# backup non-coral workspace
 cp WORKSPACE WORKSPACE.orig
-cp mediapipe/examples/coral/WORKSPACE WORKSPACE
 
+# create temps
+cp WORKSPACE WORKSPACE.1
+cp mediapipe/examples/coral/WORKSPACE.coral WORKSPACE.2
+
+# merge (shell decides concat order, unless numbered appropriately)
+cat WORKSPACE.1 WORKSPACE.2 > WORKSPACE
+
+# cleanup
+rm WORKSPACE.1 WORKSPACE.2
+
+echo 'done'

mediapipe/examples/desktop/autoflip/calculators/shot_boundary_calculator.cc

@@ -85,7 +85,7 @@ REGISTER_CALCULATOR(ShotBoundaryCalculator);
 void ShotBoundaryCalculator::ComputeHistogram(const cv::Mat& image,
                                               cv::Mat* image_histogram) {
   cv::Mat equalized_image;
-  cv::cvtColor(image.clone(), equalized_image, CV_RGB2GRAY);
+  cv::cvtColor(image.clone(), equalized_image, cv::COLOR_RGB2GRAY);
 
   double min, max;
   cv::minMaxLoc(equalized_image, &min, &max);

mediapipe/framework/BUILD

@@ -1496,13 +1496,17 @@ cc_test(
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/stream_handler:barrier_input_stream_handler",
         "//mediapipe/framework/stream_handler:early_close_input_stream_handler",
+        "//mediapipe/framework/stream_handler:fixed_size_input_stream_handler",
         "//mediapipe/framework/stream_handler:immediate_input_stream_handler",
         "//mediapipe/framework/stream_handler:mux_input_stream_handler",
+        "//mediapipe/framework/stream_handler:sync_set_input_stream_handler",
+        "//mediapipe/framework/stream_handler:timestamp_align_input_stream_handler",
         "//mediapipe/framework/tool:sink",
         "//mediapipe/framework/tool:status_util",
         "@com_google_absl//absl/container:fixed_array",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/time",
     ],
 )

mediapipe/framework/calculator_graph_test.cc

@@ -29,6 +29,7 @@
 #include "absl/memory/memory.h"
 #include "absl/strings/escaping.h"
 #include "absl/strings/str_cat.h"
+#include "absl/strings/str_format.h"
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
 #include "absl/time/clock.h"
@@ -4558,5 +4559,68 @@ TEST(CalculatorGraph, SimpleMuxCalculatorWithCustomInputStreamHandler) {
           testing::HasSubstr("ImmediateInputStreamHandler class comment")));
 }
 
+void DoTestMultipleGraphRuns(absl::string_view input_stream_handler,
+                             bool select_packet) {
+  std::string graph_proto = absl::StrFormat(R"(
+    input_stream: 'input'
+    input_stream: 'select'
+    node {
+      calculator: 'PassThroughCalculator'
+      input_stream: 'input'
+      input_stream: 'select'
+      input_stream_handler {
+        input_stream_handler: "%s"
+      }
+      output_stream: 'output'
+      output_stream: 'select_out'
+    }
+  )",
+                                            input_stream_handler.data());
+  CalculatorGraphConfig config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(graph_proto);
+  std::vector<Packet> packet_dump;
+  tool::AddVectorSink("output", &config, &packet_dump);
+
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(config));
+
+  struct Run {
+    Timestamp timestamp;
+    int value;
+  };
+  std::vector<Run> runs = {{.timestamp = Timestamp(2000), .value = 2},
+                           {.timestamp = Timestamp(1000), .value = 1}};
+  for (const Run& run : runs) {
+    MP_ASSERT_OK(graph.StartRun({}));
+
+    if (select_packet) {
+      MP_EXPECT_OK(graph.AddPacketToInputStream(
+          "select", MakePacket<int>(0).At(run.timestamp)));
+    }
+    MP_EXPECT_OK(graph.AddPacketToInputStream(
+        "input", MakePacket<int>(run.value).At(run.timestamp)));
+    MP_ASSERT_OK(graph.WaitUntilIdle());
+    ASSERT_EQ(1, packet_dump.size());
+    EXPECT_EQ(run.value, packet_dump[0].Get<int>());
+    EXPECT_EQ(run.timestamp, packet_dump[0].Timestamp());
+
+    MP_ASSERT_OK(graph.CloseAllPacketSources());
+    MP_ASSERT_OK(graph.WaitUntilDone());
+
+    packet_dump.clear();
+  }
+}
+
+TEST(CalculatorGraph, MultipleRunsWithDifferentInputStreamHandlers) {
+  DoTestMultipleGraphRuns("BarrierInputStreamHandler", true);
+  DoTestMultipleGraphRuns("DefaultInputStreamHandler", true);
+  DoTestMultipleGraphRuns("EarlyCloseInputStreamHandler", true);
+  DoTestMultipleGraphRuns("FixedSizeInputStreamHandler", true);
+  DoTestMultipleGraphRuns("ImmediateInputStreamHandler", false);
+  DoTestMultipleGraphRuns("MuxInputStreamHandler", true);
+  DoTestMultipleGraphRuns("SyncSetInputStreamHandler", true);
+  DoTestMultipleGraphRuns("TimestampAlignInputStreamHandler", true);
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/framework/calculator_node.cc

@@ -814,7 +814,8 @@ std::string CalculatorNode::DebugName() const {
         input_stream_handler_->FinalizeInputSet(input_timestamp, inputs);
         output_stream_handler_->PrepareOutputs(input_timestamp, outputs);
 
-        VLOG(2) << "Calling Calculator::Process() for node: " << DebugName();
+        VLOG(2) << "Calling Calculator::Process() for node: " << DebugName()
+                << " timestamp: " << input_timestamp;
 
         if (OutputsAreConstant(calculator_context)) {
           // Do nothing.
@@ -826,6 +827,9 @@ std::string CalculatorNode::DebugName() const {
           result = calculator_->Process(calculator_context);
         }
 
+        VLOG(2) << "Called Calculator::Process() for node: " << DebugName()
+                << " timestamp: " << input_timestamp;
+
         // Removes one packet from each shard and progresses to the next input
         // timestamp.
         input_stream_handler_->ClearCurrentInputs(calculator_context);

mediapipe/framework/calculator_profile.proto

@@ -113,7 +113,10 @@ message GraphTrace {
     optional int32 stream_id = 4;
 
     // The address of the packet contents.
-    optional int64 packet_id = 5;
+    optional int64 packet_id = 5 [deprecated = true];
+
+    // Data describing the event, such as the packet contents.
+    optional int64 event_data = 6;
   }
 
   // The kind of event recorded.
@@ -133,6 +136,7 @@ message GraphTrace {
     DSP_TASK = 12;
     TPU_TASK = 13;
     GPU_CALIBRATION = 14;
+    PACKET_QUEUED = 15;
   }
 
   // The timing for one packet set being processed at one caclulator node.

mediapipe/framework/formats/BUILD

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
 
 package(
     default_visibility = ["//visibility:private"],
@@ -24,95 +24,45 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-proto_library(
+mediapipe_proto_library(
     name = "detection_proto",
     srcs = ["detection.proto"],
     visibility = ["//visibility:public"],
     deps = ["//mediapipe/framework/formats:location_data_proto"],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "classification_proto",
     srcs = ["classification.proto"],
     visibility = ["//visibility:public"],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "image_format_proto",
     srcs = ["image_format.proto"],
     visibility = ["//visibility:public"],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "matrix_data_proto",
     srcs = ["matrix_data.proto"],
     visibility = ["//visibility:public"],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "location_data_proto",
     srcs = ["location_data.proto"],
+    portable_deps = ["//mediapipe/framework/formats/annotation:rasterization_cc_proto"],
     visibility = ["//visibility:public"],
     deps = ["//mediapipe/framework/formats/annotation:rasterization_proto"],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "time_series_header_proto",
     srcs = ["time_series_header.proto"],
     visibility = ["//visibility:public"],
 )
 
-mediapipe_cc_proto_library(
-    name = "detection_cc_proto",
-    srcs = ["detection.proto"],
-    cc_deps = [":location_data_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":detection_proto"],
-)
-
-java_lite_proto_library(
-    name = "detection_java_proto_lite",
-    strict_deps = 0,
-    visibility = ["//mediapipe:__subpackages__"],
-    deps = [":detection_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "classification_cc_proto",
-    srcs = ["classification.proto"],
-    visibility = ["//visibility:public"],
-    deps = [":classification_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "image_format_cc_proto",
-    srcs = ["image_format.proto"],
-    visibility = ["//visibility:public"],
-    deps = [":image_format_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "matrix_data_cc_proto",
-    srcs = ["matrix_data.proto"],
-    visibility = ["//visibility:public"],
-    deps = [":matrix_data_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "location_data_cc_proto",
-    srcs = ["location_data.proto"],
-    cc_deps = ["//mediapipe/framework/formats/annotation:rasterization_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":location_data_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "time_series_header_cc_proto",
-    srcs = ["time_series_header.proto"],
-    visibility = ["//visibility:public"],
-    deps = [":time_series_header_proto"],
-)
-
 cc_library(
     name = "deleting_file",
     srcs = ["deleting_file.cc"],
@@ -245,33 +195,14 @@ cc_test(
     ],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "rect_proto",
     srcs = ["rect.proto"],
-    visibility = [
+    visibility = ["//visibility:public"],
-        "//visibility:public",
+    deps = ["//mediapipe/framework/formats:location_data_proto"],
-    ],
 )
 
-mediapipe_cc_proto_library(
+mediapipe_proto_library(
-    name = "rect_cc_proto",
-    srcs = ["rect.proto"],
-    visibility = [
-        "//visibility:public",
-    ],
-    deps = [":rect_proto"],
-)
-
-java_lite_proto_library(
-    name = "rect_java_proto_lite",
-    strict_deps = 0,
-    visibility = [
-        "//mediapipe:__subpackages__",
-    ],
-    deps = [":rect_proto"],
-)
-
-proto_library(
     name = "landmark_proto",
     srcs = ["landmark.proto"],
     visibility = [
@@ -279,22 +210,6 @@ proto_library(
     ],
 )
 
-mediapipe_cc_proto_library(
-    name = "landmark_cc_proto",
-    srcs = ["landmark.proto"],
-    visibility = ["//visibility:public"],
-    deps = [":landmark_proto"],
-)
-
-java_lite_proto_library(
-    name = "landmark_java_proto_lite",
-    strict_deps = 0,
-    visibility = [
-        "//mediapipe:__subpackages__",
-    ],
-    deps = [":landmark_proto"],
-)
-
 # Expose the proto source files for building mediapipe AAR.
 filegroup(
     name = "protos_src",

mediapipe/framework/formats/annotation/BUILD

@@ -14,42 +14,28 @@
 # limitations under the License.
 #
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
+
+package(default_visibility = ["//visibility:private"])
+
 licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+mediapipe_proto_library(
-
-package(default_visibility = ["//visibility:private"])
-
-proto_library(
     name = "locus_proto",
     srcs = ["locus.proto"],
+    portable_deps = ["//mediapipe/framework/formats/annotation:rasterization_cc_proto"],
     visibility = ["//visibility:public"],
     deps = ["//mediapipe/framework/formats/annotation:rasterization_proto"],
 )
 
-proto_library(
+mediapipe_proto_library(
     name = "rasterization_proto",
     srcs = ["rasterization.proto"],
     visibility = ["//visibility:public"],
 )
 
-mediapipe_cc_proto_library(
-    name = "locus_cc_proto",
-    srcs = ["locus.proto"],
-    cc_deps = [":rasterization_cc_proto"],
-    visibility = ["//visibility:public"],
-    deps = [":locus_proto"],
-)
-
-mediapipe_cc_proto_library(
-    name = "rasterization_cc_proto",
-    srcs = ["rasterization.proto"],
-    visibility = ["//visibility:public"],
-    deps = [":rasterization_proto"],
-)
-
 # Expose the proto source files for building mediapipe AAR.
 filegroup(
     name = "protos_src",

mediapipe/framework/input_stream_handler.cc

@@ -230,8 +230,36 @@ void InputStreamHandler::FinalizeInputSet(Timestamp timestamp,
   }
 }
 
+// Returns the default CalculatorContext.
+CalculatorContext* GetCalculatorContext(CalculatorContextManager* manager) {
+  return (manager && manager->HasDefaultCalculatorContext())
+             ? manager->GetDefaultCalculatorContext()
+             : nullptr;
+}
+
+// Logs the current queue size of an input stream.
+void LogQueuedPackets(CalculatorContext* context, InputStreamManager* stream,
+                      Packet queue_tail) {
+  if (context) {
+    TraceEvent event = TraceEvent(TraceEvent::PACKET_QUEUED)
+                           .set_node_id(context->NodeId())
+                           .set_input_ts(queue_tail.Timestamp())
+                           .set_stream_id(&stream->Name())
+                           .set_event_data(stream->QueueSize() + 1);
+    ::mediapipe::LogEvent(context->GetProfilingContext(),
+                          event.set_packet_ts(queue_tail.Timestamp()));
+    Packet queue_head = stream->QueueHead();
+    if (!queue_head.IsEmpty()) {
+      ::mediapipe::LogEvent(context->GetProfilingContext(),
+                            event.set_packet_ts(queue_head.Timestamp()));
+    }
+  }
+}
+
 void InputStreamHandler::AddPackets(CollectionItemId id,
                                     const std::list<Packet>& packets) {
+  LogQueuedPackets(GetCalculatorContext(calculator_context_manager_),
+                   input_stream_managers_.Get(id), packets.back());
   bool notify = false;
   ::mediapipe::Status result =
       input_stream_managers_.Get(id)->AddPackets(packets, &notify);
@@ -245,6 +273,8 @@ void InputStreamHandler::AddPackets(CollectionItemId id,
 
 void InputStreamHandler::MovePackets(CollectionItemId id,
                                      std::list<Packet>* packets) {
+  LogQueuedPackets(GetCalculatorContext(calculator_context_manager_),
+                   input_stream_managers_.Get(id), packets->back());
   bool notify = false;
   ::mediapipe::Status result =
       input_stream_managers_.Get(id)->MovePackets(packets, &notify);
@@ -307,6 +337,8 @@ SyncSet::SyncSet(InputStreamHandler* input_stream_handler,
     : input_stream_handler_(input_stream_handler),
       stream_ids_(std::move(stream_ids)) {}
 
+void SyncSet::PrepareForRun() { last_processed_ts_ = Timestamp::Unset(); }
+
 NodeReadiness SyncSet::GetReadiness(Timestamp* min_stream_timestamp) {
   Timestamp min_bound = Timestamp::Done();
   Timestamp min_packet = Timestamp::Done();

mediapipe/framework/input_stream_handler.h

@@ -201,6 +201,9 @@ class InputStreamHandler {
     SyncSet(InputStreamHandler* input_stream_handler,
             std::vector<CollectionItemId> stream_ids);
 
+    // Reinitializes this SyncSet before each CalculatorGraph run.
+    void PrepareForRun();
+
     // Answers whether this stream is ready for Process or Close.
     NodeReadiness GetReadiness(Timestamp* min_stream_timestamp);
 

mediapipe/framework/input_stream_manager.cc

@@ -152,7 +152,7 @@ template <typename Container>
       // If the caller is MovePackets(), packet's underlying holder should be
       // transferred into queue_. Otherwise, queue_ keeps a copy of the packet.
       ++num_packets_added_;
-      VLOG(2) << "Input stream:" << name_
+      VLOG(3) << "Input stream:" << name_
               << " has added packet at time: " << packet.Timestamp();
       if (std::is_const<
               typename std::remove_reference<Container>::type>::value) {
@@ -163,15 +163,15 @@ template <typename Container>
     }
     queue_became_full = (!was_queue_full && max_queue_size_ != -1 &&
                          queue_.size() >= max_queue_size_);
-    VLOG_IF(2, queue_.size() > 1)
+    VLOG_IF(3, queue_.size() > 1)
         << "Queue size greater than 1: stream name: " << name_
         << " queue_size: " << queue_.size();
-    VLOG(2) << "Input stream:" << name_
+    VLOG(3) << "Input stream:" << name_
             << " becomes non-empty status:" << queue_became_non_empty
             << " Size: " << queue_.size();
   }
   if (queue_became_full) {
-    VLOG(2) << "Queue became full: " << Name();
+    VLOG(3) << "Queue became full: " << Name();
     becomes_full_callback_(this, &last_reported_stream_full_);
   }
   *notify = queue_became_non_empty;
@@ -257,7 +257,7 @@ Packet InputStreamManager::PopPacketAtTimestamp(Timestamp timestamp,
       next_timestamp_bound_ = timestamp.NextAllowedInStream();
     }
 
-    VLOG(2) << "Input stream " << name_
+    VLOG(3) << "Input stream " << name_
             << " selecting at timestamp:" << timestamp.Value()
             << " next timestamp bound: " << next_timestamp_bound_;
 
@@ -282,13 +282,13 @@ Packet InputStreamManager::PopPacketAtTimestamp(Timestamp timestamp,
       ++(*num_packets_dropped);
     }
 
-    VLOG(2) << "Input stream removed packets:" << name_
+    VLOG(3) << "Input stream removed packets:" << name_
             << " Size:" << queue_.size();
     queue_became_non_full = (was_queue_full && queue_.size() < max_queue_size_);
     *stream_is_done = IsDone();
   }
   if (queue_became_non_full) {
-    VLOG(2) << "Queue became non-full: " << Name();
+    VLOG(3) << "Queue became non-full: " << Name();
     becomes_not_full_callback_(this, &last_reported_stream_full_);
   }
   return packet;
@@ -302,7 +302,7 @@ Packet InputStreamManager::PopQueueHead(bool* stream_is_done) {
   {
     absl::MutexLock stream_lock(&stream_mutex_);
 
-    VLOG(2) << "Input stream " << name_ << " selecting at queue head";
+    VLOG(3) << "Input stream " << name_ << " selecting at queue head";
 
     // Check if queue is full.
     bool was_queue_full =
@@ -315,13 +315,13 @@ Packet InputStreamManager::PopQueueHead(bool* stream_is_done) {
       packet = Packet();
     }
 
-    VLOG(2) << "Input stream removed a packet:" << name_
+    VLOG(3) << "Input stream removed a packet:" << name_
             << " Size:" << queue_.size();
     queue_became_non_full = (was_queue_full && queue_.size() < max_queue_size_);
     *stream_is_done = IsDone();
   }
   if (queue_became_non_full) {
-    VLOG(2) << "Queue became non-full: " << Name();
+    VLOG(3) << "Queue became non-full: " << Name();
     becomes_not_full_callback_(this, &last_reported_stream_full_);
   }
   return packet;
@@ -349,10 +349,10 @@ void InputStreamManager::SetMaxQueueSize(int max_queue_size) {
 
   // QueueSizeCallback is called with no mutexes held.
   if (!was_full && is_full) {
-    VLOG(2) << "Queue became full: " << Name();
+    VLOG(3) << "Queue became full: " << Name();
     becomes_full_callback_(this, &last_reported_stream_full_);
   } else if (was_full && !is_full) {
-    VLOG(2) << "Queue became non-full: " << Name();
+    VLOG(3) << "Queue became non-full: " << Name();
     becomes_not_full_callback_(this, &last_reported_stream_full_);
   }
 }
@@ -382,12 +382,12 @@ void InputStreamManager::ErasePacketsEarlierThan(Timestamp timestamp) {
       queue_.pop_front();
     }
 
-    VLOG(2) << "Input stream removed packets:" << name_
+    VLOG(3) << "Input stream removed packets:" << name_
             << " Size:" << queue_.size();
     queue_became_non_full = (was_queue_full && queue_.size() < max_queue_size_);
   }
   if (queue_became_non_full) {
-    VLOG(2) << "Queue became non-full: " << Name();
+    VLOG(3) << "Queue became non-full: " << Name();
     becomes_not_full_callback_(this, &last_reported_stream_full_);
   }
 }

mediapipe/framework/output_stream_manager.cc

@@ -164,9 +164,9 @@ void OutputStreamManager::PropagateUpdatesToMirrors(
     next_timestamp_bound_ = next_timestamp_bound;
   }
   std::list<Packet>* packets_to_propagate = output_stream_shard->OutputQueue();
-  VLOG(2) << "Output stream: " << Name()
+  VLOG(3) << "Output stream: " << Name()
           << " queue size: " << packets_to_propagate->size();
-  VLOG(2) << "Output stream: " << Name()
+  VLOG(3) << "Output stream: " << Name()
           << " next timestamp: " << next_timestamp_bound;
   bool add_packets = !packets_to_propagate->empty();
   bool set_bound =

mediapipe/framework/packet.h

@@ -540,11 +540,11 @@ const Holder<T>* HolderBase::As() const {
 
 inline Packet::Packet(const Packet& packet)
     : holder_(packet.holder_), timestamp_(packet.timestamp_) {
-  VLOG(2) << "Using copy constructor of " << packet.DebugString();
+  VLOG(4) << "Using copy constructor of " << packet.DebugString();
 }
 
 inline Packet& Packet::operator=(const Packet& packet) {
-  VLOG(2) << "Using copy assignment operator of " << packet.DebugString();
+  VLOG(4) << "Using copy assignment operator of " << packet.DebugString();
   if (this != &packet) {
     holder_ = packet.holder_;
     timestamp_ = packet.timestamp_;
@@ -559,11 +559,11 @@ inline ::mediapipe::StatusOr<std::unique_ptr<T>> Packet::Consume() {
   // Clients who use this function are responsible for ensuring that no
   // other thread is doing anything with this Packet.
   if (holder_.unique()) {
-    VLOG(1) << "Consuming the data of " << DebugString();
+    VLOG(2) << "Consuming the data of " << DebugString();
     ::mediapipe::StatusOr<std::unique_ptr<T>> release_result =
         holder_->As<T>()->Release();
     if (release_result.ok()) {
-      VLOG(1) << "Setting " << DebugString() << " to empty.";
+      VLOG(2) << "Setting " << DebugString() << " to empty.";
       holder_.reset();
     }
     return release_result;
@@ -582,11 +582,11 @@ inline ::mediapipe::StatusOr<std::unique_ptr<T>> Packet::ConsumeOrCopy(
   // If holder is the sole owner of the underlying data, consumes this packet.
   if (!holder_->HolderIsOfType<packet_internal::ForeignHolder<T>>() &&
       holder_.unique()) {
-    VLOG(1) << "Consuming the data of " << DebugString();
+    VLOG(2) << "Consuming the data of " << DebugString();
     ::mediapipe::StatusOr<std::unique_ptr<T>> release_result =
         holder_->As<T>()->Release();
     if (release_result.ok()) {
-      VLOG(1) << "Setting " << DebugString() << " to empty.";
+      VLOG(2) << "Setting " << DebugString() << " to empty.";
       holder_.reset();
     }
     if (was_copied) {
@@ -594,9 +594,9 @@ inline ::mediapipe::StatusOr<std::unique_ptr<T>> Packet::ConsumeOrCopy(
     }
     return release_result;
   }
-  VLOG(1) << "Copying the data of " << DebugString();
+  VLOG(2) << "Copying the data of " << DebugString();
   std::unique_ptr<T> data_ptr = absl::make_unique<T>(Get<T>());
-  VLOG(1) << "Setting " << DebugString() << " to empty.";
+  VLOG(2) << "Setting " << DebugString() << " to empty.";
   holder_.reset();
   if (was_copied) {
     *was_copied = true;
@@ -613,11 +613,11 @@ inline ::mediapipe::StatusOr<std::unique_ptr<T>> Packet::ConsumeOrCopy(
   // If holder is the sole owner of the underlying data, consumes this packet.
   if (!holder_->HolderIsOfType<packet_internal::ForeignHolder<T>>() &&
       holder_.unique()) {
-    VLOG(1) << "Consuming the data of " << DebugString();
+    VLOG(2) << "Consuming the data of " << DebugString();
     ::mediapipe::StatusOr<std::unique_ptr<T>> release_result =
         holder_->As<T>()->Release();
     if (release_result.ok()) {
-      VLOG(1) << "Setting " << DebugString() << " to empty.";
+      VLOG(2) << "Setting " << DebugString() << " to empty.";
       holder_.reset();
     }
     if (was_copied) {
@@ -625,7 +625,7 @@ inline ::mediapipe::StatusOr<std::unique_ptr<T>> Packet::ConsumeOrCopy(
     }
     return release_result;
   }
-  VLOG(1) << "Copying the data of " << DebugString();
+  VLOG(2) << "Copying the data of " << DebugString();
   const auto& original_array = Get<T>();
   // Type T is bounded array type, such as int[N] and float[M].
   // The new operator creates a new bounded array.
@@ -633,7 +633,7 @@ inline ::mediapipe::StatusOr<std::unique_ptr<T>> Packet::ConsumeOrCopy(
   // Copies bounded array data into data_ptr.
   std::copy(std::begin(original_array), std::end(original_array),
             std::begin(*data_ptr));
-  VLOG(1) << "Setting " << DebugString() << " to empty.";
+  VLOG(2) << "Setting " << DebugString() << " to empty.";
   holder_.reset();
   if (was_copied) {
     *was_copied = true;
@@ -650,14 +650,14 @@ inline ::mediapipe::StatusOr<std::unique_ptr<T>> Packet::ConsumeOrCopy(
 }
 
 inline Packet::Packet(Packet&& packet) {
-  VLOG(2) << "Using move constructor of " << packet.DebugString();
+  VLOG(4) << "Using move constructor of " << packet.DebugString();
   holder_ = std::move(packet.holder_);
   timestamp_ = packet.timestamp_;
   packet.timestamp_ = Timestamp::Unset();
 }
 
 inline Packet& Packet::operator=(Packet&& packet) {
-  VLOG(2) << "Using move assignment operator of " << packet.DebugString();
+  VLOG(4) << "Using move assignment operator of " << packet.DebugString();
   if (this != &packet) {
     holder_ = std::move(packet.holder_);
     timestamp_ = packet.timestamp_;

mediapipe/framework/port/BUILD

@@ -246,6 +246,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":opencv_core",
+        "//mediapipe/framework:port",
         "//third_party:opencv",
     ],
 )

mediapipe/framework/port/build_config.bzl

@@ -5,6 +5,98 @@
 
 load("@com_google_protobuf//:protobuf.bzl", "cc_proto_library", "py_proto_library")
 
+def provided_args(**kwargs):
+    """Returns the keyword arguments omitting None arguments."""
+    return {k: v for k, v in kwargs.items() if v != None}
+
+def allowed_args(**kwargs):
+    """Returns the keyword arguments allowed for proto_library().
+
+    Args:
+      **kwargs: the specified keyword arguments.
+    Returns:
+      the allowed keyword arguments.
+    """
+    result = dict(kwargs)
+    result.pop("cc_api_version", None)
+    return result
+
+# TODO: load this macro from a common helper file.
+def mediapipe_proto_library(
+        name,
+        srcs,
+        deps = [],
+        visibility = None,
+        testonly = 0,
+        compatible_with = [],
+        def_proto = True,
+        def_cc_proto = True,
+        def_py_proto = True,
+        def_java_lite_proto = True,
+        def_portable_proto = True,
+        portable_deps = None):
+    """Defines the proto_library targets needed for all mediapipe platforms.
+
+    Args:
+      name: the new proto_library target name.
+      srcs: the ".proto" source files to compile.
+      deps: the proto_library targets for all referenced protobufs.
+      portable_deps: the portable_proto_library targets for all referenced protobufs.
+      visibility: visibility of this target.
+      testonly: true means the proto can be used for testing only.
+      compatible_with: see go/target-constraints.
+      def_proto: define the proto_library target
+      def_cc_proto: define the cc_proto_library target
+      def_py_proto: define the py_proto_library target
+      def_java_lite_proto: define the java_lite_proto_library target
+      def_portable_proto: define the portable_proto_library target
+    """
+    _ignore = [def_portable_proto, portable_deps]
+
+    # The proto_library targets for the compiled ".proto" source files.
+    proto_deps = [":" + name]
+
+    if def_proto:
+        native.proto_library(**allowed_args(**provided_args(
+            name = name,
+            srcs = srcs,
+            deps = deps,
+            visibility = visibility,
+            testonly = testonly,
+            cc_api_version = 2,
+            compatible_with = compatible_with,
+        )))
+
+    if def_cc_proto:
+        cc_deps = [dep.replace("_proto", "_cc_proto") for dep in deps]
+        mediapipe_cc_proto_library(**provided_args(
+            name = name.replace("_proto", "_cc_proto"),
+            srcs = srcs,
+            deps = proto_deps,
+            cc_deps = cc_deps,
+            visibility = visibility,
+            testonly = testonly,
+        ))
+
+    if def_py_proto:
+        py_deps = [dep.replace("_proto", "_py_pb2") for dep in deps]
+        mediapipe_py_proto_library(**provided_args(
+            name = name.replace("_proto", "_py_pb2"),
+            srcs = srcs,
+            proto_deps = proto_deps,
+            py_proto_deps = py_deps,
+            visibility = visibility,
+            api_version = 2,
+        ))
+
+    if def_java_lite_proto:
+        native.java_lite_proto_library(**provided_args(
+            name = name.replace("_proto", "_java_proto_lite"),
+            deps = proto_deps,
+            strict_deps = 0,
+            visibility = visibility,
+        ))
+
 def mediapipe_py_proto_library(
         name,
         srcs,
@@ -21,14 +113,14 @@ def mediapipe_py_proto_library(
       py_proto_deps: a list of dependency labels for Bazel use; must be py_proto_library.
     """
     _ignore = [api_version, proto_deps]
-    py_proto_library(
+    py_proto_library(**provided_args(
         name = name,
         srcs = srcs,
         visibility = visibility,
         default_runtime = "@com_google_protobuf//:protobuf_python",
         protoc = "@com_google_protobuf//:protoc",
         deps = py_proto_deps + ["@com_google_protobuf//:protobuf_python"],
-    )
+    ))
 
 def mediapipe_cc_proto_library(name, srcs, visibility, deps = [], cc_deps = [], testonly = 0):
     """Generate cc_proto_library for mediapipe open source version.
@@ -41,7 +133,7 @@ def mediapipe_cc_proto_library(name, srcs, visibility, deps = [], cc_deps = [],
         testonly: test only proto or not.
     """
     _ignore = [deps]
-    cc_proto_library(
+    cc_proto_library(**provided_args(
         name = name,
         srcs = srcs,
         visibility = visibility,
@@ -51,4 +143,4 @@ def mediapipe_cc_proto_library(name, srcs, visibility, deps = [], cc_deps = [],
         protoc = "@com_google_protobuf//:protoc",
         default_runtime = "@com_google_protobuf//:protobuf",
         alwayslink = 1,
-    )
+    ))

mediapipe/framework/port/opencv_video_inc.h

@@ -17,6 +17,7 @@
 
 #include <opencv2/core/version.hpp>
 
+#include "mediapipe/framework/port.h"
 #include "mediapipe/framework/port/opencv_core_inc.h"
 
 #ifdef CV_VERSION_EPOCH  // for OpenCV 2.x
@@ -83,7 +84,7 @@ inline int fourcc(char c1, char c2, char c3, char c4) {
 #include <opencv2/video.hpp>
 #include <opencv2/videoio.hpp>
 
-#if CV_VERSION_MAJOR == 4
+#if CV_VERSION_MAJOR == 4 && !defined(MEDIAPIPE_MOBILE)
 #include <opencv2/optflow.hpp>
 
 namespace cv {

mediapipe/framework/profiler/BUILD

@@ -270,17 +270,15 @@ cc_library(
     srcs = select({
         "//conditions:default": ["profiler_resource_util.cc"],
         "//mediapipe:android": ["profiler_resource_util_android.cc"],
-        "//mediapipe:apple": ["profiler_resource_util_apple.cc"],
+        "//mediapipe:ios": ["profiler_resource_util_ios.cc"],
-        "//mediapipe:macos": ["profiler_resource_util.cc"],
     }),
     hdrs = ["profiler_resource_util.h"],
     # We use Objective-C++ on iOS.
     copts = select({
         "//conditions:default": [],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "-ObjC++",
         ],
-        "//mediapipe:macos": [],
     }),
     visibility = [
         "//mediapipe/framework:mediapipe_internal",

mediapipe/framework/profiler/graph_profiler_stub.h

@@ -53,6 +53,8 @@ class TraceEvent {
     GPU_TASK,
     DSP_TASK,
     TPU_TASK,
+    GPU_CALIBRATION,
+    PACKET_QUEUED,
   };
   TraceEvent(const EventType& event_type) {}
   TraceEvent() {}
@@ -69,6 +71,7 @@ class TraceEvent {
   inline TraceEvent& set_packet_data_id(const Packet* packet) { return *this; }
   inline TraceEvent& set_thread_id(int thread_id) { return *this; }
   inline TraceEvent& set_is_finish(bool is_finish) { return *this; }
+  inline TraceEvent& set_event_data(int64 data) { return *this; }
 };
 
 // Empty implementation of ProfilingContext to be used in place of the

mediapipe/framework/profiler/graph_tracer.cc

@@ -26,6 +26,7 @@
 namespace mediapipe {
 
 namespace {
+using EventType = GraphTrace::EventType;
 
 const absl::Duration kDefaultTraceLogInterval = absl::Milliseconds(500);
 
@@ -52,14 +53,18 @@ int64 GraphTracer::GetTraceLogCapacity() {
 
 GraphTracer::GraphTracer(const ProfilerConfig& profiler_config)
     : profiler_config_(profiler_config), trace_buffer_(GetTraceLogCapacity()) {
-  event_types_disabled_.resize(static_cast<int>(GraphTrace::EventType_MAX + 1));
+  for (int disabled : profiler_config_.trace_event_types_disabled()) {
-  for (int32 event_type : profiler_config_.trace_event_types_disabled()) {
+    EventType event_type = static_cast<EventType>(disabled);
-    event_types_disabled_[event_type] = true;
+    (*trace_event_registry())[event_type].set_enabled(false);
   }
 }
 
+TraceEventRegistry* GraphTracer::trace_event_registry() {
+  return trace_builder_.trace_event_registry();
+}
+
 void GraphTracer::LogEvent(TraceEvent event) {
-  if (event_types_disabled_[static_cast<int>(event.event_type)]) {
+  if (!(*trace_event_registry())[event.event_type].enabled()) {
     return;
   }
   event.set_thread_id(GetCurrentThreadId());

mediapipe/framework/profiler/graph_tracer.h

@@ -55,6 +55,9 @@ class GraphTracer {
   // Create a tracer to record up to |capacity| recent events.
   GraphTracer(const ProfilerConfig& profiler_config);
 
+  // Returns the registry of trace event types.
+  TraceEventRegistry* trace_event_registry();
+
   // Append a TraceEvent to the TraceBuffer.
   void LogEvent(TraceEvent event);
 
@@ -85,9 +88,6 @@ class GraphTracer {
   // The settings for this tracer.
   ProfilerConfig profiler_config_;
 
-  // Indicates event types that will not be logged.
-  std::vector<bool> event_types_disabled_;
-
   // The circular buffer of TraceEvents.
   TraceBuffer trace_buffer_;
 

mediapipe/framework/profiler/graph_tracer_test.cc

@@ -161,7 +161,7 @@ TEST_F(GraphTracerTest, CalculatorTrace) {
                     finish_time: 0
                     packet_timestamp: 0
                     stream_id: 1
-                    packet_id: 1
+                    event_data: 1
                   }
                   output_trace { packet_timestamp: 0 stream_id: 2 }
                 }
@@ -236,7 +236,7 @@ TEST_F(GraphTracerTest, GraphTrace) {
                     finish_time: 0
                     packet_timestamp: 0
                     stream_id: 1
-                    packet_id: 1
+                    event_data: 1
                   }
                   output_trace { packet_timestamp: 0 stream_id: 2 }
                   output_trace { packet_timestamp: 0 stream_id: 3 }
@@ -254,7 +254,7 @@ TEST_F(GraphTracerTest, GraphTrace) {
                     finish_time: 11000
                     packet_timestamp: 0
                     stream_id: 2
-                    packet_id: 2
+                    event_data: 2
                   }
                   output_trace { packet_timestamp: 0 stream_id: 4 }
                 }
@@ -270,7 +270,7 @@ TEST_F(GraphTracerTest, GraphTrace) {
                     finish_time: 16000
                     packet_timestamp: 0
                     stream_id: 3
-                    packet_id: 3
+                    event_data: 3
                   }
                   output_trace { packet_timestamp: 0 stream_id: 5 }
                 }
@@ -286,7 +286,7 @@ TEST_F(GraphTracerTest, GraphTrace) {
                     finish_time: 38000
                     packet_timestamp: 5
                     stream_id: 3
-                    packet_id: 4
+                    event_data: 4
                   }
                   output_trace { packet_timestamp: 5 stream_id: 5 }
                 }
@@ -452,12 +452,15 @@ class GraphTracerE2ETest : public ::testing::Test {
     }
   }
   void StripDataIds(GraphTrace* trace) {
+    TraceBuilder builder;
     for (auto& ct : *trace->mutable_calculator_trace()) {
+      if ((*builder.trace_event_registry())[ct.event_type()].id_event_data()) {
         for (auto& st : *ct.mutable_input_trace()) {
-        st.clear_packet_id();
+          st.clear_event_data();
         }
         for (auto& st : *ct.mutable_output_trace()) {
-        st.clear_packet_id();
+          st.clear_event_data();
+        }
       }
     }
   }
@@ -656,63 +659,81 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) {
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
+                calculator_trace { node_id: 1 input_timestamp: 10000 }
+                calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 3 input_timestamp: 10000 }
+                calculator_trace { node_id: 3 input_timestamp: 10000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 3 input_timestamp: 10000 }
                 calculator_trace { node_id: 0 input_timestamp: 20000 }
+                calculator_trace { node_id: 1 input_timestamp: 20000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 20000 }
                 calculator_trace { node_id: 1 input_timestamp: 20000 }
+                calculator_trace { node_id: 2 input_timestamp: 20000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 2 input_timestamp: 20000 }
                 calculator_trace { node_id: 2 input_timestamp: 20000 }
+                calculator_trace { node_id: 4 input_timestamp: 20000 }
                 calculator_trace { node_id: 4 input_timestamp: 10000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 4 input_timestamp: 20000 }
                 calculator_trace { node_id: 0 input_timestamp: 30000 }
+                calculator_trace { node_id: 1 input_timestamp: 30000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 30000 }
                 calculator_trace { node_id: 1 input_timestamp: 30000 }
+                calculator_trace { node_id: 2 input_timestamp: 30000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 2 input_timestamp: 30000 }
                 calculator_trace { node_id: 2 input_timestamp: 30000 }
+                calculator_trace { node_id: 3 input_timestamp: 30000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 0 input_timestamp: 40000 }
+                calculator_trace { node_id: 1 input_timestamp: 40000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 40000 }
                 calculator_trace { node_id: 1 input_timestamp: 40000 }
+                calculator_trace { node_id: 2 input_timestamp: 40000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 2 input_timestamp: 40000 }
                 calculator_trace { node_id: 2 input_timestamp: 40000 }
+                calculator_trace { node_id: 4 input_timestamp: 40000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 3 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
+                calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 3 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
+                calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 3 input_timestamp: 30000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 0 input_timestamp: 50000 }
+                calculator_trace { node_id: 1 input_timestamp: 50000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 50000 }
                 calculator_trace { node_id: 1 input_timestamp: 50000 }
+                calculator_trace { node_id: 2 input_timestamp: 50000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 2 input_timestamp: 50000 }
                 calculator_trace { node_id: 2 input_timestamp: 50000 }
+                calculator_trace { node_id: 3 input_timestamp: 50000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 0 input_timestamp: 60000 }
+                calculator_trace { node_id: 1 input_timestamp: 60000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 60000 }
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
@@ -721,33 +742,39 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) {
                 calculator_trace { node_id: 2 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 4 input_timestamp: 20000 }
+                calculator_trace { node_id: 5 input_timestamp: 20000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 4 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 20000 }
                 calculator_trace { node_id: 5 input_timestamp: 20000 }
                 calculator_trace { node_id: 5 input_timestamp: 20000 }
+                calculator_trace { node_id: 1 input_timestamp: 20000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 4 input_timestamp: 40000 }
                 calculator_trace { node_id: 1 input_timestamp: 20000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 3 input_timestamp: 30000 }
+                calculator_trace { node_id: 5 input_timestamp: 30000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 3 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 30000 }
                 calculator_trace { node_id: 5 input_timestamp: 30000 }
                 calculator_trace { node_id: 5 input_timestamp: 30000 }
+                calculator_trace { node_id: 1 input_timestamp: 30000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 3 input_timestamp: 50000 }
                 calculator_trace { node_id: 1 input_timestamp: 30000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 3 input_timestamp: 50000 }
+                calculator_trace { node_id: 5 input_timestamp: 50000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 3 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 50000 }
                 calculator_trace { node_id: 5 input_timestamp: 50000 }
                 calculator_trace { node_id: 5 input_timestamp: 50000 }
+                calculator_trace { node_id: 1 input_timestamp: 50000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
@@ -755,16 +782,17 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) {
                 calculator_trace { node_id: 1 input_timestamp: 50000 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 4 input_timestamp: 40000 }
+                calculator_trace { node_id: 5 input_timestamp: 40000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 4 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 40000 }
                 calculator_trace { node_id: 5 input_timestamp: 40000 }
+                calculator_trace { node_id: 1 input_timestamp: 50001 }
                 calculator_trace { node_id: 1 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 5 input_timestamp: 10000 }
                 calculator_trace { node_id: 1 input_timestamp: 50001 }
-                calculator_trace { node_id: 1 input_timestamp: 10000 }
+                calculator_trace { node_id: 1 input_timestamp: 10000 })")));
-              )")));
 
   // Validate a one-timestamp slice of the event trace.
   GraphTrace trace_2;
@@ -773,7 +801,8 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) {
                                       &trace_2);
   StripThreadIds(&trace_2);
   StripDataIds(&trace_2);
-  EXPECT_THAT(trace_2,
+  EXPECT_THAT(
+      trace_2,
       EqualsProto(::mediapipe::ParseTextProtoOrDie<GraphTrace>(
           R"(
             base_time: 1544086800000000
@@ -794,6 +823,13 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) {
               finish_time: 25002
               output_trace { packet_timestamp: 0 stream_id: 5 }
             }
+            calculator_trace {
+              node_id: 5
+              input_timestamp: 0
+              event_type: PACKET_QUEUED
+              start_time: 25002
+              input_trace { packet_timestamp: 0 stream_id: 5 event_data: 1 }
+            }
             calculator_trace {
               node_id: 5
               event_type: READY_FOR_PROCESS
@@ -825,6 +861,13 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) {
               finish_time: 25002
               output_trace { packet_timestamp: 0 stream_id: 7 }
             }
+            calculator_trace {
+              node_id: 1
+              input_timestamp: 0
+              event_type: PACKET_QUEUED
+              start_time: 25002
+              input_trace { packet_timestamp: 0 stream_id: 7 event_data: 1 }
+            }
             calculator_trace {
               node_id: 1
               event_type: READY_FOR_PROCESS
@@ -861,6 +904,13 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) {
               finish_time: 25005
               output_trace { packet_timestamp: 40000 stream_id: 1 }
             }
+            calculator_trace {
+              node_id: 1
+              input_timestamp: 40000
+              event_type: PACKET_QUEUED
+              start_time: 25005
+              input_trace { packet_timestamp: 40000 stream_id: 1 event_data: 1 }
+            }
             calculator_trace {
               node_id: 1
               event_type: READY_FOR_PROCESS
@@ -880,6 +930,13 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) {
               finish_time: 25005
               output_trace { packet_timestamp: 40000 stream_id: 2 }
             }
+            calculator_trace {
+              node_id: 2
+              input_timestamp: 40000
+              event_type: PACKET_QUEUED
+              start_time: 25005
+              input_trace { packet_timestamp: 40000 stream_id: 2 event_data: 1 }
+            }
             calculator_trace {
               node_id: 2
               event_type: READY_FOR_PROCESS
@@ -904,11 +961,19 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLog) {
               finish_time: 25005
               output_trace { packet_timestamp: 40000 stream_id: 3 }
             }
+            calculator_trace {
+              node_id: 3
+              input_timestamp: 40000
+              event_type: PACKET_QUEUED
+              start_time: 25005
+              input_trace { packet_timestamp: 40000 stream_id: 3 event_data: 1 }
+            }
             calculator_trace {
               node_id: 2
               event_type: NOT_READY
               start_time: 25005
-                    })")));
+            }
+          )")));
 }
 
 // Read a GraphProfile from a file path.
@@ -931,7 +996,7 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLogFile) {
   GraphProfile profile;
   MP_EXPECT_OK(
       ReadGraphProfile(absl::StrCat(log_path, 0, ".binarypb"), &profile));
-  EXPECT_EQ(89, profile.graph_trace(0).calculator_trace().size());
+  EXPECT_EQ(111, profile.graph_trace(0).calculator_trace().size());
 }
 
 TEST_F(GraphTracerE2ETest, DemuxGraphLogFiles) {
@@ -956,7 +1021,11 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLogFiles) {
       graph_profiles.push_back(profile);
     }
   }
-  std::vector<int> expected = {37, 52, 9};
+
+  // The expected counts of calculator_trace records in each of the log files.
+  // The processing spans three 12.5ms log files, because
+  // RunDemuxInFlightGraph adds packets over 30ms.
+  std::vector<int> expected = {49, 64, 11};
   EXPECT_EQ(event_counts, expected);
   GraphProfile& profile_2 = graph_profiles[2];
   profile_2.clear_calculator_profiles();
@@ -992,6 +1061,18 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLogFiles) {
                     finish_time: 70004
                     output_trace { packet_timestamp: 40000 stream_id: 8 }
                   }
+                  calculator_trace {
+                    node_id: 5
+                    input_timestamp: 40000
+                    event_type: PACKET_QUEUED
+                    start_time: 70004
+                    input_trace {
+                      finish_time: 70004
+                      packet_timestamp: 40000
+                      stream_id: 8
+                      event_data: 1
+                    }
+                  }
                   calculator_trace {
                     node_id: 5
                     event_type: READY_FOR_PROCESS
@@ -1016,6 +1097,18 @@ TEST_F(GraphTracerE2ETest, DemuxGraphLogFiles) {
                     }
                     output_trace { packet_timestamp: 50001 stream_id: 7 }
                   }
+                  calculator_trace {
+                    node_id: 1
+                    input_timestamp: 50001
+                    event_type: PACKET_QUEUED
+                    start_time: 70004
+                    input_trace {
+                      finish_time: 70004
+                      packet_timestamp: 50001
+                      stream_id: 7
+                      event_data: 1
+                    }
+                  }
                   calculator_trace {
                     node_id: 1
                     event_type: READY_FOR_PROCESS
@@ -1199,7 +1292,7 @@ TEST_F(GraphTracerE2ETest, GpuTaskTrace) {
                                  finish_time: 0
                                  packet_timestamp: 0
                                  stream_id: 1
-                                 packet_id: 0
+                                 event_data: 0
                                }
                                output_trace { packet_timestamp: 0 stream_id: 2 }
                                thread_id: 0
@@ -1231,7 +1324,7 @@ TEST_F(GraphTracerE2ETest, GpuTaskTrace) {
               input_timestamp: 0
               event_type: PROCESS
               start_time: 0
-              input_trace { packet_timestamp: 0 stream_id: 1 packet_id: 0 }
+              input_trace { packet_timestamp: 0 stream_id: 1 event_data: 0 }
               thread_id: 0
             }
             calculator_trace {
@@ -1253,7 +1346,7 @@ TEST_F(GraphTracerE2ETest, GpuTaskTrace) {
               input_timestamp: 0
               event_type: PROCESS
               finish_time: 1000
-              output_trace { packet_timestamp: 0 stream_id: 2 packet_id: 0 }
+              output_trace { packet_timestamp: 0 stream_id: 2 event_data: 0 }
               thread_id: 0
             }
           )")));

mediapipe/framework/profiler/trace_buffer.h

@@ -23,45 +23,29 @@
 
 namespace mediapipe {
 
-// Packet content identifier.
-using PacketDataId = const void*;
-
 namespace packet_internal {
-// Returns the packet data address for a packet data holder.
+// Returns a hash of the packet data address from a packet data holder.
-inline const void* GetPacketDataId(const HolderBase* holder) {
+inline const int64 GetPacketDataId(const HolderBase* holder) {
-  return (holder == nullptr)
+  if (holder == nullptr) {
-             ? nullptr
+    return 0;
-             : &(static_cast<const Holder<int>*>(holder)->data());
+  }
+  const void* address = &(static_cast<const Holder<int>*>(holder)->data());
+  return reinterpret_cast<int64>(address);
 }
 }  // namespace packet_internal
 
 // Packet trace log event.
 struct TraceEvent {
   using EventType = GraphTrace::EventType;
-  // GraphTrace::EventType constants, repeated here to match GraphProfilerStub.
-  static constexpr EventType UNKNOWN = GraphTrace::UNKNOWN;
-  static constexpr EventType OPEN = GraphTrace::OPEN;
-  static constexpr EventType PROCESS = GraphTrace::PROCESS;
-  static constexpr EventType CLOSE = GraphTrace::CLOSE;
-  static constexpr EventType NOT_READY = GraphTrace::NOT_READY;
-  static constexpr EventType READY_FOR_PROCESS = GraphTrace::READY_FOR_PROCESS;
-  static constexpr EventType READY_FOR_CLOSE = GraphTrace::READY_FOR_CLOSE;
-  static constexpr EventType THROTTLED = GraphTrace::THROTTLED;
-  static constexpr EventType UNTHROTTLED = GraphTrace::UNTHROTTLED;
-  static constexpr EventType CPU_TASK_USER = GraphTrace::CPU_TASK_USER;
-  static constexpr EventType CPU_TASK_SYSTEM = GraphTrace::CPU_TASK_SYSTEM;
-  static constexpr EventType GPU_TASK = GraphTrace::GPU_TASK;
-  static constexpr EventType DSP_TASK = GraphTrace::DSP_TASK;
-  static constexpr EventType TPU_TASK = GraphTrace::TPU_TASK;
   absl::Time event_time;
   EventType event_type = UNKNOWN;
   bool is_finish = false;
   Timestamp input_ts = Timestamp::Unset();
   Timestamp packet_ts = Timestamp::Unset();
-  int node_id = -1;
+  int32 node_id = -1;
   const std::string* stream_id = nullptr;
-  PacketDataId packet_data_id = 0;
+  int32 thread_id = 0;
-  int thread_id = 0;
+  int64 event_data = 0;
 
   TraceEvent(const EventType& event_type) : event_type(event_type) {}
   TraceEvent() {}
@@ -91,7 +75,7 @@ struct TraceEvent {
     return *this;
   }
   inline TraceEvent& set_packet_data_id(const Packet* packet) {
-    this->packet_data_id =
+    this->event_data =
         packet_internal::GetPacketDataId(packet_internal::GetHolder(*packet));
     return *this;
   }
@@ -103,11 +87,84 @@ struct TraceEvent {
     this->is_finish = is_finish;
     return *this;
   }
+  inline TraceEvent& set_event_data(int data) {
+    this->event_data = data;
+    return *this;
+  }
+
+  // GraphTrace::EventType constants, repeated here to match GraphProfilerStub.
+  static constexpr EventType UNKNOWN = GraphTrace::UNKNOWN;
+  static constexpr EventType OPEN = GraphTrace::OPEN;
+  static constexpr EventType PROCESS = GraphTrace::PROCESS;
+  static constexpr EventType CLOSE = GraphTrace::CLOSE;
+  static constexpr EventType NOT_READY = GraphTrace::NOT_READY;
+  static constexpr EventType READY_FOR_PROCESS = GraphTrace::READY_FOR_PROCESS;
+  static constexpr EventType READY_FOR_CLOSE = GraphTrace::READY_FOR_CLOSE;
+  static constexpr EventType THROTTLED = GraphTrace::THROTTLED;
+  static constexpr EventType UNTHROTTLED = GraphTrace::UNTHROTTLED;
+  static constexpr EventType CPU_TASK_USER = GraphTrace::CPU_TASK_USER;
+  static constexpr EventType CPU_TASK_SYSTEM = GraphTrace::CPU_TASK_SYSTEM;
+  static constexpr EventType GPU_TASK = GraphTrace::GPU_TASK;
+  static constexpr EventType DSP_TASK = GraphTrace::DSP_TASK;
+  static constexpr EventType TPU_TASK = GraphTrace::TPU_TASK;
+  static constexpr EventType GPU_CALIBRATION = GraphTrace::GPU_CALIBRATION;
+  static constexpr EventType PACKET_QUEUED = GraphTrace::PACKET_QUEUED;
 };
 
 // Packet trace log buffer.
 using TraceBuffer = CircularBuffer<TraceEvent>;
 
+// TraceEvent type traits.
+class TraceEventType {
+  using EventType = TraceEvent::EventType;
+
+ public:
+  TraceEventType() {}
+  TraceEventType(EventType event_type, std::string description,
+                 bool is_packet_event = false, bool is_stream_event = false,
+                 bool id_event_data = true)
+      : event_type_(event_type),
+        description_(description),
+        is_packet_event_(is_packet_event),
+        is_stream_event_(is_stream_event),
+        id_event_data_(id_event_data) {}
+
+  // The type of event to log.
+  inline EventType event_type() const { return event_type_; }
+
+  // True if this type of event is logged.
+  inline bool enabled() const { return event_type_; }
+  inline void set_enabled(bool enabled) { enabled_ = enabled; }
+
+  // True if packet details are logged with this type of event.
+  inline bool is_packet_event() const { return is_packet_event_; }
+
+  // True if stream details are logged with this type of event.
+  inline bool is_stream_event() const { return is_stream_event_; }
+
+  // True if event_data values are assigned compact id's.
+  inline bool id_event_data() const { return id_event_data_; }
+
+ private:
+  EventType event_type_ = TraceEvent::UNKNOWN;
+  std::string description_ = "";
+  bool enabled_ = true;
+  bool is_packet_event_ = false;
+  bool is_stream_event_ = false;
+  bool id_event_data_ = true;
+};
+
+// A hash function for TraceEvent::EventType.
+struct EventTypeHash {
+  size_t operator()(const TraceEvent::EventType e) const {
+    return static_cast<size_t>(e);
+  }
+};
+
+// The registry of trace event types.
+using TraceEventRegistry =
+    std::unordered_map<TraceEvent::EventType, TraceEventType, EventTypeHash>;
+
 }  // namespace mediapipe
 
 #endif  // MEDIAPIPE_FRAMEWORK_PROFILER_TRACE_BUFFER_H_

mediapipe/framework/profiler/trace_builder.cc

@@ -57,21 +57,37 @@ namespace mediapipe {
 
 namespace {
 
-// For each event-type, whether packet details are logged.
+void BasicTraceEventTypes(TraceEventRegistry* result) {
-// The event-types are:
+  // The initializer arguments below are: event_type, description,
-//   UNKNOWN, OPEN, PROCESS, CLOSE,
+  // is_packet_event, is_stream_event, id_event_data.
-//   NOT_READY, READY_FOR_PROCESS, READY_FOR_CLOSE, THROTTLED, UNTHROTTLED
+  std::vector<TraceEventType> basic_types = {
-//   CPU_TASK_USER, CPU_TASK_SYSTEM, GPU_TASK, DSP_TASK, TPU_TASK
+      {TraceEvent::UNKNOWN, "An uninitialized trace-event."},
-constexpr bool kProfilerPacketEvents[] = {  //
+      {TraceEvent::OPEN, "A call to Calculator::Open.", true, true},
-    false, true,  true,  true,              //
+      {TraceEvent::PROCESS, "A call to Calculator::Open.", true, true},
-    false, false, false, false, false,      //
+      {TraceEvent::CLOSE, "A call to Calculator::Close.", true, true},
-    true,  true,  true,  true,  true};
 
-// For each calculator method, whether StreamTraces are desired.
+      {TraceEvent::NOT_READY, "A calculator cannot process packets yet."},
-constexpr bool kProfilerStreamEvents[] = {  //
+      {TraceEvent::READY_FOR_PROCESS, "A calculator can process packets."},
-    false, true,  true,  true,              //
+      {TraceEvent::READY_FOR_CLOSE, "A calculator is done processing packets."},
-    false, false, false, false, false,      //
+      {TraceEvent::THROTTLED, "Input is disabled due to max_queue_size."},
-    true,  true,  false, false, false};
+      {TraceEvent::UNTHROTTLED, "Input is enabled up to max_queue_size."},
+
+      {TraceEvent::CPU_TASK_USER, "User-time processing packets.", true, true},
+      {TraceEvent::CPU_TASK_SYSTEM, "System-time processing packets.", true,
+       true},
+      {TraceEvent::GPU_TASK, "GPU-time processing packets.", true, false},
+      {TraceEvent::DSP_TASK, "DSP-time processing packets.", true, false},
+      {TraceEvent::TPU_TASK, "TPU-time processing packets.", true, false},
+
+      {TraceEvent::GPU_CALIBRATION,
+       "A time measured by GPU clock and by CPU clock.", true, false},
+      {TraceEvent::PACKET_QUEUED, "An input queue size when a packet arrives.",
+       true, true, false},
+  };
+  for (TraceEventType t : basic_types) {
+    (*result)[t.event_type()] = t;
+  }
+}
 
 // A map defining int32 identifiers for std::string object pointers.
 // Lookup is fast when the same std::string object is used frequently.
@@ -106,7 +122,7 @@ class StringIdMap {
 // A map defining int32 identifiers for object pointers.
 class AddressIdMap {
  public:
-  int32 operator[](const void* id) {
+  int32 operator[](int64 id) {
     auto pointer_id = pointer_id_map_.find(id);
     if (pointer_id != pointer_id_map_.end()) {
       return pointer_id->second;
@@ -114,12 +130,10 @@ class AddressIdMap {
     return pointer_id_map_[id] = next_id++;
   }
   void clear() { pointer_id_map_.clear(); }
-  const std::unordered_map<const void*, int32>& map() {
+  const std::unordered_map<int64, int32>& map() { return pointer_id_map_; }
-    return pointer_id_map_;
-  }
 
  private:
-  std::unordered_map<const void*, int32> pointer_id_map_;
+  std::unordered_map<int64, int32> pointer_id_map_;
   int32 next_id = 0;
 };
 
@@ -147,8 +161,12 @@ class TraceBuilder::Impl {
     static std::string* empty_string = new std::string("");
     stream_id_map_[empty_string];
     packet_data_id_map_[0];
+    BasicTraceEventTypes(&trace_event_registry_);
   }
 
+  // Returns the registry of trace event types.
+  TraceEventRegistry* trace_event_registry() { return &trace_event_registry_; }
+
   static Timestamp TimestampAfter(const TraceBuffer& buffer,
                                   absl::Time begin_time) {
     Timestamp max_ts = Timestamp::Min();
@@ -176,7 +194,7 @@ class TraceBuilder::Impl {
 
     // Index TraceEvents by task-id and stream-hop-id.
     for (const TraceEvent& event : snapshot) {
-      if (!kProfilerPacketEvents[static_cast<int>(event.event_type)]) {
+      if (!trace_event_registry_[event.event_type].is_packet_event()) {
         continue;
       }
       TaskId task_id{event.node_id, event.input_ts, event.event_type};
@@ -195,7 +213,7 @@ class TraceBuilder::Impl {
     result->set_base_timestamp(base_ts_);
     std::unordered_set<TaskId> task_ids;
     for (const TraceEvent& event : snapshot) {
-      if (!kProfilerPacketEvents[static_cast<int>(event.event_type)]) {
+      if (!trace_event_registry_[event.event_type].is_packet_event()) {
         BuildEventLog(event, result->add_calculator_trace());
         continue;
       }
@@ -280,18 +298,12 @@ class TraceBuilder::Impl {
   // Construct the StreamTrace for a TraceEvent.
   void BuildStreamTrace(const TraceEvent& event,
                         GraphTrace::StreamTrace* result) {
-    if (event.is_finish) {
     result->set_stream_id(stream_id_map_[event.stream_id]);
     result->set_packet_timestamp(LogTimestamp(event.packet_ts));
-      return;
+    if (trace_event_registry_[event.event_type].id_event_data()) {
-    }
+      result->set_event_data(packet_data_id_map_[event.event_data]);
-    result->set_stream_id(stream_id_map_[event.stream_id]);
+    } else {
-    result->set_packet_timestamp(LogTimestamp(event.packet_ts));
+      result->set_event_data(event.event_data);
-    result->set_finish_time(LogTime(event.event_time));
-    result->set_packet_id(packet_data_id_map_[event.packet_data_id]);
-    const TraceEvent* output_event = FindOutputEvent(event);
-    if (output_event) {
-      result->set_start_time(LogTime(output_event->event_time));
     }
   }
 
@@ -301,10 +313,12 @@ class TraceBuilder::Impl {
     absl::Time start_time = absl::InfiniteFuture();
     absl::Time finish_time = absl::InfiniteFuture();
     for (const TraceEvent* event : task_events) {
-      if (result->input_trace().size() + result->output_trace().size() == 0) {
+      if (result->event_type() == TraceEvent::UNKNOWN) {
         result->set_node_id(event->node_id);
         result->set_event_type(event->event_type);
+        if (event->input_ts != Timestamp::Unset()) {
           result->set_input_timestamp(LogTimestamp(event->input_ts));
+        }
         result->set_thread_id(event->thread_id);
       }
       if (event->is_finish) {
@@ -312,12 +326,21 @@ class TraceBuilder::Impl {
       } else {
         start_time = std::min(start_time, event->event_time);
       }
-      if (kProfilerStreamEvents[static_cast<int>(event->event_type)]) {
+      if (trace_event_registry_[event->event_type].is_stream_event()) {
+        auto stream_trace = event->is_finish ? result->add_output_trace()
+                                             : result->add_input_trace();
         if (event->is_finish) {
-          BuildStreamTrace(*event, result->add_output_trace());
+          // Log only the packet id for each output event.
-          auto s = result->output_trace(result->output_trace_size() - 1);
+          stream_trace->set_stream_id(stream_id_map_[event->stream_id]);
+          stream_trace->set_packet_timestamp(LogTimestamp(event->packet_ts));
         } else {
-          BuildStreamTrace(*event, result->add_input_trace());
+          // Log the full stream trace for each input event.
+          BuildStreamTrace(*event, stream_trace);
+          stream_trace->set_finish_time(LogTime(event->event_time));
+          const TraceEvent* output_event = FindOutputEvent(*event);
+          if (output_event) {
+            stream_trace->set_start_time(LogTime(output_event->event_time));
+          }
         }
       }
     }
@@ -343,13 +366,11 @@ class TraceBuilder::Impl {
       result->set_input_timestamp(LogTimestamp(event.input_ts));
     }
     result->set_thread_id(event.thread_id);
-    if (kProfilerStreamEvents[static_cast<int>(event.event_type)]) {
+    if (trace_event_registry_[event.event_type].is_stream_event()) {
       if (event.stream_id) {
         auto stream_trace = event.is_finish ? result->add_output_trace()
                                             : result->add_input_trace();
-        stream_trace->set_stream_id(stream_id_map_[event.stream_id]);
+        BuildStreamTrace(event, stream_trace);
-        stream_trace->set_packet_timestamp(LogTimestamp(event.packet_ts));
-        stream_trace->set_packet_id(packet_data_id_map_[event.packet_data_id]);
       }
     }
   }
@@ -366,11 +387,17 @@ class TraceBuilder::Impl {
   int64 base_ts_ = std::numeric_limits<int64>::max();
   // The time represented as 0 in the trace.
   int64 base_time_ = std::numeric_limits<int64>::max();
+  // Indicates traits of each event type.
+  TraceEventRegistry trace_event_registry_;
 };
 
 TraceBuilder::TraceBuilder() : impl_(new Impl) {}
 TraceBuilder::~TraceBuilder() {}
 
+TraceEventRegistry* TraceBuilder::trace_event_registry() {
+  return impl_->trace_event_registry();
+}
+
 Timestamp TraceBuilder::TimestampAfter(const TraceBuffer& buffer,
                                        absl::Time begin_time) {
   return Impl::TimestampAfter(buffer, begin_time);
@@ -400,6 +427,8 @@ const TraceEvent::EventType         //
     TraceEvent::CPU_TASK_SYSTEM,    //
     TraceEvent::GPU_TASK,           //
     TraceEvent::DSP_TASK,           //
-    TraceEvent::TPU_TASK;
+    TraceEvent::TPU_TASK,           //
+    TraceEvent::GPU_CALIBRATION,    //
+    TraceEvent::PACKET_QUEUED;
 
 }  // namespace mediapipe

mediapipe/framework/profiler/trace_builder.h

@@ -28,6 +28,9 @@ class TraceBuilder {
   TraceBuilder();
   ~TraceBuilder();
 
+  // Returns the registry of trace event types.
+  TraceEventRegistry* trace_event_registry();
+
   // Returns the earliest packet timestamp appearing only after begin_time.
   static Timestamp TimestampAfter(const TraceBuffer& buffer,
                                   absl::Time begin_time);

mediapipe/framework/stream_handler/default_input_stream_handler.cc

@@ -45,6 +45,17 @@ DefaultInputStreamHandler::DefaultInputStreamHandler(
   }
 }
 
+void DefaultInputStreamHandler::PrepareForRun(
+    std::function<void()> headers_ready_callback,
+    std::function<void()> notification_callback,
+    std::function<void(CalculatorContext*)> schedule_callback,
+    std::function<void(::mediapipe::Status)> error_callback) {
+  sync_set_.PrepareForRun();
+  InputStreamHandler::PrepareForRun(
+      std::move(headers_ready_callback), std::move(notification_callback),
+      std::move(schedule_callback), std::move(error_callback));
+}
+
 NodeReadiness DefaultInputStreamHandler::GetNodeReadiness(
     Timestamp* min_stream_timestamp) {
   return sync_set_.GetReadiness(min_stream_timestamp);

mediapipe/framework/stream_handler/default_input_stream_handler.h

@@ -35,6 +35,13 @@ class DefaultInputStreamHandler : public InputStreamHandler {
                             bool calculator_run_in_parallel);
 
  protected:
+  // Reinitializes this InputStreamHandler before each CalculatorGraph run.
+  void PrepareForRun(
+      std::function<void()> headers_ready_callback,
+      std::function<void()> notification_callback,
+      std::function<void(CalculatorContext*)> schedule_callback,
+      std::function<void(::mediapipe::Status)> error_callback) override;
+
   // In DefaultInputStreamHandler, a node is "ready" if:
   // - all streams are done (need to call Close() in this case), or
   // - the minimum bound (over all empty streams) is greater than the smallest

mediapipe/framework/stream_handler/immediate_input_stream_handler.cc

@@ -40,6 +40,13 @@ class ImmediateInputStreamHandler : public InputStreamHandler {
       const MediaPipeOptions& options, bool calculator_run_in_parallel);
 
  protected:
+  // Reinitializes this InputStreamHandler before each CalculatorGraph run.
+  void PrepareForRun(
+      std::function<void()> headers_ready_callback,
+      std::function<void()> notification_callback,
+      std::function<void(CalculatorContext*)> schedule_callback,
+      std::function<void(::mediapipe::Status)> error_callback) override;
+
   // Returns kReadyForProcess whenever a Packet is available at any of
   // the input streams, or any input stream becomes done.
   NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override;
@@ -69,6 +76,23 @@ ImmediateInputStreamHandler::ImmediateInputStreamHandler(
   }
 }
 
+void ImmediateInputStreamHandler::PrepareForRun(
+    std::function<void()> headers_ready_callback,
+    std::function<void()> notification_callback,
+    std::function<void(CalculatorContext*)> schedule_callback,
+    std::function<void(::mediapipe::Status)> error_callback) {
+  {
+    absl::MutexLock lock(&mutex_);
+    for (int i = 0; i < sync_sets_.size(); ++i) {
+      sync_sets_[i].PrepareForRun();
+      ready_timestamps_[i] = Timestamp::Unset();
+    }
+  }
+  InputStreamHandler::PrepareForRun(
+      std::move(headers_ready_callback), std::move(notification_callback),
+      std::move(schedule_callback), std::move(error_callback));
+}
+
 NodeReadiness ImmediateInputStreamHandler::GetNodeReadiness(
     Timestamp* min_stream_timestamp) {
   absl::MutexLock lock(&mutex_);

mediapipe/framework/subgraph.cc

@@ -61,6 +61,13 @@ GraphRegistry::GraphRegistry(
     FunctionRegistry<std::unique_ptr<Subgraph>>* factories)
     : global_factories_(factories) {}
 
+void GraphRegistry::Register(
+    const std::string& type_name,
+    std::function<std::unique_ptr<Subgraph>()> factory) {
+  local_factories_.Register(type_name, factory);
+}
+
+// TODO: Remove this convenience function.
 void GraphRegistry::Register(const std::string& type_name,
                              const CalculatorGraphConfig& config) {
   local_factories_.Register(type_name, [config] {
@@ -69,6 +76,7 @@ void GraphRegistry::Register(const std::string& type_name,
   });
 }
 
+// TODO: Remove this convenience function.
 void GraphRegistry::Register(const std::string& type_name,
                              const CalculatorGraphTemplate& templ) {
   local_factories_.Register(type_name, [templ] {

mediapipe/framework/subgraph.h

@@ -94,6 +94,10 @@ class GraphRegistry {
   // Ownership of the specified FunctionRegistry is not transferred.
   GraphRegistry(FunctionRegistry<std::unique_ptr<Subgraph>>* factories);
 
+  // Registers a graph config builder type, using a factory function.
+  void Register(const std::string& type_name,
+                std::function<std::unique_ptr<Subgraph>()> factory);
+
   // Registers a graph config by name.
   void Register(const std::string& type_name,
                 const CalculatorGraphConfig& config);

mediapipe/gpu/BUILD

@@ -59,7 +59,7 @@ GL_BASE_LINK_OPTS = select({
         # runtime. Weak GLESv3 symbols will still be resolved if we
         # load it early enough.
     ],
-    "//mediapipe:apple": [
+    "//mediapipe:ios": [
         "-framework OpenGLES",
         "-framework CoreVideo",
     ],
@@ -111,7 +111,7 @@ cc_library(
     # Note: need the frameworks on Apple platforms to get the headers.
     linkopts = select({
         "//conditions:default": [],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "-framework OpenGLES",
             "-framework CoreVideo",
         ],
@@ -147,7 +147,7 @@ cc_library(
         "//conditions:default": [
             "gl_context_egl.cc",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "gl_context_eagl.cc",
         ],
         "//mediapipe:macos": [
@@ -214,7 +214,7 @@ cc_library(
         "//conditions:default": [
             ":gl_texture_buffer",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "//mediapipe/objc:CFHolder",
         ],
         "//mediapipe:macos": [
@@ -246,9 +246,14 @@ objc_library(
         "-Wno-shorten-64-to-32",
     ],
     sdk_frameworks = [
+        "Accelerate",
+        "CoreGraphics",
         "CoreVideo",
     ],
     visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/objc:util",
+    ],
 )
 
 objc_library(
@@ -408,7 +413,7 @@ cc_library(
         "//conditions:default": [
             "gl_texture_buffer_pool.cc",
         ],
-        "//mediapipe:apple": [],
+        "//mediapipe:ios": [],
         "//mediapipe:macos": [
             "gl_texture_buffer_pool.cc",
         ],
@@ -417,7 +422,7 @@ cc_library(
         "//conditions:default": [
             "gl_texture_buffer_pool.h",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             # The inclusions check does not see that this is provided by
             # pixel_buffer_pool_util, so we include it here too. This is
             # b/28066691.
@@ -441,7 +446,7 @@ cc_library(
         "//conditions:default": [
             ":gl_texture_buffer",
         ],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             ":pixel_buffer_pool_util",
             "//mediapipe/objc:CFHolder",
         ],

mediapipe/gpu/gl_context.cc

@@ -117,6 +117,7 @@ void* GlContext::DedicatedThread::ThreadBody(void* instance) {
 
 void GlContext::DedicatedThread::ThreadBody() {
   SetThreadName("mediapipe_gl_runner");
+
 #ifndef __EMSCRIPTEN__
   GlThreadCollector::ThreadStarting();
 #endif
@@ -276,6 +277,11 @@ bool GlContext::HasGlExtension(absl::string_view extension) const {
     absl::string_view version_string(
         reinterpret_cast<const char*>(glGetString(GL_VERSION)));
 
+    // We will decide later whether we want to use the version numbers we query
+    // for, or instead derive that information from the context creation result,
+    // which we cache here.
+    GLint gl_major_version_from_context_creation = gl_major_version_;
+
     // Let's try getting the numeric version if possible.
     glGetIntegerv(GL_MAJOR_VERSION, &gl_major_version_);
     GLenum err = glGetError();
@@ -293,6 +299,23 @@ bool GlContext::HasGlExtension(absl::string_view extension) const {
       }
     }
 
+    // If our platform-specific CreateContext already set a major GL version,
+    // then we use that.  Otherwise, we use the queried-for result. We do this
+    // as a workaround for a Swiftshader on Android bug where the ES2 context
+    // can report major version 3 instead of 2 when queried. Therefore we trust
+    // the result from context creation more than from query. See b/152519932
+    // for more details.
+    if (gl_major_version_from_context_creation > 0 &&
+        gl_major_version_ != gl_major_version_from_context_creation) {
+      LOG(WARNING) << "Requested a context with major GL version "
+                   << gl_major_version_from_context_creation
+                   << " but context reports major version " << gl_major_version_
+                   << ". Setting to " << gl_major_version_from_context_creation
+                   << ".0";
+      gl_major_version_ = gl_major_version_from_context_creation;
+      gl_minor_version_ = 0;
+    }
+
     LOG(INFO) << "GL version: " << gl_major_version_ << "." << gl_minor_version_
               << " (" << glGetString(GL_VERSION) << ")";
     if (gl_major_version_ >= 3) {
@@ -613,7 +636,17 @@ std::shared_ptr<GlSyncPoint> GlContext::CreateSyncToken() {
 #if MEDIAPIPE_DISABLE_GL_SYNC_FOR_DEBUG
   token.reset(new GlNopSyncPoint(shared_from_this()));
 #else
-  if (SymbolAvailable(&glWaitSync)) {
+
+#ifdef __EMSCRIPTEN__
+  // In Emscripten the glWaitSync function is non-null depending on linkopts,
+  // but only works in a WebGL2 context, so fall back to use Finish if it is a
+  // WebGL1/ES2 context.
+  // TODO: apply this more generally once b/152794517 is fixed.
+  bool useFenceSync = gl_major_version() > 2;
+#else
+  bool useFenceSync = SymbolAvailable(&glWaitSync);
+#endif  // __EMSCRIPTEN__
+  if (useFenceSync) {
     token.reset(new GlFenceSyncPoint(shared_from_this()));
   } else {
     token.reset(new GlFinishSyncPoint(shared_from_this()));
@@ -633,8 +666,30 @@ std::shared_ptr<GlSyncPoint> GlContext::TestOnly_CreateSpecificSyncToken(
   return nullptr;
 }
 
+// Atomically set var to the greater of its current value or target.
+template <typename T>
+static void assign_larger_value(std::atomic<T>* var, T target) {
+  T current = var->load();
+  while (current < target && !var->compare_exchange_weak(current, target)) {
+  }
+}
+
+// Note: this can get called from an arbitrary thread which is dealing with a
+// GlFinishSyncPoint originating from this context.
 void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) {
   if (gl_finish_count_ > count_to_pass) return;
+
+  // If we've been asked to do a glFinish, note the count we need to reach and
+  // signal the context our thread may currently be blocked on.
+  {
+    absl::MutexLock lock(&mutex_);
+    assign_larger_value(&gl_finish_count_target_, count_to_pass + 1);
+    wait_for_gl_finish_cv_.SignalAll();
+    if (context_waiting_on_) {
+      context_waiting_on_->wait_for_gl_finish_cv_.SignalAll();
+    }
+  }
+
   auto finish_task = [this, count_to_pass]() {
     // When a GlFinishSyncToken is created it takes the current finish count
     // from the GlContext, and we must wait for gl_finish_count_ to pass it.
@@ -646,6 +701,7 @@ void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) {
       GlFinishCalled();
     }
   };
+
   if (IsCurrent()) {
     // If we are already on the current context, we cannot call
     // RunWithoutWaiting, since that task will not run until this function
@@ -653,14 +709,54 @@ void GlContext::WaitForGlFinishCountPast(int64_t count_to_pass) {
     finish_task();
     return;
   }
+
+  std::shared_ptr<GlContext> other = GetCurrent();
+  if (other) {
+    // If another context is current, make a note that it is blocked on us, so
+    // it can signal the right condition variable if it is asked to do a
+    // glFinish.
+    absl::MutexLock other_lock(&other->mutex_);
+    DCHECK(!other->context_waiting_on_);
+    other->context_waiting_on_ = this;
+  }
   // We do not schedule this action using Run because we don't necessarily
   // want to wait for it to complete. If another job calls GlFinishCalled
   // sooner, we are done.
   RunWithoutWaiting(std::move(finish_task));
+  {
     absl::MutexLock lock(&mutex_);
     while (gl_finish_count_ <= count_to_pass) {
+      if (other && other->gl_finish_count_ < other->gl_finish_count_target_) {
+        // If another context's dedicated thread is current, it is blocked
+        // waiting for this context to issue a glFinish call. But this context
+        // may also block waiting for the other context to do the same: this can
+        // happen when two contexts are handling each other's GlFinishSyncPoints
+        // (e.g. a producer and a consumer). To avoid a deadlock a context that
+        // is waiting on another context must still service Wait calls it may
+        // receive from its own GlFinishSyncPoints.
+        //
+        // We unlock this context's mutex to avoid holding both at the same
+        // time.
+        mutex_.Unlock();
+        {
+          glFinish();
+          other->GlFinishCalled();
+        }
+        mutex_.Lock();
+        // Because we temporarily unlocked mutex_, we cannot wait on the
+        // condition variable wait away; we need to go back to re-checking the
+        // condition. Otherwise we might miss a signal.
+        continue;
+      }
       wait_for_gl_finish_cv_.Wait(&mutex_);
     }
+  }
+
+  if (other) {
+    // The other context is no longer waiting on us.
+    absl::MutexLock other_lock(&other->mutex_);
+    other->context_waiting_on_ = nullptr;
+  }
 }
 
 void GlContext::WaitSyncToken(const std::shared_ptr<GlSyncPoint>& token) {

mediapipe/gpu/gl_context.h

@@ -380,6 +380,9 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
   // Changes should be guarded by mutex_. However, we use simple atomic
   // loads for efficiency on the fast path.
   std::atomic<int64_t> gl_finish_count_ = ATOMIC_VAR_INIT(0);
+  std::atomic<int64_t> gl_finish_count_target_ = ATOMIC_VAR_INIT(0);
+
+  GlContext* context_waiting_on_ ABSL_GUARDED_BY(mutex_) = nullptr;
 
   // This mutex is held by a thread while this GL context is current on that
   // thread. Since it may be held for extended periods of time, it should not

mediapipe/gpu/gl_texture_buffer_pool.cc

@@ -26,19 +26,28 @@ GlTextureBufferPool::GlTextureBufferPool(int width, int height,
       keep_count_(keep_count) {}
 
 GlTextureBufferSharedPtr GlTextureBufferPool::GetBuffer() {
-  absl::MutexLock lock(&mutex_);
-
   std::unique_ptr<GlTextureBuffer> buffer;
+  bool reuse = false;
+
+  {
+    absl::MutexLock lock(&mutex_);
     if (available_.empty()) {
       buffer = GlTextureBuffer::Create(width_, height_, format_);
       if (!buffer) return nullptr;
     } else {
       buffer = std::move(available_.back());
       available_.pop_back();
-    buffer->Reuse();
+      reuse = true;
     }
 
     ++in_use_count_;
+  }
+
+  // This needs to wait on consumer sync points, therefore it should not be
+  // done while holding the mutex.
+  if (reuse) {
+    buffer->Reuse();
+  }
 
   // Return a shared_ptr with a custom deleter that adds the buffer back
   // to our available list.
@@ -60,15 +69,24 @@ std::pair<int, int> GlTextureBufferPool::GetInUseAndAvailableCounts() {
 }
 
 void GlTextureBufferPool::Return(GlTextureBuffer* buf) {
+  std::vector<std::unique_ptr<GlTextureBuffer>> trimmed;
+  {
     absl::MutexLock lock(&mutex_);
     --in_use_count_;
     available_.emplace_back(buf);
-  TrimAvailable();
+    TrimAvailable(&trimmed);
+  }
+  // The trimmed buffers will be released without holding the lock.
 }
 
-void GlTextureBufferPool::TrimAvailable() {
+void GlTextureBufferPool::TrimAvailable(
+    std::vector<std::unique_ptr<GlTextureBuffer>>* trimmed) {
   int keep = std::max(keep_count_ - in_use_count_, 0);
   if (available_.size() > keep) {
+    auto trim_it = std::next(available_.begin(), keep);
+    if (trimmed) {
+      std::move(available_.begin(), trim_it, std::back_inserter(*trimmed));
+    }
     available_.resize(keep);
   }
 }

mediapipe/gpu/gl_texture_buffer_pool.h

@@ -60,7 +60,8 @@ class GlTextureBufferPool
 
   // If the total number of buffers is greater than keep_count, destroys any
   // surplus buffers that are no longer in use.
-  void TrimAvailable() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+  void TrimAvailable(std::vector<std::unique_ptr<GlTextureBuffer>>* trimmed)
+      ABSL_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
 
   const int width_;
   const int height_;

mediapipe/gpu/pixel_buffer_pool_util.mm

@@ -16,6 +16,8 @@
 
 #import <Foundation/Foundation.h>
 
+#include "mediapipe/objc/util.h"
+
 #if !defined(ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK) && !defined(NDEBUG)
 #define ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK 1
 #endif  // defined(ENABLE_MEDIAPIPE_GPU_BUFFER_THRESHOLD_CHECK)
@@ -27,17 +29,13 @@ CVPixelBufferPoolRef CreateCVPixelBufferPool(
     CFTimeInterval maxAge) {
   CVPixelBufferPoolRef pool = NULL;
 
-  NSDictionary *sourcePixelBufferOptions = @{
+  NSMutableDictionary *sourcePixelBufferOptions =
+      [(__bridge NSDictionary*)GetCVPixelBufferAttributesForGlCompatibility() mutableCopy];
+  [sourcePixelBufferOptions addEntriesFromDictionary:@{
     (id)kCVPixelBufferPixelFormatTypeKey : @(pixelFormat),
     (id)kCVPixelBufferWidthKey : @(width),
     (id)kCVPixelBufferHeightKey : @(height),
-#if TARGET_OS_OSX
+  }];
-      (id)kCVPixelFormatOpenGLCompatibility : @(YES),
-#else
-      (id)kCVPixelFormatOpenGLESCompatibility : @(YES),
-#endif  // TARGET_OS_OSX
-      (id)kCVPixelBufferIOSurfacePropertiesKey : @{ /*empty dictionary*/ }
-  };
 
   NSMutableDictionary *pixelBufferPoolOptions = [[NSMutableDictionary alloc] init];
   pixelBufferPoolOptions[(id)kCVPixelBufferPoolMinimumBufferCountKey] = @(keepCount);
@@ -131,14 +129,6 @@ static void FreeRefConReleaseCallback(void* refCon, const void* baseAddress) {
 
 CVReturn CreateCVPixelBufferWithoutPool(
     int width, int height, OSType pixelFormat, CVPixelBufferRef* outBuffer) {
-  NSDictionary *attributes = @{
-#if TARGET_OS_OSX
-      (id)kCVPixelFormatOpenGLCompatibility : @(YES),
-#else
-      (id)kCVPixelFormatOpenGLESCompatibility : @(YES),
-#endif  // TARGET_OS_OSX
-      (id)kCVPixelBufferIOSurfacePropertiesKey : @{ /*empty dictionary*/ }
-  };
 #if TARGET_IPHONE_SIMULATOR
   // On the simulator, syncing the texture with the pixelbuffer does not work,
   // and we have to use glReadPixels. Since GL_UNPACK_ROW_LENGTH is not
@@ -151,12 +141,12 @@ CVReturn CreateCVPixelBufferWithoutPool(
   void* data = malloc(bytes_per_row * height);
   return CVPixelBufferCreateWithBytes(
       kCFAllocatorDefault, width, height, pixelFormat, data, bytes_per_row,
-      FreeRefConReleaseCallback, data, (__bridge CFDictionaryRef)attributes,
+      FreeRefConReleaseCallback, data, GetCVPixelBufferAttributesForGlCompatibility(),
       outBuffer);
 #else
   return CVPixelBufferCreate(
       kCFAllocatorDefault, width, height, pixelFormat,
-      (__bridge CFDictionaryRef)attributes, outBuffer);
+      GetCVPixelBufferAttributesForGlCompatibility(), outBuffer);
 #endif
 }
 

mediapipe/java/com/google/mediapipe/components/BUILD

@@ -31,8 +31,8 @@ android_library(
         "//third_party:androidx_core",
         "//third_party:androidx_legacy_support_v4",
         "//third_party:androidx_recyclerview",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 
@@ -50,10 +50,10 @@ android_library(
         "//third_party:androidx_legacy_support_v4",
         "//third_party:camera2",
         "//third_party:camerax_core",
-        "@androidx_concurrent_futures//jar",
+        "@maven//:androidx_concurrent_concurrent_futures",
-        "@androidx_lifecycle//jar",
+        "@maven//:androidx_lifecycle_lifecycle_common",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 
@@ -67,8 +67,8 @@ android_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/java/com/google/mediapipe/framework/BUILD

@@ -59,10 +59,10 @@ android_library(
         ":android_core",
         "//third_party:androidx_annotation",
         "//third_party:androidx_legacy_support_v4",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_common_flogger//jar",
+        "@maven//:com_google_flogger_flogger",
-        "@com_google_common_flogger_system_backend//jar",
+        "@maven//:com_google_flogger_flogger_system_backend",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 
@@ -85,10 +85,10 @@ android_library(
         "//mediapipe/framework:calculator_java_proto_lite",
         "//mediapipe/framework:calculator_profile_java_proto_lite",
         "//mediapipe/framework/tool:calculator_graph_template_java_proto_lite",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_common_flogger//jar",
+        "@maven//:com_google_flogger_flogger",
-        "@com_google_common_flogger_system_backend//jar",
+        "@maven//:com_google_flogger_flogger_system_backend",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/java/com/google/mediapipe/glutil/BUILD

@@ -24,10 +24,10 @@ android_library(
     visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/java/com/google/mediapipe/framework:android_framework_no_proguard",
-        "@com_google_code_findbugs//jar",
+        "@maven//:com_google_code_findbugs_jsr305",
-        "@com_google_common_flogger//jar",
+        "@maven//:com_google_flogger_flogger",
-        "@com_google_common_flogger_system_backend//jar",
+        "@maven//:com_google_flogger_flogger_system_backend",
-        "@com_google_guava_android//jar",
+        "@maven//:com_google_guava_guava",
     ],
 )
 

mediapipe/models/README.md

@@ -8,17 +8,24 @@ Here are descriptions of the models used in the [example applications](../docs/e
 
 ### Face Detection
   * [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/face_detection_front.tflite)
-  * Paper: ["BlazeFace: Sub-millisecond Neural Face Detection on Mobile GPUs"](https://sites.google.com/corp/view/perception-cv4arvr/blazeface)
+  * Paper: ["BlazeFace: Sub-millisecond Neural Face Detection on Mobile GPUs"](https://arxiv.org/abs/1907.05047)
   * [Model card](https://sites.google.com/corp/view/perception-cv4arvr/blazeface#h.p_21ojPZDx3cqq)
 
+### Face Mesh
+  * [TF.js model](https://tfhub.dev/mediapipe/facemesh/1)
+  * Paper: ["Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs"](https://arxiv.org/abs/1907.06724)
+  * [TensorFlow Blog post](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
+  * [Model card](https://drive.google.com/file/d/1VFC_wIpw4O7xBOiTgUldl79d9LA-LsnA/view)
+
 ### Hand Detection and Tracking
-  * [Palm detection TfLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite)
+  * Palm detection: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/palm_detection.tflite), [TF.js model](https://tfhub.dev/mediapipe/handdetector/1)
-  * [2D hand landmark TfLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite)
+  * 2D hand landmark: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark.tflite)
-  * [3D hand landmark TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark_3d.tflite)
+  * 3D hand landmark: [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hand_landmark_3d.tflite), [TF.js model](https://tfhub.dev/mediapipe/handskeleton/1)
   * [Google AI Blog post](https://mediapipe.page.link/handgoogleaiblog)
+  * [TensorFlow Blog post](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
   * [Model card](https://mediapipe.page.link/handmc)
 
 ### Hair Segmentation
   * [TFLite model](https://github.com/google/mediapipe/tree/master/mediapipe/models/hair_segmentation.tflite)
-  * Paper: ["Real-time Hair segmentation and recoloring on Mobile GPUs"](https://sites.google.com/corp/view/perception-cv4arvr/hair-segmentation)
+  * Paper: ["Real-time Hair segmentation and recoloring on Mobile GPUs"](https://arxiv.org/abs/1907.06740)
   * [Model card](https://sites.google.com/corp/view/perception-cv4arvr/hair-segmentation#h.p_NimuO7PgHxlY)

mediapipe/objc/BUILD

@@ -19,9 +19,14 @@ cc_library(
     visibility = ["//mediapipe/framework:mediapipe_internal"],
     deps = [
         ":CFHolder",
-        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:packet",
         "//mediapipe/framework/formats:image_frame",
+        "//mediapipe/framework/port:logging",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:source_location",
         "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/memory",
     ],
 )
 

mediapipe/objc/MPPGraph.mm

@@ -26,6 +26,7 @@
 #include "mediapipe/gpu/MPPGraphGPUData.h"
 #include "mediapipe/gpu/gl_base.h"
 #include "mediapipe/gpu/gpu_shared_data_internal.h"
+#include "mediapipe/objc/util.h"
 
 #import "mediapipe/objc/NSError+util_status.h"
 #import "GTMDefines.h"
@@ -116,14 +117,10 @@ void CallFrameDelegate(void* wrapperVoid, const std::string& streamName,
       if (format == mediapipe::ImageFormat::SRGBA ||
           format == mediapipe::ImageFormat::GRAY8) {
         CVPixelBufferRef pixelBuffer;
-        // To ensure compatibility with CVOpenGLESTextureCache, this attribute should be present.
-        NSDictionary* attributes = @{
-          (id)kCVPixelBufferIOSurfacePropertiesKey : @{},
-        };
         // If kCVPixelFormatType_32RGBA does not work, it returns kCVReturnInvalidPixelFormat.
         CVReturn error = CVPixelBufferCreate(
             NULL, frame.Width(), frame.Height(), kCVPixelFormatType_32BGRA,
-            (__bridge CFDictionaryRef)attributes, &pixelBuffer);
+            GetCVPixelBufferAttributesForGlCompatibility(), &pixelBuffer);
         _GTMDevAssert(error == kCVReturnSuccess, @"CVPixelBufferCreate failed: %d", error);
         error = CVPixelBufferLockBaseAddress(pixelBuffer, 0);
         _GTMDevAssert(error == kCVReturnSuccess, @"CVPixelBufferLockBaseAddress failed: %d", error);

mediapipe/objc/util.cc

@@ -517,17 +517,29 @@ CFDictionaryRef GetCVPixelBufferAttributesForGlCompatibility() {
     CFDictionaryRef empty_dict = CFDictionaryCreate(
         kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks,
         &kCFTypeDictionaryValueCallBacks);
+
     // To ensure compatibility with CVOpenGLESTextureCache, these attributes
-    // should be present.
+    // should be present. However, on simulator this IOSurface attribute
+    // actually causes CVOpenGLESTextureCache to fail. b/144850076
     const void* keys[] = {
+#if !TARGET_IPHONE_SIMULATOR
       kCVPixelBufferIOSurfacePropertiesKey,
+#endif  // !TARGET_IPHONE_SIMULATOR
+
 #if TARGET_OS_OSX
       kCVPixelFormatOpenGLCompatibility,
 #else
       kCVPixelFormatOpenGLESCompatibility,
 #endif  // TARGET_OS_OSX
     };
-    const void* values[] = {empty_dict, kCFBooleanTrue};
+
+    const void* values[] = {
+#if !TARGET_IPHONE_SIMULATOR
+      empty_dict,
+#endif  // !TARGET_IPHONE_SIMULATOR
+      kCFBooleanTrue
+    };
+
     attrs = CFDictionaryCreate(
         kCFAllocatorDefault, keys, values, ABSL_ARRAYSIZE(values),
         &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks);

mediapipe/util/BUILD

@@ -173,14 +173,14 @@ cc_library(
     srcs = select({
         "//conditions:default": ["resource_util.cc"],
         "//mediapipe:android": ["resource_util_android.cc"],
-        "//mediapipe:apple": ["resource_util_apple.cc"],
+        "//mediapipe:ios": ["resource_util_apple.cc"],
         "//mediapipe:macos": ["resource_util.cc"],
     }),
     hdrs = ["resource_util.h"],
     # We use Objective-C++ on iOS.
     copts = select({
         "//conditions:default": [],
-        "//mediapipe:apple": [
+        "//mediapipe:ios": [
             "-ObjC++",
         ],
         "//mediapipe:macos": [],
@@ -201,7 +201,7 @@ cc_library(
             "//mediapipe/util/android:asset_manager_util",
             "//mediapipe/util/android/file/base",
         ],
-        "//mediapipe:apple": [],
+        "//mediapipe:ios": [],
         "//mediapipe:macos": [
             "//mediapipe/framework/port:file_helpers",
         ],