Project import generated by Copybara.

GitOrigin-RevId: 5aca6b3f07b67e09988a901f50f595ca5f566e67

.bazelrc

@@ -3,7 +3,7 @@
 # Basic build settings
 build --jobs 128
 build --define='absl=1'
-build --cxxopt='-std=c++11'
+build --cxxopt='-std=c++14'
 build --copt='-Wno-sign-compare'
 build --copt='-Wno-unused-function'
 build --copt='-Wno-uninitialized'

README.md

@@ -41,7 +41,10 @@ A web-based visualizer is hosted on [viz.mediapipe.dev](https://viz.mediapipe.de
 * [MediaPipe: A Framework for Building Perception Pipelines](https://arxiv.org/abs/1906.08172)
 
 ## Events
-* [ML Conference, Berlin 9-11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
+* [MediaPipe Madrid Meetup, 16 Dec 2019](https://www.meetup.com/Madrid-AI-Developers-Group/events/266329088/)
+* [MediaPipe London Meetup, Google 123 Building, 12 Dec 2019](https://www.meetup.com/London-AI-Tech-Talk/events/266329038)
+* [ML Conference, Berlin, 11 Dec 2019](https://mlconference.ai/machine-learning-advanced-development/mediapipe-building-real-time-cross-platform-mobile-web-edge-desktop-video-audio-ml-pipelines/)
+* [MediaPipe Berlin Meetup, Google Berlin, 11 Dec 2019](https://www.meetup.com/Berlin-AI-Tech-Talk/events/266328794/)
 * [The 3rd Workshop on YouTube-8M Large Scale Video Understanding Workshop](https://research.google.com/youtube8m/workshop2019/index.html) Seoul, Korea ICCV 2019
 * [AI DevWorld 2019](https://aidevworld.com) on Oct 10 in San Jose, California
 * [Google Industry Workshop at ICIP 2019](http://2019.ieeeicip.org/?action=page4&id=14#Google) [Presentation](https://docs.google.com/presentation/d/e/2PACX-1vRIBBbO_LO9v2YmvbHHEt1cwyqH6EjDxiILjuT0foXy1E7g6uyh4CesB2DkkEwlRDO9_lWfuKMZx98T/pub?start=false&loop=false&delayms=3000&slide=id.g556cc1a659_0_5) on Sept 24 in Taipei, Taiwan

WORKSPACE

@@ -10,8 +10,7 @@ http_archive(
     sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
 )
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "0.24.1",
+versions.check(minimum_bazel_version = "0.24.1")
-               maximum_bazel_version = "0.29.1")
 
 # ABSL cpp library.
 http_archive(
@@ -104,9 +103,9 @@ http_archive(
     ],
 )
 
-# 2019-08-15
+# 2019-11-12
-_TENSORFLOW_GIT_COMMIT = "67def62936e28f97c16182dfcc467d8d1cae02b4"
+_TENSORFLOW_GIT_COMMIT = "a5f9bcd64453ff3d1f64cb4da4786db3d2da7f82"
-_TENSORFLOW_SHA256= "ddd4e3c056e7c0ff2ef29133b30fa62781dfbf8a903e99efb91a02d292fa9562"
+_TENSORFLOW_SHA256= "f2b6f2ab2ffe63e86eccd3ce4bea6b7197383d726638dfeeebcdc1e7de73f075"
 http_archive(
     name = "org_tensorflow",
     urls = [
@@ -115,13 +114,6 @@ http_archive(
     ],
     strip_prefix = "tensorflow-%s" % _TENSORFLOW_GIT_COMMIT,
     sha256 = _TENSORFLOW_SHA256,
-    patches = [
-        "@//third_party:tensorflow_065c20bf79253257c87bd4614bb9a7fdef015cbb.diff",
-        "@//third_party:tensorflow_f67fcbefce906cd419e4657f0d41e21019b71abd.diff",
-    ],
-    patch_args = [
-        "-p1",
-    ],
 )
 
 load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
@@ -255,18 +247,11 @@ android_sdk_repository(
 
 # iOS basic build deps.
 
-load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+http_archive(
-
-git_repository(
     name = "build_bazel_rules_apple",
-    remote = "https://github.com/bazelbuild/rules_apple.git",
+    sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e",
-    tag = "0.18.0",
+    strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3",
-    patches = [
+    url = "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz",
-        "@//third_party:rules_apple_c0863d0596ae6b769a29fa3fb72ff036444fd249.diff",
-    ],
-    patch_args = [
-        "-p1",
-    ],
 )
 
 load(

mediapipe/calculators/core/BUILD

@@ -13,12 +13,12 @@
 # limitations under the License.
 #
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:private"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-
 proto_library(
     name = "concatenate_vector_calculator_proto",
     srcs = ["concatenate_vector_calculator.proto"],
@@ -79,6 +79,13 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "clip_vector_size_calculator_proto",
+    srcs = ["clip_vector_size_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = ["//mediapipe/framework:calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "packet_cloner_calculator_cc_proto",
     srcs = ["packet_cloner_calculator.proto"],
@@ -111,6 +118,14 @@ mediapipe_cc_proto_library(
     deps = [":concatenate_vector_calculator_proto"],
 )
 
+mediapipe_cc_proto_library(
+    name = "clip_vector_size_calculator_cc_proto",
+    srcs = ["clip_vector_size_calculator.proto"],
+    cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
+    visibility = ["//visibility:public"],
+    deps = [":clip_vector_size_calculator_proto"],
+)
+
 mediapipe_cc_proto_library(
     name = "dequantize_byte_array_calculator_cc_proto",
     srcs = ["dequantize_byte_array_calculator.proto"],
@@ -169,6 +184,66 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "begin_loop_calculator",
+    srcs = ["begin_loop_calculator.cc"],
+    hdrs = ["begin_loop_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_contract",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:collection_item_id",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "end_loop_calculator",
+    srcs = ["end_loop_calculator.cc"],
+    hdrs = ["end_loop_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_contract",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:collection_item_id",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/util:render_data_cc_proto",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "begin_end_loop_calculator_graph_test",
+    srcs = ["begin_end_loop_calculator_graph_test.cc"],
+    deps = [
+        ":begin_loop_calculator",
+        ":end_loop_calculator",
+        "//mediapipe/calculators/core:packet_cloner_calculator",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_contract",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+)
+
 cc_library(
     name = "concatenate_vector_calculator",
     srcs = ["concatenate_vector_calculator.cc"],
@@ -219,6 +294,50 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "clip_vector_size_calculator",
+    srcs = ["clip_vector_size_calculator.cc"],
+    hdrs = ["clip_vector_size_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":clip_vector_size_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@org_tensorflow//tensorflow/lite:framework",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "clip_detection_vector_size_calculator",
+    srcs = ["clip_detection_vector_size_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":clip_vector_size_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "clip_vector_size_calculator_test",
+    srcs = ["clip_vector_size_calculator_test.cc"],
+    deps = [
+        ":clip_vector_size_calculator",
+        "//mediapipe/calculators/core:packet_resampler_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:timestamp",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_library(
     name = "counting_source_calculator",
     srcs = ["counting_source_calculator.cc"],
@@ -300,7 +419,7 @@ cc_library(
         "//visibility:public",
     ],
     deps = [
-        "//mediapipe/calculators/core:packet_cloner_calculator_cc_proto",
+        ":packet_cloner_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "@com_google_absl//absl/strings",
     ],

mediapipe/calculators/core/begin_end_loop_calculator_graph_test.cc

@@ -0,0 +1,335 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/calculators/core/begin_loop_calculator.h"
+#include "mediapipe/calculators/core/end_loop_calculator.h"
+#include "mediapipe/framework/calculator_contract.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+
+namespace mediapipe {
+namespace {
+
+typedef BeginLoopCalculator<std::vector<int>> BeginLoopIntegerCalculator;
+REGISTER_CALCULATOR(BeginLoopIntegerCalculator);
+
+class IncrementCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<int>();
+    cc->Outputs().Index(0).Set<int>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    const int& input_int = cc->Inputs().Index(0).Get<int>();
+    auto output_int = absl::make_unique<int>(input_int + 1);
+    cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+};
+
+REGISTER_CALCULATOR(IncrementCalculator);
+
+typedef EndLoopCalculator<std::vector<int>> EndLoopIntegersCalculator;
+REGISTER_CALCULATOR(EndLoopIntegersCalculator);
+
+class BeginEndLoopCalculatorGraphTest : public ::testing::Test {
+ protected:
+  BeginEndLoopCalculatorGraphTest() {
+    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
+        R"(
+          num_threads: 4
+          input_stream: "ints"
+          node {
+            calculator: "BeginLoopIntegerCalculator"
+            input_stream: "ITERABLE:ints"
+            output_stream: "ITEM:int"
+            output_stream: "BATCH_END:timestamp"
+          }
+          node {
+            calculator: "IncrementCalculator"
+            input_stream: "int"
+            output_stream: "int_plus_one"
+          }
+          node {
+            calculator: "EndLoopIntegersCalculator"
+            input_stream: "ITEM:int_plus_one"
+            input_stream: "BATCH_END:timestamp"
+            output_stream: "ITERABLE:ints_plus_one"
+          }
+        )");
+    tool::AddVectorSink("ints_plus_one", &graph_config_, &output_packets_);
+  }
+
+  CalculatorGraphConfig graph_config_;
+  std::vector<Packet> output_packets_;
+};
+
+TEST_F(BeginEndLoopCalculatorGraphTest, SingleEmptyVector) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+  auto input_vector = absl::make_unique<std::vector<int>>();
+  Timestamp input_timestamp = Timestamp(0);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector.release()).At(input_timestamp)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // EndLoopCalc will forward the timestamp bound because there are no elements
+  // in collection to output.
+  ASSERT_EQ(0, output_packets_.size());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(BeginEndLoopCalculatorGraphTest, SingleNonEmptyVector) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+  auto input_vector = absl::make_unique<std::vector<int>>();
+  input_vector->emplace_back(0);
+  input_vector->emplace_back(1);
+  input_vector->emplace_back(2);
+  Timestamp input_timestamp = Timestamp(0);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector.release()).At(input_timestamp)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ASSERT_EQ(1, output_packets_.size());
+  EXPECT_EQ(input_timestamp, output_packets_[0].Timestamp());
+  std::vector<int> expected_output_vector = {1, 2, 3};
+  EXPECT_EQ(expected_output_vector, output_packets_[0].Get<std::vector<int>>());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(BeginEndLoopCalculatorGraphTest, MultipleVectors) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  auto input_vector0 = absl::make_unique<std::vector<int>>();
+  input_vector0->emplace_back(0);
+  input_vector0->emplace_back(1);
+  Timestamp input_timestamp0 = Timestamp(0);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector0.release()).At(input_timestamp0)));
+
+  auto input_vector1 = absl::make_unique<std::vector<int>>();
+  Timestamp input_timestamp1 = Timestamp(1);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector1.release()).At(input_timestamp1)));
+
+  auto input_vector2 = absl::make_unique<std::vector<int>>();
+  input_vector2->emplace_back(2);
+  input_vector2->emplace_back(3);
+  Timestamp input_timestamp2 = Timestamp(2);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector2.release()).At(input_timestamp2)));
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  ASSERT_EQ(2, output_packets_.size());
+
+  EXPECT_EQ(input_timestamp0, output_packets_[0].Timestamp());
+  std::vector<int> expected_output_vector0 = {1, 2};
+  EXPECT_EQ(expected_output_vector0,
+            output_packets_[0].Get<std::vector<int>>());
+
+  // At input_timestamp1, EndLoopCalc will forward timestamp bound as there are
+  // no elements in vector to process.
+
+  EXPECT_EQ(input_timestamp2, output_packets_[1].Timestamp());
+  std::vector<int> expected_output_vector2 = {3, 4};
+  EXPECT_EQ(expected_output_vector2,
+            output_packets_[1].Get<std::vector<int>>());
+}
+
+class MultiplierCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->Inputs().Index(0).Set<int>();
+    cc->Inputs().Index(1).Set<int>();
+    cc->Outputs().Index(0).Set<int>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    const int& input_int = cc->Inputs().Index(0).Get<int>();
+    const int& multiplier_int = cc->Inputs().Index(1).Get<int>();
+    auto output_int = absl::make_unique<int>(input_int * multiplier_int);
+    cc->Outputs().Index(0).Add(output_int.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+};
+
+REGISTER_CALCULATOR(MultiplierCalculator);
+
+class BeginEndLoopCalculatorGraphWithClonedInputsTest : public ::testing::Test {
+ protected:
+  BeginEndLoopCalculatorGraphWithClonedInputsTest() {
+    graph_config_ = ParseTextProtoOrDie<CalculatorGraphConfig>(
+        R"(
+          num_threads: 4
+          input_stream: "ints"
+          input_stream: "multiplier"
+          node {
+            calculator: "BeginLoopIntegerCalculator"
+            input_stream: "ITERABLE:ints"
+            input_stream: "CLONE:multiplier"
+            output_stream: "ITEM:int_at_loop"
+            output_stream: "CLONE:multiplier_cloned_at_loop"
+            output_stream: "BATCH_END:timestamp"
+          }
+          node {
+            calculator: "MultiplierCalculator"
+            input_stream: "int_at_loop"
+            input_stream: "multiplier_cloned_at_loop"
+            output_stream: "multiplied_int_at_loop"
+          }
+          node {
+            calculator: "EndLoopIntegersCalculator"
+            input_stream: "ITEM:multiplied_int_at_loop"
+            input_stream: "BATCH_END:timestamp"
+            output_stream: "ITERABLE:multiplied_ints"
+          }
+        )");
+    tool::AddVectorSink("multiplied_ints", &graph_config_, &output_packets_);
+  }
+
+  CalculatorGraphConfig graph_config_;
+  std::vector<Packet> output_packets_;
+};
+
+TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleEmptyVector) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+  auto input_vector = absl::make_unique<std::vector<int>>();
+  Timestamp input_timestamp = Timestamp(42);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector.release()).At(input_timestamp)));
+  auto multiplier = absl::make_unique<int>(2);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "multiplier", Adopt(multiplier.release()).At(input_timestamp)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  // EndLoopCalc will forward the timestamp bound because there are no elements
+  // in collection to output.
+  ASSERT_EQ(0, output_packets_.size());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, SingleNonEmptyVector) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+  auto input_vector = absl::make_unique<std::vector<int>>();
+  input_vector->emplace_back(0);
+  input_vector->emplace_back(1);
+  input_vector->emplace_back(2);
+  Timestamp input_timestamp = Timestamp(42);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector.release()).At(input_timestamp)));
+  auto multiplier = absl::make_unique<int>(2);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "multiplier", Adopt(multiplier.release()).At(input_timestamp)));
+  MP_ASSERT_OK(graph.WaitUntilIdle());
+
+  ASSERT_EQ(1, output_packets_.size());
+  EXPECT_EQ(input_timestamp, output_packets_[0].Timestamp());
+  std::vector<int> expected_output_vector = {0, 2, 4};
+  EXPECT_EQ(expected_output_vector, output_packets_[0].Get<std::vector<int>>());
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+}
+
+TEST_F(BeginEndLoopCalculatorGraphWithClonedInputsTest, MultipleVectors) {
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config_));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  auto input_vector0 = absl::make_unique<std::vector<int>>();
+  input_vector0->emplace_back(0);
+  input_vector0->emplace_back(1);
+  Timestamp input_timestamp0 = Timestamp(42);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector0.release()).At(input_timestamp0)));
+  auto multiplier0 = absl::make_unique<int>(2);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "multiplier", Adopt(multiplier0.release()).At(input_timestamp0)));
+
+  auto input_vector1 = absl::make_unique<std::vector<int>>();
+  Timestamp input_timestamp1 = Timestamp(43);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector1.release()).At(input_timestamp1)));
+  auto multiplier1 = absl::make_unique<int>(2);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "multiplier", Adopt(multiplier1.release()).At(input_timestamp1)));
+
+  auto input_vector2 = absl::make_unique<std::vector<int>>();
+  input_vector2->emplace_back(2);
+  input_vector2->emplace_back(3);
+  Timestamp input_timestamp2 = Timestamp(44);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "ints", Adopt(input_vector2.release()).At(input_timestamp2)));
+  auto multiplier2 = absl::make_unique<int>(3);
+  MP_ASSERT_OK(graph.AddPacketToInputStream(
+      "multiplier", Adopt(multiplier2.release()).At(input_timestamp2)));
+
+  MP_ASSERT_OK(graph.CloseAllPacketSources());
+  MP_ASSERT_OK(graph.WaitUntilDone());
+
+  ASSERT_EQ(2, output_packets_.size());
+
+  EXPECT_EQ(input_timestamp0, output_packets_[0].Timestamp());
+  std::vector<int> expected_output_vector0 = {0, 2};
+  EXPECT_EQ(expected_output_vector0,
+            output_packets_[0].Get<std::vector<int>>());
+
+  // At input_timestamp1, EndLoopCalc will forward timestamp bound as there are
+  // no elements in vector to process.
+
+  EXPECT_EQ(input_timestamp2, output_packets_[1].Timestamp());
+  std::vector<int> expected_output_vector2 = {6, 9};
+  EXPECT_EQ(expected_output_vector2,
+            output_packets_[1].Get<std::vector<int>>());
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/core/begin_loop_calculator.cc

@@ -0,0 +1,40 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/begin_loop_calculator.h"
+
+#include <vector>
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+
+namespace mediapipe {
+
+// A calculator to process std::vector<NormalizedLandmark>.
+typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
+    BeginLoopNormalizedLandmarkCalculator;
+REGISTER_CALCULATOR(BeginLoopNormalizedLandmarkCalculator);
+
+// A calculator to process std::vector<std::vector<NormalizedLandmark>>.
+typedef BeginLoopCalculator<
+    std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
+    BeginLoopNormalizedLandmarksVectorCalculator;
+REGISTER_CALCULATOR(BeginLoopNormalizedLandmarksVectorCalculator);
+
+// A calculator to process std::vector<NormalizedRect>.
+typedef BeginLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
+    BeginLoopNormalizedRectCalculator;
+REGISTER_CALCULATOR(BeginLoopNormalizedRectCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/begin_loop_calculator.h

@@ -0,0 +1,157 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_
+
+#include "absl/memory/memory.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_contract.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/collection_item_id.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Calculator for implementing loops on iterable collections inside a MediaPipe
+// graph.
+//
+// It is designed to be used like:
+//
+// node {
+//   calculator:    "BeginLoopWithIterableCalculator"
+//   input_stream:  "ITERABLE:input_iterable"      # IterableT @ext_ts
+//   output_stream: "ITEM:input_element"           # ItemT     @loop_internal_ts
+//   output_stream: "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+// }
+//
+// node {
+//   calculator:    "ElementToBlaConverterSubgraph"
+//   input_stream:  "ITEM:input_to_loop_body"      # ItemT     @loop_internal_ts
+//   output_stream: "BLA:output_of_loop_body"      # ItemU     @loop_internal_ts
+// }
+//
+// node {
+//   calculator:    "EndLoopWithOutputCalculator"
+//   input_stream:  "ITEM:output_of_loop_body"     # ItemU     @loop_internal_ts
+//   input_stream:  "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+//   output_stream: "OUTPUT:aggregated_result"     # IterableU @ext_ts
+// }
+//
+// BeginLoopCalculator accepts an optional input stream tagged with "TICK"
+// which if non-empty, wakes up the calculator and calls
+// BeginLoopCalculator::Process(). Input streams tagged with "CLONE" are cloned
+// to the corresponding output streams at loop timestamps. This ensures that a
+// MediaPipe graph or sub-graph can run multiple times, once per element in the
+// "ITERABLE" for each pakcet clone of the packets in the "CLONE" input streams.
+template <typename IterableT>
+class BeginLoopCalculator : public CalculatorBase {
+  using ItemT = typename IterableT::value_type;
+
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    // A non-empty packet in the optional "TICK" input stream wakes up the
+    // calculator.
+    if (cc->Inputs().HasTag("TICK")) {
+      cc->Inputs().Tag("TICK").SetAny();
+    }
+
+    // An iterable collection in the input stream.
+    RET_CHECK(cc->Inputs().HasTag("ITERABLE"));
+    cc->Inputs().Tag("ITERABLE").Set<IterableT>();
+
+    // An element from the collection.
+    RET_CHECK(cc->Outputs().HasTag("ITEM"));
+    cc->Outputs().Tag("ITEM").Set<ItemT>();
+
+    RET_CHECK(cc->Outputs().HasTag("BATCH_END"));
+    cc->Outputs()
+        .Tag("BATCH_END")
+        .Set<Timestamp>(
+            // A flush signal to the corresponding EndLoopCalculator for it to
+            // emit the aggregated result with the timestamp contained in this
+            // flush signal packet.
+        );
+
+    // Input streams tagged with "CLONE" are cloned to the corresponding
+    // "CLONE" output streams at loop timestamps.
+    RET_CHECK(cc->Inputs().NumEntries("CLONE") ==
+              cc->Outputs().NumEntries("CLONE"));
+    if (cc->Inputs().NumEntries("CLONE") > 0) {
+      for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) {
+        cc->Inputs().Get("CLONE", i).SetAny();
+        cc->Outputs().Get("CLONE", i).SetSameAs(&cc->Inputs().Get("CLONE", i));
+      }
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) final {
+    Timestamp last_timestamp = loop_internal_timestamp_;
+    if (!cc->Inputs().Tag("ITERABLE").IsEmpty()) {
+      const IterableT& collection =
+          cc->Inputs().Tag("ITERABLE").template Get<IterableT>();
+      for (const auto& item : collection) {
+        cc->Outputs().Tag("ITEM").AddPacket(
+            MakePacket<ItemT>(item).At(loop_internal_timestamp_));
+        ForwardClonePackets(cc, loop_internal_timestamp_);
+        ++loop_internal_timestamp_;
+      }
+    }
+
+    // The collection was empty and nothing was processed.
+    if (last_timestamp == loop_internal_timestamp_) {
+      // Increment loop_internal_timestamp_ because it is used up now.
+      ++loop_internal_timestamp_;
+      for (auto it = cc->Outputs().begin(); it < cc->Outputs().end(); ++it) {
+        it->SetNextTimestampBound(loop_internal_timestamp_);
+      }
+    }
+
+    // The for loop processing the input collection already incremented
+    // loop_internal_timestamp_. To emit BATCH_END packet along the last
+    // non-BATCH_END packet, decrement by one.
+    cc->Outputs()
+        .Tag("BATCH_END")
+        .AddPacket(MakePacket<Timestamp>(cc->InputTimestamp())
+                       .At(Timestamp(loop_internal_timestamp_ - 1)));
+
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  void ForwardClonePackets(CalculatorContext* cc, Timestamp output_timestamp) {
+    if (cc->Inputs().NumEntries("CLONE") > 0) {
+      for (int i = 0; i < cc->Inputs().NumEntries("CLONE"); ++i) {
+        if (!cc->Inputs().Get("CLONE", i).IsEmpty()) {
+          auto input_packet = cc->Inputs().Get("CLONE", i).Value();
+          cc->Outputs()
+              .Get("CLONE", i)
+              .AddPacket(std::move(input_packet).At(output_timestamp));
+        }
+      }
+    }
+  }
+
+  // Fake timestamps generated per element in collection.
+  Timestamp loop_internal_timestamp_ = Timestamp(0);
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_BEGIN_LOOP_CALCULATOR_H_

mediapipe/calculators/core/clip_detection_vector_size_calculator.cc

@@ -0,0 +1,26 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <vector>
+
+#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+
+namespace mediapipe {
+
+typedef ClipVectorSizeCalculator<::mediapipe::Detection>
+    ClipDetectionVectorSizeCalculator;
+REGISTER_CALCULATOR(ClipDetectionVectorSizeCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/clip_vector_size_calculator.cc

@@ -0,0 +1,28 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
+
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+
+namespace mediapipe {
+
+typedef ClipVectorSizeCalculator<::mediapipe::NormalizedRect>
+    ClipNormalizedRectVectorSizeCalculator;
+REGISTER_CALCULATOR(ClipNormalizedRectVectorSizeCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/clip_vector_size_calculator.h

@@ -0,0 +1,137 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_
+
+#include <type_traits>
+#include <vector>
+
+#include "mediapipe/calculators/core/clip_vector_size_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Clips the size of the input vector of type T to a specified max_vec_size.
+// In a graph it will be used as:
+// node {
+//   calculator: "ClipIntVectorSizeCalculator"
+//   input_stream: "input_vector"
+//   output_stream: "output_vector"
+//   options {
+//     [mediapipe.ClipIntVectorSizeCalculatorOptions.ext] {
+//       max_vec_size: 5
+//     }
+//   }
+// }
+template <typename T>
+class ClipVectorSizeCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().NumEntries() == 1);
+    RET_CHECK(cc->Outputs().NumEntries() == 1);
+
+    if (cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
+            .max_vec_size() < 1) {
+      return ::mediapipe::InternalError(
+          "max_vec_size should be greater than or equal to 1.");
+    }
+
+    cc->Inputs().Index(0).Set<std::vector<T>>();
+    cc->Outputs().Index(0).Set<std::vector<T>>();
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    max_vec_size_ = cc->Options<::mediapipe::ClipVectorSizeCalculatorOptions>()
+                        .max_vec_size();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (max_vec_size_ < 1) {
+      return ::mediapipe::InternalError(
+          "max_vec_size should be greater than or equal to 1.");
+    }
+    if (cc->Inputs().Index(0).IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    return ClipVectorSize<T>(std::is_copy_constructible<T>(), cc);
+  }
+
+  template <typename U>
+  ::mediapipe::Status ClipVectorSize(std::true_type, CalculatorContext* cc) {
+    auto output = absl::make_unique<std::vector<U>>();
+    const std::vector<U>& input_vector =
+        cc->Inputs().Index(0).Get<std::vector<U>>();
+    if (max_vec_size_ >= input_vector.size()) {
+      output->insert(output->end(), input_vector.begin(), input_vector.end());
+    } else {
+      for (int i = 0; i < max_vec_size_; ++i) {
+        output->push_back(input_vector[i]);
+      }
+    }
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+
+  template <typename U>
+  ::mediapipe::Status ClipVectorSize(std::false_type, CalculatorContext* cc) {
+    return ConsumeAndClipVectorSize<T>(std::is_move_constructible<U>(), cc);
+  }
+
+  template <typename U>
+  ::mediapipe::Status ConsumeAndClipVectorSize(std::true_type,
+                                               CalculatorContext* cc) {
+    auto output = absl::make_unique<std::vector<U>>();
+    ::mediapipe::StatusOr<std::unique_ptr<std::vector<U>>> input_status =
+        cc->Inputs().Index(0).Value().Consume<std::vector<U>>();
+
+    if (input_status.ok()) {
+      std::unique_ptr<std::vector<U>> input_vector =
+          std::move(input_status).ValueOrDie();
+      auto begin_it = input_vector->begin();
+      auto end_it = input_vector->end();
+      if (max_vec_size_ < input_vector->size()) {
+        end_it = input_vector->begin() + max_vec_size_;
+      }
+      output->insert(output->end(), std::make_move_iterator(begin_it),
+                     std::make_move_iterator(end_it));
+    } else {
+      return input_status.status();
+    }
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+
+  template <typename U>
+  ::mediapipe::Status ConsumeAndClipVectorSize(std::false_type,
+                                               CalculatorContext* cc) {
+    return ::mediapipe::InternalError(
+        "Cannot copy or move input vectors and clip their size.");
+  }
+
+ private:
+  int max_vec_size_ = 0;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_CLIP_VECTOR_SIZE_CALCULATOR_H_

mediapipe/calculators/core/clip_vector_size_calculator.proto

@@ -0,0 +1,28 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message ClipVectorSizeCalculatorOptions {
+  extend CalculatorOptions {
+    optional ClipVectorSizeCalculatorOptions ext = 274674998;
+  }
+
+  // Maximum size of output vector.
+  optional int32 max_vec_size = 1 [default = 1];
+}

mediapipe/calculators/core/clip_vector_size_calculator_test.cc

@@ -0,0 +1,179 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/clip_vector_size_calculator.h"
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"  // NOLINT
+
+namespace mediapipe {
+
+typedef ClipVectorSizeCalculator<int> TestClipIntVectorSizeCalculator;
+REGISTER_CALCULATOR(TestClipIntVectorSizeCalculator);
+
+void AddInputVector(const std::vector<int>& input, int64 timestamp,
+                    CalculatorRunner* runner) {
+  runner->MutableInputs()->Index(0).packets.push_back(
+      MakePacket<std::vector<int>>(input).At(Timestamp(timestamp)));
+}
+
+TEST(TestClipIntVectorSizeCalculatorTest, EmptyVectorInput) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "TestClipIntVectorSizeCalculator"
+        input_stream: "input_vector"
+        output_stream: "output_vector"
+        options {
+          [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 1 }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+
+  std::vector<int> input = {};
+  AddInputVector(input, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  EXPECT_TRUE(outputs[0].Get<std::vector<int>>().empty());
+}
+
+TEST(TestClipIntVectorSizeCalculatorTest, OneTimestamp) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "TestClipIntVectorSizeCalculator"
+        input_stream: "input_vector"
+        output_stream: "output_vector"
+        options {
+          [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 2 }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+
+  std::vector<int> input = {0, 1, 2, 3};
+  AddInputVector(input, /*timestamp=*/1, &runner);
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const std::vector<int>& output = outputs[0].Get<std::vector<int>>();
+  EXPECT_EQ(2, output.size());
+  std::vector<int> expected_vector = {0, 1};
+  EXPECT_EQ(expected_vector, output);
+}
+
+TEST(TestClipIntVectorSizeCalculatorTest, TwoInputsAtTwoTimestamps) {
+  CalculatorGraphConfig::Node node_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+        calculator: "TestClipIntVectorSizeCalculator"
+        input_stream: "input_vector"
+        output_stream: "output_vector"
+        options {
+          [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
+        }
+      )");
+  CalculatorRunner runner(node_config);
+
+  {
+    std::vector<int> input = {0, 1, 2, 3};
+    AddInputVector(input, /*timestamp=*/1, &runner);
+  }
+  {
+    std::vector<int> input = {2, 3, 4, 5};
+    AddInputVector(input, /*timestamp=*/2, &runner);
+  }
+  MP_ASSERT_OK(runner.Run());
+
+  const std::vector<Packet>& outputs = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(2, outputs.size());
+  {
+    EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+    const std::vector<int>& output = outputs[0].Get<std::vector<int>>();
+    EXPECT_EQ(3, output.size());
+    std::vector<int> expected_vector = {0, 1, 2};
+    EXPECT_EQ(expected_vector, output);
+  }
+  {
+    EXPECT_EQ(Timestamp(2), outputs[1].Timestamp());
+    const std::vector<int>& output = outputs[1].Get<std::vector<int>>();
+    EXPECT_EQ(3, output.size());
+    std::vector<int> expected_vector = {2, 3, 4};
+    EXPECT_EQ(expected_vector, output);
+  }
+}
+
+typedef ClipVectorSizeCalculator<std::unique_ptr<int>>
+    TestClipUniqueIntPtrVectorSizeCalculator;
+REGISTER_CALCULATOR(TestClipUniqueIntPtrVectorSizeCalculator);
+
+TEST(TestClipUniqueIntPtrVectorSizeCalculatorTest, ConsumeOneTimestamp) {
+  /* Note: We don't use CalculatorRunner for this test because it keeps copies
+   * of input packets, so packets sent to the graph don't have sole ownership.
+   * The test needs to send packets that own the data.
+   */
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: "input_vector"
+        node {
+          calculator: "TestClipUniqueIntPtrVectorSizeCalculator"
+          input_stream: "input_vector"
+          output_stream: "output_vector"
+          options {
+            [mediapipe.ClipVectorSizeCalculatorOptions.ext] { max_vec_size: 3 }
+          }
+        }
+      )");
+
+  std::vector<Packet> outputs;
+  tool::AddVectorSink("output_vector", &graph_config, &outputs);
+
+  CalculatorGraph graph;
+  MP_EXPECT_OK(graph.Initialize(graph_config));
+  MP_EXPECT_OK(graph.StartRun({}));
+
+  // input1 : {0, 1, 2, 3, 4, 5}
+  auto input_vector = absl::make_unique<std::vector<std::unique_ptr<int>>>(6);
+  for (int i = 0; i < 6; ++i) {
+    input_vector->at(i) = absl::make_unique<int>(i);
+  }
+
+  MP_EXPECT_OK(graph.AddPacketToInputStream(
+      "input_vector", Adopt(input_vector.release()).At(Timestamp(1))));
+
+  MP_EXPECT_OK(graph.WaitUntilIdle());
+  MP_EXPECT_OK(graph.CloseAllPacketSources());
+  MP_EXPECT_OK(graph.WaitUntilDone());
+
+  EXPECT_EQ(1, outputs.size());
+  EXPECT_EQ(Timestamp(1), outputs[0].Timestamp());
+  const std::vector<std::unique_ptr<int>>& result =
+      outputs[0].Get<std::vector<std::unique_ptr<int>>>();
+  EXPECT_EQ(3, result.size());
+  for (int i = 0; i < 3; ++i) {
+    const std::unique_ptr<int>& v = result[i];
+    EXPECT_EQ(i, *v);
+  }
+}
+
+}  // namespace mediapipe

mediapipe/calculators/core/end_loop_calculator.cc

@@ -0,0 +1,45 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/core/end_loop_calculator.h"
+
+#include <vector>
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/util/render_data.pb.h"
+
+namespace mediapipe {
+
+typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedRect>>
+    EndLoopNormalizedRectCalculator;
+REGISTER_CALCULATOR(EndLoopNormalizedRectCalculator);
+
+typedef EndLoopCalculator<std::vector<::mediapipe::NormalizedLandmark>>
+    EndLoopNormalizedLandmarkCalculator;
+REGISTER_CALCULATOR(EndLoopNormalizedLandmarkCalculator);
+
+typedef EndLoopCalculator<
+    std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
+    EndLoopNormalizedLandmarksVectorCalculator;
+REGISTER_CALCULATOR(EndLoopNormalizedLandmarksVectorCalculator);
+
+typedef EndLoopCalculator<std::vector<bool>> EndLoopBooleanCalculator;
+REGISTER_CALCULATOR(EndLoopBooleanCalculator);
+
+typedef EndLoopCalculator<std::vector<::mediapipe::RenderData>>
+    EndLoopRenderDataCalculator;
+REGISTER_CALCULATOR(EndLoopRenderDataCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/core/end_loop_calculator.h

@@ -0,0 +1,106 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_
+
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_contract.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/collection_item_id.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Calculator for completing the processing of loops on iterable collections
+// inside a MediaPipe graph. The EndLoopCalculator collects all input packets
+// from ITEM input_stream into a collection and upon receiving the flush signal
+// from the "BATCH_END" tagged input stream, it emits the aggregated results
+// at the original timestamp contained in the "BATCH_END" input stream.
+//
+// It is designed to be used like:
+//
+// node {
+//   calculator:    "BeginLoopWithIterableCalculator"
+//   input_stream:  "ITERABLE:input_iterable"      # IterableT @ext_ts
+//   output_stream: "ITEM:input_element"           # ItemT     @loop_internal_ts
+//   output_stream: "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+// }
+//
+// node {
+//   calculator:    "ElementToBlaConverterSubgraph"
+//   input_stream:  "ITEM:input_to_loop_body"      # ItemT     @loop_internal_ts
+//   output_stream: "BLA:output_of_loop_body"      # ItemU     @loop_internal_ts
+// }
+//
+// node {
+//   calculator:    "EndLoopWithOutputCalculator"
+//   input_stream:  "ITEM:output_of_loop_body"     # ItemU     @loop_internal_ts
+//   input_stream:  "BATCH_END:ext_ts"             # Timestamp @loop_internal_ts
+//   output_stream: "OUTPUT:aggregated_result"     # IterableU @ext_ts
+// }
+template <typename IterableT>
+class EndLoopCalculator : public CalculatorBase {
+  using ItemT = typename IterableT::value_type;
+
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().HasTag("BATCH_END"))
+        << "Missing BATCH_END tagged input_stream.";
+    cc->Inputs().Tag("BATCH_END").Set<Timestamp>();
+
+    RET_CHECK(cc->Inputs().HasTag("ITEM"));
+    cc->Inputs().Tag("ITEM").Set<ItemT>();
+
+    RET_CHECK(cc->Outputs().HasTag("ITERABLE"));
+    cc->Outputs().Tag("ITERABLE").Set<IterableT>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (!cc->Inputs().Tag("ITEM").IsEmpty()) {
+      if (!input_stream_collection_) {
+        input_stream_collection_.reset(new IterableT);
+      }
+      input_stream_collection_->push_back(
+          cc->Inputs().Tag("ITEM").template Get<ItemT>());
+    }
+
+    if (!cc->Inputs().Tag("BATCH_END").Value().IsEmpty()) {  // flush signal
+      Timestamp loop_control_ts =
+          cc->Inputs().Tag("BATCH_END").template Get<Timestamp>();
+      if (input_stream_collection_) {
+        cc->Outputs()
+            .Tag("ITERABLE")
+            .Add(input_stream_collection_.release(), loop_control_ts);
+      } else {
+        // Since there is no collection, inform downstream calculators to not
+        // expect any packet by updating the timestamp bounds.
+        cc->Outputs()
+            .Tag("ITERABLE")
+            .SetNextTimestampBound(Timestamp(loop_control_ts.Value() + 1));
+      }
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  std::unique_ptr<IterableT> input_stream_collection_;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_CORE_END_LOOP_CALCULATOR_H_

mediapipe/calculators/core/packet_resampler_calculator.cc

@@ -74,6 +74,12 @@ class PacketResamplerCalculator : public CalculatorBase {
   ::mediapipe::Status Process(CalculatorContext* cc) override;
 
  private:
+  // Calculates the first sampled timestamp that incorporates a jittering
+  // offset.
+  void InitializeNextOutputTimestampWithJitter();
+  // Calculates the next sampled timestamp that incorporates a jittering offset.
+  void UpdateNextOutputTimestampWithJitter();
+
   // Logic for Process() when jitter_ != 0.0.
   ::mediapipe::Status ProcessWithJitter(CalculatorContext* cc);
 
@@ -233,6 +239,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
       << Timestamp::kTimestampUnitsPerSecond;
 
   frame_time_usec_ = static_cast<int64>(1000000.0 / frame_rate_);
+
   video_header_.frame_rate = frame_rate_;
 
   if (resampler_options.output_header() !=
@@ -295,6 +302,17 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
   return ::mediapipe::OkStatus();
 }
 
+void PacketResamplerCalculator::InitializeNextOutputTimestampWithJitter() {
+  next_output_timestamp_ =
+      first_timestamp_ + frame_time_usec_ * random_->RandFloat();
+}
+
+void PacketResamplerCalculator::UpdateNextOutputTimestampWithJitter() {
+  next_output_timestamp_ +=
+      frame_time_usec_ *
+      ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
+}
+
 ::mediapipe::Status PacketResamplerCalculator::ProcessWithJitter(
     CalculatorContext* cc) {
   RET_CHECK_GT(cc->InputTimestamp(), Timestamp::PreStream());
@@ -302,8 +320,13 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
 
   if (first_timestamp_ == Timestamp::Unset()) {
     first_timestamp_ = cc->InputTimestamp();
-    next_output_timestamp_ =
+    InitializeNextOutputTimestampWithJitter();
-        first_timestamp_ + frame_time_usec_ * random_->RandFloat();
+    if (first_timestamp_ == next_output_timestamp_) {
+      OutputWithinLimits(
+          cc,
+          cc->Inputs().Get(input_data_id_).Value().At(next_output_timestamp_));
+      UpdateNextOutputTimestampWithJitter();
+    }
     return ::mediapipe::OkStatus();
   }
 
@@ -322,9 +345,7 @@ TimestampDiff TimestampDiffFromSeconds(double seconds) {
                               ? last_packet_
                               : cc->Inputs().Get(input_data_id_).Value())
                              .At(next_output_timestamp_));
-  next_output_timestamp_ +=
+  UpdateNextOutputTimestampWithJitter();
-      frame_time_usec_ *
-      ((1.0 - jitter_) + 2.0 * jitter_ * random_->RandFloat());
   return ::mediapipe::OkStatus();
 }
 

mediapipe/calculators/image/BUILD

@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:private"])
 
 exports_files(["LICENSE"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-
 proto_library(
     name = "opencv_image_encoder_calculator_proto",
     srcs = ["opencv_image_encoder_calculator.proto"],

mediapipe/calculators/tensorflow/BUILD

@@ -13,12 +13,12 @@
 # limitations under the License.
 #
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:private"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-
 proto_library(
     name = "graph_tensors_packet_generator_proto",
     srcs = ["graph_tensors_packet_generator.proto"],
@@ -138,7 +138,7 @@ mediapipe_cc_proto_library(
     srcs = ["image_frame_to_tensor_calculator.proto"],
     cc_deps = [
         "//mediapipe/framework:calculator_cc_proto",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
     visibility = ["//visibility:public"],
     deps = [":image_frame_to_tensor_calculator_proto"],
@@ -173,7 +173,7 @@ mediapipe_cc_proto_library(
     srcs = ["pack_media_sequence_calculator.proto"],
     cc_deps = [
         "//mediapipe/framework:calculator_cc_proto",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
     visibility = ["//visibility:public"],
     deps = [":pack_media_sequence_calculator_proto"],
@@ -192,7 +192,7 @@ mediapipe_cc_proto_library(
     srcs = ["tensorflow_session_from_frozen_graph_generator.proto"],
     cc_deps = [
         "//mediapipe/framework:packet_generator_cc_proto",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
     visibility = ["//visibility:public"],
     deps = [":tensorflow_session_from_frozen_graph_generator_proto"],
@@ -203,7 +203,7 @@ mediapipe_cc_proto_library(
     srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"],
     cc_deps = [
         "//mediapipe/framework:calculator_cc_proto",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
     visibility = ["//visibility:public"],
     deps = [":tensorflow_session_from_frozen_graph_calculator_proto"],
@@ -277,7 +277,7 @@ mediapipe_cc_proto_library(
     srcs = ["vector_int_to_tensor_calculator_options.proto"],
     cc_deps = [
         "//mediapipe/framework:calculator_cc_proto",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
     visibility = ["//visibility:public"],
     deps = [":vector_int_to_tensor_calculator_options_proto"],
@@ -296,7 +296,7 @@ cc_library(
     srcs = ["graph_tensors_packet_generator.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        "//mediapipe/calculators/tensorflow:graph_tensors_packet_generator_cc_proto",
+        ":graph_tensors_packet_generator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
@@ -311,7 +311,7 @@ cc_library(
     srcs = ["image_frame_to_tensor_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        "//mediapipe/calculators/tensorflow:image_frame_to_tensor_calculator_cc_proto",
+        ":image_frame_to_tensor_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/port:ret_check",
@@ -333,7 +333,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework/formats:time_series_header_cc_proto",
-        "//mediapipe/calculators/tensorflow:matrix_to_tensor_calculator_options_cc_proto",
+        ":matrix_to_tensor_calculator_options_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:status",
@@ -354,7 +354,7 @@ cc_library(
     srcs = ["lapped_tensor_buffer_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        "//mediapipe/calculators/tensorflow:lapped_tensor_buffer_calculator_cc_proto",
+        ":lapped_tensor_buffer_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
@@ -408,7 +408,7 @@ cc_library(
         "//mediapipe/util/sequence:media_sequence",
         "//mediapipe/util/sequence:media_sequence_util",
         "@com_google_absl//absl/strings",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
     alwayslink = 1,
 )
@@ -423,7 +423,7 @@ cc_library(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
     alwayslink = 1,
 )
@@ -436,7 +436,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":tensorflow_session",
-        "//mediapipe/calculators/tensorflow:tensorflow_inference_calculator_cc_proto",
+        ":tensorflow_inference_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/tool:status_util",
         "@com_google_absl//absl/strings",
@@ -514,7 +514,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":tensorflow_session",
-        "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_generator_cc_proto",
+        ":tensorflow_session_from_frozen_graph_generator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/tool:status_util",
         "//mediapipe/framework/port:status",
@@ -573,7 +573,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":tensorflow_session",
-        "//mediapipe/calculators/tensorflow:tensorflow_session_from_saved_model_generator_cc_proto",
+        ":tensorflow_session_from_saved_model_generator_cc_proto",
         "//mediapipe/framework:packet_generator",
         "//mediapipe/framework:packet_type",
         "//mediapipe/framework/tool:status_util",
@@ -597,7 +597,7 @@ cc_library(
     srcs = ["tensor_squeeze_dimensions_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        "//mediapipe/calculators/tensorflow:tensor_squeeze_dimensions_calculator_cc_proto",
+        ":tensor_squeeze_dimensions_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
@@ -611,7 +611,7 @@ cc_library(
     srcs = ["tensor_to_image_frame_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        "//mediapipe/calculators/tensorflow:tensor_to_image_frame_calculator_cc_proto",
+        ":tensor_to_image_frame_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/port:ret_check",
@@ -627,7 +627,7 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "//mediapipe/framework/formats:time_series_header_cc_proto",
-        "//mediapipe/calculators/tensorflow:tensor_to_matrix_calculator_cc_proto",
+        ":tensor_to_matrix_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/port:status",
@@ -654,7 +654,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "@org_tensorflow//tensorflow/core:lib",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
     alwayslink = 1,
 )
@@ -667,7 +667,7 @@ cc_library(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:ret_check",
-        "//mediapipe/calculators/tensorflow:tensor_to_vector_float_calculator_options_cc_proto",
+        ":tensor_to_vector_float_calculator_options_cc_proto",
     ] + select({
         "//conditions:default": [
             "@org_tensorflow//tensorflow/core:framework",
@@ -695,7 +695,7 @@ cc_library(
         "//mediapipe/util:audio_decoder_cc_proto",
         "//mediapipe/util/sequence:media_sequence",
         "@com_google_absl//absl/strings",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
     alwayslink = 1,
 )
@@ -719,7 +719,7 @@ cc_library(
     srcs = ["vector_float_to_tensor_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        "//mediapipe/calculators/tensorflow:vector_float_to_tensor_calculator_options_cc_proto",
+        ":vector_float_to_tensor_calculator_options_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
@@ -733,11 +733,11 @@ cc_library(
     srcs = ["unpack_yt8m_sequence_example_calculator.cc"],
     visibility = ["//visibility:public"],
     deps = [
-        "//mediapipe/calculators/tensorflow:lapped_tensor_buffer_calculator_cc_proto",
+        ":lapped_tensor_buffer_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:packet",
         "//mediapipe/framework/port:status",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
     alwayslink = 1,
 )
@@ -747,7 +747,7 @@ cc_test(
     srcs = ["graph_tensors_packet_generator_test.cc"],
     deps = [
         ":graph_tensors_packet_generator",
-        "//mediapipe/calculators/tensorflow:graph_tensors_packet_generator_cc_proto",
+        ":graph_tensors_packet_generator_cc_proto",
         "//mediapipe/framework:packet",
         "//mediapipe/framework:packet_generator_cc_proto",
         "//mediapipe/framework:packet_set",
@@ -779,7 +779,7 @@ cc_test(
     srcs = ["matrix_to_tensor_calculator_test.cc"],
     deps = [
         ":matrix_to_tensor_calculator",
-        "//mediapipe/calculators/tensorflow:matrix_to_tensor_calculator_options_cc_proto",
+        ":matrix_to_tensor_calculator_options_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/formats:matrix",
@@ -795,13 +795,13 @@ cc_test(
     srcs = ["lapped_tensor_buffer_calculator_test.cc"],
     deps = [
         ":lapped_tensor_buffer_calculator",
-        "//mediapipe/calculators/tensorflow:lapped_tensor_buffer_calculator_cc_proto",
+        ":lapped_tensor_buffer_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/port:gtest_main",
         "@com_google_absl//absl/memory",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
 )
 
@@ -840,7 +840,7 @@ cc_test(
         "//mediapipe/util/sequence:media_sequence",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
 )
 
@@ -867,7 +867,7 @@ cc_test(
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:direct_session",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
         "@org_tensorflow//tensorflow/core:testlib",
         "@org_tensorflow//tensorflow/core/kernels:conv_ops",
         "@org_tensorflow//tensorflow/core/kernels:math",
@@ -883,7 +883,7 @@ cc_test(
         ":tensorflow_inference_calculator",
         ":tensorflow_session",
         ":tensorflow_session_from_frozen_graph_generator",
-        "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_generator_cc_proto",
+        ":tensorflow_session_from_frozen_graph_generator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:packet",
         "//mediapipe/framework:packet_generator_cc_proto",
@@ -897,7 +897,7 @@ cc_test(
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:direct_session",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
         "@org_tensorflow//tensorflow/core:testlib",
         "@org_tensorflow//tensorflow/core/kernels:conv_ops",
         "@org_tensorflow//tensorflow/core/kernels:math",
@@ -913,7 +913,7 @@ cc_test(
         ":tensorflow_inference_calculator",
         ":tensorflow_session",
         ":tensorflow_session_from_saved_model_generator",
-        "//mediapipe/calculators/tensorflow:tensorflow_session_from_saved_model_generator_cc_proto",
+        ":tensorflow_session_from_saved_model_generator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:packet",
         "//mediapipe/framework:packet_generator_cc_proto",
@@ -923,14 +923,8 @@ cc_test(
         "//mediapipe/framework/tool:tag_map_helper",
         "//mediapipe/framework/tool:validate_type",
         "@com_google_absl//absl/strings",
+        "@org_tensorflow//tensorflow/core:all_kernels",
         "@org_tensorflow//tensorflow/core:direct_session",
-        "@org_tensorflow//tensorflow/core/kernels:array",
-        "@org_tensorflow//tensorflow/core/kernels:bitcast_op",
-        "@org_tensorflow//tensorflow/core/kernels:conv_ops",
-        "@org_tensorflow//tensorflow/core/kernels:io",
-        "@org_tensorflow//tensorflow/core/kernels:state",
-        "@org_tensorflow//tensorflow/core/kernels:string",
-        "@org_tensorflow//tensorflow/core/kernels/data:tensor_dataset_op",
     ],
 )
 
@@ -954,14 +948,8 @@ cc_test(
         "//mediapipe/framework/tool:tag_map_helper",
         "//mediapipe/framework/tool:validate_type",
         "@com_google_absl//absl/strings",
+        "@org_tensorflow//tensorflow/core:all_kernels",
         "@org_tensorflow//tensorflow/core:direct_session",
-        "@org_tensorflow//tensorflow/core/kernels:array",
-        "@org_tensorflow//tensorflow/core/kernels:bitcast_op",
-        "@org_tensorflow//tensorflow/core/kernels:conv_ops",
-        "@org_tensorflow//tensorflow/core/kernels:io",
-        "@org_tensorflow//tensorflow/core/kernels:state",
-        "@org_tensorflow//tensorflow/core/kernels:string",
-        "@org_tensorflow//tensorflow/core/kernels/data:tensor_dataset_op",
     ],
 )
 
@@ -970,12 +958,12 @@ cc_test(
     srcs = ["tensor_squeeze_dimensions_calculator_test.cc"],
     deps = [
         ":tensor_squeeze_dimensions_calculator",
-        "//mediapipe/calculators/tensorflow:tensor_squeeze_dimensions_calculator_cc_proto",
+        ":tensor_squeeze_dimensions_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
 )
 
@@ -985,13 +973,13 @@ cc_test(
     srcs = ["tensor_to_image_frame_calculator_test.cc"],
     deps = [
         ":tensor_to_image_frame_calculator",
-        "//mediapipe/calculators/tensorflow:tensor_to_image_frame_calculator_cc_proto",
+        ":tensor_to_image_frame_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
 )
 
@@ -1001,14 +989,14 @@ cc_test(
     srcs = ["tensor_to_matrix_calculator_test.cc"],
     deps = [
         ":tensor_to_matrix_calculator",
-        "//mediapipe/calculators/tensorflow:tensor_to_matrix_calculator_cc_proto",
+        ":tensor_to_matrix_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/formats:matrix",
         "//mediapipe/framework/formats:time_series_header_cc_proto",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
 )
 
@@ -1017,12 +1005,12 @@ cc_test(
     srcs = ["tensor_to_vector_float_calculator_test.cc"],
     deps = [
         ":tensor_to_vector_float_calculator",
-        "//mediapipe/calculators/tensorflow:tensor_to_vector_float_calculator_options_cc_proto",
+        ":tensor_to_vector_float_calculator_options_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
 )
 
@@ -1042,7 +1030,7 @@ cc_test(
         "//mediapipe/util/sequence:media_sequence",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
 )
 
@@ -1056,7 +1044,7 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
 )
 
@@ -1065,12 +1053,12 @@ cc_test(
     srcs = ["vector_float_to_tensor_calculator_test.cc"],
     deps = [
         ":vector_float_to_tensor_calculator",
-        "//mediapipe/calculators/tensorflow:vector_float_to_tensor_calculator_options_cc_proto",
+        ":vector_float_to_tensor_calculator_options_cc_proto",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
 )
 
@@ -1094,7 +1082,7 @@ cc_test(
         ":tensorflow_session",
         ":tensorflow_inference_calculator",
         ":tensorflow_session_from_frozen_graph_generator",
-        "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_generator_cc_proto",
+        ":tensorflow_session_from_frozen_graph_generator_cc_proto",
         "//mediapipe/framework/deps:file_path",
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:calculator_runner",

mediapipe/calculators/tflite/BUILD

@@ -238,6 +238,7 @@ cc_library(
             "@org_tensorflow//tensorflow/lite/delegates/gpu/common:shape",
             "@org_tensorflow//tensorflow/lite/delegates/gpu/metal:buffer_convert",
             "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate",
+            "@org_tensorflow//tensorflow/lite/delegates/gpu:metal_delegate_internal",
         ],
         "//conditions:default": [
             "//mediapipe/gpu:gl_calculator_helper",

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -27,8 +27,7 @@
 #include "tensorflow/lite/kernels/register.h"
 #include "tensorflow/lite/model.h"
 
-#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) && \
+#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
-    !defined(__APPLE__)
 #include "mediapipe/gpu/gl_calculator_helper.h"
 #include "mediapipe/gpu/gpu_buffer.h"
 #include "tensorflow/lite/delegates/gpu/common/shape.h"
@@ -49,6 +48,7 @@
 #include "tensorflow/lite/delegates/gpu/common/shape.h"
 #include "tensorflow/lite/delegates/gpu/metal/buffer_convert.h"
 #include "tensorflow/lite/delegates/gpu/metal_delegate.h"
+#include "tensorflow/lite/delegates/gpu/metal_delegate_internal.h"
 #endif  // iOS
 
 namespace {
@@ -553,9 +553,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
 #if defined(__APPLE__) && !TARGET_OS_OSX  // iOS
   // Configure and create the delegate.
-  GpuDelegateOptions options;
+  TFLGpuDelegateOptions options;
   options.allow_precision_loss = false;  // Must match converter, F=float/T=half
-  options.wait_type = GpuDelegateOptions::WaitType::kPassive;
+  options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive;
   if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options);
   id<MTLDevice> device = gpu_helper_.mtlDevice;
 

mediapipe/calculators/tflite/tflite_tensors_to_detections_calculator.cc

@@ -172,13 +172,11 @@ class TfLiteTensorsToDetectionsCalculator : public CalculatorBase {
       const int* detection_classes, std::vector<Detection>* output_detections);
   Detection ConvertToDetection(float box_ymin, float box_xmin, float box_ymax,
                                float box_xmax, float score, int class_id,
-                               int detection_id, bool flip_vertically);
+                               bool flip_vertically);
 
   int num_classes_ = 0;
   int num_boxes_ = 0;
   int num_coords_ = 0;
-  // Unique detection ID per new detection.
-  static int next_detection_id_;
   std::set<int> ignore_classes_;
 
   ::mediapipe::TfLiteTensorsToDetectionsCalculatorOptions options_;
@@ -199,10 +197,6 @@ class TfLiteTensorsToDetectionsCalculator : public CalculatorBase {
 };
 REGISTER_CALCULATOR(TfLiteTensorsToDetectionsCalculator);
 
-// Initialization of non-const static member should happen outside class
-// definition.
-int TfLiteTensorsToDetectionsCalculator::next_detection_id_ = 0;
-
 ::mediapipe::Status TfLiteTensorsToDetectionsCalculator::GetContract(
     CalculatorContract* cc) {
   RET_CHECK(!cc->Inputs().GetTags().empty());
@@ -686,10 +680,7 @@ int TfLiteTensorsToDetectionsCalculator::next_detection_id_ = 0;
     Detection detection = ConvertToDetection(
         detection_boxes[box_offset + 0], detection_boxes[box_offset + 1],
         detection_boxes[box_offset + 2], detection_boxes[box_offset + 3],
-        detection_scores[i], detection_classes[i], next_detection_id_,
+        detection_scores[i], detection_classes[i], options_.flip_vertically());
-        options_.flip_vertically());
-    // Increment to get next unique detection ID.
-    ++next_detection_id_;
     // Add keypoints.
     if (options_.num_keypoints() > 0) {
       auto* location_data = detection.mutable_location_data();
@@ -712,11 +703,10 @@ int TfLiteTensorsToDetectionsCalculator::next_detection_id_ = 0;
 
 Detection TfLiteTensorsToDetectionsCalculator::ConvertToDetection(
     float box_ymin, float box_xmin, float box_ymax, float box_xmax, float score,
-    int class_id, int detection_id, bool flip_vertically) {
+    int class_id, bool flip_vertically) {
   Detection detection;
   detection.add_score(score);
   detection.add_label_id(class_id);
-  detection.set_detection_id(detection_id);
 
   LocationData* location_data = detection.mutable_location_data();
   location_data->set_format(LocationData::RELATIVE_BOUNDING_BOX);

mediapipe/calculators/util/BUILD

@@ -12,14 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:public"])
 
 exports_files(["LICENSE"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-
 proto_library(
     name = "annotation_overlay_calculator_proto",
     srcs = ["annotation_overlay_calculator.proto"],
@@ -72,6 +72,24 @@ proto_library(
     ],
 )
 
+proto_library(
+    name = "collection_has_min_size_calculator_proto",
+    srcs = ["collection_has_min_size_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
+proto_library(
+    name = "association_calculator_proto",
+    srcs = ["association_calculator.proto"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_proto",
+    ],
+)
+
 mediapipe_cc_proto_library(
     name = "annotation_overlay_calculator_cc_proto",
     srcs = ["annotation_overlay_calculator.proto"],
@@ -141,6 +159,26 @@ mediapipe_cc_proto_library(
     ],
 )
 
+mediapipe_cc_proto_library(
+    name = "collection_has_min_size_calculator_cc_proto",
+    srcs = ["collection_has_min_size_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+    ],
+    visibility = ["//mediapipe:__subpackages__"],
+    deps = [":collection_has_min_size_calculator_proto"],
+)
+
+mediapipe_cc_proto_library(
+    name = "association_calculator_cc_proto",
+    srcs = ["association_calculator.proto"],
+    cc_deps = [
+        "//mediapipe/framework:calculator_cc_proto",
+    ],
+    visibility = ["//mediapipe:__subpackages__"],
+    deps = [":association_calculator_proto"],
+)
+
 cc_library(
     name = "packet_frequency_calculator",
     srcs = ["packet_frequency_calculator.cc"],
@@ -847,3 +885,101 @@ cc_library(
     ],
     alwayslink = 1,
 )
+
+cc_library(
+    name = "filter_collection_calculator",
+    srcs = ["filter_collection_calculator.cc"],
+    hdrs = ["filter_collection_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/strings",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "collection_has_min_size_calculator",
+    srcs = ["collection_has_min_size_calculator.cc"],
+    hdrs = ["collection_has_min_size_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":collection_has_min_size_calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "association_calculator",
+    hdrs = ["association_calculator.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":association_calculator_cc_proto",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:collection_item_id",
+        "//mediapipe/framework/port:rectangle",
+        "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/memory",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "association_norm_rect_calculator",
+    srcs = ["association_norm_rect_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":association_calculator",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:rectangle",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "association_detection_calculator",
+    srcs = ["association_detection_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":association_calculator",
+        "//mediapipe/framework:calculator_context",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location",
+        "//mediapipe/framework/port:rectangle",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
+cc_test(
+    name = "association_calculator_test",
+    srcs = ["association_calculator_test.cc"],
+    deps = [
+        ":association_detection_calculator",
+        ":association_norm_rect_calculator",
+        "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework:calculator_runner",
+        "//mediapipe/framework:collection_item_id",
+        "//mediapipe/framework:packet",
+        "//mediapipe/framework/deps:message_matchers",
+        "//mediapipe/framework/formats:detection_cc_proto",
+        "//mediapipe/framework/formats:location_data_cc_proto",
+        "//mediapipe/framework/formats:rect_cc_proto",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:parse_text_proto",
+    ],
+)

mediapipe/calculators/util/annotation_overlay_calculator.cc

@@ -148,9 +148,6 @@ class AnnotationOverlayCalculator : public CalculatorBase {
   // Underlying helper renderer library.
   std::unique_ptr<AnnotationRenderer> renderer_;
 
-  // Number of input streams with render data.
-  int num_render_streams_;
-
   // Indicates if image frame is available as input.
   bool image_frame_available_ = false;
 
@@ -181,20 +178,15 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
     return ::mediapipe::InternalError("GPU output must have GPU input.");
   }
 
-  // Assume all inputs are render streams; adjust below.
-  int num_render_streams = cc->Inputs().NumEntries();
-
   // Input image to render onto copy of.
 #if !defined(MEDIAPIPE_DISABLE_GPU)
   if (cc->Inputs().HasTag(kInputFrameTagGpu)) {
     cc->Inputs().Tag(kInputFrameTagGpu).Set<mediapipe::GpuBuffer>();
-    num_render_streams = cc->Inputs().NumEntries() - 1;
     use_gpu |= true;
   }
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   if (cc->Inputs().HasTag(kInputFrameTag)) {
     cc->Inputs().Tag(kInputFrameTag).Set<ImageFrame>();
-    num_render_streams = cc->Inputs().NumEntries() - 1;
   }
 
   // Data streams to render.
@@ -246,12 +238,10 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
   if (cc->Inputs().HasTag(kInputFrameTagGpu) ||
       cc->Inputs().HasTag(kInputFrameTag)) {
     image_frame_available_ = true;
-    num_render_streams_ = cc->Inputs().NumEntries() - 1;
   } else {
     image_frame_available_ = false;
     RET_CHECK(options_.has_canvas_width_px());
     RET_CHECK(options_.has_canvas_height_px());
-    num_render_streams_ = cc->Inputs().NumEntries();
   }
 
   // Initialize the helper renderer library.

mediapipe/calculators/util/association_calculator.h

@@ -0,0 +1,259 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_
+
+#include <memory>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "mediapipe/calculators/util/association_calculator.pb.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/collection_item_id.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/rectangle.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Computes the overlap similarity based on Intersection over Union (IoU) of
+// two rectangles.
+inline float OverlapSimilarity(const Rectangle_f& rect1,
+                               const Rectangle_f& rect2) {
+  if (!rect1.Intersects(rect2)) return 0.0f;
+  // Compute IoU similarity score.
+  const float intersection_area = Rectangle_f(rect1).Intersect(rect2).Area();
+  const float normalization = rect1.Area() + rect2.Area() - intersection_area;
+  return normalization > 0.0f ? intersection_area / normalization : 0.0f;
+}
+
+// AssocationCalculator<T> accepts multiple inputs of vectors of type T that can
+// be converted to Rectangle_f. The output is a vector of type T that contains
+// elements from the input vectors that don't overlap with each other. When
+// two elements overlap, the element that comes in from a later input stream
+// is kept in the output. This association operation is useful for multiple
+// instance inference pipelines in MediaPipe.
+// If an input stream is tagged with "PREV" tag, IDs of overlapping elements
+// from "PREV" input stream are propagated to the output. Elements in the "PREV"
+// input stream that don't overlap with other elements are not added to the
+// output. This stream is designed to take detections from previous timestamp,
+// e.g. output of PreviousLoopbackCalculator to provide temporal association.
+// See AssociationDetectionCalculator and AssociationNormRectCalculator for
+// example uses.
+template <typename T>
+class AssociationCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    // Atmost one input stream can be tagged with "PREV".
+    RET_CHECK_LE(cc->Inputs().NumEntries("PREV"), 1);
+
+    if (cc->Inputs().HasTag("PREV")) {
+      RET_CHECK_GE(cc->Inputs().NumEntries(), 2);
+    }
+
+    for (CollectionItemId id = cc->Inputs().BeginId();
+         id < cc->Inputs().EndId(); ++id) {
+      cc->Inputs().Get(id).Set<std::vector<T>>();
+    }
+
+    cc->Outputs().Index(0).Set<std::vector<T>>();
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+
+    has_prev_input_stream_ = cc->Inputs().HasTag("PREV");
+    if (has_prev_input_stream_) {
+      prev_input_stream_id_ = cc->Inputs().GetId("PREV", 0);
+    }
+    options_ = cc->Options<::mediapipe::AssociationCalculatorOptions>();
+    CHECK_GE(options_.min_similarity_threshold(), 0);
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    auto get_non_overlapping_elements = GetNonOverlappingElements(cc);
+    if (!get_non_overlapping_elements.ok()) {
+      return get_non_overlapping_elements.status();
+    }
+    std::list<T> result = get_non_overlapping_elements.ValueOrDie();
+
+    if (has_prev_input_stream_ &&
+        !cc->Inputs().Get(prev_input_stream_id_).IsEmpty()) {
+      // Processed all regular input streams. Now compare the result list
+      // elements with those in the PREV input stream, and propagate IDs from
+      // PREV input stream as appropriate.
+      const std::vector<T>& prev_input_vec =
+          cc->Inputs()
+              .Get(prev_input_stream_id_)
+              .template Get<std::vector<T>>();
+
+      MP_RETURN_IF_ERROR(
+          PropagateIdsFromPreviousToCurrent(prev_input_vec, &result));
+    }
+
+    auto output = absl::make_unique<std::vector<T>>();
+    for (auto it = result.begin(); it != result.end(); ++it) {
+      output->push_back(*it);
+    }
+    cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
+
+    return ::mediapipe::OkStatus();
+  }
+
+ protected:
+  ::mediapipe::AssociationCalculatorOptions options_;
+
+  bool has_prev_input_stream_;
+  CollectionItemId prev_input_stream_id_;
+
+  virtual ::mediapipe::StatusOr<Rectangle_f> GetRectangle(const T& input) {
+    return ::mediapipe::OkStatus();
+  }
+
+  virtual std::pair<bool, int> GetId(const T& input) { return {false, -1}; }
+
+  virtual void SetId(T* input, int id) {}
+
+ private:
+  // Get a list of non-overlapping elements from all input streams, with
+  // increasing order of priority based on input stream index.
+  mediapipe::StatusOr<std::list<T>> GetNonOverlappingElements(
+      CalculatorContext* cc) {
+    std::list<T> result;
+
+    // Initialize result with the first non-empty input vector.
+    CollectionItemId non_empty_id = cc->Inputs().BeginId();
+    for (CollectionItemId id = cc->Inputs().BeginId();
+         id < cc->Inputs().EndId(); ++id) {
+      if (id == prev_input_stream_id_ || cc->Inputs().Get(id).IsEmpty()) {
+        continue;
+      }
+      const std::vector<T>& input_vec =
+          cc->Inputs().Get(id).Get<std::vector<T>>();
+      if (!input_vec.empty()) {
+        non_empty_id = id;
+        result.push_back(input_vec[0]);
+        for (int j = 1; j < input_vec.size(); ++j) {
+          MP_RETURN_IF_ERROR(AddElementToList(input_vec[j], &result));
+        }
+        break;
+      }
+    }
+
+    // Compare remaining input vectors with the non-empty result vector,
+    // remove lower-priority overlapping elements from the result vector and
+    // had corresponding higher-priority elements as necessary.
+    for (CollectionItemId id = non_empty_id + 1; id < cc->Inputs().EndId();
+         ++id) {
+      if (id == prev_input_stream_id_ || cc->Inputs().Get(id).IsEmpty()) {
+        continue;
+      }
+      const std::vector<T>& input_vec =
+          cc->Inputs().Get(id).Get<std::vector<T>>();
+
+      for (int vi = 0; vi < input_vec.size(); ++vi) {
+        MP_RETURN_IF_ERROR(AddElementToList(input_vec[vi], &result));
+      }
+    }
+
+    return result;
+  }
+
+  ::mediapipe::Status AddElementToList(T element, std::list<T>* current) {
+    // Compare this element with elements of the input collection. If this
+    // element has high overlap with elements of the collection, remove
+    // those elements from the collection and add this element.
+    ASSIGN_OR_RETURN(auto cur_rect, GetRectangle(element));
+
+    bool change_id = false;
+    int new_elem_id = -1;
+
+    for (auto uit = current->begin(); uit != current->end();) {
+      ASSIGN_OR_RETURN(auto prev_rect, GetRectangle(*uit));
+      if (OverlapSimilarity(cur_rect, prev_rect) >
+          options_.min_similarity_threshold()) {
+        std::pair<bool, int> prev_id = GetId(*uit);
+        // If prev_id.first is false when some element doesn't have an ID,
+        // change_id and new_elem_id will not be updated.
+        if (prev_id.first) {
+          change_id = prev_id.first;
+          new_elem_id = prev_id.second;
+        }
+        uit = current->erase(uit);
+      } else {
+        ++uit;
+      }
+    }
+
+    if (change_id) {
+      SetId(&element, new_elem_id);
+    }
+    current->push_back(element);
+
+    return ::mediapipe::OkStatus();
+  }
+
+  // Compare elements of the current list with elements in from the collection
+  // of elements from the previous input stream, and propagate IDs from the
+  // previous input stream as appropriate.
+  ::mediapipe::Status PropagateIdsFromPreviousToCurrent(
+      const std::vector<T>& prev_input_vec, std::list<T>* current) {
+    for (auto vit = current->begin(); vit != current->end(); ++vit) {
+      auto get_cur_rectangle = GetRectangle(*vit);
+      if (!get_cur_rectangle.ok()) {
+        return get_cur_rectangle.status();
+      }
+      const Rectangle_f& cur_rect = get_cur_rectangle.ValueOrDie();
+
+      bool change_id = false;
+      int id_for_vi = -1;
+
+      for (int ui = 0; ui < prev_input_vec.size(); ++ui) {
+        auto get_prev_rectangle = GetRectangle(prev_input_vec[ui]);
+        if (!get_prev_rectangle.ok()) {
+          return get_prev_rectangle.status();
+        }
+        const Rectangle_f& prev_rect = get_prev_rectangle.ValueOrDie();
+
+        if (OverlapSimilarity(cur_rect, prev_rect) >
+            options_.min_similarity_threshold()) {
+          std::pair<bool, int> prev_id = GetId(prev_input_vec[ui]);
+          // If prev_id.first is false when some element doesn't have an ID,
+          // change_id and id_for_vi will not be updated.
+          if (prev_id.first) {
+            change_id = prev_id.first;
+            id_for_vi = prev_id.second;
+          }
+        }
+      }
+
+      if (change_id) {
+        T element = *vit;
+        SetId(&element, id_for_vi);
+        *vit = element;
+      }
+    }
+    return ::mediapipe::OkStatus();
+  }
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_ASSOCIATION_CALCULATOR_H_

mediapipe/calculators/util/association_calculator.proto

@@ -0,0 +1,27 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message AssociationCalculatorOptions {
+  extend CalculatorOptions {
+    optional AssociationCalculatorOptions ext = 275124847;
+  }
+
+  optional float min_similarity_threshold = 1 [default = 1.0];
+}

mediapipe/calculators/util/association_calculator_test.cc

@@ -0,0 +1,476 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/framework/calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/calculator_runner.h"
+#include "mediapipe/framework/collection_item_id.h"
+#include "mediapipe/framework/deps/message_matchers.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/location_data.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/packet.h"
+#include "mediapipe/framework/port/gmock.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status_matchers.h"
+
+namespace mediapipe {
+
+namespace {
+
+::mediapipe::Detection DetectionWithRelativeLocationData(double xmin,
+                                                         double ymin,
+                                                         double width,
+                                                         double height) {
+  ::mediapipe::Detection detection;
+  ::mediapipe::LocationData* location_data = detection.mutable_location_data();
+  location_data->set_format(::mediapipe::LocationData::RELATIVE_BOUNDING_BOX);
+  location_data->mutable_relative_bounding_box()->set_xmin(xmin);
+  location_data->mutable_relative_bounding_box()->set_ymin(ymin);
+  location_data->mutable_relative_bounding_box()->set_width(width);
+  location_data->mutable_relative_bounding_box()->set_height(height);
+  return detection;
+}
+
+}  // namespace
+
+class AssociationDetectionCalculatorTest : public ::testing::Test {
+ protected:
+  AssociationDetectionCalculatorTest() {
+    //  0.4                                         ================
+    //                                              |    |    |    |
+    //  0.3 =====================                   |  DET2   |    |
+    //      |    |    |   DET1  |                   |    |   DET4  |
+    //  0.2 |   DET0  |    ===========              ================
+    //      |    |    |    |    |    |
+    //  0.1 =====|===============    |
+    //           |    DET3 |    |    |
+    //  0.0      ================    |
+    //                     |   DET5  |
+    // -0.1                ===========
+    //     0.1  0.2  0.3  0.4  0.5  0.6  0.7  0.8  0.9  1.0  1.1  1.2
+
+    // Detection det_0.
+    det_0 = DetectionWithRelativeLocationData(/*xmin=*/0.1, /*ymin=*/0.1,
+                                              /*width=*/0.2, /*height=*/0.2);
+    det_0.set_detection_id(0);
+
+    // Detection det_1.
+    det_1 = DetectionWithRelativeLocationData(/*xmin=*/0.3, /*ymin=*/0.1,
+                                              /*width=*/0.2, /*height=*/0.2);
+    det_1.set_detection_id(1);
+
+    // Detection det_2.
+    det_2 = DetectionWithRelativeLocationData(/*xmin=*/0.9, /*ymin=*/0.2,
+                                              /*width=*/0.2, /*height=*/0.2);
+    det_2.set_detection_id(2);
+
+    // Detection det_3.
+    det_3 = DetectionWithRelativeLocationData(/*xmin=*/0.2, /*ymin=*/0.0,
+                                              /*width=*/0.3, /*height=*/0.3);
+    det_3.set_detection_id(3);
+
+    // Detection det_4.
+    det_4 = DetectionWithRelativeLocationData(/*xmin=*/1.0, /*ymin=*/0.2,
+                                              /*width=*/0.2, /*height=*/0.2);
+    det_4.set_detection_id(4);
+
+    // Detection det_5.
+    det_5 = DetectionWithRelativeLocationData(/*xmin=*/0.3, /*ymin=*/-0.1,
+                                              /*width=*/0.3, /*height=*/0.3);
+    det_5.set_detection_id(5);
+  }
+
+  ::mediapipe::Detection det_0, det_1, det_2, det_3, det_4, det_5;
+};
+
+TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTest) {
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    calculator: "AssociationDetectionCalculator"
+    input_stream: "input_vec_0"
+    input_stream: "input_vec_1"
+    input_stream: "input_vec_2"
+    output_stream: "output_vec"
+    options {
+      [mediapipe.AssociationCalculatorOptions.ext] {
+        min_similarity_threshold: 0.1
+      }
+    }
+  )"));
+
+  // Input Stream 0: det_0, det_1, det_2.
+  auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
+  input_vec_0->push_back(det_0);
+  input_vec_0->push_back(det_1);
+  input_vec_0->push_back(det_2);
+  runner.MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_vec_0.release()).At(Timestamp(1)));
+
+  // Input Stream 1: det_3, det_4.
+  auto input_vec_1 = absl::make_unique<std::vector<::mediapipe::Detection>>();
+  input_vec_1->push_back(det_3);
+  input_vec_1->push_back(det_4);
+  runner.MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_vec_1.release()).At(Timestamp(1)));
+
+  // Input Stream 2: det_5.
+  auto input_vec_2 = absl::make_unique<std::vector<::mediapipe::Detection>>();
+  input_vec_2->push_back(det_5);
+  runner.MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_vec_2.release()).At(Timestamp(1)));
+
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, output.size());
+  const auto& assoc_rects =
+      output[0].Get<std::vector<::mediapipe::Detection>>();
+
+  // det_3 overlaps with det_0, det_1 and det_5 overlaps with det_3. Since det_5
+  // is in the highest priority, we remove other rects. det_4 overlaps with
+  // det_2, and det_4 is higher priority, so we keep it. The final output
+  // therefore contains 2 elements.
+  EXPECT_EQ(2, assoc_rects.size());
+  // Outputs are in order of inputs, so det_4 is before det_5 in output vector.
+
+  // det_4 overlaps with det_2, so new id for det_4 is 2.
+  EXPECT_TRUE(assoc_rects[0].has_detection_id());
+  EXPECT_EQ(2, assoc_rects[0].detection_id());
+  det_4.set_detection_id(2);
+  EXPECT_THAT(assoc_rects[0], EqualsProto(det_4));
+
+  // det_3 overlaps with det_0, so new id for det_3 is 0.
+  // det_3 overlaps with det_1, so new id for det_3 is 1.
+  // det_5 overlaps with det_3, so new id for det_5 is 1.
+  EXPECT_TRUE(assoc_rects[1].has_detection_id());
+  EXPECT_EQ(1, assoc_rects[1].detection_id());
+  det_5.set_detection_id(1);
+  EXPECT_THAT(assoc_rects[1], EqualsProto(det_5));
+}
+
+TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestWithPrev) {
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    calculator: "AssociationDetectionCalculator"
+    input_stream: "PREV:input_vec_0"
+    input_stream: "input_vec_1"
+    output_stream: "output_vec"
+    options {
+      [mediapipe.AssociationCalculatorOptions.ext] {
+        min_similarity_threshold: 0.1
+      }
+    }
+  )"));
+
+  // Input Stream 0: det_3, det_4.
+  auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
+  input_vec_0->push_back(det_3);
+  input_vec_0->push_back(det_4);
+  CollectionItemId prev_input_stream_id =
+      runner.MutableInputs()->GetId("PREV", 0);
+  runner.MutableInputs()
+      ->Get(prev_input_stream_id)
+      .packets.push_back(Adopt(input_vec_0.release()).At(Timestamp(1)));
+
+  // Input Stream 1: det_5.
+  auto input_vec_1 = absl::make_unique<std::vector<::mediapipe::Detection>>();
+  input_vec_1->push_back(det_5);
+  CollectionItemId input_stream_id = runner.MutableInputs()->GetId("", 0);
+  runner.MutableInputs()
+      ->Get(input_stream_id)
+      .packets.push_back(Adopt(input_vec_1.release()).At(Timestamp(1)));
+
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, output.size());
+  const auto& assoc_rects =
+      output[0].Get<std::vector<::mediapipe::Detection>>();
+
+  // det_5 overlaps with det_3 and doesn't overlap with det_4. Since det_4 is
+  // in the PREV input stream, it doesn't get copied to the output, so the final
+  // output contains 1 element.
+  EXPECT_EQ(1, assoc_rects.size());
+
+  // det_5 overlaps with det_3, det_3 is in PREV, so new id for det_5 is 3.
+  EXPECT_TRUE(assoc_rects[0].has_detection_id());
+  EXPECT_EQ(3, assoc_rects[0].detection_id());
+  det_5.set_detection_id(3);
+  EXPECT_THAT(assoc_rects[0], EqualsProto(det_5));
+}
+
+TEST_F(AssociationDetectionCalculatorTest, DetectionAssocTestReverse) {
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    calculator: "AssociationDetectionCalculator"
+    input_stream: "input_vec_0"
+    input_stream: "input_vec_1"
+    input_stream: "input_vec_2"
+    output_stream: "output_vec"
+    options {
+      [mediapipe.AssociationCalculatorOptions.ext] {
+        min_similarity_threshold: 0.1
+      }
+    }
+  )"));
+
+  // Input Stream 0: det_5.
+  auto input_vec_0 = absl::make_unique<std::vector<::mediapipe::Detection>>();
+  input_vec_0->push_back(det_5);
+  runner.MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_vec_0.release()).At(Timestamp(1)));
+
+  // Input Stream 1: det_3, det_4.
+  auto input_vec_1 = absl::make_unique<std::vector<::mediapipe::Detection>>();
+  input_vec_1->push_back(det_3);
+  input_vec_1->push_back(det_4);
+  runner.MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_vec_1.release()).At(Timestamp(1)));
+
+  // Input Stream 2: det_0, det_1, det_2.
+  auto input_vec_2 = absl::make_unique<std::vector<::mediapipe::Detection>>();
+  input_vec_2->push_back(det_0);
+  input_vec_2->push_back(det_1);
+  input_vec_2->push_back(det_2);
+  runner.MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_vec_2.release()).At(Timestamp(1)));
+
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, output.size());
+  const auto& assoc_rects =
+      output[0].Get<std::vector<::mediapipe::Detection>>();
+
+  // det_3 overlaps with det_5, so det_5 is removed. det_0 overlaps with det_3,
+  // so det_3 is removed as det_0 is in higher priority for keeping. det_2
+  // overlaps with det_4 so det_4 is removed as det_2 is higher priority for
+  // keeping. The final output therefore contains 3 elements.
+  EXPECT_EQ(3, assoc_rects.size());
+  // Outputs are in same order as inputs.
+
+  // det_3 overlaps with det_5, so new id for det_3 is 5.
+  // det_0 overlaps with det_3, so new id for det_0 is 5.
+  EXPECT_TRUE(assoc_rects[0].has_detection_id());
+  EXPECT_EQ(5, assoc_rects[0].detection_id());
+  det_0.set_detection_id(5);
+  EXPECT_THAT(assoc_rects[0], EqualsProto(det_0));
+
+  // det_1 stays with id 1.
+  EXPECT_TRUE(assoc_rects[1].has_detection_id());
+  EXPECT_EQ(1, assoc_rects[1].detection_id());
+  EXPECT_THAT(assoc_rects[1], EqualsProto(det_1));
+
+  // det_2 overlaps with det_4, so new id for det_2 is 4.
+  EXPECT_TRUE(assoc_rects[2].has_detection_id());
+  EXPECT_EQ(4, assoc_rects[2].detection_id());
+  det_2.set_detection_id(4);
+  EXPECT_THAT(assoc_rects[2], EqualsProto(det_2));
+}
+
+class AssociationNormRectCalculatorTest : public ::testing::Test {
+ protected:
+  AssociationNormRectCalculatorTest() {
+    //  0.4                                         ================
+    //                                              |    |    |    |
+    //  0.3 =====================                   |   NR2   |    |
+    //      |    |    |   NR1   |                   |    |    NR4  |
+    //  0.2 |   NR0   |    ===========              ================
+    //      |    |    |    |    |    |
+    //  0.1 =====|===============    |
+    //           |    NR3  |    |    |
+    //  0.0      ================    |
+    //                     |   NR5   |
+    // -0.1                ===========
+    //     0.1  0.2  0.3  0.4  0.5  0.6  0.7  0.8  0.9  1.0  1.1  1.2
+
+    // NormalizedRect nr_0.
+    nr_0.set_x_center(0.2);
+    nr_0.set_y_center(0.2);
+    nr_0.set_width(0.2);
+    nr_0.set_height(0.2);
+
+    // NormalizedRect nr_1.
+    nr_1.set_x_center(0.4);
+    nr_1.set_y_center(0.2);
+    nr_1.set_width(0.2);
+    nr_1.set_height(0.2);
+
+    // NormalizedRect nr_2.
+    nr_2.set_x_center(1.0);
+    nr_2.set_y_center(0.3);
+    nr_2.set_width(0.2);
+    nr_2.set_height(0.2);
+
+    // NormalizedRect nr_3.
+    nr_3.set_x_center(0.35);
+    nr_3.set_y_center(0.15);
+    nr_3.set_width(0.3);
+    nr_3.set_height(0.3);
+
+    // NormalizedRect nr_4.
+    nr_4.set_x_center(1.1);
+    nr_4.set_y_center(0.3);
+    nr_4.set_width(0.2);
+    nr_4.set_height(0.2);
+
+    // NormalizedRect nr_5.
+    nr_5.set_x_center(0.45);
+    nr_5.set_y_center(0.05);
+    nr_5.set_width(0.3);
+    nr_5.set_height(0.3);
+  }
+
+  ::mediapipe::NormalizedRect nr_0, nr_1, nr_2, nr_3, nr_4, nr_5;
+};
+
+TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTest) {
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    calculator: "AssociationNormRectCalculator"
+    input_stream: "input_vec_0"
+    input_stream: "input_vec_1"
+    input_stream: "input_vec_2"
+    output_stream: "output_vec"
+    options {
+      [mediapipe.AssociationCalculatorOptions.ext] {
+        min_similarity_threshold: 0.1
+      }
+    }
+  )"));
+
+  // Input Stream 0: nr_0, nr_1, nr_2.
+  auto input_vec_0 =
+      absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
+  input_vec_0->push_back(nr_0);
+  input_vec_0->push_back(nr_1);
+  input_vec_0->push_back(nr_2);
+  runner.MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_vec_0.release()).At(Timestamp(1)));
+
+  // Input Stream 1: nr_3, nr_4.
+  auto input_vec_1 =
+      absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
+  input_vec_1->push_back(nr_3);
+  input_vec_1->push_back(nr_4);
+  runner.MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_vec_1.release()).At(Timestamp(1)));
+
+  // Input Stream 2: nr_5.
+  auto input_vec_2 =
+      absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
+  input_vec_2->push_back(nr_5);
+  runner.MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_vec_2.release()).At(Timestamp(1)));
+
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, output.size());
+  const auto& assoc_rects =
+      output[0].Get<std::vector<::mediapipe::NormalizedRect>>();
+
+  // nr_3 overlaps with nr_0, nr_1 and nr_5 overlaps with nr_3. Since nr_5 is
+  // in the highest priority, we remove other rects.
+  // nr_4 overlaps with nr_2, and nr_4 is higher priority, so we keep it.
+  // The final output therefore contains 2 elements.
+  EXPECT_EQ(2, assoc_rects.size());
+  // Outputs are in order of inputs, so nr_4 is before nr_5 in output vector.
+  EXPECT_THAT(assoc_rects[0], EqualsProto(nr_4));
+  EXPECT_THAT(assoc_rects[1], EqualsProto(nr_5));
+}
+
+TEST_F(AssociationNormRectCalculatorTest, NormRectAssocTestReverse) {
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    calculator: "AssociationNormRectCalculator"
+    input_stream: "input_vec_0"
+    input_stream: "input_vec_1"
+    input_stream: "input_vec_2"
+    output_stream: "output_vec"
+    options {
+      [mediapipe.AssociationCalculatorOptions.ext] {
+        min_similarity_threshold: 0.1
+      }
+    }
+  )"));
+
+  // Input Stream 0: nr_5.
+  auto input_vec_0 =
+      absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
+  input_vec_0->push_back(nr_5);
+  runner.MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_vec_0.release()).At(Timestamp(1)));
+
+  // Input Stream 1: nr_3, nr_4.
+  auto input_vec_1 =
+      absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
+  input_vec_1->push_back(nr_3);
+  input_vec_1->push_back(nr_4);
+  runner.MutableInputs()->Index(1).packets.push_back(
+      Adopt(input_vec_1.release()).At(Timestamp(1)));
+
+  // Input Stream 2: nr_0, nr_1, nr_2.
+  auto input_vec_2 =
+      absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
+  input_vec_2->push_back(nr_0);
+  input_vec_2->push_back(nr_1);
+  input_vec_2->push_back(nr_2);
+  runner.MutableInputs()->Index(2).packets.push_back(
+      Adopt(input_vec_2.release()).At(Timestamp(1)));
+
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, output.size());
+  const auto& assoc_rects =
+      output[0].Get<std::vector<::mediapipe::NormalizedRect>>();
+
+  // nr_3 overlaps with nr_5, so nr_5 is removed. nr_0 overlaps with nr_3, so
+  // nr_3 is removed as nr_0 is in higher priority for keeping. nr_2 overlaps
+  // with nr_4 so nr_4 is removed as nr_2 is higher priority for keeping.
+  // The final output therefore contains 3 elements.
+  EXPECT_EQ(3, assoc_rects.size());
+  // Outputs are in same order as inputs.
+  EXPECT_THAT(assoc_rects[0], EqualsProto(nr_0));
+  EXPECT_THAT(assoc_rects[1], EqualsProto(nr_1));
+  EXPECT_THAT(assoc_rects[2], EqualsProto(nr_2));
+}
+
+TEST_F(AssociationNormRectCalculatorTest, NormRectAssocSingleInputStream) {
+  CalculatorRunner runner(ParseTextProtoOrDie<CalculatorGraphConfig::Node>(R"(
+    calculator: "AssociationNormRectCalculator"
+    input_stream: "input_vec"
+    output_stream: "output_vec"
+    options {
+      [mediapipe.AssociationCalculatorOptions.ext] {
+        min_similarity_threshold: 0.1
+      }
+    }
+  )"));
+
+  // Input Stream : nr_3, nr_5.
+  auto input_vec =
+      absl::make_unique<std::vector<::mediapipe::NormalizedRect>>();
+  input_vec->push_back(nr_3);
+  input_vec->push_back(nr_5);
+  runner.MutableInputs()->Index(0).packets.push_back(
+      Adopt(input_vec.release()).At(Timestamp(1)));
+
+  MP_ASSERT_OK(runner.Run()) << "Calculator execution failed.";
+  const std::vector<Packet>& output = runner.Outputs().Index(0).packets;
+  EXPECT_EQ(1, output.size());
+  const auto& assoc_rects =
+      output[0].Get<std::vector<::mediapipe::NormalizedRect>>();
+
+  // nr_5 overlaps with nr_3. Since nr_5 is after nr_3 in the same input stream
+  // we remove nr_3 and keep nr_5.
+  // The final output therefore contains 1 elements.
+  EXPECT_EQ(1, assoc_rects.size());
+  EXPECT_THAT(assoc_rects[0], EqualsProto(nr_5));
+}
+
+}  // namespace mediapipe

mediapipe/calculators/util/association_detection_calculator.cc

@@ -0,0 +1,77 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/association_calculator.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/detection.pb.h"
+#include "mediapipe/framework/formats/location.h"
+#include "mediapipe/framework/port/rectangle.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// A subclass of AssociationCalculator<T> for Detection. Example:
+// node {
+//   calculator: "AssociationDetectionCalculator"
+//   input_stream: "PREV:input_vec_0"
+//   input_stream: "input_vec_1"
+//   input_stream: "input_vec_2"
+//   output_stream: "output_vec"
+//   options {
+//     [mediapipe.AssociationCalculatorOptions.ext] {
+//       min_similarity_threshold: 0.1
+//     }
+// }
+class AssociationDetectionCalculator
+    : public AssociationCalculator<::mediapipe::Detection> {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    return AssociationCalculator<::mediapipe::Detection>::GetContract(cc);
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    return AssociationCalculator<::mediapipe::Detection>::Open(cc);
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    return AssociationCalculator<::mediapipe::Detection>::Process(cc);
+  }
+
+  ::mediapipe::Status Close(CalculatorContext* cc) override {
+    return AssociationCalculator<::mediapipe::Detection>::Close(cc);
+  }
+
+ protected:
+  ::mediapipe::StatusOr<Rectangle_f> GetRectangle(
+      const ::mediapipe::Detection& input) override {
+    if (!input.has_location_data()) {
+      return ::mediapipe::InternalError("Missing location_data in Detection");
+    }
+    const Location location(input.location_data());
+    return location.GetRelativeBBox();
+  }
+
+  std::pair<bool, int> GetId(const ::mediapipe::Detection& input) override {
+    return {input.has_detection_id(), input.detection_id()};
+  }
+
+  void SetId(::mediapipe::Detection* input, int id) override {
+    input->set_detection_id(id);
+  }
+};
+
+REGISTER_CALCULATOR(AssociationDetectionCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/util/association_norm_rect_calculator.cc

@@ -0,0 +1,72 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/association_calculator.h"
+#include "mediapipe/framework/calculator_context.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+#include "mediapipe/framework/port/rectangle.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// A subclass of AssociationCalculator<T> for NormalizedRect. Example use case:
+// node {
+//   calculator: "AssociationNormRectCalculator"
+//   input_stream: "input_vec_0"
+//   input_stream: "input_vec_1"
+//   input_stream: "input_vec_2"
+//   output_stream: "output_vec"
+//   options {
+//     [mediapipe.AssociationCalculatorOptions.ext] {
+//       min_similarity_threshold: 0.1
+//     }
+// }
+class AssociationNormRectCalculator
+    : public AssociationCalculator<::mediapipe::NormalizedRect> {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    return AssociationCalculator<::mediapipe::NormalizedRect>::GetContract(cc);
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    return AssociationCalculator<::mediapipe::NormalizedRect>::Open(cc);
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    return AssociationCalculator<::mediapipe::NormalizedRect>::Process(cc);
+  }
+
+  ::mediapipe::Status Close(CalculatorContext* cc) override {
+    return AssociationCalculator<::mediapipe::NormalizedRect>::Close(cc);
+  }
+
+ protected:
+  ::mediapipe::StatusOr<Rectangle_f> GetRectangle(
+      const ::mediapipe::NormalizedRect& input) override {
+    if (!input.has_x_center() || !input.has_y_center() || !input.has_width() ||
+        !input.has_height()) {
+      return ::mediapipe::InternalError(
+          "Missing dimensions in NormalizedRect.");
+    }
+    const float xmin = input.x_center() - input.width() / 2.0;
+    const float ymin = input.y_center() - input.height() / 2.0;
+    // TODO: Support rotation for rectangle.
+    return Rectangle_f(xmin, ymin, input.width(), input.height());
+  }
+};
+
+REGISTER_CALCULATOR(AssociationNormRectCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/util/collection_has_min_size_calculator.cc

@@ -0,0 +1,26 @@
+
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/collection_has_min_size_calculator.h"
+
+#include "mediapipe/framework/formats/rect.pb.h"
+
+namespace mediapipe {
+
+typedef CollectionHasMinSizeCalculator<std::vector<::mediapipe::NormalizedRect>>
+    NormalizedRectVectorHasMinSizeCalculator;
+REGISTER_CALCULATOR(NormalizedRectVectorHasMinSizeCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/util/collection_has_min_size_calculator.h

@@ -0,0 +1,84 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_
+
+#include <vector>
+
+#include "mediapipe/calculators/util/collection_has_min_size_calculator.pb.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// Deterimines if an input iterable collection has a minimum size, specified
+// in CollectionHasMinSizeCalculatorOptions. Example usage:
+// node {
+//   calculator: "IntVectorHasMinSizeCalculator"
+//   input_stream: "ITERABLE:input_int_vector"
+//   output_stream: "has_min_ints"
+//  options {
+//    [mediapipe.CollectionHasMinSizeCalculatorOptions.ext] {
+//      min_size: 2
+//    }
+//   }
+// }
+template <typename IterableT>
+class CollectionHasMinSizeCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().HasTag("ITERABLE"));
+    RET_CHECK_EQ(1, cc->Inputs().NumEntries());
+
+    RET_CHECK_EQ(1, cc->Outputs().NumEntries());
+
+    RET_CHECK_GE(
+        cc->Options<::mediapipe::CollectionHasMinSizeCalculatorOptions>()
+            .min_size(),
+        0);
+
+    cc->Inputs().Tag("ITERABLE").Set<IterableT>();
+    cc->Outputs().Index(0).Set<bool>();
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    min_size_ =
+        cc->Options<::mediapipe::CollectionHasMinSizeCalculatorOptions>()
+            .min_size();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    const IterableT& input = cc->Inputs().Tag("ITERABLE").Get<IterableT>();
+    bool has_min_size = input.size() >= min_size_;
+
+    cc->Outputs().Index(0).AddPacket(
+        MakePacket<bool>(has_min_size).At(cc->InputTimestamp()));
+
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  int min_size_ = 0;
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_COLLECTION_HAS_MIN_SIZE_CALCULATOR_H_

mediapipe/calculators/util/collection_has_min_size_calculator.proto

@@ -0,0 +1,29 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe;
+
+import "mediapipe/framework/calculator.proto";
+
+message CollectionHasMinSizeCalculatorOptions {
+  extend CalculatorOptions {
+    optional CollectionHasMinSizeCalculatorOptions ext = 259397840;
+  }
+
+  // The minimum size an input iterable collection should have for the
+  // calculator to output true.
+  optional int32 min_size = 1 [default = 0];
+}

mediapipe/calculators/util/filter_collection_calculator.cc

@@ -0,0 +1,34 @@
+
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/filter_collection_calculator.h"
+
+#include <vector>
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+#include "mediapipe/framework/formats/rect.pb.h"
+
+namespace mediapipe {
+
+typedef FilterCollectionCalculator<std::vector<::mediapipe::NormalizedRect>>
+    FilterNormalizedRectCollectionCalculator;
+REGISTER_CALCULATOR(FilterNormalizedRectCollectionCalculator);
+
+typedef FilterCollectionCalculator<
+    std::vector<std::vector<::mediapipe::NormalizedLandmark>>>
+    FilterLandmarksCollectionCalculator;
+REGISTER_CALCULATOR(FilterLandmarksCollectionCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/util/filter_collection_calculator.h

@@ -0,0 +1,109 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_
+#define MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_
+
+#include <vector>
+
+#include "absl/strings/str_cat.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/canonical_errors.h"
+#include "mediapipe/framework/port/ret_check.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+
+// A calculator that gates elements of an input collection based on
+// corresponding boolean values of the "CONDITION" vector. If there is no input
+// collection or "CONDITION" vector, the calculator forwards timestamp bounds
+// for downstream calculators. If the "CONDITION" vector has false values for
+// all elements of the input collection, the calculator outputs a packet
+// containing an empty collection.
+// Example usage:
+// node {
+//   calculator: "FilterCollectionCalculator"
+//   input_stream: "ITERABLE:input_collection"
+//   input_stream: "CONDITION:condition_vector"
+//   output_stream: "ITERABLE:output_collection"
+// }
+// This calculator is able to handle collections of copyable types T.
+template <typename IterableT>
+class FilterCollectionCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    RET_CHECK(cc->Inputs().HasTag("ITERABLE"));
+    RET_CHECK(cc->Inputs().HasTag("CONDITION"));
+    RET_CHECK(cc->Outputs().HasTag("ITERABLE"));
+
+    cc->Inputs().Tag("ITERABLE").Set<IterableT>();
+    cc->Inputs().Tag("CONDITION").Set<std::vector<bool>>();
+
+    cc->Outputs().Tag("ITERABLE").Set<IterableT>();
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (cc->Inputs().Tag("ITERABLE").IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+    if (cc->Inputs().Tag("CONDITION").IsEmpty()) {
+      return ::mediapipe::OkStatus();
+    }
+
+    const std::vector<bool>& filter_by =
+        cc->Inputs().Tag("CONDITION").Get<std::vector<bool>>();
+
+    return FilterCollection<IterableT>(
+        std::is_copy_constructible<typename IterableT::value_type>(), cc,
+        filter_by);
+  }
+
+  template <typename IterableU>
+  ::mediapipe::Status FilterCollection(std::true_type, CalculatorContext* cc,
+                                       const std::vector<bool>& filter_by) {
+    const IterableU& input = cc->Inputs().Tag("ITERABLE").Get<IterableU>();
+    if (input.size() != filter_by.size()) {
+      return ::mediapipe::InternalError(absl::StrCat(
+          "Input vector size: ", input.size(),
+          " doesn't mach condition vector size: ", filter_by.size()));
+    }
+
+    auto output = absl::make_unique<IterableU>();
+    for (int i = 0; i < input.size(); ++i) {
+      if (filter_by[i]) {
+        output->push_back(input[i]);
+      }
+    }
+    cc->Outputs().Tag("ITERABLE").Add(output.release(), cc->InputTimestamp());
+    return ::mediapipe::OkStatus();
+  }
+
+  template <typename IterableU>
+  ::mediapipe::Status FilterCollection(std::false_type, CalculatorContext* cc,
+                                       const std::vector<bool>& filter_by) {
+    return ::mediapipe::InternalError(
+        "Cannot copy input collection to filter it.");
+  }
+};
+
+}  // namespace mediapipe
+
+#endif  // MEDIAPIPE_CALCULATORS_UTIL_FILTER_VECTOR_CALCULATOR_H_

mediapipe/calculators/util/rect_transformation_calculator.cc

@@ -23,7 +23,9 @@ namespace mediapipe {
 namespace {
 
 constexpr char kNormRectTag[] = "NORM_RECT";
+constexpr char kNormRectsTag[] = "NORM_RECTS";
 constexpr char kRectTag[] = "RECT";
+constexpr char kRectsTag[] = "RECTS";
 constexpr char kImageSizeTag[] = "IMAGE_SIZE";
 
 // Wraps around an angle in radians to within -M_PI and M_PI.
@@ -72,17 +74,31 @@ REGISTER_CALCULATOR(RectTransformationCalculator);
 
 ::mediapipe::Status RectTransformationCalculator::GetContract(
     CalculatorContract* cc) {
-  RET_CHECK(cc->Inputs().HasTag(kNormRectTag) ^ cc->Inputs().HasTag(kRectTag));
+  RET_CHECK_EQ((cc->Inputs().HasTag(kNormRectTag) ? 1 : 0) +
+                   (cc->Inputs().HasTag(kNormRectsTag) ? 1 : 0) +
+                   (cc->Inputs().HasTag(kRectTag) ? 1 : 0) +
+                   (cc->Inputs().HasTag(kRectsTag) ? 1 : 0),
+               1);
   if (cc->Inputs().HasTag(kRectTag)) {
     cc->Inputs().Tag(kRectTag).Set<Rect>();
     cc->Outputs().Index(0).Set<Rect>();
   }
+  if (cc->Inputs().HasTag(kRectsTag)) {
+    cc->Inputs().Tag(kRectsTag).Set<std::vector<Rect>>();
+    cc->Outputs().Index(0).Set<std::vector<Rect>>();
+  }
   if (cc->Inputs().HasTag(kNormRectTag)) {
     RET_CHECK(cc->Inputs().HasTag(kImageSizeTag));
     cc->Inputs().Tag(kNormRectTag).Set<NormalizedRect>();
     cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
     cc->Outputs().Index(0).Set<NormalizedRect>();
   }
+  if (cc->Inputs().HasTag(kNormRectsTag)) {
+    RET_CHECK(cc->Inputs().HasTag(kImageSizeTag));
+    cc->Inputs().Tag(kNormRectsTag).Set<std::vector<NormalizedRect>>();
+    cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
+    cc->Outputs().Index(0).Set<std::vector<NormalizedRect>>();
+  }
 
   return ::mediapipe::OkStatus();
 }
@@ -105,7 +121,17 @@ REGISTER_CALCULATOR(RectTransformationCalculator);
     cc->Outputs().Index(0).AddPacket(
         MakePacket<Rect>(rect).At(cc->InputTimestamp()));
   }
-
+  if (cc->Inputs().HasTag(kRectsTag) &&
+      !cc->Inputs().Tag(kRectsTag).IsEmpty()) {
+    auto rects = cc->Inputs().Tag(kRectsTag).Get<std::vector<Rect>>();
+    auto output_rects = absl::make_unique<std::vector<Rect>>(rects.size());
+    for (int i = 0; i < rects.size(); ++i) {
+      output_rects->at(i) = rects[i];
+      auto it = output_rects->begin() + i;
+      TransformRect(&(*it));
+    }
+    cc->Outputs().Index(0).Add(output_rects.release(), cc->InputTimestamp());
+  }
   if (cc->Inputs().HasTag(kNormRectTag) &&
       !cc->Inputs().Tag(kNormRectTag).IsEmpty()) {
     auto rect = cc->Inputs().Tag(kNormRectTag).Get<NormalizedRect>();
@@ -115,6 +141,21 @@ REGISTER_CALCULATOR(RectTransformationCalculator);
     cc->Outputs().Index(0).AddPacket(
         MakePacket<NormalizedRect>(rect).At(cc->InputTimestamp()));
   }
+  if (cc->Inputs().HasTag(kNormRectsTag) &&
+      !cc->Inputs().Tag(kNormRectsTag).IsEmpty()) {
+    auto rects =
+        cc->Inputs().Tag(kNormRectsTag).Get<std::vector<NormalizedRect>>();
+    const auto& image_size =
+        cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
+    auto output_rects =
+        absl::make_unique<std::vector<NormalizedRect>>(rects.size());
+    for (int i = 0; i < rects.size(); ++i) {
+      output_rects->at(i) = rects[i];
+      auto it = output_rects->begin() + i;
+      TransformNormalizedRect(&(*it), image_size.first, image_size.second);
+    }
+    cc->Outputs().Index(0).Add(output_rects.release(), cc->InputTimestamp());
+  }
 
   return ::mediapipe::OkStatus();
 }

mediapipe/calculators/video/BUILD

@@ -13,12 +13,16 @@
 # limitations under the License.
 #
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+load(
+    "//mediapipe/framework/tool:mediapipe_graph.bzl",
+    "mediapipe_binary_graph",
+)
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:private"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-
 proto_library(
     name = "flow_to_image_calculator_proto",
     srcs = ["flow_to_image_calculator.proto"],
@@ -52,9 +56,7 @@ mediapipe_cc_proto_library(
 cc_library(
     name = "flow_to_image_calculator",
     srcs = ["flow_to_image_calculator.cc"],
-    visibility = [
+    visibility = ["//visibility:public"],
-        "//visibility:public",
-    ],
     deps = [
         "//mediapipe/calculators/video:flow_to_image_calculator_cc_proto",
         "//mediapipe/calculators/video/tool:flow_quantizer_model",
@@ -129,10 +131,20 @@ cc_library(
     alwayslink = 1,
 )
 
+filegroup(
+    name = "test_videos",
+    srcs = [
+        "testdata/format_FLV_H264_AAC.video",
+        "testdata/format_MKV_VP8_VORBIS.video",
+        "testdata/format_MP4_AVC720P_AAC.video",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 cc_test(
     name = "opencv_video_decoder_calculator_test",
     srcs = ["opencv_video_decoder_calculator_test.cc"],
-    data = ["//mediapipe/calculators/video/testdata:test_videos"],
+    data = [":test_videos"],
     deps = [
         ":opencv_video_decoder_calculator",
         "//mediapipe/framework:calculator_runner",
@@ -151,7 +163,7 @@ cc_test(
 cc_test(
     name = "opencv_video_encoder_calculator_test",
     srcs = ["opencv_video_encoder_calculator_test.cc"],
-    data = ["//mediapipe/calculators/video/testdata:test_videos"],
+    data = [":test_videos"],
     deps = [
         ":opencv_video_decoder_calculator",
         ":opencv_video_encoder_calculator",
@@ -175,7 +187,6 @@ cc_test(
 cc_test(
     name = "tvl1_optical_flow_calculator_test",
     srcs = ["tvl1_optical_flow_calculator_test.cc"],
-    data = ["//mediapipe/calculators/image/testdata:test_images"],
     deps = [
         ":tvl1_optical_flow_calculator",
         "//mediapipe/framework:calculator_framework",

mediapipe/calculators/video/opencv_video_decoder_calculator.cc

@@ -123,6 +123,7 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
       cc->Outputs()
           .Tag("VIDEO_PRESTREAM")
           .Add(header.release(), Timestamp::PreStream());
+      cc->Outputs().Tag("VIDEO_PRESTREAM").Close();
     }
     // Rewind to the very first frame.
     cap_->set(cv::CAP_PROP_POS_AVI_RATIO, 0);

mediapipe/calculators/video/testdata/BUILD

@@ -1,26 +0,0 @@
-# Copyright 2019 The MediaPipe Authors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-licenses(["notice"])  # Apache 2.0
-
-filegroup(
-    name = "test_videos",
-    srcs = [
-        "format_FLV_H264_AAC.video",
-        "format_MKV_VP8_VORBIS.video",
-        "format_MP4_AVC720P_AAC.video",
-    ],
-    visibility = ["//visibility:public"],
-)

mediapipe/calculators/video/tool/BUILD

@@ -13,24 +13,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//mediapipe/calculators/video:__subpackages__"])
 
 exports_files(["LICENSE"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-
 proto_library(
     name = "flow_quantizer_model_proto",
     srcs = ["flow_quantizer_model.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
 )
 
 mediapipe_cc_proto_library(
     name = "flow_quantizer_model_cc_proto",
     srcs = ["flow_quantizer_model.proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":flow_quantizer_model_proto"],
 )
 

mediapipe/docs/examples.md

@@ -76,6 +76,15 @@ MediaPipe with a TFLite model for hand tracking in a GPU-accelerated pipeline.
 *   [Android](./hand_tracking_mobile_gpu.md)
 *   [iOS](./hand_tracking_mobile_gpu.md)
 
+### Multi-Hand Tracking with GPU
+
+[Multi-Hand Tracking with GPU](./multi_hand_tracking_mobile_gpu.md) illustrates
+how to use MediaPipe with a TFLite model for multi-hand tracking in a
+GPU-accelerated pipeline.
+
+*   [Android](./multi_hand_tracking_mobile_gpu.md)
+*   [iOS](./multi_hand_tracking_mobile_gpu.md)
+
 ### Hair Segmentation with GPU
 
 [Hair Segmentation on GPU](./hair_segmentation_mobile_gpu.md) illustrates how to
@@ -132,6 +141,15 @@ with live video from a webcam.
 *   [Desktop GPU](./hand_tracking_desktop.md)
 *   [Desktop CPU](./hand_tracking_desktop.md)
 
+### Multi-Hand Tracking on Desktop with Webcam
+
+[Multi-Hand Tracking on Desktop with Webcam](./multi_hand_tracking_desktop.md)
+shows how to use MediaPipe with a TFLite model for multi-hand tracking on
+desktop using CPU or GPU with live video from a webcam.
+
+*   [Desktop GPU](./multi_hand_tracking_desktop.md)
+*   [Desktop CPU](./multi_hand_tracking_desktop.md)
+
 ### Hair Segmentation on Desktop with Webcam
 
 [Hair Segmentation on Desktop with Webcam](./hair_segmentation_desktop.md) shows

mediapipe/docs/face_detection_desktop.md

@@ -77,7 +77,7 @@ below and paste it into
 ```bash
 # MediaPipe graph that performs face detection with TensorFlow Lite on CPU & GPU.
 # Used in the examples in
-# mediapipie/examples/desktop/face_detection:face_detection_cpu.
+# mediapipe/examples/desktop/face_detection:face_detection_cpu.
 
 # Images on CPU coming into and out of the graph.
 input_stream: "input_video"

mediapipe/docs/hair_segmentation_desktop.md

@@ -31,8 +31,6 @@ $ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \
 #INFO: Found 1 target...
 #Target //mediapipe/examples/desktop/hair_segmentation:hair_segmentation_gpu up-to-date:
 #  bazel-bin/mediapipe/examples/desktop/hair_segmentation/hair_segmentation_gpu
-#INFO: Elapsed time: 18.209s, Forge stats: 13026/13057 actions cached, 20.8s CPU used, 0.0s queue time, 89.3 MB ObjFS output (novel bytes: 87.4 MB), 0.0 MB local output, Critical Path: 11.88s, Remote (86.01% of the time): [queue: 0.00%, network: 16.83%, setup: 4.59%, process: 38.92%]
-#INFO: Streaming build results to: http://sponge2/37d5a184-293b-4e98-a43e-b22084db3142
 #INFO: Build completed successfully, 12210 total actions
 
 # This will open up your webcam as long as it is connected and on
@@ -53,7 +51,7 @@ below and paste it into
 ```bash
 # MediaPipe graph that performs hair segmentation with TensorFlow Lite on GPU.
 # Used in the example in
-# mediapipie/examples/android/src/java/com/mediapipe/apps/hairsegmentationgpu.
+# mediapipe/examples/android/src/java/com/mediapipe/apps/hairsegmentationgpu.
 
 # Images on GPU coming into and out of the graph.
 input_stream: "input_video"

mediapipe/docs/hair_segmentation_mobile_gpu.md

@@ -29,7 +29,7 @@ below and paste it into [MediaPipe Visualizer](https://viz.mediapipe.dev/).
 ```bash
 # MediaPipe graph that performs hair segmentation with TensorFlow Lite on GPU.
 # Used in the example in
-# mediapipie/examples/android/src/java/com/mediapipe/apps/hairsegmentationgpu.
+# mediapipe/examples/android/src/java/com/mediapipe/apps/hairsegmentationgpu.
 
 # Images on GPU coming into and out of the graph.
 input_stream: "input_video"

mediapipe/docs/hand_tracking_desktop.md

@@ -31,8 +31,6 @@ $ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
 # It should print:
 #Target //mediapipe/examples/desktop/hand_tracking:hand_tracking_cpu up-to-date:
 #  bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_cpu
-#INFO: Elapsed time: 22.645s, Forge stats: 13356/13463 actions cached, 1.5m CPU used, 0.0s queue time, 819.8 MB ObjFS output (novel bytes: 85.6 MB), 0.0 MB local output, Critical Path: 14.43s, Remote (87.25% of the time): [queue: 0.00%, network: 14.88%, setup: 4.80%, process: 39.80%, fetch: 18.15%]
-#INFO: Streaming build results to: http://sponge2/360196b9-33ab-44b1-84a7-1022b5043307
 #INFO: Build completed successfully, 12517 total actions
 
 # This will open up your webcam as long as it is connected and on
@@ -54,8 +52,6 @@ $ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \
 # It should print:
 # Target //mediapipe/examples/desktop/hand_tracking:hand_tracking_gpu up-to-date:
 #  bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_gpu
-#INFO: Elapsed time: 84.055s, Forge stats: 6858/19343 actions cached, 1.6h CPU used, 0.9s queue time, 1.68 GB ObjFS output (novel bytes: 485.1 MB), 0.0 MB local output, Critical Path: 48.14s, Remote (99.40% of the time): [queue: 0.00%, setup: 5.59%, process: 74.44%]
-#INFO: Streaming build results to: http://sponge2/00c7f95f-6fbc-432d-8978-f5d361efca3b
 #INFO: Build completed successfully, 22455 total actions
 
 # This will open up your webcam as long as it is connected and on
@@ -77,7 +73,7 @@ below and paste it into
 # MediaPipe graph that performs hand tracking on desktop with TensorFlow Lite
 # on CPU & GPU.
 # Used in the example in
-# mediapipie/examples/desktop/hand_tracking:hand_tracking_cpu.
+# mediapipe/examples/desktop/hand_tracking:hand_tracking_cpu.
 
 # Images coming into and out of the graph.
 input_stream: "input_video"

mediapipe/docs/hand_tracking_mobile_gpu.md

@@ -100,8 +100,8 @@ see the Visualizing Subgraphs section in the
 ```bash
 # MediaPipe graph that performs hand tracking with TensorFlow Lite on GPU.
 # Used in the examples in
-# mediapipie/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and
+# mediapipe/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and
-# mediapipie/examples/ios/handtrackinggpu.
+# mediapipe/examples/ios/handtrackinggpu.
 
 # Images coming into and out of the graph.
 input_stream: "input_video"

mediapipe/docs/install.md

@@ -7,11 +7,8 @@ future.
 Note: If you plan to use TensorFlow calculators and example apps, there is a
 known issue with gcc and g++ version 6.3 and 7.3. Please use other versions.
 
-Note: While Mediapipe configures TensorFlow, if you see the
+Note: To make Mediapipe work with TensorFlow, please install the python "future"
-following error:
+library and the python "six" library using `pip install --user future six`.
-`"...git_configure.bzl", line 14, in _fail fail(("%sGit Configuration
-Error:%s %...)))`,
-please install the python future library using: `$ pip install --user future`.
 
 Choose your operating system:
 
@@ -42,11 +39,19 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (version between 0.24.1 and 0.29.1).
+2.  Install Bazel (0.24.1 and above required).
 
-    Follow the official
+    Option 1. Use package manager tool to install the latest version of Bazel.
-    [documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
+
-    to install Bazel manually. Note that MediaPipe doesn't support Bazel 1.0.0+ yet.
+    ```bash
+    $ sudo apt-get install bazel
+
+    # Run 'bazel version' to check version of bazel installed
+    ```
+
+    Option 2. Follow the official
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
+    to install any version of Bazel manually.
 
 3.  Install OpenCV and FFmpeg.
 
@@ -152,11 +157,11 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (version between 0.24.1 and 0.29.1).
+2.  Install Bazel (0.24.1 and above required).
 
     Follow the official
-    [documentation](https://docs.bazel.build/versions/master/install-redhat.html)
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
-    to install Bazel manually. Note that MediaPipe doesn't support Bazel 1.0.0+ yet.
+    to install Bazel manually.
 
 3.  Install OpenCV.
 
@@ -240,24 +245,19 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-3.  Install Bazel (version between 0.24.1 and 0.29.1).
+3.  Install Bazel (0.24.1 and above required).
 
-     Option 1. Use package manager tool to install Bazel 0.29.1
+    Option 1. Use package manager tool to install the latest version of Bazel.
 
     ```bash
-    # If Bazel 1.0.0+ was installed.
+    $ brew install bazel
-    $ brew uninstall bazel
-
-    # Install Bazel 0.29.1
-    $ brew install https://raw.githubusercontent.com/bazelbuild/homebrew-tap/223ffb570c21c0a2af251afc6df9dec0214c6e74/Formula/bazel.rb
-    $ brew link bazel
 
     # Run 'bazel version' to check version of bazel installed
     ```
 
     Option 2. Follow the official
-    [documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
+    [Bazel documentation](https://docs.bazel.build/versions/master/install-os-x.html#install-with-installer-mac-os-x)
-    to install Bazel manually. Note that MediaPipe doesn't support Bazel 1.0.0+ yet.
+    to install any version of Bazel manually.
 
 4.  Install OpenCV and FFmpeg.
 

mediapipe/docs/multi_hand_tracking_desktop.md

@@ -0,0 +1,177 @@
+## Multi-Hand Tracking on Desktop
+
+This is an example of using MediaPipe to run hand tracking models (TensorFlow
+Lite) and render bounding boxes on the detected hand instances (for multiple
+hands). To know more about the hand tracking models, please refer to the model
+[`README file`]. Moreover, if you are interested in running the same TensorfFlow
+Lite model on Android/iOS, please see the
+[Mulit-Hand Tracking on GPU on Android/iOS](multi_hand_tracking_mobile_gpu.md)
+and
+
+We show the hand tracking demos with TensorFlow Lite model using the Webcam:
+
+-   [TensorFlow Lite Multi-Hand Tracking Demo with Webcam (CPU)](#tensorflow-lite-multi-hand-tracking-demo-with-webcam-cpu)
+
+-   [TensorFlow Lite Multi-Hand Tracking Demo with Webcam (GPU)](#tensorflow-lite-multi-hand-tracking-demo-with-webcam-gpu)
+
+Note: Desktop GPU works only on Linux. Mesa drivers need to be installed. Please
+see
+[step 4 of "Installing on Debian and Ubuntu" in the installation guide](./install.md).
+
+Note: If MediaPipe depends on OpenCV 2, please see the
+[known issues with OpenCV 2](#known-issues-with-opencv-2) section.
+
+### TensorFlow Lite Multi-Hand Tracking Demo with Webcam (CPU)
+
+To build and run the TensorFlow Lite example on desktop (CPU) with Webcam, run:
+
+```bash
+# Video from webcam running on desktop CPU
+$ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
+    mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_cpu
+
+# It should print:
+#Target //mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_cpu up-to-date:
+#  bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_cpu
+
+# This will open up your webcam as long as it is connected and on
+# Any errors is likely due to your webcam being not accessible
+$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_cpu \
+    --calculator_graph_config_file=mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt
+```
+
+### TensorFlow Lite Multi-Hand Tracking Demo with Webcam (GPU)
+
+To build and run the TensorFlow Lite example on desktop (GPU) with Webcam, run:
+
+```bash
+# Video from webcam running on desktop GPU
+# This works only for linux currently
+$ bazel build -c opt --copt -DMESA_EGL_NO_X11_HEADERS \
+    mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_gpu
+
+# It should print:
+# Target //mediapipe/examples/desktop/multi_hand_tracking:multi_hand_tracking_gpu up-to-date:
+#  bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_gpu
+
+# This will open up your webcam as long as it is connected and on
+# Any errors is likely due to your webcam being not accessible,
+# or GPU drivers not setup properly.
+$ GLOG_logtostderr=1 bazel-bin/mediapipe/examples/desktop/multi_hand_tracking/multi_hand_tracking_gpu \
+    --calculator_graph_config_file=mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt
+```
+
+#### Graph
+
+![graph visualization](images/multi_hand_tracking_desktop.png)
+
+To visualize the graph as shown above, copy the text specification of the graph
+below and paste it into [MediaPipe Visualizer](https://viz.mediapipe.dev).
+
+```bash
+# MediaPipe graph that performs multi-hand tracking on desktop with TensorFlow
+# Lite on CPU.
+# Used in the example in
+# mediapipie/examples/desktop/multi_hand_tracking:multi_hand_tracking_cpu.
+
+# Images coming into and out of the graph.
+input_stream: "input_video"
+output_stream: "output_video"
+
+# Determines if an input vector of NormalizedRect has a size greater than or
+# equal to the provided min_size.
+node {
+  calculator: "NormalizedRectVectorHasMinSizeCalculator"
+  input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks"
+  output_stream: "prev_has_enough_hands"
+  node_options: {
+    [type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
+      # This value can be changed to support tracking arbitrary number of hands.
+      # Please also remember to modify max_vec_size in
+      # ClipVectorSizeCalculatorOptions in
+      # mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt
+      min_size: 2
+    }
+  }
+}
+
+# Drops the incoming image if the previous frame had at least N hands.
+# Otherwise, passes the incoming image through to trigger a new round of hand
+# detection in MultiHandDetectionSubgraph.
+node {
+  calculator: "GateCalculator"
+  input_stream: "input_video"
+  input_stream: "DISALLOW:prev_has_enough_hands"
+  output_stream: "multi_hand_detection_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.GateCalculatorOptions] {
+      empty_packets_as_allow: true
+    }
+  }
+}
+
+# Subgraph that detections hands (see multi_hand_detection_cpu.pbtxt).
+node {
+  calculator: "MultiHandDetectionSubgraph"
+  input_stream: "multi_hand_detection_input_video"
+  output_stream: "DETECTIONS:multi_palm_detections"
+  output_stream: "NORM_RECTS:multi_palm_rects"
+}
+
+# Subgraph that localizes hand landmarks for multiple hands (see
+# multi_hand_landmark.pbtxt).
+node {
+  calculator: "MultiHandLandmarkSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_stream: "NORM_RECTS:multi_hand_rects"
+  output_stream: "LANDMARKS:multi_hand_landmarks"
+  output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks"
+}
+
+# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the
+# arrival of the next input image sends out the cached rectangle with the
+# timestamp replaced by that of the input image, essentially generating a packet
+# that carries the previous hand rectangle. Note that upon the arrival of the
+# very first input image, an empty packet is sent out to jump start the
+# feedback loop.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:input_video"
+  input_stream: "LOOP:multi_hand_rects_from_landmarks"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks"
+}
+
+# Performs association between NormalizedRect vector elements from previous
+# frame and those from the current frame if MultiHandDetectionSubgraph runs.
+# This calculator ensures that the output multi_hand_rects vector doesn't
+# contain overlapping regions based on the specified min_similarity_threshold.
+node {
+  calculator: "AssociationNormRectCalculator"
+  input_stream: "prev_multi_hand_rects_from_landmarks"
+  input_stream: "multi_palm_rects"
+  output_stream: "multi_hand_rects"
+  node_options: {
+    [type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
+      min_similarity_threshold: 0.1
+    }
+  }
+}
+
+# Subgraph that renders annotations and overlays them on top of the input
+# images (see multi_hand_renderer_cpu.pbtxt).
+node {
+  calculator: "MultiHandRendererSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_stream: "DETECTIONS:multi_palm_detections"
+  input_stream: "LANDMARKS:multi_hand_landmarks"
+  input_stream: "NORM_RECTS:0:multi_palm_rects"
+  input_stream: "NORM_RECTS:1:multi_hand_rects"
+  output_stream: "IMAGE:output_video"
+}
+```
+
+[`README file`]:https://github.com/google/mediapipe/tree/master/mediapipe/README.md

mediapipe/docs/multi_hand_tracking_mobile_gpu.md

@@ -0,0 +1,755 @@
+# Multi-Hand Tracking (GPU)
+
+This doc focuses on the
+[example graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt)
+that performs multi-hand tracking with TensorFlow Lite on GPU. It is related to
+the [hand_tracking_example](./hand_tracking_mobile_gpu.md), and we recommend
+users to review the (single) hand tracking example first.
+
+![multi_hand_tracking_android_gpu.gif](images/mobile/multi_hand_tracking_android_gpu.gif)
+
+In the visualization above, the red dots represent the hand landmarks and the
+green lines are simply connections between selected landmark paris for
+visualization of the hand skeleton. When there are fewer than `N` hands (`N=2`
+in the graphs here), the purple box represents a hand rectangle that covers the
+entire hand, derived from hand detection (see
+[hand_detection_example](./hand_detection_mobile_gpu.md)). When there are `N`
+hands (i.e. 2 hands for the graphs here), the red boxes represent hand
+rectangles for each of the hands, derived from the previous round of hand
+landmark localization using an ML model (see also
+[model card](https://mediapipe.page.link/handmc)). Hand landmark localization
+for each hand is performed only within the hand rectangle for computational
+efficiency and accuracy. Hand detection is only invoked whenever there are fewer
+than `N` hands in the previous iteration.
+
+This example can also run a model that localizes hand landmarks in 3D (i.e.,
+estimating an extra z coordinate):
+
+![multi_hand_tracking_3d_android_gpu.gif](images/mobile/multi_hand_tracking_3d_android_gpu.gif)
+
+In the visualization above, the localized hand landmarks are represented by dots
+in different shades, with the brighter ones denoting landmarks closer to the
+camera.
+
+## Android
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu)
+
+To build the app yourself, run:
+
+```bash
+bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu
+```
+
+To build for the 3D mode, run:
+
+```bash
+bazel build -c opt --config=android_arm64 --define 3D=true mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu
+```
+
+Once the app is built, install it on Android device with:
+
+```bash
+adb install bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/multihandtrackinggpu.apk
+```
+
+## iOS
+
+[Source](https://github.com/google/mediapipe/tree/master/mediapipe/examples/ios/multihandtrackinggpu).
+
+See the general [instructions](./mediapipe_ios_setup.md) for building iOS
+examples and generating an Xcode project. This will be the HandDetectionGpuApp
+target.
+
+To build on the command line:
+
+```bash
+bazel build -c opt --config=ios_arm64 mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp
+```
+
+To build for the 3D mode, run:
+
+```bash
+bazel build -c opt --config=ios_arm64 --define 3D=true mediapipe/examples/ios/multihandtrackinggpu:MultiHandTrackingGpuApp
+```
+
+## Graph
+
+The multi-hand tracking [main graph](#main-graph) internal utilizes a
+[multi_hand_detection_subgraph](#multi-hand-detection-subgraph), a
+[multi_hand_landmark_subgraph](#multi-hand-landmark-subgraph), and a
+[multi_hand_renderer_subgraph](#multi-hand-renderer-subgraph).
+
+The subgraphs show up in the main graph visualization as nodes colored in
+purple, and the subgraph itself can also be visualized just like a regular
+graph. For more information on how to visualize a graph that includes subgraphs,
+see the Visualizing Subgraphs section in the
+[visualizer documentation](./visualizer.md).
+
+### Main Graph
+
+![multi_hand_tracking_mobile_graph](images/mobile/multi_hand_tracking_mobile.png)
+
+There are two key differences between this graph and the
+[single_hand_tracking_mobile_graph](./hand_tracking_mobile_gpu.md).
+
+1.  There is a `NormalizedRectVectorHasMinSize` calculator, that checks if in
+    input vector of `NormalizedRect` objects has a minimum size equal to `N`. In
+    this graph, if the vector contains fewer than `N` objects,
+    `MultiHandDetection` subgraph runs. Otherwise, the `GateCalculator` doesn't
+    send any image packets to the `MultiHandDetection` subgraph. This way, the
+    main graph is efficient in that it avoids running the costly hand detection
+    step when there are already `N` hands in the frame.
+2.  The `MergeCalculator` has been replaced by the `AssociationNormRect`
+    calculator. This `AssociationNormRect` takes as input a vector of
+    `NormalizedRect` objects from the `MultiHandDetection` subgraph on the
+    current frame, and a vector of `NormalizedRect` objects from the
+    `MultiHandLandmark` subgraph from the previous frame, and performs an
+    association operation between these objects. This calculator ensures that
+    the output vector doesn't contain overlapping regions based on the specified
+    `min_similarity_threshold`.
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt)
+
+```bash
+# MediaPipe graph that performs multi-hand tracking with TensorFlow Lite on GPU.
+# Used in the examples in
+# mediapipie/examples/android/src/java/com/mediapipe/apps/multihandtrackinggpu.
+
+# Images coming into and out of the graph.
+input_stream: "input_video"
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:multi_hand_rects"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Determines if an input vector of NormalizedRect has a size greater than or
+# equal to the provided min_size.
+node {
+  calculator: "NormalizedRectVectorHasMinSizeCalculator"
+  input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks"
+  output_stream: "prev_has_enough_hands"
+  node_options: {
+    [type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
+      # This value can be changed to support tracking arbitrary number of hands.
+      # Please also remember to modify max_vec_size in
+      # ClipVectorSizeCalculatorOptions in
+      # mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt
+      min_size: 2
+    }
+  }
+}
+
+# Drops the incoming image if the previous frame had at least N hands.
+# Otherwise, passes the incoming image through to trigger a new round of hand
+# detection in MultiHandDetectionSubgraph.
+node {
+  calculator: "GateCalculator"
+  input_stream: "throttled_input_video"
+  input_stream: "DISALLOW:prev_has_enough_hands"
+  output_stream: "multi_hand_detection_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.GateCalculatorOptions] {
+      empty_packets_as_allow: true
+    }
+  }
+}
+
+# Subgraph that detections hands (see multi_hand_detection_gpu.pbtxt).
+node {
+  calculator: "MultiHandDetectionSubgraph"
+  input_stream: "multi_hand_detection_input_video"
+  output_stream: "DETECTIONS:multi_palm_detections"
+  output_stream: "NORM_RECTS:multi_palm_rects"
+}
+
+# Subgraph that localizes hand landmarks for multiple hands (see
+# multi_hand_landmark.pbtxt).
+node {
+  calculator: "MultiHandLandmarkSubgraph"
+  input_stream: "IMAGE:throttled_input_video"
+  input_stream: "NORM_RECTS:multi_hand_rects"
+  output_stream: "LANDMARKS:multi_hand_landmarks"
+  output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks"
+}
+
+# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the
+# arrival of the next input image sends out the cached rectangle with the
+# timestamp replaced by that of the input image, essentially generating a packet
+# that carries the previous hand rectangle. Note that upon the arrival of the
+# very first input image, an empty packet is sent out to jump start the
+# feedback loop.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:throttled_input_video"
+  input_stream: "LOOP:multi_hand_rects_from_landmarks"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks"
+}
+
+# Performs association between NormalizedRect vector elements from previous
+# frame and those from the current frame if MultiHandDetectionSubgraph runs.
+# This calculator ensures that the output multi_hand_rects vector doesn't
+# contain overlapping regions based on the specified min_similarity_threshold.
+node {
+  calculator: "AssociationNormRectCalculator"
+  input_stream: "prev_multi_hand_rects_from_landmarks"
+  input_stream: "multi_palm_rects"
+  output_stream: "multi_hand_rects"
+  node_options: {
+    [type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
+      min_similarity_threshold: 0.1
+    }
+  }
+}
+
+# Subgraph that renders annotations and overlays them on top of the input
+# images (see multi_hand_renderer_gpu.pbtxt).
+node {
+  calculator: "MultiHandRendererSubgraph"
+  input_stream: "IMAGE:throttled_input_video"
+  input_stream: "DETECTIONS:multi_palm_detections"
+  input_stream: "LANDMARKS:multi_hand_landmarks"
+  input_stream: "NORM_RECTS:0:multi_palm_rects"
+  input_stream: "NORM_RECTS:1:multi_hand_rects"
+  output_stream: "IMAGE:output_video"
+}
+```
+
+### Multi-Hand Detection Subgraph
+
+![multi_hand_detection_gpu_subgraph](images/mobile/multi_hand_detection_gpu_subgraph.png)
+
+This graph outputs a vector of `NormalizedRect` objects corresponding to each of
+the hand instances visible in the frame. Note that at the end of this graph,
+there is a `ClipNormalizedRectVectorSizeCalculator`. This calculator clips the
+size of the input vector to a maximum size `N`. This implies that the
+`MultiHandDetection` subgraph outputs a vector of maximum `N` hand instance
+locations.
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt)
+
+```bash
+# MediaPipe multi-hand detection subgraph.
+
+type: "MultiHandDetectionSubgraph"
+
+input_stream: "input_video"
+output_stream: "DETECTIONS:palm_detections"
+output_stream: "NORM_RECTS:clipped_hand_rects_from_palm_detections"
+
+# Transforms the input image on GPU to a 256x256 image. To scale the input
+# image, the scale_mode option is set to FIT to preserve the aspect ratio,
+# resulting in potential letterboxing in the transformed image.
+node: {
+  calculator: "ImageTransformationCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "IMAGE_GPU:transformed_input_video"
+  output_stream: "LETTERBOX_PADDING:letterbox_padding"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageTransformationCalculatorOptions] {
+      output_width: 256
+      output_height: 256
+      scale_mode: FIT
+    }
+  }
+}
+
+# Generates a single side packet containing a TensorFlow Lite op resolver that
+# supports custom ops needed by the model used in this graph.
+node {
+  calculator: "TfLiteCustomOpResolverCalculator"
+  output_side_packet: "opresolver"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteCustomOpResolverCalculatorOptions] {
+      use_gpu: true
+    }
+  }
+}
+
+# Converts the transformed input image on GPU into an image tensor stored as a
+# TfLiteTensor.
+node {
+  calculator: "TfLiteConverterCalculator"
+  input_stream: "IMAGE_GPU:transformed_input_video"
+  output_stream: "TENSORS_GPU:image_tensor"
+}
+
+# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
+# vector of tensors representing, for instance, detection boxes/keypoints and
+# scores.
+node {
+  calculator: "TfLiteInferenceCalculator"
+  input_stream: "TENSORS_GPU:image_tensor"
+  output_stream: "TENSORS_GPU:detection_tensors"
+  input_side_packet: "CUSTOM_OP_RESOLVER:opresolver"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteInferenceCalculatorOptions] {
+      model_path: "mediapipe/models/palm_detection.tflite"
+      use_gpu: true
+    }
+  }
+}
+
+# Generates a single side packet containing a vector of SSD anchors based on
+# the specification in the options.
+node {
+  calculator: "SsdAnchorsCalculator"
+  output_side_packet: "anchors"
+  node_options: {
+    [type.googleapis.com/mediapipe.SsdAnchorsCalculatorOptions] {
+      num_layers: 5
+      min_scale: 0.1171875
+      max_scale: 0.75
+      input_size_height: 256
+      input_size_width: 256
+      anchor_offset_x: 0.5
+      anchor_offset_y: 0.5
+      strides: 8
+      strides: 16
+      strides: 32
+      strides: 32
+      strides: 32
+      aspect_ratios: 1.0
+      fixed_anchor_size: true
+    }
+  }
+}
+
+# Decodes the detection tensors generated by the TensorFlow Lite model, based on
+# the SSD anchors and the specification in the options, into a vector of
+# detections. Each detection describes a detected object.
+node {
+  calculator: "TfLiteTensorsToDetectionsCalculator"
+  input_stream: "TENSORS_GPU:detection_tensors"
+  input_side_packet: "ANCHORS:anchors"
+  output_stream: "DETECTIONS:detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.TfLiteTensorsToDetectionsCalculatorOptions] {
+      num_classes: 1
+      num_boxes: 2944
+      num_coords: 18
+      box_coord_offset: 0
+      keypoint_coord_offset: 4
+      num_keypoints: 7
+      num_values_per_keypoint: 2
+      sigmoid_score: true
+      score_clipping_thresh: 100.0
+      reverse_output_order: true
+
+      x_scale: 256.0
+      y_scale: 256.0
+      h_scale: 256.0
+      w_scale: 256.0
+      min_score_thresh: 0.7
+    }
+  }
+}
+
+# Performs non-max suppression to remove excessive detections.
+node {
+  calculator: "NonMaxSuppressionCalculator"
+  input_stream: "detections"
+  output_stream: "filtered_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.NonMaxSuppressionCalculatorOptions] {
+      min_suppression_threshold: 0.3
+      overlap_type: INTERSECTION_OVER_UNION
+      algorithm: WEIGHTED
+      return_empty_detections: true
+    }
+  }
+}
+
+# Maps detection label IDs to the corresponding label text ("Palm"). The label
+# map is provided in the label_map_path option.
+node {
+  calculator: "DetectionLabelIdToTextCalculator"
+  input_stream: "filtered_detections"
+  output_stream: "labeled_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionLabelIdToTextCalculatorOptions] {
+      label_map_path: "mediapipe/models/palm_detection_labelmap.txt"
+    }
+  }
+}
+
+# Adjusts detection locations (already normalized to [0.f, 1.f]) on the
+# letterboxed image (after image transformation with the FIT scale mode) to the
+# corresponding locations on the same image with the letterbox removed (the
+# input image to the graph before image transformation).
+node {
+  calculator: "DetectionLetterboxRemovalCalculator"
+  input_stream: "DETECTIONS:labeled_detections"
+  input_stream: "LETTERBOX_PADDING:letterbox_padding"
+  output_stream: "DETECTIONS:palm_detections"
+}
+
+# Extracts image size from the input images.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE_GPU:input_video"
+  output_stream: "SIZE:image_size"
+}
+
+# Converts each palm detection into a rectangle (normalized by image size)
+# that encloses the palm and is rotated such that the line connecting center of
+# the wrist and MCP of the middle finger is aligned with the Y-axis of the
+# rectangle.
+node {
+  calculator: "DetectionsToRectsCalculator"
+  input_stream: "DETECTIONS:palm_detections"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "NORM_RECTS:palm_rects"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionsToRectsCalculatorOptions] {
+      rotation_vector_start_keypoint_index: 0  # Center of wrist.
+      rotation_vector_end_keypoint_index: 2  # MCP of middle finger.
+      rotation_vector_target_angle_degrees: 90
+      output_zero_rect_for_empty_detections: true
+    }
+  }
+}
+
+# Expands and shifts the rectangle that contains the palm so that it's likely
+# to cover the entire hand.
+node {
+  calculator: "RectTransformationCalculator"
+  input_stream: "NORM_RECTS:palm_rects"
+  input_stream: "IMAGE_SIZE:image_size"
+  output_stream: "hand_rects_from_palm_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.RectTransformationCalculatorOptions] {
+      scale_x: 2.6
+      scale_y: 2.6
+      shift_y: -0.5
+      square_long: true
+    }
+  }
+}
+
+# Clips the size of the input vector to the provided max_vec_size. This
+# determines the maximum number of hand instances this graph outputs.
+# Note that the performance gain of clipping detections earlier in this graph is
+# minimal because NMS will minimize overlapping detections and the number of
+# detections isn't expected to exceed 5-10.
+node {
+  calculator: "ClipNormalizedRectVectorSizeCalculator"
+  input_stream: "hand_rects_from_palm_detections"
+  output_stream: "clipped_hand_rects_from_palm_detections"
+  node_options: {
+    [type.googleapis.com/mediapipe.ClipVectorSizeCalculatorOptions] {
+      # This value can be changed to support tracking arbitrary number of hands.
+      # Please also remember to modify min_size in
+      # CollectionHsMinSizeCalculatorOptions in
+      # mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt and
+      # mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt.
+      max_vec_size: 2
+    }
+  }
+}
+```
+
+### Multi-Hand Landmark Subgraph
+
+![multi_hand_landmark_subgraph.pbtxt](images/mobile/multi_hand_landmark_subgraph.png)
+
+This graph accepts as input a vector of `NormalizedRect` objects, corresponding
+the the region of each hand instance in the input image. For each
+`NormalizedRect` object, the graph runs the existing `HandLandmark` subgraph and
+collect the outputs of this subgraph into vectors. This is enabled by
+`BeginLoop` and `EndLoop` calculators.
+
+The `BeginLoop` calculator accepts as input a packet containing an iterable
+collection of elements. This calculator is templatized (see
+[begin_loop_calculator.h](https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/begin_loop_calculator.h)).
+If the input packet arrived at a timestamp `ts`, this calculator outputs each
+element in the collection at a fake timestamp `internal_ts`. At the end of the
+collection, the calculator outputs the arrival timestamp `ts` in the output
+stream tagged with `BATCH_END`.
+
+The nodes between the `BeginLoop` calculator and the corresponding `EndLoop`
+calculator process individual packets at the fake timestamps `internal_ts`.
+After each element is processed, it is sent to the `EndLoop` calculator (see
+[end_loop_calculator.h](https://github.com/google/mediapipe/tree/master/mediapipe/calculators/core/end_loop_calculator.h)),
+which collects these elements in an output collection. The `EndLoop` calculator
+listens for packets from the `BATCH_END` output stream of the `BeginLoop`
+calculator. When the `BATCH_END` packet containing the real timestamp `ts`
+arrives at the `EndLoop` calculator, the `EndLoop` calculator outputs a packet
+containing the collection of processed elements at the real timestamp `ts`.
+
+In the multi-hand landmark subgraph, the `EndLoop` calculators collect the
+output vector of hand landmarks per hand instance, the boolean values indicating
+the presence of each hand and the `NormalizedRect` objects corresponding to the
+regions surrounding each hand into vectors.
+
+Finally, based on the hand presence boolean value, the graph filters the
+collections of hand landmarks and `NormalizdRect` objects corresponding to each
+hand instance.
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_landmark.pbtxt)
+
+```bash
+# MediaPipe hand landmark localization subgraph.
+
+type: "MultiHandLandmarkSubgraph"
+
+input_stream: "IMAGE:input_video"
+# A vector of NormalizedRect, one per each hand detected.
+input_stream: "NORM_RECTS:multi_hand_rects"
+# A vector of NormalizedLandmarks, one set per each hand.
+output_stream: "LANDMARKS:filtered_multi_hand_landmarks"
+# A vector of NormalizedRect, one per each hand.
+output_stream: "NORM_RECTS:filtered_multi_hand_rects_for_next_frame"
+
+# Outputs each element of multi_hand_rects at a fake timestamp for the rest
+# of the graph to process. Clones the input_video packet for each
+# single_hand_rect at the fake timestamp. At the end of the loop,
+# outputs the BATCH_END timestamp for downstream calculators to inform them
+# that all elements in the vector have been processed.
+node {
+  calculator: "BeginLoopNormalizedRectCalculator"
+  input_stream: "ITERABLE:multi_hand_rects"
+  input_stream: "CLONE:input_video"
+  output_stream: "ITEM:single_hand_rect"
+  output_stream: "CLONE:input_video_cloned"
+  output_stream: "BATCH_END:single_hand_rect_timestamp"
+}
+
+node {
+  calculator: "HandLandmarkSubgraph"
+  input_stream: "IMAGE:input_video_cloned"
+  input_stream: "NORM_RECT:single_hand_rect"
+  output_stream: "LANDMARKS:single_hand_landmarks"
+  output_stream: "NORM_RECT:single_hand_rect_from_landmarks"
+  output_stream: "PRESENCE:single_hand_presence"
+}
+
+# Collects the boolean presence value for each single hand into a vector. Upon
+# receiving the BATCH_END timestamp, outputs a vector of boolean values at the
+# BATCH_END timestamp.
+node {
+  calculator: "EndLoopBooleanCalculator"
+  input_stream: "ITEM:single_hand_presence"
+  input_stream: "BATCH_END:single_hand_rect_timestamp"
+  output_stream: "ITERABLE:multi_hand_presence"
+}
+
+# Collects a set of landmarks for each hand into a vector. Upon receiving the
+# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
+# timestamp.
+node {
+  calculator: "EndLoopNormalizedLandmarksVectorCalculator"
+  input_stream: "ITEM:single_hand_landmarks"
+  input_stream: "BATCH_END:single_hand_rect_timestamp"
+  output_stream: "ITERABLE:multi_hand_landmarks"
+}
+
+# Collects a NormalizedRect for each hand into a vector. Upon receiving the
+# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
+# timestamp.
+node {
+  calculator: "EndLoopNormalizedRectCalculator"
+  input_stream: "ITEM:single_hand_rect_from_landmarks"
+  input_stream: "BATCH_END:single_hand_rect_timestamp"
+  output_stream: "ITERABLE:multi_hand_rects_for_next_frame"
+}
+
+# Filters the input vector of landmarks based on hand presence value for each
+# hand. If the hand presence for hand #i is false, the set of landmarks
+# corresponding to that hand are dropped from the vector.
+node {
+  calculator: "FilterLandmarksCollectionCalculator"
+  input_stream: "ITERABLE:multi_hand_landmarks"
+  input_stream: "CONDITION:multi_hand_presence"
+  output_stream: "ITERABLE:filtered_multi_hand_landmarks"
+}
+
+# Filters the input vector of NormalizedRect based on hand presence value for
+# each hand. If the hand presence for hand #i is false, the NormalizedRect
+# corresponding to that hand are dropped from the vector.
+node {
+  calculator: "FilterNormalizedRectCollectionCalculator"
+  input_stream: "ITERABLE:multi_hand_rects_for_next_frame"
+  input_stream: "CONDITION:multi_hand_presence"
+  output_stream: "ITERABLE:filtered_multi_hand_rects_for_next_frame"
+}
+```
+
+### Multi-Hand Renderer Subgraph
+
+![multi_hand_renderer_gpu_subgraph.pbtxt](images/mobile/multi_hand_renderer_gpu_subgraph.png)
+
+This graph also uses `BeginLoop` and `EndLoop` calculators to iteratively
+convert a set of hand landmarks per hand instance into corresponding
+`RenderData` objects.
+
+[Source pbtxt file](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt)
+
+```bash
+# MediaPipe multi-hand tracking rendering subgraph.
+
+type: "MultiHandRendererSubgraph"
+
+input_stream: "IMAGE:input_image"
+# A vector of NormalizedLandmarks, one for each hand.
+input_stream: "LANDMARKS:multi_hand_landmarks"
+# A vector of NormalizedRect, one for each hand.
+input_stream: "NORM_RECTS:0:multi_palm_rects"
+# A vector of NormalizedRect, one for each hand.
+input_stream: "NORM_RECTS:1:multi_hand_rects"
+# A vector of Detection, one for each hand.
+input_stream: "DETECTIONS:palm_detections"
+output_stream: "IMAGE:output_image"
+
+# Converts detections to drawing primitives for annotation overlay.
+node {
+  calculator: "DetectionsToRenderDataCalculator"
+  input_stream: "DETECTIONS:palm_detections"
+  output_stream: "RENDER_DATA:detection_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] {
+      thickness: 4.0
+      color { r: 0 g: 255 b: 0 }
+    }
+  }
+}
+
+# Converts normalized rects to drawing primitives for annotation overlay.
+node {
+  calculator: "RectToRenderDataCalculator"
+  input_stream: "NORM_RECTS:multi_hand_rects"
+  output_stream: "RENDER_DATA:multi_hand_rects_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
+      filled: false
+      color { r: 255 g: 0 b: 0 }
+      thickness: 4.0
+    }
+  }
+}
+
+# Converts normalized rects to drawing primitives for annotation overlay.
+node {
+  calculator: "RectToRenderDataCalculator"
+  input_stream: "NORM_RECTS:multi_palm_rects"
+  output_stream: "RENDER_DATA:multi_palm_rects_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.RectToRenderDataCalculatorOptions] {
+      filled: false
+      color { r: 125 g: 0 b: 122 }
+      thickness: 4.0
+    }
+  }
+}
+
+# Outputs each element of multi_palm_landmarks at a fake timestamp for the rest
+# of the graph to process. At the end of the loop, outputs the BATCH_END
+# timestamp for downstream calculators to inform them that all elements in the
+# vector have been processed.
+node {
+  calculator: "BeginLoopNormalizedLandmarksVectorCalculator"
+  input_stream: "ITERABLE:multi_hand_landmarks"
+  output_stream: "ITEM:single_hand_landmarks"
+  output_stream: "BATCH_END:landmark_timestamp"
+}
+
+# Converts landmarks to drawing primitives for annotation overlay.
+node {
+  calculator: "LandmarksToRenderDataCalculator"
+  input_stream: "NORM_LANDMARKS:single_hand_landmarks"
+  output_stream: "RENDER_DATA:single_hand_landmark_render_data"
+  node_options: {
+    [type.googleapis.com/mediapipe.LandmarksToRenderDataCalculatorOptions] {
+      landmark_connections: 0
+      landmark_connections: 1
+      landmark_connections: 1
+      landmark_connections: 2
+      landmark_connections: 2
+      landmark_connections: 3
+      landmark_connections: 3
+      landmark_connections: 4
+      landmark_connections: 0
+      landmark_connections: 5
+      landmark_connections: 5
+      landmark_connections: 6
+      landmark_connections: 6
+      landmark_connections: 7
+      landmark_connections: 7
+      landmark_connections: 8
+      landmark_connections: 5
+      landmark_connections: 9
+      landmark_connections: 9
+      landmark_connections: 10
+      landmark_connections: 10
+      landmark_connections: 11
+      landmark_connections: 11
+      landmark_connections: 12
+      landmark_connections: 9
+      landmark_connections: 13
+      landmark_connections: 13
+      landmark_connections: 14
+      landmark_connections: 14
+      landmark_connections: 15
+      landmark_connections: 15
+      landmark_connections: 16
+      landmark_connections: 13
+      landmark_connections: 17
+      landmark_connections: 0
+      landmark_connections: 17
+      landmark_connections: 17
+      landmark_connections: 18
+      landmark_connections: 18
+      landmark_connections: 19
+      landmark_connections: 19
+      landmark_connections: 20
+      landmark_color { r: 255 g: 0 b: 0 }
+      connection_color { r: 0 g: 255 b: 0 }
+      thickness: 4.0
+    }
+  }
+}
+
+# Collects a RenderData object for each hand into a vector. Upon receiving the
+# BATCH_END timestamp, outputs the vector of RenderData at the BATCH_END
+# timestamp.
+node {
+  calculator: "EndLoopRenderDataCalculator"
+  input_stream: "ITEM:single_hand_landmark_render_data"
+  input_stream: "BATCH_END:landmark_timestamp"
+  output_stream: "ITERABLE:multi_hand_landmarks_render_data"
+}
+
+# Draws annotations and overlays them on top of the input images. Consumes
+# a vector of RenderData objects and draws each of them on the input frame.
+node {
+  calculator: "AnnotationOverlayCalculator"
+  input_stream: "INPUT_FRAME_GPU:input_image"
+  input_stream: "detection_render_data"
+  input_stream: "multi_hand_rects_render_data"
+  input_stream: "multi_palm_rects_render_data"
+  input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
+  output_stream: "OUTPUT_FRAME_GPU:output_image"
+}
+```

mediapipe/docs/object_detection_desktop.md

@@ -54,7 +54,7 @@ below and paste it into
 # MediaPipe graph that performs object detection on desktop with TensorFlow
 # on CPU.
 # Used in the example in
-# mediapipie/examples/desktop/object_detection:object_detection_tensorflow.
+# mediapipe/examples/desktop/object_detection:object_detection_tensorflow.
 
 # Decodes an input video file into images and a video header.
 node {
@@ -218,8 +218,6 @@ $ bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 \
 # It should print:
 #Target //mediapipe/examples/desktop/object_detection:object_detection_cpu up-to-date:
 #  bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_cpu
-#INFO: Elapsed time: 16.020s, Forge stats: 13001/13003 actions cached, 2.1s CPU used, 0.0s queue time, 89.0 MB ObjFS output (novel bytes: 88.0 MB), 0.0 MB local output, Critical Path: 10.01s, Remote (41.42% of the time): [queue: 0.00%, setup: 4.21%, process: 12.48%]
-#INFO: Streaming build results to: http://sponge2/1824d4cc-ba63-4350-bdc0-aacbd45b902b
 #INFO: Build completed successfully, 12154 total actions
 
 # This will open up your webcam as long as it is connected and on
@@ -240,7 +238,7 @@ below and paste it into
 # MediaPipe graph that performs object detection on desktop with TensorFlow Lite
 # on CPU.
 # Used in the example in
-# mediapipie/examples/desktop/object_detection:object_detection_tflite.
+# mediapipe/examples/desktop/object_detection:object_detection_tflite.
 
 # max_queue_size limits the number of packets enqueued on any input stream
 # by throttling inputs to the graph. This makes the graph only process one

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/AndroidManifest.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="com.google.mediapipe.apps.multihandtrackinggpu">
+
+  <uses-sdk
+      android:minSdkVersion="21"
+      android:targetSdkVersion="27" />
+
+  <!-- For using the camera -->
+  <uses-permission android:name="android.permission.CAMERA" />
+  <uses-feature android:name="android.hardware.camera" />
+  <uses-feature android:name="android.hardware.camera.autofocus" />
+  <!-- For MediaPipe -->
+  <uses-feature android:glEsVersion="0x00020000" android:required="true" />
+
+
+  <application
+      android:allowBackup="true"
+      android:label="@string/app_name"
+      android:supportsRtl="true"
+      android:theme="@style/AppTheme">
+      <activity
+          android:name=".MainActivity"
+          android:exported="true"
+          android:screenOrientation="portrait">
+          <intent-filter>
+              <action android:name="android.intent.action.MAIN" />
+              <category android:name="android.intent.category.LAUNCHER" />
+          </intent-filter>
+      </activity>
+  </application>
+
+</manifest>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/BUILD

@@ -0,0 +1,103 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:private"])
+
+cc_binary(
+    name = "libmediapipe_jni.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = [
+        "//mediapipe/graphs/hand_tracking:multi_hand_mobile_calculators",
+        "//mediapipe/java/com/google/mediapipe/framework/jni:mediapipe_framework_jni",
+    ],
+)
+
+cc_library(
+    name = "mediapipe_jni_lib",
+    srcs = [":libmediapipe_jni.so"],
+    alwayslink = 1,
+)
+
+# Maps the binary graph to an alias (e.g., the app name) for convenience so that the alias can be
+# easily incorporated into the app via, for example,
+# MainActivity.BINARY_GRAPH_NAME = "appname.binarypb".
+genrule(
+    name = "binary_graph",
+    srcs = ["//mediapipe/graphs/hand_tracking:multi_hand_tracking_mobile_gpu_binary_graph"],
+    outs = ["multihandtrackinggpu.binarypb"],
+    cmd = "cp $< $@",
+)
+
+# To use the 3D model instead of the default 2D model, add "--define 3D=true" to the
+# bazel build command.
+config_setting(
+    name = "use_3d_model",
+    define_values = {
+        "3D": "true",
+    },
+)
+
+genrule(
+    name = "model",
+    srcs = select({
+        "//conditions:default": ["//mediapipe/models:hand_landmark.tflite"],
+        ":use_3d_model": ["//mediapipe/models:hand_landmark_3d.tflite"],
+    }),
+    outs = ["hand_landmark.tflite"],
+    cmd = "cp $< $@",
+)
+
+android_library(
+    name = "mediapipe_lib",
+    srcs = glob(["*.java"]),
+    assets = [
+        ":binary_graph",
+        ":model",
+        "//mediapipe/models:palm_detection.tflite",
+        "//mediapipe/models:palm_detection_labelmap.txt",
+    ],
+    assets_dir = "",
+    manifest = "AndroidManifest.xml",
+    resource_files = glob(["res/**"]),
+    deps = [
+        ":mediapipe_jni_lib",
+        "//mediapipe/java/com/google/mediapipe/components:android_camerax_helper",
+        "//mediapipe/java/com/google/mediapipe/components:android_components",
+        "//mediapipe/java/com/google/mediapipe/framework:android_framework",
+        "//mediapipe/java/com/google/mediapipe/glutil",
+        "//third_party:androidx_appcompat",
+        "//third_party:androidx_constraint_layout",
+        "//third_party:androidx_legacy_support_v4",
+        "//third_party:androidx_material",
+        "//third_party:androidx_recyclerview",
+        "//third_party:opencv",
+        "@androidx_concurrent_futures//jar",
+        "@androidx_lifecycle//jar",
+        "@com_google_code_findbugs//jar",
+        "@com_google_guava_android//jar",
+    ],
+)
+
+android_binary(
+    name = "multihandtrackinggpu",
+    manifest = "AndroidManifest.xml",
+    manifest_values = {"applicationId": "com.google.mediapipe.apps.multihandtrackinggpu"},
+    multidex = "native",
+    deps = [
+        ":mediapipe_lib",
+    ],
+)

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/MainActivity.java

@@ -0,0 +1,167 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package com.google.mediapipe.apps.multihandtrackinggpu;
+
+import android.graphics.SurfaceTexture;
+import android.os.Bundle;
+import androidx.appcompat.app.AppCompatActivity;
+import android.util.Size;
+import android.view.SurfaceHolder;
+import android.view.SurfaceView;
+import android.view.View;
+import android.view.ViewGroup;
+import com.google.mediapipe.components.CameraHelper;
+import com.google.mediapipe.components.CameraXPreviewHelper;
+import com.google.mediapipe.components.ExternalTextureConverter;
+import com.google.mediapipe.components.FrameProcessor;
+import com.google.mediapipe.components.PermissionHelper;
+import com.google.mediapipe.framework.AndroidAssetUtil;
+import com.google.mediapipe.glutil.EglManager;
+
+/** Main activity of MediaPipe example apps. */
+public class MainActivity extends AppCompatActivity {
+  private static final String TAG = "MainActivity";
+
+  private static final String BINARY_GRAPH_NAME = "multihandtrackinggpu.binarypb";
+  private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
+  private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
+  private static final CameraHelper.CameraFacing CAMERA_FACING = CameraHelper.CameraFacing.FRONT;
+
+  // Flips the camera-preview frames vertically before sending them into FrameProcessor to be
+  // processed in a MediaPipe graph, and flips the processed frames back when they are displayed.
+  // This is needed because OpenGL represents images assuming the image origin is at the bottom-left
+  // corner, whereas MediaPipe in general assumes the image origin is at top-left.
+  private static final boolean FLIP_FRAMES_VERTICALLY = true;
+
+  static {
+    // Load all native libraries needed by the app.
+    System.loadLibrary("mediapipe_jni");
+    System.loadLibrary("opencv_java4");
+  }
+
+  // {@link SurfaceTexture} where the camera-preview frames can be accessed.
+  private SurfaceTexture previewFrameTexture;
+  // {@link SurfaceView} that displays the camera-preview frames processed by a MediaPipe graph.
+  private SurfaceView previewDisplayView;
+
+  // Creates and manages an {@link EGLContext}.
+  private EglManager eglManager;
+  // Sends camera-preview frames into a MediaPipe graph for processing, and displays the processed
+  // frames onto a {@link Surface}.
+  private FrameProcessor processor;
+  // Converts the GL_TEXTURE_EXTERNAL_OES texture from Android camera into a regular texture to be
+  // consumed by {@link FrameProcessor} and the underlying MediaPipe graph.
+  private ExternalTextureConverter converter;
+
+  // Handles camera access via the {@link CameraX} Jetpack support library.
+  private CameraXPreviewHelper cameraHelper;
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    super.onCreate(savedInstanceState);
+    setContentView(R.layout.activity_main);
+
+    previewDisplayView = new SurfaceView(this);
+    setupPreviewDisplayView();
+
+    // Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
+    // binary graphs.
+    AndroidAssetUtil.initializeNativeAssetManager(this);
+
+    eglManager = new EglManager(null);
+    processor =
+        new FrameProcessor(
+            this,
+            eglManager.getNativeContext(),
+            BINARY_GRAPH_NAME,
+            INPUT_VIDEO_STREAM_NAME,
+            OUTPUT_VIDEO_STREAM_NAME);
+    processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
+
+    PermissionHelper.checkAndRequestCameraPermissions(this);
+  }
+
+  @Override
+  protected void onResume() {
+    super.onResume();
+    converter = new ExternalTextureConverter(eglManager.getContext());
+    converter.setFlipY(FLIP_FRAMES_VERTICALLY);
+    converter.setConsumer(processor);
+    if (PermissionHelper.cameraPermissionsGranted(this)) {
+      startCamera();
+    }
+  }
+
+  @Override
+  protected void onPause() {
+    super.onPause();
+    converter.close();
+  }
+
+  @Override
+  public void onRequestPermissionsResult(
+      int requestCode, String[] permissions, int[] grantResults) {
+    super.onRequestPermissionsResult(requestCode, permissions, grantResults);
+    PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
+  }
+
+  private void setupPreviewDisplayView() {
+    previewDisplayView.setVisibility(View.GONE);
+    ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
+    viewGroup.addView(previewDisplayView);
+
+    previewDisplayView
+        .getHolder()
+        .addCallback(
+            new SurfaceHolder.Callback() {
+              @Override
+              public void surfaceCreated(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
+              }
+
+              @Override
+              public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
+                // (Re-)Compute the ideal size of the camera-preview display (the area that the
+                // camera-preview frames get rendered onto, potentially with scaling and rotation)
+                // based on the size of the SurfaceView that contains the display.
+                Size viewSize = new Size(width, height);
+                Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
+
+                // Connect the converter to the camera-preview frames as its input (via
+                // previewFrameTexture), and configure the output width and height as the computed
+                // display size.
+                converter.setSurfaceTextureAndAttachToGLContext(
+                    previewFrameTexture, displaySize.getWidth(), displaySize.getHeight());
+              }
+
+              @Override
+              public void surfaceDestroyed(SurfaceHolder holder) {
+                processor.getVideoSurfaceOutput().setSurface(null);
+              }
+            });
+  }
+
+  private void startCamera() {
+    cameraHelper = new CameraXPreviewHelper();
+    cameraHelper.setOnCameraStartedListener(
+        surfaceTexture -> {
+          previewFrameTexture = surfaceTexture;
+          // Make the display view visible to start showing the preview. This triggers the
+          // SurfaceHolder.Callback added to (the holder of) previewDisplayView.
+          previewDisplayView.setVisibility(View.VISIBLE);
+        });
+    cameraHelper.startCamera(this, CAMERA_FACING, /*surfaceTexture=*/ null);
+  }
+}

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/layout/activity_main.xml

@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+    <FrameLayout
+        android:id="@+id/preview_display_layout"
+        android:layout_width="fill_parent"
+        android:layout_height="fill_parent"
+        android:layout_weight="1">
+        <TextView
+            android:id="@+id/no_camera_access_view"
+            android:layout_height="fill_parent"
+            android:layout_width="fill_parent"
+            android:gravity="center"
+            android:text="@string/no_camera_access" />
+    </FrameLayout>
+</androidx.constraintlayout.widget.ConstraintLayout>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/colors.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <color name="colorPrimary">#008577</color>
+    <color name="colorPrimaryDark">#00574B</color>
+    <color name="colorAccent">#D81B60</color>
+</resources>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/strings.xml

@@ -0,0 +1,4 @@
+<resources>
+    <string name="app_name" translatable="false">Multi-Hand Tracking GPU</string>
+    <string name="no_camera_access" translatable="false">Please grant camera permissions.</string>
+</resources>

mediapipe/examples/android/src/java/com/google/mediapipe/apps/multihandtrackinggpu/res/values/styles.xml

@@ -0,0 +1,11 @@
+<resources>
+
+    <!-- Base application theme. -->
+    <style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
+        <!-- Customize your theme here. -->
+        <item name="colorPrimary">@color/colorPrimary</item>
+        <item name="colorPrimaryDark">@color/colorPrimaryDark</item>
+        <item name="colorAccent">@color/colorAccent</item>
+    </style>
+
+</resources>

mediapipe/examples/desktop/media_sequence/kinetics_dataset.py

@@ -68,13 +68,15 @@ import os
 import random
 import subprocess
 import sys
+import tarfile
 import tempfile
 import urllib
-import zipfile
+
 from absl import app
 from absl import flags
 from absl import logging
 import tensorflow as tf
+
 from mediapipe.util.sequence import media_sequence as ms
 
 CITATION = r"""@article{kay2017kinetics,
@@ -84,21 +86,28 @@ CITATION = r"""@article{kay2017kinetics,
   year={2017},
   url = {https://deepmind.com/research/open-source/kinetics},
 }"""
-ANNOTATION_URL = "https://storage.googleapis.com/deepmind-media/research/Kinetics_700.zip"
+ANNOTATION_URL = "https://storage.googleapis.com/deepmind-media/Datasets/kinetics700.tar.gz"
 SECONDS_TO_MICROSECONDS = 1000000
 GRAPHS = ["tvl1_flow_and_rgb_from_file.pbtxt"]
 FILEPATTERN = "kinetics_700_%s_25fps_rgb_flow"
 SPLITS = {
     "train": {
         "shards": 1000,
-        "examples": 545317},
+        "examples": 541632
-    "val": {"shards": 100,
+    },
-            "examples": 35000},
+    "validate": {
-    "test": {"shards": 100,
+        "shards": 100,
-             "examples": 70000},
+        "examples": 34727
-    "custom": {"csv": None,  # Add a CSV for your own data here.
+    },
+    "test": {
+        "shards": 100,
+        "examples": 69347
+    },
+    "custom": {
+        "csv": None,  # Add a CSV for your own data here.
         "shards": 1,  # Change this number to increase sharding.
-               "examples": -1},  # Negative 1 allows any number of examples.
+        "examples": -1
+    },  # Negative 1 allows any number of examples.
 }
 NUM_CLASSES = 700
 
@@ -312,18 +321,16 @@ class Kinetics(object):
     logging.info("Downloading annotations.")
     paths = {}
     if download_labels_for_map:
-      zip_path = os.path.join(self.path_to_data, ANNOTATION_URL.split("/")[-1])
+      tar_path = os.path.join(self.path_to_data, ANNOTATION_URL.split("/")[-1])
-      if not tf.io.gfile.exists(zip_path):
+      if not tf.io.gfile.exists(tar_path):
-        urlretrieve(ANNOTATION_URL, zip_path)
+        urlretrieve(ANNOTATION_URL, tar_path)
-        with zipfile.ZipFile(zip_path) as annotations_zip:
+        with tarfile.open(tar_path) as annotations_tar:
-          annotations_zip.extractall(self.path_to_data)
+          annotations_tar.extractall(self.path_to_data)
-      for split in ["train", "test", "val"]:
+      for split in ["train", "test", "validate"]:
-        zip_path = os.path.join(self.path_to_data,
+        csv_path = os.path.join(self.path_to_data, "kinetics700/%s.csv" % split)
-                                "kinetics_700_%s.zip" % split)
-        csv_path = zip_path.replace(".zip", ".csv")
         if not tf.io.gfile.exists(csv_path):
-          with zipfile.ZipFile(zip_path) as annotations_zip:
+          with tarfile.open(tar_path) as annotations_tar:
-            annotations_zip.extractall(self.path_to_data)
+            annotations_tar.extractall(self.path_to_data)
         paths[split] = csv_path
     for split, contents in SPLITS.items():
       if "csv" in contents and contents["csv"]:

mediapipe/examples/desktop/multi_hand_tracking/BUILD

@@ -0,0 +1,42 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//mediapipe/examples:__subpackages__"])
+
+cc_binary(
+    name = "multi_hand_tracking_tflite",
+    deps = [
+        "//mediapipe/examples/desktop:simple_run_graph_main",
+        "//mediapipe/graphs/hand_tracking:multi_hand_desktop_tflite_calculators",
+    ],
+)
+
+cc_binary(
+    name = "multi_hand_tracking_cpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main",
+        "//mediapipe/graphs/hand_tracking:multi_hand_desktop_tflite_calculators",
+    ],
+)
+
+# Linux only
+cc_binary(
+    name = "multi_hand_tracking_gpu",
+    deps = [
+        "//mediapipe/examples/desktop:demo_run_graph_main_gpu",
+        "//mediapipe/graphs/hand_tracking:multi_hand_mobile_calculators",
+    ],
+)

mediapipe/examples/desktop/object_detection/BUILD

@@ -16,51 +16,12 @@ licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//mediapipe/examples:__subpackages__"])
 
-cc_library(
-    name = "object_detection_tensorflow_deps",
-    deps = [
-        "@org_tensorflow//tensorflow/c/kernels:bitcast_op",
-        "@org_tensorflow//tensorflow/core:direct_session",
-        "@org_tensorflow//tensorflow/core/kernels:argmax_op",
-        "@org_tensorflow//tensorflow/core/kernels:bias_op",
-        "@org_tensorflow//tensorflow/core/kernels:cast_op",
-        "@org_tensorflow//tensorflow/core/kernels:concat_op",
-        "@org_tensorflow//tensorflow/core/kernels:constant_op",
-        "@org_tensorflow//tensorflow/core/kernels:control_flow_ops",
-        "@org_tensorflow//tensorflow/core/kernels:conv_ops",
-        "@org_tensorflow//tensorflow/core/kernels:cwise_op",
-        "@org_tensorflow//tensorflow/core/kernels:depthwise_conv_op",
-        "@org_tensorflow//tensorflow/core/kernels:fused_batch_norm_op",
-        "@org_tensorflow//tensorflow/core/kernels:gather_op",
-        "@org_tensorflow//tensorflow/core/kernels:identity_op",
-        "@org_tensorflow//tensorflow/core/kernels:logging_ops",
-        "@org_tensorflow//tensorflow/core/kernels:matmul_op",
-        "@org_tensorflow//tensorflow/core/kernels:non_max_suppression_op",
-        "@org_tensorflow//tensorflow/core/kernels:pack_op",
-        "@org_tensorflow//tensorflow/core/kernels:reduction_ops",
-        "@org_tensorflow//tensorflow/core/kernels:relu_op",
-        "@org_tensorflow//tensorflow/core/kernels:reshape_op",
-        "@org_tensorflow//tensorflow/core/kernels:resize_bilinear_op",
-        "@org_tensorflow//tensorflow/core/kernels:sequence_ops",
-        "@org_tensorflow//tensorflow/core/kernels:shape_ops",
-        "@org_tensorflow//tensorflow/core/kernels:slice_op",
-        "@org_tensorflow//tensorflow/core/kernels:split_op",
-        "@org_tensorflow//tensorflow/core/kernels:tensor_array_ops",
-        "@org_tensorflow//tensorflow/core/kernels:tile_ops",
-        "@org_tensorflow//tensorflow/core/kernels:topk_op",
-        "@org_tensorflow//tensorflow/core/kernels:transpose_op",
-        "@org_tensorflow//tensorflow/core/kernels:unpack_op",
-        "@org_tensorflow//tensorflow/core/kernels/data:tensor_dataset_op",
-    ],
-    alwayslink = 1,
-)
-
 cc_binary(
     name = "object_detection_tensorflow",
     deps = [
-        ":object_detection_tensorflow_deps",
         "//mediapipe/examples/desktop:simple_run_graph_main",
         "//mediapipe/graphs/object_detection:desktop_tensorflow_calculators",
+        "@org_tensorflow//tensorflow/core:all_kernels",
         "@org_tensorflow//tensorflow/core:direct_session",
     ],
 )

mediapipe/examples/ios/multihandtrackinggpu/AppDelegate.h

@@ -0,0 +1,21 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface AppDelegate : UIResponder <UIApplicationDelegate>
+
+@property(strong, nonatomic) UIWindow *window;
+
+@end

mediapipe/examples/ios/multihandtrackinggpu/AppDelegate.m

@@ -0,0 +1,59 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "AppDelegate.h"
+
+@interface AppDelegate ()
+
+@end
+
+@implementation AppDelegate
+
+- (BOOL)application:(UIApplication *)application
+    didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
+  // Override point for customization after application launch.
+  return YES;
+}
+
+- (void)applicationWillResignActive:(UIApplication *)application {
+  // Sent when the application is about to move from active to inactive state. This can occur for
+  // certain types of temporary interruptions (such as an incoming phone call or SMS message) or
+  // when the user quits the application and it begins the transition to the background state. Use
+  // this method to pause ongoing tasks, disable timers, and invalidate graphics rendering
+  // callbacks. Games should use this method to pause the game.
+}
+
+- (void)applicationDidEnterBackground:(UIApplication *)application {
+  // Use this method to release shared resources, save user data, invalidate timers, and store
+  // enough application state information to restore your application to its current state in case
+  // it is terminated later. If your application supports background execution, this method is
+  // called instead of applicationWillTerminate: when the user quits.
+}
+
+- (void)applicationWillEnterForeground:(UIApplication *)application {
+  // Called as part of the transition from the background to the active state; here you can undo
+  // many of the changes made on entering the background.
+}
+
+- (void)applicationDidBecomeActive:(UIApplication *)application {
+  // Restart any tasks that were paused (or not yet started) while the application was inactive. If
+  // the application was previously in the background, optionally refresh the user interface.
+}
+
+- (void)applicationWillTerminate:(UIApplication *)application {
+  // Called when the application is about to terminate. Save data if appropriate. See also
+  // applicationDidEnterBackground:.
+}
+
+@end

mediapipe/examples/ios/multihandtrackinggpu/Assets.xcassets/AppIcon.appiconset/Contents.json

@@ -0,0 +1,99 @@
+{
+  "images" : [
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "20x20",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "29x29",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "40x40",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "iphone",
+      "size" : "60x60",
+      "scale" : "3x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "20x20",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "29x29",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "40x40",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "76x76",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ipad",
+      "size" : "83.5x83.5",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "ios-marketing",
+      "size" : "1024x1024",
+      "scale" : "1x"
+    }
+  ],
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
+

mediapipe/examples/ios/multihandtrackinggpu/Assets.xcassets/Contents.json

@@ -0,0 +1,7 @@
+{
+  "info" : {
+    "version" : 1,
+    "author" : "xcode"
+  }
+}
+

mediapipe/examples/ios/multihandtrackinggpu/BUILD

@@ -0,0 +1,95 @@
+# Copyright 2019 The MediaPipe Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+licenses(["notice"])  # Apache 2.0
+
+MIN_IOS_VERSION = "10.0"
+
+load(
+    "@build_bazel_rules_apple//apple:ios.bzl",
+    "ios_application",
+)
+
+# To use the 3D model instead of the default 2D model, add "--define 3D=true" to the
+# bazel build command.
+config_setting(
+    name = "use_3d_model",
+    define_values = {
+        "3D": "true",
+    },
+)
+
+genrule(
+    name = "model",
+    srcs = select({
+        "//conditions:default": ["//mediapipe/models:hand_landmark.tflite"],
+        ":use_3d_model": ["//mediapipe/models:hand_landmark_3d.tflite"],
+    }),
+    outs = ["hand_landmark.tflite"],
+    cmd = "cp $< $@",
+)
+
+ios_application(
+    name = "MultiHandTrackingGpuApp",
+    bundle_id = "com.google.mediapipe.MultiHandTrackingGpu",
+    families = [
+        "iphone",
+        "ipad",
+    ],
+    infoplists = ["Info.plist"],
+    minimum_os_version = MIN_IOS_VERSION,
+    provisioning_profile = "//mediapipe/examples/ios:provisioning_profile",
+    deps = [
+        ":MultiHandTrackingGpuAppLibrary",
+        "@ios_opencv//:OpencvFramework",
+    ],
+)
+
+objc_library(
+    name = "MultiHandTrackingGpuAppLibrary",
+    srcs = [
+        "AppDelegate.m",
+        "ViewController.mm",
+        "main.m",
+    ],
+    hdrs = [
+        "AppDelegate.h",
+        "ViewController.h",
+    ],
+    data = [
+        "Base.lproj/LaunchScreen.storyboard",
+        "Base.lproj/Main.storyboard",
+        ":model",
+        "//mediapipe/graphs/hand_tracking:multi_hand_tracking_mobile_gpu_binary_graph",
+        "//mediapipe/models:palm_detection.tflite",
+        "//mediapipe/models:palm_detection_labelmap.txt",
+    ],
+    sdk_frameworks = [
+        "AVFoundation",
+        "CoreGraphics",
+        "CoreMedia",
+        "UIKit",
+    ],
+    deps = [
+        "//mediapipe/objc:mediapipe_framework_ios",
+        "//mediapipe/objc:mediapipe_input_sources_ios",
+        "//mediapipe/objc:mediapipe_layer_renderer",
+    ] + select({
+        "//mediapipe:ios_i386": [],
+        "//mediapipe:ios_x86_64": [],
+        "//conditions:default": [
+            "//mediapipe/graphs/hand_tracking:multi_hand_mobile_calculators",
+        ],
+    }),
+)

mediapipe/examples/ios/multihandtrackinggpu/Base.lproj/LaunchScreen.storyboard

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="13122.16" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="13104.12"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="53" y="375"/>
+        </scene>
+    </scenes>
+</document>

mediapipe/examples/ios/multihandtrackinggpu/Base.lproj/Main.storyboard

@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="14490.70" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="BYZ-38-t0r">
+    <device id="retina4_7" orientation="portrait">
+        <adaptation id="fullscreen"/>
+    </device>
+    <dependencies>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="14490.49"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="ViewController" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="375" height="667"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="EfB-xq-knP">
+                                <rect key="frame" x="0.0" y="20" width="375" height="647"/>
+                                <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                                <subviews>
+                                    <label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="emf-N5-sEd">
+                                        <rect key="frame" x="57" y="248" width="260" height="151"/>
+                                        <autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
+                                        <fontDescription key="fontDescription" type="system" pointSize="17"/>
+                                        <color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                        <nil key="highlightedColor"/>
+                                    </label>
+                                </subviews>
+                                <color key="backgroundColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+                                <accessibility key="accessibilityConfiguration" label="PreviewDisplayView">
+                                    <bool key="isElement" value="YES"/>
+                                </accessibility>
+                            </view>
+                        </subviews>
+                        <color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                    </view>
+                    <connections>
+                        <outlet property="_liveView" destination="EfB-xq-knP" id="JQp-2n-q9q"/>
+                        <outlet property="_noCameraLabel" destination="emf-N5-sEd" id="91G-3Z-cU3"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="48.799999999999997" y="20.239880059970016"/>
+        </scene>
+    </scenes>
+</document>

mediapipe/examples/ios/multihandtrackinggpu/Info.plist

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>NSCameraUsageDescription</key>
+  <string>This app uses the camera to demonstrate live video processing.</string>
+  <key>CFBundleDevelopmentRegion</key>
+  <string>en</string>
+  <key>CFBundleExecutable</key>
+  <string>$(EXECUTABLE_NAME)</string>
+  <key>CFBundleIdentifier</key>
+  <string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+  <key>CFBundleInfoDictionaryVersion</key>
+  <string>6.0</string>
+  <key>CFBundleName</key>
+  <string>$(PRODUCT_NAME)</string>
+  <key>CFBundlePackageType</key>
+  <string>APPL</string>
+  <key>CFBundleShortVersionString</key>
+  <string>1.0</string>
+  <key>CFBundleVersion</key>
+  <string>1</string>
+  <key>LSRequiresIPhoneOS</key>
+  <true/>
+  <key>UILaunchStoryboardName</key>
+  <string>LaunchScreen</string>
+  <key>UIMainStoryboardFile</key>
+  <string>Main</string>
+  <key>UIRequiredDeviceCapabilities</key>
+  <array>
+    <string>armv7</string>
+  </array>
+  <key>UISupportedInterfaceOrientations</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+  <key>UISupportedInterfaceOrientations~ipad</key>
+  <array>
+    <string>UIInterfaceOrientationPortrait</string>
+  </array>
+</dict>
+</plist>

mediapipe/examples/ios/multihandtrackinggpu/ViewController.h

@@ -0,0 +1,19 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+
+@interface ViewController : UIViewController
+
+@end

mediapipe/examples/ios/multihandtrackinggpu/ViewController.mm

@@ -0,0 +1,178 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "ViewController.h"
+
+#import "mediapipe/objc/MPPGraph.h"
+#import "mediapipe/objc/MPPCameraInputSource.h"
+#import "mediapipe/objc/MPPLayerRenderer.h"
+
+static NSString* const kGraphName = @"multi_hand_tracking_mobile_gpu";
+
+static const char* kInputStream = "input_video";
+static const char* kOutputStream = "output_video";
+static const char* kVideoQueueLabel = "com.google.mediapipe.example.videoQueue";
+
+@interface ViewController () <MPPGraphDelegate, MPPInputSourceDelegate>
+
+// The MediaPipe graph currently in use. Initialized in viewDidLoad, started in viewWillAppear: and
+// sent video frames on _videoQueue.
+@property(nonatomic) MPPGraph* mediapipeGraph;
+
+@end
+
+@implementation ViewController {
+  /// Handles camera access via AVCaptureSession library.
+  MPPCameraInputSource* _cameraSource;
+
+  /// Inform the user when camera is unavailable.
+  IBOutlet UILabel* _noCameraLabel;
+  /// Display the camera preview frames.
+  IBOutlet UIView* _liveView;
+  /// Render frames in a layer.
+  MPPLayerRenderer* _renderer;
+
+  /// Process camera frames on this queue.
+  dispatch_queue_t _videoQueue;
+}
+
+#pragma mark - Cleanup methods
+
+- (void)dealloc {
+  self.mediapipeGraph.delegate = nil;
+  [self.mediapipeGraph cancel];
+  // Ignore errors since we're cleaning up.
+  [self.mediapipeGraph closeAllInputStreamsWithError:nil];
+  [self.mediapipeGraph waitUntilDoneWithError:nil];
+}
+
+#pragma mark - MediaPipe graph methods
+
++ (MPPGraph*)loadGraphFromResource:(NSString*)resource {
+  // Load the graph config resource.
+  NSError* configLoadError = nil;
+  NSBundle* bundle = [NSBundle bundleForClass:[self class]];
+  if (!resource || resource.length == 0) {
+    return nil;
+  }
+  NSURL* graphURL = [bundle URLForResource:resource withExtension:@"binarypb"];
+  NSData* data = [NSData dataWithContentsOfURL:graphURL options:0 error:&configLoadError];
+  if (!data) {
+    NSLog(@"Failed to load MediaPipe graph config: %@", configLoadError);
+    return nil;
+  }
+
+  // Parse the graph config resource into mediapipe::CalculatorGraphConfig proto object.
+  mediapipe::CalculatorGraphConfig config;
+  config.ParseFromArray(data.bytes, data.length);
+
+  // Create MediaPipe graph with mediapipe::CalculatorGraphConfig proto object.
+  MPPGraph* newGraph = [[MPPGraph alloc] initWithGraphConfig:config];
+  [newGraph addFrameOutputStream:kOutputStream outputPacketType:MPPPacketTypePixelBuffer];
+  return newGraph;
+}
+
+#pragma mark - UIViewController methods
+
+- (void)viewDidLoad {
+  [super viewDidLoad];
+
+  _renderer = [[MPPLayerRenderer alloc] init];
+  _renderer.layer.frame = _liveView.layer.bounds;
+  [_liveView.layer addSublayer:_renderer.layer];
+  _renderer.frameScaleMode = MPPFrameScaleModeFillAndCrop;
+  // When using the front camera, mirror the input for a more natural look.
+  _renderer.mirrored = YES;
+
+  dispatch_queue_attr_t qosAttribute = dispatch_queue_attr_make_with_qos_class(
+      DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, /*relative_priority=*/0);
+  _videoQueue = dispatch_queue_create(kVideoQueueLabel, qosAttribute);
+
+  _cameraSource = [[MPPCameraInputSource alloc] init];
+  [_cameraSource setDelegate:self queue:_videoQueue];
+  _cameraSource.sessionPreset = AVCaptureSessionPresetHigh;
+  _cameraSource.cameraPosition = AVCaptureDevicePositionFront;
+  // The frame's native format is rotated with respect to the portrait orientation.
+  _cameraSource.orientation = AVCaptureVideoOrientationPortrait;
+
+  self.mediapipeGraph = [[self class] loadGraphFromResource:kGraphName];
+  self.mediapipeGraph.delegate = self;
+  // Set maxFramesInFlight to a small value to avoid memory contention for real-time processing.
+  self.mediapipeGraph.maxFramesInFlight = 2;
+}
+
+// In this application, there is only one ViewController which has no navigation to other view
+// controllers, and there is only one View with live display showing the result of running the
+// MediaPipe graph on the live video feed. If more view controllers are needed later, the graph
+// setup/teardown and camera start/stop logic should be updated appropriately in response to the
+// appearance/disappearance of this ViewController, as viewWillAppear: can be invoked multiple times
+// depending on the application navigation flow in that case.
+- (void)viewWillAppear:(BOOL)animated {
+  [super viewWillAppear:animated];
+
+  [_cameraSource requestCameraAccessWithCompletionHandler:^void(BOOL granted) {
+    if (granted) {
+      [self startGraphAndCamera];
+      dispatch_async(dispatch_get_main_queue(), ^{
+        _noCameraLabel.hidden = YES;
+      });
+    }
+  }];
+}
+
+- (void)startGraphAndCamera {
+  // Start running self.mediapipeGraph.
+  NSError* error;
+  if (![self.mediapipeGraph startWithError:&error]) {
+    NSLog(@"Failed to start graph: %@", error);
+  }
+
+  // Start fetching frames from the camera.
+  dispatch_async(_videoQueue, ^{
+    [_cameraSource start];
+  });
+}
+
+#pragma mark - MPPGraphDelegate methods
+
+// Receives CVPixelBufferRef from the MediaPipe graph. Invoked on a MediaPipe worker thread.
+- (void)mediapipeGraph:(MPPGraph*)graph
+    didOutputPixelBuffer:(CVPixelBufferRef)pixelBuffer
+              fromStream:(const std::string&)streamName {
+  if (streamName == kOutputStream) {
+    // Display the captured image on the screen.
+    CVPixelBufferRetain(pixelBuffer);
+    dispatch_async(dispatch_get_main_queue(), ^{
+      [_renderer renderPixelBuffer:pixelBuffer];
+      CVPixelBufferRelease(pixelBuffer);
+    });
+  }
+}
+
+#pragma mark - MPPInputSourceDelegate methods
+
+// Must be invoked on _videoQueue.
+- (void)processVideoFrame:(CVPixelBufferRef)imageBuffer
+                timestamp:(CMTime)timestamp
+               fromSource:(MPPInputSource*)source {
+  if (source != _cameraSource) {
+    NSLog(@"Unknown source: %@", source);
+    return;
+  }
+  [self.mediapipeGraph sendPixelBuffer:imageBuffer
+                            intoStream:kInputStream
+                            packetType:MPPPacketTypePixelBuffer];
+}
+
+@end

mediapipe/examples/ios/multihandtrackinggpu/main.m

@@ -0,0 +1,22 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <UIKit/UIKit.h>
+#import "AppDelegate.h"
+
+int main(int argc, char * argv[]) {
+  @autoreleasepool {
+      return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class]));
+  }
+}

mediapipe/framework/BUILD

@@ -14,13 +14,12 @@
 # limitations under the License.
 #
 
+load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_py_proto_library")
+
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:private"])
 
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
-load("//mediapipe/framework/port:build_config.bzl", "mediapipe_py_proto_library")
-
 package_group(
     name = "mediapipe_internal",
     packages = [
@@ -464,6 +463,8 @@ cc_library(
         "//mediapipe/framework:packet_generator_cc_proto",
         "//mediapipe/framework:status_handler_cc_proto",
         "//mediapipe/framework:thread_pool_executor_cc_proto",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/container:flat_hash_set",
         "//mediapipe/gpu:graph_support",
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/container:fixed_array",
@@ -1272,6 +1273,7 @@ cc_library(
         "//mediapipe/framework/tool:validate",
         "//mediapipe/framework/tool:validate_name",
         "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/container:flat_hash_set",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
     ],

mediapipe/framework/calculator_graph.cc

@@ -24,6 +24,7 @@
 #include <vector>
 
 #include "absl/container/fixed_array.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_format.h"
@@ -1017,8 +1018,8 @@ void CalculatorGraph::UpdateThrottledNodes(InputStreamManager* stream,
   // TODO Change the throttling code to use the index directly
   // rather than looking up a stream name.
   int node_index = validated_graph_->OutputStreamToNode(stream->Name());
-  std::unordered_set<int> owned_set;
+  absl::flat_hash_set<int> owned_set;
-  const std::unordered_set<int>* upstream_nodes;
+  const absl::flat_hash_set<int>* upstream_nodes;
   if (node_index >= validated_graph_->CalculatorInfos().size()) {
     // TODO just create a NodeTypeInfo object for each virtual node.
     owned_set.insert(node_index);
@@ -1100,10 +1101,10 @@ bool CalculatorGraph::UnthrottleSources() {
   // This is a sufficient because succesfully growing at least one full input
   // stream during each call to UnthrottleSources will eventually resolve
   // each deadlock.
-  std::unordered_set<InputStreamManager*> full_streams;
+  absl::flat_hash_set<InputStreamManager*> full_streams;
   {
     absl::MutexLock lock(&full_input_streams_mutex_);
-    for (std::unordered_set<InputStreamManager*>& s : full_input_streams_) {
+    for (absl::flat_hash_set<InputStreamManager*>& s : full_input_streams_) {
       if (!s.empty()) {
         full_streams.insert(s.begin(), s.end());
       }

mediapipe/framework/calculator_graph.h

@@ -23,13 +23,13 @@
 #include <map>
 #include <memory>
 #include <string>
-#include <unordered_map>
-#include <unordered_set>
 #include <utility>
 #include <vector>
 
 #include "absl/base/macros.h"
 #include "absl/container/fixed_array.h"
+#include "absl/container/flat_hash_map.h"
+#include "absl/container/flat_hash_set.h"
 #include "absl/synchronization/mutex.h"
 #include "mediapipe/framework/calculator.pb.h"
 #include "mediapipe/framework/calculator_base.h"
@@ -579,18 +579,18 @@ class CalculatorGraph {
   // A node is scheduled only if this set is empty.  Similarly, a packet
   // is added to a graph input stream only if this set is empty.
   // Note that this vector contains an unused entry for each non-source node.
-  std::vector<std::unordered_set<InputStreamManager*>> full_input_streams_
+  std::vector<absl::flat_hash_set<InputStreamManager*>> full_input_streams_
       GUARDED_BY(full_input_streams_mutex_);
 
   // Maps stream names to graph input stream objects.
-  std::unordered_map<std::string, std::unique_ptr<GraphInputStream>>
+  absl::flat_hash_map<std::string, std::unique_ptr<GraphInputStream>>
       graph_input_streams_;
 
   // Maps graph input streams to their virtual node ids.
-  std::unordered_map<std::string, int> graph_input_stream_node_ids_;
+  absl::flat_hash_map<std::string, int> graph_input_stream_node_ids_;
 
   // Maps graph input streams to their max queue size.
-  std::unordered_map<std::string, int> graph_input_stream_max_queue_size_;
+  absl::flat_hash_map<std::string, int> graph_input_stream_max_queue_size_;
 
   // The factory for making counters associated with this graph.
   std::unique_ptr<CounterFactory> counter_factory_;

mediapipe/framework/calculator_graph_side_packet_test.cc

@@ -68,6 +68,7 @@ class CountAndOutputSummarySidePacketInCloseCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Close(CalculatorContext* cc) final {
+    absl::SleepFor(absl::Milliseconds(300));  // For GetOutputSidePacket test.
     cc->OutputSidePackets().Index(0).Set(
         MakePacket<int>(count_).At(Timestamp::Unset()));
     return ::mediapipe::OkStatus();

mediapipe/framework/graph_validation_test.cc

@@ -32,7 +32,7 @@ namespace mediapipe {
 namespace {
 
 // Shows validation success for a graph and a subgraph.
-TEST(ValidatedGraphConfigTest, InitializeGraphFromProtos) {
+TEST(GraphValidationTest, InitializeGraphFromProtos) {
   auto config_1 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
     type: "PassThroughGraph"
     input_stream: "INPUT:stream_1"
@@ -102,7 +102,7 @@ TEST(ValidatedGraphConfigTest, InitializeGraphFromProtos) {
 }
 
 // Shows validation failure due to an unregistered subgraph.
-TEST(ValidatedGraphConfigTest, InitializeGraphFromLinker) {
+TEST(GraphValidationTest, InitializeGraphFromLinker) {
   EXPECT_FALSE(SubgraphRegistry::IsRegistered("DubQuadTestSubgraph"));
   ValidatedGraphConfig builder_1;
   ::mediapipe::Status status_1 =
@@ -114,7 +114,7 @@ TEST(ValidatedGraphConfigTest, InitializeGraphFromLinker) {
 }
 
 // Shows validation success for a graph and a template subgraph.
-TEST(ValidatedGraphConfigTest, InitializeTemplateFromProtos) {
+TEST(GraphValidationTest, InitializeTemplateFromProtos) {
   mediapipe::tool::TemplateParser::Parser parser;
   CalculatorGraphTemplate config_1;
   CHECK(parser.ParseFromString(R"(
@@ -210,5 +210,109 @@ TEST(ValidatedGraphConfigTest, InitializeTemplateFromProtos) {
       )")));
 }
 
+// Shows passing validation of optional subgraph inputs and output streams.
+TEST(GraphValidationTest, OptionalSubgraphStreams) {
+  // A subgraph defining two optional input streams
+  // and two optional output streams.
+  auto config_1 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    type: "PassThroughGraph"
+    input_stream: "INPUT:input_0"
+    input_stream: "INPUT:1:input_1"
+    output_stream: "OUTPUT:output_0"
+    output_stream: "OUTPUT:1:output_1"
+    node {
+      calculator: "PassThroughCalculator"
+      input_stream: "input_0"    # Any Type.
+      input_stream: "input_1"    # Any Type.
+      output_stream: "output_0"  # Same as input.
+    }
+  )");
+
+  // An enclosing graph that specifies one of the two optional input streams
+  // and one of the two optional output streams.
+  auto config_2 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "INPUT:foo_in"
+    output_stream: "OUTPUT:foo_out"
+    node {
+      calculator: "PassThroughCalculator"
+      input_stream: "foo_in"    # Any Type.
+      output_stream: "foo_bar"  # Same as input.
+    }
+    node {
+      calculator: "PassThroughGraph"
+      input_stream: "INPUT:foo_bar"    # Any Type.
+      output_stream: "OUTPUT:foo_out"  # Same as input.
+    }
+  )");
+
+  GraphValidation validation_1;
+  MP_EXPECT_OK(validation_1.Validate({config_1, config_2}, {}));
+  CalculatorGraph graph_1;
+  MP_EXPECT_OK(graph_1.Initialize({config_1, config_2}, {}));
+  EXPECT_THAT(
+      graph_1.Config(),
+
+      // The result includes only the requested input and output streams.
+      EqualsProto(::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_stream: "INPUT:foo_in"
+        output_stream: "OUTPUT:foo_out"
+        node {
+          calculator: "PassThroughCalculator"
+          input_stream: "foo_in"
+          output_stream: "foo_bar"
+        }
+        node {
+          calculator: "PassThroughCalculator"
+          input_stream: "foo_bar"
+          output_stream: "foo_out"
+        }
+        executor {}
+      )")));
+}
+
+// Shows failing validation of optional subgraph inputs and output streams.
+TEST(GraphValidationTest, OptionalSubgraphStreamsMismatched) {
+  // A subgraph defining two optional input streams
+  // and two optional output streams.
+  auto config_1 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    type: "PassThroughGraph"
+    input_stream: "INPUT:input_0"
+    input_stream: "INPUT:1:input_1"
+    output_stream: "OUTPUT:output_0"
+    output_stream: "OUTPUT:1:output_1"
+    node {
+      calculator: "PassThroughCalculator"
+      input_stream: "input_0"    # Any Type.
+      input_stream: "input_1"    # Any Type.
+      output_stream: "output_0"  # Same as input.
+    }
+  )");
+
+  // An enclosing graph that specifies one of the two optional input streams
+  // and both of the two optional output streams.
+  auto config_2 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_stream: "INPUT:foo_in"
+    output_stream: "OUTPUT:foo_out"
+    node {
+      calculator: "PassThroughCalculator"
+      input_stream: "foo_in"    # Any Type.
+      output_stream: "foo_bar"  # Same as input.
+    }
+    node {
+      calculator: "PassThroughGraph"
+      input_stream: "INPUT:foo_bar"    # Any Type.
+      input_stream: "INPUT:1:foo_bar"  # Any Type.
+      output_stream: "OUTPUT:foo_out"  # Same as input.
+    }
+  )");
+
+  GraphValidation validation_1;
+  mediapipe::Status status = validation_1.Validate({config_1, config_2}, {});
+  ASSERT_EQ(status.code(), ::mediapipe::StatusCode::kInvalidArgument);
+  ASSERT_THAT(status.ToString(),
+              testing::HasSubstr(
+                  "PassThroughCalculator must use matching tags and indexes"));
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/framework/packet.h

@@ -85,7 +85,7 @@ class Packet {
   // given timestamp. Does not modify *this.
   Packet At(class Timestamp timestamp) const&;
 
-  // The rvalue reference overload of Packet's memeber function
+  // The rvalue reference overload of Packet's member function
   // Packet::At(class Timestamp). Moves *this to a new Packet and returns
   // the new Packet with the given timestamp.
   Packet At(class Timestamp timestamp) &&;

mediapipe/framework/port/BUILD

@@ -247,6 +247,26 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "opencv_features2d",
+    hdrs = ["opencv_features2d_inc.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":opencv_core",
+        "//third_party:opencv",
+    ],
+)
+
+cc_library(
+    name = "opencv_calib3d",
+    hdrs = ["opencv_calib3d_inc.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":opencv_core",
+        "//third_party:opencv",
+    ],
+)
+
 cc_library(
     name = "parse_text_proto",
     hdrs = [

mediapipe/framework/port/opencv_calib3d_inc.h

@@ -0,0 +1,26 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_FRAMEWORK_PORT_OPENCV_CALIB3D_INC_H_
+#define MEDIAPIPE_FRAMEWORK_PORT_OPENCV_CALIB3D_INC_H_
+
+#include <opencv2/core/version.hpp>
+
+#ifdef CV_VERSION_EPOCH  // for OpenCV 2.x
+#include <opencv2/calib3d/calib3d.hpp>
+#else
+#include <opencv2/calib3d.hpp>
+#endif
+
+#endif  // MEDIAPIPE_FRAMEWORK_PORT_OPENCV_CALIB3D_INC_H_

mediapipe/framework/port/opencv_features2d_inc.h

@@ -0,0 +1,26 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_FRAMEWORK_PORT_OPENCV_FEATURES2D_INC_H_
+#define MEDIAPIPE_FRAMEWORK_PORT_OPENCV_FEATURES2D_INC_H_
+
+#include <opencv2/core/version.hpp>
+
+#ifdef CV_VERSION_EPOCH  // for OpenCV 2.x
+#include <opencv2/features2d/features2d.hpp>
+#else
+#include <opencv2/features2d.hpp>
+#endif
+
+#endif  // MEDIAPIPE_FRAMEWORK_PORT_OPENCV_FEATURES2D_INC_H_

mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc

@@ -155,7 +155,7 @@ class FixedSizeInputStreamHandler : public DefaultInputStreamHandler {
     return (fixed_min_size_) ? EraseAllSurplus() : EraseAnySurplus(keep_one);
   }
 
-  NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) {
+  NodeReadiness GetNodeReadiness(Timestamp* min_stream_timestamp) override {
     DCHECK(min_stream_timestamp);
     absl::MutexLock lock(&erase_mutex_);
     // kReadyForProcess is returned only once until FillInputSet completes.

mediapipe/framework/testdata/BUILD

@@ -31,7 +31,7 @@ mediapipe_cc_proto_library(
     name = "sky_light_calculator_cc_proto",
     srcs = ["sky_light_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":sky_light_calculator_proto"],
 )
 
@@ -45,7 +45,7 @@ mediapipe_cc_proto_library(
     name = "night_light_calculator_cc_proto",
     srcs = ["night_light_calculator.proto"],
     cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
-    visibility = ["//mediapipe:__subpackages__"],
+    visibility = ["//visibility:public"],
     deps = [":night_light_calculator_proto"],
 )
 

mediapipe/framework/tool/BUILD

@@ -177,7 +177,7 @@ cc_library(
     deps = [
         "//mediapipe/framework:packet",
         "//mediapipe/framework/port:statusor",
-        "@org_tensorflow//tensorflow/core:protos_all_cc",
+        "@org_tensorflow//tensorflow/core:protos_all",
     ],
 )
 

mediapipe/framework/tool/mediapipe_graph.bzl

@@ -110,7 +110,7 @@ def mediapipe_simple_subgraph(
       testonly: pass 1 if the graph is to be used only for tests.
       **kwargs: Remaining keyword args, forwarded to cc_library.
     """
-    graph_base_name = graph.replace(":", "/").split("/")[-1].rsplit(".", 1)[0]
+    graph_base_name = name
     mediapipe_binary_graph(
         name = name + "_graph",
         graph = graph,

mediapipe/framework/tool/subgraph_expansion.cc

@@ -52,6 +52,31 @@ namespace tool {
   return ::mediapipe::OkStatus();
 }
 
+// Returns subgraph streams not requested by a subgraph-node.
+::mediapipe::Status FindIgnoredStreams(
+    const proto_ns::RepeatedPtrField<ProtoString>& src_streams,
+    const proto_ns::RepeatedPtrField<ProtoString>& dst_streams,
+    std::set<std::string>* result) {
+  ASSIGN_OR_RETURN(auto src_map, tool::TagMap::Create(src_streams));
+  ASSIGN_OR_RETURN(auto dst_map, tool::TagMap::Create(dst_streams));
+  std::set_difference(src_map->Names().begin(), src_map->Names().end(),
+                      dst_map->Names().begin(), dst_map->Names().end(),
+                      std::inserter(*result, result->begin()));
+  return ::mediapipe::OkStatus();
+}
+
+// Removes subgraph streams not requested by a subgraph-node.
+::mediapipe::Status RemoveIgnoredStreams(
+    proto_ns::RepeatedPtrField<ProtoString>* streams,
+    const std::set<std::string>& missing_streams) {
+  for (int i = streams->size() - 1; i >= 0; --i) {
+    if (missing_streams.count(streams->Get(i)) > 0) {
+      streams->DeleteSubrange(i, 1);
+    }
+  }
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status TransformNames(
     CalculatorGraphConfig* config,
     const std::function<std::string(absl::string_view)>& transform) {
@@ -190,6 +215,14 @@ static ::mediapipe::Status PrefixNames(int subgraph_index,
           .SetPrepend()
       << "while processing the output side packets of subgraph node "
       << subgraph_node.calculator() << ": ";
+  std::set<std::string> ignored_input_streams;
+  MP_RETURN_IF_ERROR(FindIgnoredStreams(subgraph_config->input_stream(),
+                                        subgraph_node.input_stream(),
+                                        &ignored_input_streams));
+  std::set<std::string> ignored_input_side_packets;
+  MP_RETURN_IF_ERROR(FindIgnoredStreams(subgraph_config->input_side_packet(),
+                                        subgraph_node.input_side_packet(),
+                                        &ignored_input_side_packets));
   std::map<std::string, std::string>* name_map;
   auto replace_names = [&name_map](absl::string_view s) {
     std::string original(s);
@@ -207,6 +240,12 @@ static ::mediapipe::Status PrefixNames(int subgraph_index,
         TransformStreamNames(node.mutable_input_side_packet(), replace_names));
     MP_RETURN_IF_ERROR(
         TransformStreamNames(node.mutable_output_side_packet(), replace_names));
+
+    // Remove input streams and side packets ignored by the subgraph-node.
+    MP_RETURN_IF_ERROR(RemoveIgnoredStreams(node.mutable_input_stream(),
+                                            ignored_input_streams));
+    MP_RETURN_IF_ERROR(RemoveIgnoredStreams(node.mutable_input_side_packet(),
+                                            ignored_input_side_packets));
   }
   name_map = &side_packet_map;
   for (auto& generator : *subgraph_config->mutable_packet_generator()) {

mediapipe/framework/validated_graph_config.cc

@@ -14,8 +14,7 @@
 
 #include "mediapipe/framework/validated_graph_config.h"
 
-#include <unordered_set>
+#include "absl/container/flat_hash_set.h"
-
 #include "absl/memory/memory.h"
 #include "absl/strings/str_cat.h"
 #include "absl/strings/str_join.h"
@@ -934,7 +933,7 @@ NodeTypeInfo::NodeRef ValidatedGraphConfig::NodeForSorterIndex(
 }
 
 ::mediapipe::Status ValidatedGraphConfig::ValidateExecutors() {
-  std::unordered_set<ProtoString> declared_names;
+  absl::flat_hash_set<ProtoString> declared_names;
   for (const ExecutorConfig& executor_config : config_.executor()) {
     if (IsReservedExecutorName(executor_config.name())) {
       return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
@@ -964,7 +963,7 @@ NodeTypeInfo::NodeRef ValidatedGraphConfig::NodeForSorterIndex(
              << "\"" << executor_name << "\" is a reserved executor name.";
     }
     // The executor must be declared in an ExecutorConfig.
-    if (declared_names.find(executor_name) == declared_names.end()) {
+    if (!declared_names.contains(executor_name)) {
       return ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
              << "The executor \"" << executor_name
              << "\" is not declared in an ExecutorConfig.";

mediapipe/framework/validated_graph_config.h

@@ -16,9 +16,9 @@
 #define MEDIAPIPE_FRAMEWORK_VALIDATED_GRAPH_CONFIG_H_
 
 #include <map>
-#include <unordered_set>
 #include <vector>
 
+#include "absl/container/flat_hash_set.h"
 #include "mediapipe/framework/calculator.pb.h"
 #include "mediapipe/framework/calculator_contract.h"
 #include "mediapipe/framework/packet_generator.pb.h"
@@ -169,7 +169,7 @@ class NodeTypeInfo {
   // be a virtual node corresponding to a graph input stream (which are
   // listed by index contiguously after all calculators).
   // This function is only valid for a NodeTypeInfo of NodeType CALCULATOR.
-  const std::unordered_set<int>& AncestorSources() const {
+  const absl::flat_hash_set<int>& AncestorSources() const {
     return ancestor_sources_;
   }
   // Returns True if the source was not already there.
@@ -213,7 +213,7 @@ class NodeTypeInfo {
   NodeRef node_;
 
   // The set of sources which affect this node.
-  std::unordered_set<int> ancestor_sources_;
+  absl::flat_hash_set<int> ancestor_sources_;
 };
 
 // Information for either the input or output side of an edge.  An edge

mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt

@@ -1,6 +1,6 @@
 # MediaPipe graph that performs face detection with TensorFlow Lite on CPU.
 # Used in the examples in
-# mediapipie/examples/desktop/face_detection:face_detection_cpu.
+# mediapipe/examples/desktop/face_detection:face_detection_cpu.
 
 # Images on GPU coming into and out of the graph.
 input_stream: "input_video"

mediapipe/graphs/hand_tracking/BUILD

@@ -12,26 +12,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-licenses(["notice"])  # Apache 2.0
-
-package(default_visibility = ["//visibility:public"])
-
 load(
     "//mediapipe/framework/tool:mediapipe_graph.bzl",
     "mediapipe_binary_graph",
 )
 
+licenses(["notice"])  # Apache 2.0
+
+package(default_visibility = ["//visibility:public"])
+
 cc_library(
-    name = "desktop_tflite_calculators",
+    name = "desktop_offline_calculators",
     deps = [
         "//mediapipe/calculators/core:flow_limiter_calculator",
         "//mediapipe/calculators/core:gate_calculator",
         "//mediapipe/calculators/core:immediate_mux_calculator",
-        "//mediapipe/calculators/core:merge_calculator",
         "//mediapipe/calculators/core:packet_inner_join_calculator",
         "//mediapipe/calculators/core:previous_loopback_calculator",
         "//mediapipe/calculators/video:opencv_video_decoder_calculator",
         "//mediapipe/calculators/video:opencv_video_encoder_calculator",
+    ],
+)
+
+cc_library(
+    name = "desktop_tflite_calculators",
+    deps = [
+        ":desktop_offline_calculators",
+        "//mediapipe/calculators/core:merge_calculator",
         "//mediapipe/graphs/hand_tracking/subgraphs:hand_detection_cpu",
         "//mediapipe/graphs/hand_tracking/subgraphs:hand_landmark_cpu",
         "//mediapipe/graphs/hand_tracking/subgraphs:renderer_cpu",
@@ -58,6 +65,39 @@ mediapipe_binary_graph(
     deps = [":mobile_calculators"],
 )
 
+cc_library(
+    name = "multi_hand_desktop_tflite_calculators",
+    deps = [
+        ":desktop_offline_calculators",
+        "//mediapipe/calculators/util:association_norm_rect_calculator",
+        "//mediapipe/calculators/util:collection_has_min_size_calculator",
+        "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_detection_cpu",
+        "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_landmark_cpu",
+        "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_renderer_cpu",
+    ],
+)
+
+cc_library(
+    name = "multi_hand_mobile_calculators",
+    deps = [
+        "//mediapipe/calculators/core:flow_limiter_calculator",
+        "//mediapipe/calculators/core:gate_calculator",
+        "//mediapipe/calculators/core:previous_loopback_calculator",
+        "//mediapipe/calculators/util:association_norm_rect_calculator",
+        "//mediapipe/calculators/util:collection_has_min_size_calculator",
+        "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_detection_gpu",
+        "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_landmark_gpu",
+        "//mediapipe/graphs/hand_tracking/subgraphs:multi_hand_renderer_gpu",
+    ],
+)
+
+mediapipe_binary_graph(
+    name = "multi_hand_tracking_mobile_gpu_binary_graph",
+    graph = "multi_hand_tracking_mobile.pbtxt",
+    output_name = "multi_hand_tracking_mobile_gpu.binarypb",
+    deps = [":multi_hand_mobile_calculators"],
+)
+
 cc_library(
     name = "detection_mobile_calculators",
     deps = [

mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt

@@ -1,7 +1,7 @@
 # MediaPipe graph that performs hand detection on desktop with TensorFlow Lite
 # on CPU.
 # Used in the example in
-# mediapipie/examples/desktop/hand_tracking:hand_detection_cpu.
+# mediapipe/examples/desktop/hand_tracking:hand_detection_cpu.
 
 # Images coming into and out of the graph.
 input_stream: "input_video"

mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt

@@ -1,7 +1,7 @@
 # MediaPipe graph that performs hand detection with TensorFlow Lite on GPU.
 # Used in the examples in
-# mediapipie/examples/android/src/java/com/mediapipe/apps/handdetectiongpu and
+# mediapipe/examples/android/src/java/com/mediapipe/apps/handdetectiongpu and
-# mediapipie/examples/ios/handdetectiongpu.
+# mediapipe/examples/ios/handdetectiongpu.
 
 # Images coming into and out of the graph.
 input_stream: "input_video"

mediapipe/graphs/hand_tracking/hand_tracking_desktop.pbtxt

@@ -1,7 +1,7 @@
 # MediaPipe graph that performs hand tracking on desktop with TensorFlow Lite
 # on CPU.
 # Used in the example in
-# mediapipie/examples/desktop/hand_tracking:hand_tracking_tflite.
+# mediapipe/examples/desktop/hand_tracking:hand_tracking_tflite.
 
 # max_queue_size limits the number of packets enqueued on any input stream
 # by throttling inputs to the graph. This makes the graph only process one

mediapipe/graphs/hand_tracking/hand_tracking_mobile.pbtxt

@@ -1,7 +1,7 @@
 # MediaPipe graph that performs hand tracking with TensorFlow Lite on GPU.
 # Used in the examples in
-# mediapipie/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and
+# mediapipe/examples/android/src/java/com/mediapipe/apps/handtrackinggpu and
-# mediapipie/examples/ios/handtrackinggpu.
+# mediapipe/examples/ios/handtrackinggpu.
 
 # Images coming into and out of the graph.
 input_stream: "input_video"

mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop.pbtxt

@@ -0,0 +1,127 @@
+# MediaPipe graph that performs multi-hand tracking on desktop with TensorFlow
+# Lite on CPU.
+# Used in the example in
+# mediapipie/examples/desktop/hand_tracking:multi_hand_tracking_tflite.
+
+# max_queue_size limits the number of packets enqueued on any input stream
+# by throttling inputs to the graph. This makes the graph only process one
+# frame per time.
+max_queue_size: 1
+
+# Decodes an input video file into images and a video header.
+node {
+  calculator: "OpenCvVideoDecoderCalculator"
+  input_side_packet: "INPUT_FILE_PATH:input_video_path"
+  output_stream: "VIDEO:input_video"
+  output_stream: "VIDEO_PRESTREAM:input_video_header"
+}
+
+# Determines if an input vector of NormalizedRect has a size greater than or
+# equal to the provided min_size.
+node {
+  calculator: "NormalizedRectVectorHasMinSizeCalculator"
+  input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks"
+  output_stream: "prev_has_enough_hands"
+  node_options: {
+    [type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
+      # This value can be changed to support tracking arbitrary number of hands.
+      # Please also remember to modify max_vec_size in
+      # ClipVectorSizeCalculatorOptions in
+      # mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_cpu.pbtxt
+      min_size: 2
+    }
+  }
+}
+
+# Drops the incoming image if the previous frame had at least N hands.
+# Otherwise, passes the incoming image through to trigger a new round of hand
+# detection in MultiHandDetectionSubgraph.
+node {
+  calculator: "GateCalculator"
+  input_stream: "input_video"
+  input_stream: "DISALLOW:prev_has_enough_hands"
+  output_stream: "multi_hand_detection_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.GateCalculatorOptions] {
+      empty_packets_as_allow: true
+    }
+  }
+}
+
+# Subgraph that detections hands (see multi_hand_detection_cpu.pbtxt).
+node {
+  calculator: "MultiHandDetectionSubgraph"
+  input_stream: "multi_hand_detection_input_video"
+  output_stream: "DETECTIONS:multi_palm_detections"
+  output_stream: "NORM_RECTS:multi_palm_rects"
+}
+
+# Subgraph that localizes hand landmarks for multiple hands (see
+# multi_hand_landmark.pbtxt).
+node {
+  calculator: "MultiHandLandmarkSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_stream: "NORM_RECTS:multi_hand_rects"
+  output_stream: "LANDMARKS:multi_hand_landmarks"
+  output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks"
+}
+
+# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the
+# arrival of the next input image sends out the cached rectangle with the
+# timestamp replaced by that of the input image, essentially generating a packet
+# that carries the previous hand rectangle. Note that upon the arrival of the
+# very first input image, an empty packet is sent out to jump start the
+# feedback loop.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:input_video"
+  input_stream: "LOOP:multi_hand_rects_from_landmarks"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks"
+}
+
+# Performs association between NormalizedRect vector elements from previous
+# frame and those from the current frame if MultiHandDetectionSubgraph runs.
+# This calculator ensures that the output multi_hand_rects vector doesn't
+# contain overlapping regions based on the specified min_similarity_threshold.
+node {
+  calculator: "AssociationNormRectCalculator"
+  input_stream: "prev_multi_hand_rects_from_landmarks"
+  input_stream: "multi_palm_rects"
+  output_stream: "multi_hand_rects"
+  node_options: {
+    [type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
+      min_similarity_threshold: 0.1
+    }
+  }
+}
+
+# Subgraph that renders annotations and overlays them on top of the input
+# images (see multi_hand_renderer_cpu.pbtxt).
+node {
+  calculator: "MultiHandRendererSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_stream: "DETECTIONS:multi_palm_detections"
+  input_stream: "LANDMARKS:multi_hand_landmarks"
+  input_stream: "NORM_RECTS:0:multi_palm_rects"
+  input_stream: "NORM_RECTS:1:multi_hand_rects"
+  output_stream: "IMAGE:output_video"
+}
+
+# Encodes the annotated images into a video file, adopting properties specified
+# in the input video header, e.g., video framerate.
+node {
+  calculator: "OpenCvVideoEncoderCalculator"
+  input_stream: "VIDEO:output_video"
+  input_stream: "VIDEO_PRESTREAM:input_video_header"
+  input_side_packet: "OUTPUT_FILE_PATH:output_video_path"
+  node_options: {
+    [type.googleapis.com/mediapipe.OpenCvVideoEncoderCalculatorOptions]: {
+      codec: "avc1"
+      video_format: "mp4"
+    }
+  }
+}

mediapipe/graphs/hand_tracking/multi_hand_tracking_desktop_live.pbtxt

@@ -0,0 +1,103 @@
+# MediaPipe graph that performs multi-hand tracking on desktop with TensorFlow
+# Lite on CPU.
+# Used in the example in
+# mediapipie/examples/desktop/hand_tracking:multi_hand_tracking_cpu.
+
+# Images coming into and out of the graph.
+input_stream: "input_video"
+output_stream: "output_video"
+
+# Determines if an input vector of NormalizedRect has a size greater than or
+# equal to the provided min_size.
+node {
+  calculator: "NormalizedRectVectorHasMinSizeCalculator"
+  input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks"
+  output_stream: "prev_has_enough_hands"
+  node_options: {
+    [type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
+      # This value can be changed to support tracking arbitrary number of hands.
+      # Please also remember to modify max_vec_size in
+      # ClipVectorSizeCalculatorOptions in
+      # mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt
+      min_size: 2
+    }
+  }
+}
+
+# Drops the incoming image if the previous frame had at least N hands.
+# Otherwise, passes the incoming image through to trigger a new round of hand
+# detection in MultiHandDetectionSubgraph.
+node {
+  calculator: "GateCalculator"
+  input_stream: "input_video"
+  input_stream: "DISALLOW:prev_has_enough_hands"
+  output_stream: "multi_hand_detection_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.GateCalculatorOptions] {
+      empty_packets_as_allow: true
+    }
+  }
+}
+
+# Subgraph that detections hands (see multi_hand_detection_cpu.pbtxt).
+node {
+  calculator: "MultiHandDetectionSubgraph"
+  input_stream: "multi_hand_detection_input_video"
+  output_stream: "DETECTIONS:multi_palm_detections"
+  output_stream: "NORM_RECTS:multi_palm_rects"
+}
+
+# Subgraph that localizes hand landmarks for multiple hands (see
+# multi_hand_landmark.pbtxt).
+node {
+  calculator: "MultiHandLandmarkSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_stream: "NORM_RECTS:multi_hand_rects"
+  output_stream: "LANDMARKS:multi_hand_landmarks"
+  output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks"
+}
+
+# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the
+# arrival of the next input image sends out the cached rectangle with the
+# timestamp replaced by that of the input image, essentially generating a packet
+# that carries the previous hand rectangle. Note that upon the arrival of the
+# very first input image, an empty packet is sent out to jump start the
+# feedback loop.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:input_video"
+  input_stream: "LOOP:multi_hand_rects_from_landmarks"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks"
+}
+
+# Performs association between NormalizedRect vector elements from previous
+# frame and those from the current frame if MultiHandDetectionSubgraph runs.
+# This calculator ensures that the output multi_hand_rects vector doesn't
+# contain overlapping regions based on the specified min_similarity_threshold.
+node {
+  calculator: "AssociationNormRectCalculator"
+  input_stream: "prev_multi_hand_rects_from_landmarks"
+  input_stream: "multi_palm_rects"
+  output_stream: "multi_hand_rects"
+  node_options: {
+    [type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
+      min_similarity_threshold: 0.1
+    }
+  }
+}
+
+# Subgraph that renders annotations and overlays them on top of the input
+# images (see multi_hand_renderer_cpu.pbtxt).
+node {
+  calculator: "MultiHandRendererSubgraph"
+  input_stream: "IMAGE:input_video"
+  input_stream: "DETECTIONS:multi_palm_detections"
+  input_stream: "LANDMARKS:multi_hand_landmarks"
+  input_stream: "NORM_RECTS:0:multi_palm_rects"
+  input_stream: "NORM_RECTS:1:multi_hand_rects"
+  output_stream: "IMAGE:output_video"
+}

mediapipe/graphs/hand_tracking/multi_hand_tracking_mobile.pbtxt

@@ -0,0 +1,123 @@
+# MediaPipe graph that performs multi-hand tracking with TensorFlow Lite on GPU.
+# Used in the examples in
+# mediapipe/examples/android/src/java/com/mediapipe/apps/multihandtrackinggpu.
+
+# Images coming into and out of the graph.
+input_stream: "input_video"
+output_stream: "output_video"
+
+# Throttles the images flowing downstream for flow control. It passes through
+# the very first incoming image unaltered, and waits for downstream nodes
+# (calculators and subgraphs) in the graph to finish their tasks before it
+# passes through another image. All images that come in while waiting are
+# dropped, limiting the number of in-flight images in most part of the graph to
+# 1. This prevents the downstream nodes from queuing up incoming images and data
+# excessively, which leads to increased latency and memory usage, unwanted in
+# real-time mobile applications. It also eliminates unnecessarily computation,
+# e.g., the output produced by a node may get dropped downstream if the
+# subsequent nodes are still busy processing previous inputs.
+node {
+  calculator: "FlowLimiterCalculator"
+  input_stream: "input_video"
+  input_stream: "FINISHED:multi_hand_rects"
+  input_stream_info: {
+    tag_index: "FINISHED"
+    back_edge: true
+  }
+  output_stream: "throttled_input_video"
+}
+
+# Determines if an input vector of NormalizedRect has a size greater than or
+# equal to the provided min_size.
+node {
+  calculator: "NormalizedRectVectorHasMinSizeCalculator"
+  input_stream: "ITERABLE:prev_multi_hand_rects_from_landmarks"
+  output_stream: "prev_has_enough_hands"
+  node_options: {
+    [type.googleapis.com/mediapipe.CollectionHasMinSizeCalculatorOptions] {
+      # This value can be changed to support tracking arbitrary number of hands.
+      # Please also remember to modify max_vec_size in
+      # ClipVectorSizeCalculatorOptions in
+      # mediapipe/graphs/hand_tracking/subgraphs/multi_hand_detection_gpu.pbtxt
+      min_size: 2
+    }
+  }
+}
+
+# Drops the incoming image if the previous frame had at least N hands.
+# Otherwise, passes the incoming image through to trigger a new round of hand
+# detection in MultiHandDetectionSubgraph.
+node {
+  calculator: "GateCalculator"
+  input_stream: "throttled_input_video"
+  input_stream: "DISALLOW:prev_has_enough_hands"
+  output_stream: "multi_hand_detection_input_video"
+  node_options: {
+    [type.googleapis.com/mediapipe.GateCalculatorOptions] {
+      empty_packets_as_allow: true
+    }
+  }
+}
+
+# Subgraph that detections hands (see multi_hand_detection_gpu.pbtxt).
+node {
+  calculator: "MultiHandDetectionSubgraph"
+  input_stream: "multi_hand_detection_input_video"
+  output_stream: "DETECTIONS:multi_palm_detections"
+  output_stream: "NORM_RECTS:multi_palm_rects"
+}
+
+# Subgraph that localizes hand landmarks for multiple hands (see
+# multi_hand_landmark.pbtxt).
+node {
+  calculator: "MultiHandLandmarkSubgraph"
+  input_stream: "IMAGE:throttled_input_video"
+  input_stream: "NORM_RECTS:multi_hand_rects"
+  output_stream: "LANDMARKS:multi_hand_landmarks"
+  output_stream: "NORM_RECTS:multi_hand_rects_from_landmarks"
+}
+
+# Caches a hand rectangle fed back from MultiHandLandmarkSubgraph, and upon the
+# arrival of the next input image sends out the cached rectangle with the
+# timestamp replaced by that of the input image, essentially generating a packet
+# that carries the previous hand rectangle. Note that upon the arrival of the
+# very first input image, an empty packet is sent out to jump start the
+# feedback loop.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:throttled_input_video"
+  input_stream: "LOOP:multi_hand_rects_from_landmarks"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_multi_hand_rects_from_landmarks"
+}
+
+# Performs association between NormalizedRect vector elements from previous
+# frame and those from the current frame if MultiHandDetectionSubgraph runs.
+# This calculator ensures that the output multi_hand_rects vector doesn't
+# contain overlapping regions based on the specified min_similarity_threshold.
+node {
+  calculator: "AssociationNormRectCalculator"
+  input_stream: "prev_multi_hand_rects_from_landmarks"
+  input_stream: "multi_palm_rects"
+  output_stream: "multi_hand_rects"
+  node_options: {
+    [type.googleapis.com/mediapipe.AssociationCalculatorOptions] {
+      min_similarity_threshold: 0.1
+    }
+  }
+}
+
+# Subgraph that renders annotations and overlays them on top of the input
+# images (see multi_hand_renderer_gpu.pbtxt).
+node {
+  calculator: "MultiHandRendererSubgraph"
+  input_stream: "IMAGE:throttled_input_video"
+  input_stream: "DETECTIONS:multi_palm_detections"
+  input_stream: "LANDMARKS:multi_hand_landmarks"
+  input_stream: "NORM_RECTS:0:multi_palm_rects"
+  input_stream: "NORM_RECTS:1:multi_hand_rects"
+  output_stream: "IMAGE:output_video"
+}