Project import generated by Copybara.

GitOrigin-RevId: 6f964e58d874e47fb6207aa97d060a4cd6428527

WORKSPACE

@@ -10,15 +10,15 @@ http_archive(
     sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
 )
 load("@bazel_skylib//lib:versions.bzl", "versions")
-versions.check(minimum_bazel_version = "0.24.1",
+versions.check(minimum_bazel_version = "1.0.0",
                maximum_bazel_version = "1.2.1")
 
 
-# ABSL cpp library lts_2019_08_08.
+# ABSL cpp library lts_2020_02_25
 http_archive(
     name = "com_google_absl",
     urls = [
-        "https://github.com/abseil/abseil-cpp/archive/20190808.tar.gz",
+        "https://github.com/abseil/abseil-cpp/archive/20200225.tar.gz",
     ],
     # Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
     patches = [
@@ -27,8 +27,8 @@ http_archive(
     patch_args = [
         "-p1",
     ],
-    strip_prefix = "abseil-cpp-20190808",
+    strip_prefix = "abseil-cpp-20200225",
-    sha256 = "8100085dada279bf3ee00cd064d43b5f55e5d913be0dfe2906f06f8f28d5b37e"
+    sha256 = "728a813291bdec2aa46eab8356ace9f75ac2ed9dfe2df5ab603c4e6c09f1c353"
 )
 
 http_archive(
@@ -117,18 +117,19 @@ http_archive(
     ],
 )
 
-# 2019-11-21
+# 2020-02-12
-_TENSORFLOW_GIT_COMMIT = "f482488b481a799ca07e7e2d153cf47b8e91a60c"
+# The last commit before TensorFlow switched to Bazel 2.0
-_TENSORFLOW_SHA256= "8d9118c2ce186c7e1403f04b96982fe72c184060c7f7a93e30a28dca358694f0"
+_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328"
+_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13"
 http_archive(
     name = "org_tensorflow",
     urls = [
       "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
       "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
     ],
-    # Patch https://github.com/tensorflow/tensorflow/commit/e3a7bdbebb99352351a19e2e403136166aa52934
+    # A compatibility patch
     patches = [
-        "@//third_party:org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff"
+        "@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff"
     ],
     patch_args = [
         "-p1",

mediapipe/calculators/core/BUILD

@@ -610,6 +610,22 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_test(
+    name = "side_packet_to_stream_calculator_test",
+    srcs = ["side_packet_to_stream_calculator_test.cc"],
+    deps = [
+        ":side_packet_to_stream_calculator",
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:gtest_main",
+        "//mediapipe/framework/port:integral_types",
+        "//mediapipe/framework/port:parse_text_proto",
+        "//mediapipe/framework/port:status",
+        "//mediapipe/framework/tool:options_util",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
 cc_test(
     name = "immediate_mux_calculator_test",
     srcs = ["immediate_mux_calculator_test.cc"],

mediapipe/calculators/core/side_packet_to_stream_calculator.cc

@@ -28,55 +28,133 @@ using mediapipe::PacketTypeSet;
 using mediapipe::Timestamp;
 
 namespace {
+
+constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
+constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
+constexpr char kTagAtZero[] = "AT_ZERO";
+constexpr char kTagAtTick[] = "AT_TICK";
+constexpr char kTagTick[] = "TICK";
+
 static std::map<std::string, Timestamp>* kTimestampMap = []() {
   auto* res = new std::map<std::string, Timestamp>();
-  res->emplace("AT_PRESTREAM", Timestamp::PreStream());
+  res->emplace(kTagAtPreStream, Timestamp::PreStream());
-  res->emplace("AT_POSTSTREAM", Timestamp::PostStream());
+  res->emplace(kTagAtPostStream, Timestamp::PostStream());
-  res->emplace("AT_ZERO", Timestamp(0));
+  res->emplace(kTagAtZero, Timestamp(0));
+  res->emplace(kTagAtTick, Timestamp::Unset());
   return res;
 }();
 
+template <typename CC>
+std::string GetOutputTag(const CC& cc) {
+  // Single output tag only is required by contract.
+  return *cc.Outputs().GetTags().begin();
+}
+
 }  // namespace
 
-// Outputs the single input_side_packet at the timestamp specified in the
+// Outputs side packet(s) in corresponding output stream(s) with a particular
-// output_stream tag. Valid tags are AT_PRESTREAM, AT_POSTSTREAM and AT_ZERO.
+// timestamp, depending on the tag used to define output stream(s). (One tag can
+// be used only.)
+//
+// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK and
+// corresponding timestamps are Timestamp::PreStream(), Timestamp::PostStream(),
+// Timestamp(0) and timestamp of a packet received in TICK input.
+//
+// Examples:
+// node {
+//   calculator: "SidePacketToStreamCalculator"
+//   input_side_packet: "side_packet"
+//   output_stream: "AT_PRESTREAM:packet"
+// }
+//
+// node {
+//   calculator: "SidePacketToStreamCalculator"
+//   input_stream: "TICK:tick"
+//   input_side_packet: "side_packet"
+//   output_stream: "AT_TICK:packet"
+// }
 class SidePacketToStreamCalculator : public CalculatorBase {
  public:
   SidePacketToStreamCalculator() = default;
   ~SidePacketToStreamCalculator() override = default;
 
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
+  ::mediapipe::Status Open(CalculatorContext* cc) override;
   ::mediapipe::Status Process(CalculatorContext* cc) override;
   ::mediapipe::Status Close(CalculatorContext* cc) override;
+
+ private:
+  bool is_tick_processing_ = false;
+  std::string output_tag_;
 };
 REGISTER_CALCULATOR(SidePacketToStreamCalculator);
 
 ::mediapipe::Status SidePacketToStreamCalculator::GetContract(
     CalculatorContract* cc) {
-  cc->InputSidePackets().Index(0).SetAny();
+  const auto& tags = cc->Outputs().GetTags();
+  RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
+      << "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK tags is "
+         "allowed and required to specify output stream(s).";
+  RET_CHECK(
+      (cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) ||
+      (!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick)))
+      << "Either both of TICK and AT_TICK should be used or none of them.";
+  const std::string output_tag = GetOutputTag(*cc);
+  const int num_entries = cc->Outputs().NumEntries(output_tag);
+  RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries())
+      << "Same number of input side packets and output streams is required.";
+  for (int i = 0; i < num_entries; ++i) {
+    cc->InputSidePackets().Index(i).SetAny();
+    cc->Outputs()
+        .Get(output_tag, i)
+        .SetSameAs(cc->InputSidePackets().Index(i).GetSameAs());
+  }
 
-  std::set<std::string> tags = cc->Outputs().GetTags();
+  if (cc->Inputs().HasTag(kTagTick)) {
-  RET_CHECK_EQ(tags.size(), 1);
+    cc->Inputs().Tag(kTagTick).SetAny();
+  }
 
-  RET_CHECK_EQ(kTimestampMap->count(*tags.begin()), 1);
+  return ::mediapipe::OkStatus();
-  cc->Outputs().Tag(*tags.begin()).SetAny();
+}
 
+::mediapipe::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
+  output_tag_ = GetOutputTag(*cc);
+  if (cc->Inputs().HasTag(kTagTick)) {
+    is_tick_processing_ = true;
+    // Set offset, so output timestamp bounds are updated in response to TICK
+    // timestamp bound update.
+    cc->SetOffset(TimestampDiff(0));
+  }
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status SidePacketToStreamCalculator::Process(
     CalculatorContext* cc) {
-  return mediapipe::tool::StatusStop();
+  if (is_tick_processing_) {
+    // TICK input is guaranteed to be non-empty, as it's the only input stream
+    // for this calculator.
+    const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
+    for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
+      cc->Outputs()
+          .Get(output_tag_, i)
+          .AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
+    }
+
+    return ::mediapipe::OkStatus();
+  }
+
+  return ::mediapipe::tool::StatusStop();
 }
 
 ::mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
-  std::set<std::string> tags = cc->Outputs().GetTags();
+  if (!cc->Outputs().HasTag(kTagAtTick)) {
-  RET_CHECK_EQ(tags.size(), 1);
+    const auto& timestamp = kTimestampMap->at(output_tag_);
-  const std::string& tag = *tags.begin();
+    for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
-  RET_CHECK_EQ(kTimestampMap->count(tag), 1);
+      cc->Outputs()
-  cc->Outputs().Tag(tag).AddPacket(
+          .Get(output_tag_, i)
-      cc->InputSidePackets().Index(0).At(kTimestampMap->at(tag)));
+          .AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
-
+    }
+  }
   return ::mediapipe::OkStatus();
 }
 

mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc

@@ -0,0 +1,275 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "absl/strings/match.h"
+#include "absl/strings/str_replace.h"
+#include "absl/strings/string_view.h"
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/gtest.h"
+#include "mediapipe/framework/port/integral_types.h"
+#include "mediapipe/framework/port/parse_text_proto.h"
+#include "mediapipe/framework/port/status.h"
+#include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/framework/tool/options_util.h"
+
+namespace mediapipe {
+namespace {
+
+TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tick"
+            input_side_packet: "side_packet"
+            output_stream: "packet"
+            node {
+              calculator: "SidePacketToStreamCalculator"
+              input_side_packet: "side_packet"
+              output_stream: "AT_TICK:packet"
+            }
+          )");
+  CalculatorGraph graph;
+  auto status = graph.Initialize(graph_config);
+  EXPECT_FALSE(status.ok());
+  EXPECT_PRED2(
+      absl::StrContains, status.message(),
+      "Either both of TICK and AT_TICK should be used or none of them.");
+}
+
+TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tick"
+            input_side_packet: "side_packet"
+            output_stream: "packet"
+            node {
+              calculator: "SidePacketToStreamCalculator"
+              input_side_packet: "side_packet"
+              output_stream: "DOES_NOT_EXIST:packet"
+            }
+          )");
+  CalculatorGraph graph;
+  auto status = graph.Initialize(graph_config);
+  EXPECT_FALSE(status.ok());
+  EXPECT_PRED2(absl::StrContains, status.message(),
+               "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK "
+               "tags is allowed and required to specify output stream(s).");
+}
+
+TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tick"
+            input_side_packet: "side_packet0"
+            input_side_packet: "side_packet1"
+            node {
+              calculator: "SidePacketToStreamCalculator"
+              input_side_packet: "side_packet0"
+              input_side_packet: "side_packet1"
+              output_stream: "AT_TICK:packet0"
+              output_stream: "AT_PRE_STREAM:packet1"
+            }
+          )");
+  CalculatorGraph graph;
+  auto status = graph.Initialize(graph_config);
+  EXPECT_FALSE(status.ok());
+  EXPECT_PRED2(absl::StrContains, status.message(),
+               "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK "
+               "tags is allowed and required to specify output stream(s).");
+}
+
+TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_side_packet: "side_packet0"
+            input_side_packet: "side_packet1"
+            node {
+              calculator: "SidePacketToStreamCalculator"
+              input_side_packet: "side_packet0"
+              output_stream: "AT_PRESTREAM:0:packet0"
+              output_stream: "AT_PRESTREAM:1:packet1"
+            }
+          )");
+  CalculatorGraph graph;
+  auto status = graph.Initialize(graph_config);
+  EXPECT_FALSE(status.ok());
+  EXPECT_PRED2(
+      absl::StrContains, status.message(),
+      "Same number of input side packets and output streams is required.");
+}
+
+TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_side_packet: "side_packet0"
+            input_side_packet: "side_packet1"
+            node {
+              calculator: "SidePacketToStreamCalculator"
+              input_side_packet: "side_packet0"
+              input_side_packet: "side_packet1"
+              output_stream: "AT_PRESTREAM:packet0"
+            }
+          )");
+  CalculatorGraph graph;
+  auto status = graph.Initialize(graph_config);
+  EXPECT_FALSE(status.ok());
+  EXPECT_PRED2(
+      absl::StrContains, status.message(),
+      "Same number of input side packets and output streams is required.");
+}
+
+void DoTestNonAtTickOutputTag(absl::string_view tag,
+                              Timestamp expected_timestamp) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(absl::StrReplaceAll(
+          R"(
+            input_side_packet: "side_packet"
+            output_stream: "packet"
+            node {
+              calculator: "SidePacketToStreamCalculator"
+              input_side_packet: "side_packet"
+              output_stream: "$tag:packet"
+            }
+          )",
+          {{"$tag", tag}}));
+  CalculatorGraph graph;
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  const int expected_value = 10;
+  std::vector<Packet> output_packets;
+  MP_ASSERT_OK(graph.ObserveOutputStream(
+      "packet", [&output_packets](const Packet& packet) {
+        output_packets.push_back(packet);
+        return ::mediapipe::OkStatus();
+      }));
+  MP_ASSERT_OK(
+      graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
+  MP_ASSERT_OK(graph.WaitForObservedOutput());
+
+  ASSERT_FALSE(output_packets.empty());
+  EXPECT_EQ(expected_timestamp, output_packets.back().Timestamp());
+  EXPECT_EQ(expected_value, output_packets.back().Get<int>());
+}
+
+TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) {
+  DoTestNonAtTickOutputTag("AT_PRESTREAM", Timestamp::PreStream());
+  DoTestNonAtTickOutputTag("AT_POSTSTREAM", Timestamp::PostStream());
+  DoTestNonAtTickOutputTag("AT_ZERO", Timestamp(0));
+}
+
+TEST(SidePacketToStreamCalculator, AtTick) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tick"
+            input_side_packet: "side_packet"
+            output_stream: "packet"
+            node {
+              calculator: "SidePacketToStreamCalculator"
+              input_stream: "TICK:tick"
+              input_side_packet: "side_packet"
+              output_stream: "AT_TICK:packet"
+            }
+          )");
+  std::vector<Packet> output_packets;
+  tool::AddVectorSink("packet", &graph_config, &output_packets);
+  CalculatorGraph graph;
+
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  const int expected_value = 20;
+  MP_ASSERT_OK(
+      graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
+
+  auto tick_and_verify = [&graph, &output_packets,
+                          expected_value](int at_timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "tick",
+        MakePacket<int>(/*doesn't matter*/ 1).At(Timestamp(at_timestamp))));
+    MP_ASSERT_OK(graph.WaitUntilIdle());
+
+    ASSERT_FALSE(output_packets.empty());
+    EXPECT_EQ(Timestamp(at_timestamp), output_packets.back().Timestamp());
+    EXPECT_EQ(expected_value, output_packets.back().Get<int>());
+  };
+
+  tick_and_verify(/*at_timestamp=*/0);
+  tick_and_verify(/*at_timestamp=*/1);
+  tick_and_verify(/*at_timestamp=*/128);
+  tick_and_verify(/*at_timestamp=*/1024);
+  tick_and_verify(/*at_timestamp=*/1025);
+}
+
+TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
+  CalculatorGraphConfig graph_config =
+      ParseTextProtoOrDie<CalculatorGraphConfig>(
+          R"(
+            input_stream: "tick"
+            input_side_packet: "side_packet0"
+            input_side_packet: "side_packet1"
+            output_stream: "packet0"
+            output_stream: "packet1"
+            node {
+              calculator: "SidePacketToStreamCalculator"
+              input_stream: "TICK:tick"
+              input_side_packet: "side_packet0"
+              input_side_packet: "side_packet1"
+              output_stream: "AT_TICK:0:packet0"
+              output_stream: "AT_TICK:1:packet1"
+            }
+          )");
+  std::vector<Packet> output_packets0;
+  tool::AddVectorSink("packet0", &graph_config, &output_packets0);
+  std::vector<Packet> output_packets1;
+  tool::AddVectorSink("packet1", &graph_config, &output_packets1);
+  CalculatorGraph graph;
+
+  MP_ASSERT_OK(graph.Initialize(graph_config));
+  const int expected_value0 = 20;
+  const int expected_value1 = 128;
+  MP_ASSERT_OK(
+      graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
+                      {"side_packet1", MakePacket<int>(expected_value1)}}));
+
+  auto tick_and_verify = [&graph, &output_packets0, &output_packets1,
+                          expected_value0, expected_value1](int at_timestamp) {
+    MP_ASSERT_OK(graph.AddPacketToInputStream(
+        "tick",
+        MakePacket<int>(/*doesn't matter*/ 1).At(Timestamp(at_timestamp))));
+    MP_ASSERT_OK(graph.WaitUntilIdle());
+
+    ASSERT_FALSE(output_packets0.empty());
+    ASSERT_FALSE(output_packets1.empty());
+
+    EXPECT_EQ(Timestamp(at_timestamp), output_packets0.back().Timestamp());
+    EXPECT_EQ(expected_value0, output_packets0.back().Get<int>());
+    EXPECT_EQ(Timestamp(at_timestamp), output_packets1.back().Timestamp());
+    EXPECT_EQ(expected_value1, output_packets1.back().Get<int>());
+  };
+
+  tick_and_verify(/*at_timestamp=*/0);
+  tick_and_verify(/*at_timestamp=*/1);
+  tick_and_verify(/*at_timestamp=*/128);
+  tick_and_verify(/*at_timestamp=*/1024);
+  tick_and_verify(/*at_timestamp=*/1025);
+}
+
+}  // namespace
+}  // namespace mediapipe

mediapipe/calculators/image/BUILD

@@ -346,9 +346,7 @@ cc_library(
         ],
         "//conditions:default": [],
     }),
-    visibility = [
+    visibility = ["//visibility:public"],
-        "//visibility:public",
-    ],
     deps = [
         ":image_cropping_calculator_cc_proto",
         "//mediapipe/framework:calculator_framework",

mediapipe/calculators/image/color_convert_calculator.cc

@@ -75,6 +75,11 @@ class ColorConvertCalculator : public CalculatorBase {
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
   ::mediapipe::Status Process(CalculatorContext* cc) override;
 
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    cc->SetOffset(TimestampDiff(0));
+    return ::mediapipe::OkStatus();
+  }
+
  private:
   // Wrangles the appropriate inputs and outputs to perform the color
   // conversion. The ImageFrame on input_tag is converted using the

mediapipe/calculators/image/image_cropping_calculator.cc

@@ -119,6 +119,13 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   }
 
+  // Validate border mode.
+  if (use_gpu_) {
+    MP_RETURN_IF_ERROR(ValidateBorderModeForGPU(cc));
+  } else {
+    MP_RETURN_IF_ERROR(ValidateBorderModeForCPU(cc));
+  }
+
   return ::mediapipe::OkStatus();
 }
 
@@ -162,6 +169,32 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForCPU(
+    CalculatorContext* cc) {
+  int border_mode;
+  return GetBorderModeForOpenCV(cc, &border_mode);
+}
+
+::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
+    CalculatorContext* cc) {
+  mediapipe::ImageCroppingCalculatorOptions options =
+      cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
+
+  switch (options.border_mode()) {
+    case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
+      LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU "
+                   << "implementation and will fall back into BORDER_REPLICATE";
+      break;
+    case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
+      break;
+    default:
+      RET_CHECK_FAIL() << "Unsupported border mode for GPU: "
+                       << options.border_mode();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) {
   if (cc->Inputs().Tag(kImageTag).IsEmpty()) {
     return ::mediapipe::OkStatus();
@@ -172,6 +205,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
   auto [target_width, target_height, rect_center_x, rect_center_y, rotation] =
       GetCropSpecs(cc, input_img.Width(), input_img.Height());
 
+  // Get border mode and value for OpenCV.
+  int border_mode;
+  MP_RETURN_IF_ERROR(GetBorderModeForOpenCV(cc, &border_mode));
+
   const cv::RotatedRect min_rect(cv::Point2f(rect_center_x, rect_center_y),
                                  cv::Size2f(target_width, target_height),
                                  rotation * 180.f / M_PI);
@@ -191,7 +228,9 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
       cv::getPerspectiveTransform(src_points, dst_points);
   cv::Mat cropped_image;
   cv::warpPerspective(input_mat, cropped_image, projection_matrix,
-                      cv::Size(min_rect.size.width, min_rect.size.height));
+                      cv::Size(min_rect.size.width, min_rect.size.height),
+                      /* flags = */ 0,
+                      /* borderMode = */ border_mode);
 
   std::unique_ptr<ImageFrame> output_frame(new ImageFrame(
       input_img.Format(), cropped_image.cols, cropped_image.rows));
@@ -453,6 +492,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
       rotation = options.rotation();
     }
   }
+
   return {
       .width = crop_width,
       .height = crop_height,
@@ -462,4 +502,24 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
   };
 }
 
+::mediapipe::Status ImageCroppingCalculator::GetBorderModeForOpenCV(
+    CalculatorContext* cc, int* border_mode) {
+  mediapipe::ImageCroppingCalculatorOptions options =
+      cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
+
+  switch (options.border_mode()) {
+    case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
+      *border_mode = cv::BORDER_CONSTANT;
+      break;
+    case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
+      *border_mode = cv::BORDER_REPLICATE;
+      break;
+    default:
+      RET_CHECK_FAIL() << "Unsupported border mode for CPU: "
+                       << options.border_mode();
+  }
+
+  return ::mediapipe::OkStatus();
+}
+
 }  // namespace mediapipe

mediapipe/calculators/image/image_cropping_calculator.h

@@ -36,6 +36,7 @@
 // Note: input_stream values take precedence over options defined in the graph.
 //
 namespace mediapipe {
+
 struct RectSpec {
   int width;
   int height;
@@ -63,12 +64,16 @@ class ImageCroppingCalculator : public CalculatorBase {
                                int src_height);
 
  private:
+  ::mediapipe::Status ValidateBorderModeForCPU(CalculatorContext* cc);
+  ::mediapipe::Status ValidateBorderModeForGPU(CalculatorContext* cc);
   ::mediapipe::Status RenderCpu(CalculatorContext* cc);
   ::mediapipe::Status RenderGpu(CalculatorContext* cc);
   ::mediapipe::Status InitGpu(CalculatorContext* cc);
   void GlRender();
   void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height,
                            int* dst_width, int* dst_height);
+  ::mediapipe::Status GetBorderModeForOpenCV(CalculatorContext* cc,
+                                             int* border_mode);
 
   mediapipe::ImageCroppingCalculatorOptions options_;
 

mediapipe/calculators/image/image_cropping_calculator.proto

@@ -40,4 +40,15 @@ message ImageCroppingCalculatorOptions {
   // The (0, 0) point is at the (top, left) corner.
   optional float norm_center_x = 6 [default = 0];
   optional float norm_center_y = 7 [default = 0];
+
+  enum BorderMode {
+    // First unspecified value is required by the guideline. See details here:
+    // https://developers.google.com/protocol-buffers/docs/style#enums
+    BORDER_UNSPECIFIED = 0;
+    BORDER_ZERO = 1;
+    BORDER_REPLICATE = 2;
+  }
+
+  // Specifies behaviour for crops that go beyond image borders.
+  optional BorderMode border_mode = 8 [default = BORDER_ZERO];
 }

mediapipe/calculators/image/image_cropping_calculator_test.cc

@@ -56,10 +56,10 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
             }
           )");
 
-  auto calculator_state =
+  auto calculator_state = absl::make_unique<CalculatorState>(
-      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+      "Node", 0, "Calculator", calculator_node, nullptr);
-  auto cc =
+  auto cc = absl::make_unique<CalculatorContext>(
-      CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
+      calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(),
       tool::CreateTagMap({}).ValueOrDie());
 
   RectSpec expectRect = {
@@ -69,8 +69,8 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
       .center_y = 50,
       .rotation = 0.3,
   };
-  EXPECT_EQ(
+  EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
-      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+                                                  input_height),
             expectRect);
 }  // TEST
 
@@ -96,10 +96,10 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
             }
           )");
 
-  auto calculator_state =
+  auto calculator_state = absl::make_unique<CalculatorState>(
-      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+      "Node", 0, "Calculator", calculator_node, nullptr);
-  auto cc =
+  auto cc = absl::make_unique<CalculatorContext>(
-      CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
+      calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(),
       tool::CreateTagMap({}).ValueOrDie());
   RectSpec expectRect = {
       .width = 50,
@@ -108,8 +108,8 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
       .center_y = 50,
       .rotation = 0.3,
   };
-  EXPECT_EQ(
+  EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
-      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+                                                  input_height),
             expectRect);
 }  // TEST
 
@@ -138,16 +138,16 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
             }
           )");
 
-  auto calculator_state =
+  auto calculator_state = absl::make_unique<CalculatorState>(
-      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+      "Node", 0, "Calculator", calculator_node, nullptr);
   auto inputTags = tool::CreateTagMap({
                                           "HEIGHT:0:crop_height",
                                           "WIDTH:0:crop_width",
                                       })
                        .ValueOrDie();
-  auto cc = CalculatorContext(&calculator_state, inputTags,
+  auto cc = absl::make_unique<CalculatorContext>(
-                              tool::CreateTagMap({}).ValueOrDie());
+      calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie());
-  auto& inputs = cc.Inputs();
+  auto& inputs = cc->Inputs();
   inputs.Tag(kHeightTag).Value() = MakePacket<int>(1);
   inputs.Tag(kWidthTag).Value() = MakePacket<int>(1);
   RectSpec expectRect = {
@@ -157,8 +157,8 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
       .center_y = 50,
       .rotation = 0.3,
   };
-  EXPECT_EQ(
+  EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
-      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+                                                  input_height),
             expectRect);
 }  // TEST
 
@@ -186,15 +186,15 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
             }
           )");
 
-  auto calculator_state =
+  auto calculator_state = absl::make_unique<CalculatorState>(
-      CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
+      "Node", 0, "Calculator", calculator_node, nullptr);
   auto inputTags = tool::CreateTagMap({
                                           "RECT:0:rect",
                                       })
                        .ValueOrDie();
-  auto cc = CalculatorContext(&calculator_state, inputTags,
+  auto cc = absl::make_unique<CalculatorContext>(
-                              tool::CreateTagMap({}).ValueOrDie());
+      calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie());
-  auto& inputs = cc.Inputs();
+  auto& inputs = cc->Inputs();
   mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
       R"(
         width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
@@ -207,8 +207,8 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
       .center_y = 40,
       .rotation = 0.5,
   };
-  EXPECT_EQ(
+  EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
-      ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
+                                                  input_height),
             expectRect);
 }  // TEST
 

mediapipe/calculators/image/image_transformation_calculator.cc

@@ -104,6 +104,14 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 //   to be a multiple of 90 degrees. If provided, it overrides the
 //   ROTATION_DEGREES input side packet.
 //
+//   FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not. If
+//   provided, it overrides the FLIP_HORIZONTALLY input side packet and/or
+//   corresponding field in the calculator options.
+//
+//   FLIP_VERTICALLY (optional): Whether to flip image vertically or not. If
+//   provided, it overrides the FLIP_VERTICALLY input side packet and/or
+//   corresponding field in the calculator options.
+//
 // Output:
 //   One of the following two tags:
 //   IMAGE - ImageFrame representing the output image.
@@ -129,6 +137,12 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
 //   degrees. It has to be a multiple of 90 degrees. It overrides the
 //   corresponding field in the calculator options.
 //
+//   FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not.
+//   It overrides the corresponding field in the calculator options.
+//
+//   FLIP_VERTICALLY (optional): Whether to flip image vertically or not.
+//   It overrides the corresponding field in the calculator options.
+//
 // Calculator options (see image_transformation_calculator.proto):
 //   output_width, output_height - (optional) Desired scaled image size.
 //   rotation_mode - (optional) Rotation in multiples of 90 degrees.
@@ -167,6 +181,8 @@ class ImageTransformationCalculator : public CalculatorBase {
   int output_height_ = 0;
   mediapipe::RotationMode_Mode rotation_;
   mediapipe::ScaleMode_Mode scale_mode_;
+  bool flip_horizontally_ = false;
+  bool flip_vertically_ = false;
 
   bool use_gpu_ = false;
 #if !defined(MEDIAPIPE_DISABLE_GPU)
@@ -203,6 +219,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   if (cc->Inputs().HasTag("ROTATION_DEGREES")) {
     cc->Inputs().Tag("ROTATION_DEGREES").Set<int>();
   }
+  if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) {
+    cc->Inputs().Tag("FLIP_HORIZONTALLY").Set<bool>();
+  }
+  if (cc->Inputs().HasTag("FLIP_VERTICALLY")) {
+    cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
+  }
 
   if (cc->InputSidePackets().HasTag("OUTPUT_DIMENSIONS")) {
     cc->InputSidePackets().Tag("OUTPUT_DIMENSIONS").Set<DimensionsPacketType>();
@@ -210,6 +232,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
     cc->InputSidePackets().Tag("ROTATION_DEGREES").Set<int>();
   }
+  if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
+    cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set<bool>();
+  }
+  if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
+    cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set<bool>();
+  }
 
   if (cc->Outputs().HasTag("LETTERBOX_PADDING")) {
     cc->Outputs().Tag("LETTERBOX_PADDING").Set<std::array<float, 4>>();
@@ -245,6 +273,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
     output_width_ = options_.output_width();
     output_height_ = options_.output_height();
   }
+
   if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
     rotation_ = DegreesToRotationMode(
         cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
@@ -252,6 +281,20 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
     rotation_ = options_.rotation_mode();
   }
 
+  if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
+    flip_horizontally_ =
+        cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>();
+  } else {
+    flip_horizontally_ = options_.flip_horizontally();
+  }
+
+  if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
+    flip_vertically_ =
+        cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>();
+  } else {
+    flip_vertically_ = options_.flip_vertically();
+  }
+
   scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE);
 
   if (use_gpu_) {
@@ -268,12 +311,37 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
 ::mediapipe::Status ImageTransformationCalculator::Process(
     CalculatorContext* cc) {
+  // Override values if specified so.
+  if (cc->Inputs().HasTag("ROTATION_DEGREES") &&
+      !cc->Inputs().Tag("ROTATION_DEGREES").IsEmpty()) {
+    rotation_ =
+        DegreesToRotationMode(cc->Inputs().Tag("ROTATION_DEGREES").Get<int>());
+  }
+  if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") &&
+      !cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) {
+    flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get<bool>();
+  }
+  if (cc->Inputs().HasTag("FLIP_VERTICALLY") &&
+      !cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
+    flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
+  }
+
   if (use_gpu_) {
 #if !defined(MEDIAPIPE_DISABLE_GPU)
+    if (cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
+      // Image is missing, hence no way to produce output image. (Timestamp
+      // bound will be updated automatically.)
+      return ::mediapipe::OkStatus();
+    }
     return helper_.RunInGlContext(
         [this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); });
 #endif  //  !MEDIAPIPE_DISABLE_GPU
   } else {
+    if (cc->Inputs().Tag("IMAGE").IsEmpty()) {
+      // Image is missing, hence no way to produce output image. (Timestamp
+      // bound will be updated automatically.)
+      return ::mediapipe::OkStatus();
+    }
     return RenderCpu(cc);
   }
   return ::mediapipe::OkStatus();
@@ -360,11 +428,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
         .Add(padding.release(), cc->InputTimestamp());
   }
 
-  if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
-    rotation_ = DegreesToRotationMode(
-        cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
-  }
-
   cv::Mat rotated_mat;
   const int angle = RotationModeToDegrees(rotation_);
   cv::Point2f src_center(scaled_mat.cols / 2.0, scaled_mat.rows / 2.0);
@@ -372,11 +435,9 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   cv::warpAffine(scaled_mat, rotated_mat, rotation_mat, scaled_mat.size());
 
   cv::Mat flipped_mat;
-  if (options_.flip_horizontally() || options_.flip_vertically()) {
+  if (flip_horizontally_ || flip_vertically_) {
     const int flip_code =
-        options_.flip_horizontally() && options_.flip_vertically()
+        flip_horizontally_ && flip_vertically_ ? -1 : flip_horizontally_;
-            ? -1
-            : options_.flip_horizontally();
     cv::flip(rotated_mat, flipped_mat, flip_code);
   } else {
     flipped_mat = rotated_mat;
@@ -450,11 +511,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
   }
   RET_CHECK(renderer) << "Unsupported input texture type";
 
-  if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
-    rotation_ = DegreesToRotationMode(
-        cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
-  }
-
   mediapipe::FrameScaleMode scale_mode = mediapipe::FrameScaleModeFromProto(
       scale_mode_, mediapipe::FrameScaleMode::kStretch);
   mediapipe::FrameRotation rotation =
@@ -469,7 +525,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
 
   MP_RETURN_IF_ERROR(renderer->GlRender(
       src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
-      rotation, options_.flip_horizontally(), options_.flip_vertically(),
+      rotation, flip_horizontally_, flip_vertically_,
       /*flip_texture=*/false));
 
   glActiveTexture(GL_TEXTURE1);

mediapipe/calculators/image/scale_image_calculator.cc

@@ -264,7 +264,7 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
                                         options_.target_width(),           //
                                         options_.target_height(),          //
                                         options_.preserve_aspect_ratio(),  //
-                                        options_.scale_to_multiple_of_two(),  //
+                                        options_.scale_to_multiple_of(),   //
                                         &output_width_, &output_height_));
   MP_RETURN_IF_ERROR(FindInterpolationAlgorithm(options_.algorithm(),
                                                 &interpolation_algorithm_));
@@ -361,17 +361,21 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
       output_format_ = input_format_;
     }
 
+    const bool is_positive_and_even =
+        (options_.scale_to_multiple_of() >= 1) &&
+        (options_.scale_to_multiple_of() % 2 == 0);
+
     if (output_format_ == ImageFormat::YCBCR420P) {
-      RET_CHECK(options_.scale_to_multiple_of_two())
+      RET_CHECK(is_positive_and_even)
           << "ScaleImageCalculator always outputs width and height that are "
              "divisible by 2 when output format is YCbCr420P. To scale to "
              "width and height of odd numbers, the output format must be SRGB.";
     } else if (options_.preserve_aspect_ratio()) {
-      RET_CHECK(options_.scale_to_multiple_of_two())
+      RET_CHECK(options_.scale_to_multiple_of() == 2)
           << "ScaleImageCalculator always outputs width and height that are "
-             "divisible by 2 when perserving aspect ratio. To scale to width "
+             "divisible by 2 when preserving aspect ratio. If you'd like to "
-             "and height of odd numbers, please set "
+             "set scale_to_multiple_of to something other than 2, please "
-             "preserve_aspect_ratio to false.";
+             "set preserve_aspect_ratio to false.";
     }
 
     if (input_width_ > 0 && input_height_ > 0 &&

mediapipe/calculators/image/scale_image_calculator.proto

@@ -11,9 +11,10 @@ import "mediapipe/framework/formats/image_format.proto";
 // 2) Scale and convert the image to fit inside target_width x target_height
 //    using the specified scaling algorithm.  (maintaining the aspect
 //    ratio if preserve_aspect_ratio is true).
-// The output width and height will be divisible by 2. It is possible to output
+// The output width and height will be divisible by 2, by default. It is
-// width and height that are odd number when the output format is SRGB and not
+// possible to output width and height that are odd numbers when the output
-// perserving the aspect ratio. See scale_to_multiple_of_two option for details.
+// format is SRGB and the aspect ratio is left unpreserved. See
+// scale_to_multiple_of for details.
 message ScaleImageCalculatorOptions {
   extend CalculatorOptions {
     optional ScaleImageCalculatorOptions ext = 66237115;
@@ -23,7 +24,7 @@ message ScaleImageCalculatorOptions {
   // depending on the other options below.  If unset, use the same width
   // or height as the input.  If only one is set then determine the other
   // from the aspect ratio (after cropping).  The output width and height
-  // will be divisible by 2.
+  // will be divisible by 2, by default.
   optional int32 target_width = 1;
   optional int32 target_height = 2;
 
@@ -31,7 +32,8 @@ message ScaleImageCalculatorOptions {
   // fits inside the box represented by target_width and target_height.
   // Otherwise it is scaled to fit target_width and target_height
   // completely.  In any case, the aspect ratio that is preserved is
-  // that after cropping to the minimum/maximum aspect ratio.
+  // that after cropping to the minimum/maximum aspect ratio. Additionally, if
+  // true, the output width and height will be divisible by 2.
   optional bool preserve_aspect_ratio = 3 [default = true];
 
   // If ratio is positive, crop the image to this minimum and maximum
@@ -95,11 +97,13 @@ message ScaleImageCalculatorOptions {
   // SRGB or YCBCR420P.
   optional ImageFormat.Format input_format = 12;
 
-  // If true, the output width and height will be divisible by 2. Otherwise it
+  // If set to 2, the target width and height will be rounded-down
-  // will use the exact specified output width and height, which is only
+  // to the nearest even number. If set to any positive value other than 2,
-  // supported when the output format is SRGB and preserve_aspect_ratio option
+  // preserve_aspect_ratio must be false and the target width and height will be
-  // is set to false.
+  // rounded-down to multiples of the given value. If set to any value less than
-  optional bool scale_to_multiple_of_two = 13 [default = true];
+  // 1, it will be treated like 1.
+  // NOTE: If set to an odd number, the output format must be SRGB.
+  optional int32 scale_to_multiple_of = 13 [default = 2];
 
   // If true, assume the input YUV is BT.709 (this is the HDTV standard, so most
   // content is likely using it). If false use the previous assumption of BT.601

mediapipe/calculators/image/scale_image_utils.cc

@@ -93,12 +93,22 @@ double ParseRational(const std::string& rational) {
                                          int target_width,            //
                                          int target_height,           //
                                          bool preserve_aspect_ratio,  //
-                                         bool scale_to_multiple_of_two,  //
+                                         int scale_to_multiple_of,    //
                                          int* output_width,
                                          int* output_height) {
   CHECK(output_width);
   CHECK(output_height);
 
+  if (preserve_aspect_ratio) {
+    RET_CHECK(scale_to_multiple_of == 2)
+        << "FindOutputDimensions always outputs width and height that are "
+           "divisible by 2 when preserving aspect ratio. If you'd like to "
+           "set scale_to_multiple_of to something other than 2, please "
+           "set preserve_aspect_ratio to false.";
+  }
+
+  if (scale_to_multiple_of < 1) scale_to_multiple_of = 1;
+
   if (!preserve_aspect_ratio || (target_width <= 0 && target_height <= 0)) {
     if (target_width <= 0) {
       target_width = input_width;
@@ -106,13 +116,13 @@ double ParseRational(const std::string& rational) {
     if (target_height <= 0) {
       target_height = input_height;
     }
-    if (scale_to_multiple_of_two) {
+
-      *output_width = (target_width / 2) * 2;
+    target_width -= target_width % scale_to_multiple_of;
-      *output_height = (target_height / 2) * 2;
+    target_height -= target_height % scale_to_multiple_of;
-    } else {
+
     *output_width = target_width;
     *output_height = target_height;
-    }
+
     return ::mediapipe::OkStatus();
   }
 

mediapipe/calculators/image/scale_image_utils.h

@@ -35,17 +35,19 @@ namespace scale_image {
                                        int* col_start, int* row_start);
 
 // Given an input width and height, a target width and height, whether to
-// preserve the aspect ratio, and whether to round down to a multiple of 2,
+// preserve the aspect ratio, and whether to round-down to the multiple of a
-// determine the output width and height. If target_width or target_height is
+// given number nearest to the targets, determine the output width and height.
-// non-positive, then they will be set to the input_width and input_height
+// If target_width or target_height is non-positive, then they will be set to
-// respectively. The output_width and output_height will be reduced as necessary
+// the input_width and input_height respectively. If scale_to_multiple_of is
-// to preserve_aspect_ratio and to scale_to_multipe_of_two if these options are
+// less than 1, it will be treated like 1. The output_width and
-// specified.
+// output_height will be reduced as necessary to preserve_aspect_ratio if the
+// option is specified. If preserving the aspect ratio is desired, you must set
+// scale_to_multiple_of to 2.
 ::mediapipe::Status FindOutputDimensions(int input_width, int input_height,  //
                                          int target_width,
                                          int target_height,           //
                                          bool preserve_aspect_ratio,  //
-                                         bool scale_to_multiple_of_two,  //
+                                         int scale_to_multiple_of,    //
                                          int* output_width, int* output_height);
 
 }  // namespace scale_image

mediapipe/calculators/image/scale_image_utils_test.cc

@@ -79,49 +79,49 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) {
   int output_width;
   int output_height;
   // Not scale.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, true, &output_width,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, 2, &output_width,
                                     &output_height));
   EXPECT_EQ(200, output_width);
   EXPECT_EQ(100, output_height);
   // Not scale with odd input size.
-  MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, false,
+  MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, 1, &output_width,
-                                    &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(201, output_width);
   EXPECT_EQ(101, output_height);
   // Scale down by 1/2.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, 2, &output_width,
-                                    &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale up, doubling dimensions.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, 2, &output_width,
-                                    &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(400, output_width);
   EXPECT_EQ(200, output_height);
   // Fits a 2:1 image into a 150 x 150 box. Output dimensions are always
   // visible by 2.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, 2, &output_width,
-                                    &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(150, output_width);
   EXPECT_EQ(74, output_height);
   // Fits a 2:1 image into a 400 x 50 box.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, 2, &output_width,
-                                    &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale to multiple number with odd targe size.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
-                                    &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale to multiple number with odd targe size.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, false,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
-                                    &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(50, output_height);
   // Scale to odd size.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, false,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, 1, &output_width,
-                                    &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(151, output_width);
   EXPECT_EQ(101, output_height);
 }
@@ -131,22 +131,62 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsNoAspectRatio) {
   int output_width;
   int output_height;
   // Scale width only.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 2, &output_width,
-                                    &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(100, output_width);
   EXPECT_EQ(100, output_height);
   // Scale height only.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, 2, &output_width,
-                                    &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(200, output_width);
   EXPECT_EQ(200, output_height);
   // Scale both dimensions.
-  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, true,
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 2, &output_width,
-                                    &output_width, &output_height));
+                                    &output_height));
   EXPECT_EQ(150, output_width);
   EXPECT_EQ(200, output_height);
 }
 
+// Tests scale_to_multiple_of.
+TEST(ScaleImageUtilsTest, FindOutputDimensionsDownScaleToMultipleOf) {
+  int output_width;
+  int output_height;
+  // Set no targets, downscale to a multiple of 8.
+  MP_ASSERT_OK(FindOutputDimensions(100, 100, -1, -1, false, 8, &output_width,
+                                    &output_height));
+  EXPECT_EQ(96, output_width);
+  EXPECT_EQ(96, output_height);
+  // Set width target, downscale to a multiple of 8.
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 8, &output_width,
+                                    &output_height));
+  EXPECT_EQ(96, output_width);
+  EXPECT_EQ(96, output_height);
+  // Set height target, downscale to a multiple of 8.
+  MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, 201, false, 8, &output_width,
+                                    &output_height));
+  EXPECT_EQ(200, output_width);
+  EXPECT_EQ(200, output_height);
+  // Set both targets, downscale to a multiple of 8.
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 8, &output_width,
+                                    &output_height));
+  EXPECT_EQ(144, output_width);
+  EXPECT_EQ(200, output_height);
+  // Doesn't throw error if keep aspect is true and downscale multiple is 2.
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 200, true, 2, &output_width,
+                                    &output_height));
+  EXPECT_EQ(400, output_width);
+  EXPECT_EQ(200, output_height);
+  // Throws error if keep aspect is true, but downscale multiple is not 2.
+  ASSERT_THAT(FindOutputDimensions(200, 100, 400, 200, true, 4, &output_width,
+                                   &output_height),
+              testing::Not(testing::status::IsOk()));
+  // Downscaling to multiple ignored if multiple is less than 2.
+  MP_ASSERT_OK(FindOutputDimensions(200, 100, 401, 201, false, 1, &output_width,
+                                    &output_height));
+  EXPECT_EQ(401, output_width);
+  EXPECT_EQ(201, output_height);
+}
+
 }  // namespace
 }  // namespace scale_image
 }  // namespace mediapipe

mediapipe/calculators/tensorflow/BUILD

@@ -138,7 +138,7 @@ mediapipe_cc_proto_library(
     srcs = ["image_frame_to_tensor_calculator.proto"],
     cc_deps = [
         "//mediapipe/framework:calculator_cc_proto",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
     visibility = ["//visibility:public"],
     deps = [":image_frame_to_tensor_calculator_proto"],
@@ -173,7 +173,7 @@ mediapipe_cc_proto_library(
     srcs = ["pack_media_sequence_calculator.proto"],
     cc_deps = [
         "//mediapipe/framework:calculator_cc_proto",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
     visibility = ["//visibility:public"],
     deps = [":pack_media_sequence_calculator_proto"],
@@ -192,7 +192,7 @@ mediapipe_cc_proto_library(
     srcs = ["tensorflow_session_from_frozen_graph_generator.proto"],
     cc_deps = [
         "//mediapipe/framework:packet_generator_cc_proto",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
     visibility = ["//visibility:public"],
     deps = [":tensorflow_session_from_frozen_graph_generator_proto"],
@@ -203,7 +203,7 @@ mediapipe_cc_proto_library(
     srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"],
     cc_deps = [
         "//mediapipe/framework:calculator_cc_proto",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
     visibility = ["//visibility:public"],
     deps = [":tensorflow_session_from_frozen_graph_calculator_proto"],
@@ -277,7 +277,7 @@ mediapipe_cc_proto_library(
     srcs = ["vector_int_to_tensor_calculator_options.proto"],
     cc_deps = [
         "//mediapipe/framework:calculator_cc_proto",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
     visibility = ["//visibility:public"],
     deps = [":vector_int_to_tensor_calculator_options_proto"],
@@ -408,7 +408,7 @@ cc_library(
         "//mediapipe/util/sequence:media_sequence",
         "//mediapipe/util/sequence:media_sequence_util",
         "@com_google_absl//absl/strings",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
     alwayslink = 1,
 )
@@ -423,7 +423,7 @@ cc_library(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
     alwayslink = 1,
 )
@@ -654,7 +654,7 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "@org_tensorflow//tensorflow/core:lib",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
     alwayslink = 1,
 )
@@ -695,7 +695,7 @@ cc_library(
         "//mediapipe/util:audio_decoder_cc_proto",
         "//mediapipe/util/sequence:media_sequence",
         "@com_google_absl//absl/strings",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
     alwayslink = 1,
 )
@@ -737,7 +737,7 @@ cc_library(
         "//mediapipe/framework:calculator_framework",
         "//mediapipe/framework:packet",
         "//mediapipe/framework/port:status",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
     alwayslink = 1,
 )
@@ -745,6 +745,7 @@ cc_library(
 cc_test(
     name = "graph_tensors_packet_generator_test",
     srcs = ["graph_tensors_packet_generator_test.cc"],
+    linkstatic = 1,
     deps = [
         ":graph_tensors_packet_generator",
         ":graph_tensors_packet_generator_cc_proto",
@@ -761,6 +762,7 @@ cc_test(
     name = "image_frame_to_tensor_calculator_test",
     size = "small",
     srcs = ["image_frame_to_tensor_calculator_test.cc"],
+    linkstatic = 1,
     deps = [
         ":image_frame_to_tensor_calculator",
         "//mediapipe/framework:calculator_framework",
@@ -777,6 +779,7 @@ cc_test(
     name = "matrix_to_tensor_calculator_test",
     size = "small",
     srcs = ["matrix_to_tensor_calculator_test.cc"],
+    linkstatic = 1,
     deps = [
         ":matrix_to_tensor_calculator",
         ":matrix_to_tensor_calculator_options_cc_proto",
@@ -793,6 +796,7 @@ cc_test(
     name = "lapped_tensor_buffer_calculator_test",
     size = "small",
     srcs = ["lapped_tensor_buffer_calculator_test.cc"],
+    linkstatic = 1,
     deps = [
         ":lapped_tensor_buffer_calculator",
         ":lapped_tensor_buffer_calculator_cc_proto",
@@ -801,7 +805,7 @@ cc_test(
         "//mediapipe/framework/port:gtest_main",
         "@com_google_absl//absl/memory",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
 )
 
@@ -840,7 +844,7 @@ cc_test(
         "//mediapipe/util/sequence:media_sequence",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
 )
 
@@ -867,7 +871,7 @@ cc_test(
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:direct_session",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
         "@org_tensorflow//tensorflow/core:testlib",
         "@org_tensorflow//tensorflow/core/kernels:conv_ops",
         "@org_tensorflow//tensorflow/core/kernels:math",
@@ -897,7 +901,7 @@ cc_test(
         "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/core:direct_session",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
         "@org_tensorflow//tensorflow/core:testlib",
         "@org_tensorflow//tensorflow/core/kernels:conv_ops",
         "@org_tensorflow//tensorflow/core/kernels:math",
@@ -956,6 +960,7 @@ cc_test(
 cc_test(
     name = "tensor_squeeze_dimensions_calculator_test",
     srcs = ["tensor_squeeze_dimensions_calculator_test.cc"],
+    linkstatic = 1,
     deps = [
         ":tensor_squeeze_dimensions_calculator",
         ":tensor_squeeze_dimensions_calculator_cc_proto",
@@ -963,7 +968,7 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
 )
 
@@ -971,6 +976,7 @@ cc_test(
     name = "tensor_to_image_frame_calculator_test",
     size = "small",
     srcs = ["tensor_to_image_frame_calculator_test.cc"],
+    linkstatic = 1,
     deps = [
         ":tensor_to_image_frame_calculator",
         ":tensor_to_image_frame_calculator_cc_proto",
@@ -979,7 +985,7 @@ cc_test(
         "//mediapipe/framework/formats:image_frame",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
 )
 
@@ -987,6 +993,7 @@ cc_test(
     name = "tensor_to_matrix_calculator_test",
     size = "small",
     srcs = ["tensor_to_matrix_calculator_test.cc"],
+    linkstatic = 1,
     deps = [
         ":tensor_to_matrix_calculator",
         ":tensor_to_matrix_calculator_cc_proto",
@@ -996,13 +1003,14 @@ cc_test(
         "//mediapipe/framework/formats:time_series_header_cc_proto",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
 )
 
 cc_test(
     name = "tensor_to_vector_float_calculator_test",
     srcs = ["tensor_to_vector_float_calculator_test.cc"],
+    linkstatic = 1,
     deps = [
         ":tensor_to_vector_float_calculator",
         ":tensor_to_vector_float_calculator_options_cc_proto",
@@ -1010,7 +1018,7 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
 )
 
@@ -1030,13 +1038,14 @@ cc_test(
         "//mediapipe/util/sequence:media_sequence",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
 )
 
 cc_test(
     name = "vector_int_to_tensor_calculator_test",
     srcs = ["vector_int_to_tensor_calculator_test.cc"],
+    linkstatic = 1,
     deps = [
         ":vector_int_to_tensor_calculator",
         ":vector_int_to_tensor_calculator_options_cc_proto",
@@ -1044,13 +1053,14 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
 )
 
 cc_test(
     name = "vector_float_to_tensor_calculator_test",
     srcs = ["vector_float_to_tensor_calculator_test.cc"],
+    linkstatic = 1,
     deps = [
         ":vector_float_to_tensor_calculator",
         ":vector_float_to_tensor_calculator_options_cc_proto",
@@ -1058,7 +1068,7 @@ cc_test(
         "//mediapipe/framework:calculator_runner",
         "//mediapipe/framework/port:gtest_main",
         "@org_tensorflow//tensorflow/core:framework",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
 )
 

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc

@@ -17,7 +17,7 @@
 #if !defined(__ANDROID__)
 #include "mediapipe/framework/port/file_helpers.h"
 #endif
-#include "absl/strings/substitute.h"
+#include "absl/strings/str_replace.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
@@ -63,7 +63,7 @@ const std::string MaybeConvertSignatureToTag(
     output.resize(name.length());
     std::transform(name.begin(), name.end(), output.begin(),
                    [](unsigned char c) { return std::toupper(c); });
-    output = absl::Substitute(output, "/", "_");
+    output = absl::StrReplaceAll(output, {{"/", "_"}});
     return output;
   } else {
     return name;

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "absl/strings/substitute.h"
+#include "absl/strings/str_replace.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
 #include "mediapipe/framework/calculator.pb.h"

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc

@@ -17,7 +17,7 @@
 #if !defined(__ANDROID__)
 #include "mediapipe/framework/port/file_helpers.h"
 #endif
-#include "absl/strings/substitute.h"
+#include "absl/strings/str_replace.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
 #include "mediapipe/framework/deps/file_path.h"
@@ -65,7 +65,7 @@ const std::string MaybeConvertSignatureToTag(
     output.resize(name.length());
     std::transform(name.begin(), name.end(), output.begin(),
                    [](unsigned char c) { return std::toupper(c); });
-    output = absl::Substitute(output, "/", "_");
+    output = absl::StrReplaceAll(output, {{"/", "_"}});
     return output;
   } else {
     return name;

mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "absl/strings/substitute.h"
+#include "absl/strings/str_replace.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session.h"
 #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"

mediapipe/calculators/tflite/BUILD

@@ -485,6 +485,7 @@ cc_test(
         "//mediapipe/framework/port:integral_types",
         "//mediapipe/framework/port:parse_text_proto",
         "//mediapipe/framework/tool:validate_type",
+        "@com_google_absl//absl/strings",
         "@org_tensorflow//tensorflow/lite:framework",
         "@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
     ],

mediapipe/calculators/tflite/tflite_inference_calculator.cc

@@ -148,7 +148,7 @@ struct GPUData {
 //   options: {
 //     [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
 //       model_path: "modelname.tflite"
-//       use_gpu: true
+//       delegate { gpu {} }
 //     }
 //   }
 // }
@@ -163,6 +163,9 @@ struct GPUData {
 //
 class TfLiteInferenceCalculator : public CalculatorBase {
  public:
+  using TfLiteDelegatePtr =
+      std::unique_ptr<TfLiteDelegate, std::function<void(TfLiteDelegate*)>>;
+
   static ::mediapipe::Status GetContract(CalculatorContract* cc);
 
   ::mediapipe::Status Open(CalculatorContext* cc) override;
@@ -176,7 +179,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
 
   std::unique_ptr<tflite::Interpreter> interpreter_;
   std::unique_ptr<tflite::FlatBufferModel> model_;
-  TfLiteDelegate* delegate_ = nullptr;
+  TfLiteDelegatePtr delegate_;
 
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   mediapipe::GlCalculatorHelper gpu_helper_;
@@ -212,12 +215,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
             cc->Outputs().HasTag("TENSORS_GPU"));
 
-  bool use_gpu = false;
+  const auto& options =
+      cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
+  bool use_gpu =
+      options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
 
   if (cc->Inputs().HasTag("TENSORS"))
     cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
   if (cc->Inputs().HasTag("TENSORS_GPU")) {
+    RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
+        << "GPU input is compatible with GPU delegate only.";
+
     cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
@@ -227,6 +236,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
 #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
   if (cc->Outputs().HasTag("TENSORS_GPU")) {
+    RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
+        << "GPU output is compatible with GPU delegate only.";
+
     cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
     use_gpu |= true;
   }
@@ -238,10 +250,6 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
         .Set<tflite::ops::builtin::BuiltinOpResolver>();
   }
 
-  const auto& options =
-      cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
-  use_gpu |= options.use_gpu();
-
   if (use_gpu) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
     MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
@@ -454,7 +462,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     if (gpu_inference_) {
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
       MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
-        TfLiteGpuDelegateDelete(delegate_);
+        delegate_ = nullptr;
         for (int i = 0; i < gpu_data_in_.size(); ++i) {
           gpu_data_in_[i].reset();
         }
@@ -464,7 +472,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
         return ::mediapipe::OkStatus();
       }));
 #elif defined(MEDIAPIPE_IOS)
-      TFLGpuDelegateDelete(delegate_);
+      delegate_ = nullptr;
       for (int i = 0; i < gpu_data_in_.size(); ++i) {
         gpu_data_in_[i].reset();
       }
@@ -472,9 +480,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
         gpu_data_out_[i].reset();
       }
 #endif
-    }
+    } else {
       delegate_ = nullptr;
     }
+  }
 #if defined(MEDIAPIPE_EDGE_TPU)
   edgetpu_context_.reset();
 #endif
@@ -501,7 +510,8 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   }
 
   // Get execution modes.
-  gpu_inference_ = options.use_gpu();
+  gpu_inference_ =
+      options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
 
   return ::mediapipe::OkStatus();
 }
@@ -526,8 +536,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
   RET_CHECK(interpreter_);
 
-#if defined(__EMSCRIPTEN__)
+#if defined(__EMSCRIPTEN__) || defined(MEDIAPIPE_EDGE_TPU)
   interpreter_->SetNumThreads(1);
+#else
+  interpreter_->SetNumThreads(
+      cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
+          .cpu_num_thread());
 #endif  // __EMSCRIPTEN__
 
   if (gpu_output_) {
@@ -545,20 +559,37 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
 
 ::mediapipe::Status TfLiteInferenceCalculator::LoadDelegate(
     CalculatorContext* cc) {
-#if defined(MEDIAPIPE_ANDROID)
+  const auto& calculator_opts =
+      cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>();
+  if (calculator_opts.has_delegate() &&
+      calculator_opts.delegate().has_tflite()) {
+    // Default tflite inference requeqsted - no need to modify graph.
+    return ::mediapipe::OkStatus();
+  }
+
   if (!gpu_inference_) {
-    if (cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
+#if defined(MEDIAPIPE_ANDROID)
-            .use_nnapi()) {
+    const bool nnapi_requested = calculator_opts.has_delegate()
+                                     ? calculator_opts.delegate().has_nnapi()
+                                     : calculator_opts.use_nnapi();
+    if (nnapi_requested) {
       // Attempt to use NNAPI.
       // If not supported, the default CPU delegate will be created and used.
       interpreter_->SetAllowFp16PrecisionForFp32(1);
-      delegate_ = tflite::NnApiDelegate();
+      delegate_ =
-      RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
+          TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) {
+            // No need to free according to tflite::NnApiDelegate()
+            // documentation.
+          });
+      RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
+                   kTfLiteOk);
+      return ::mediapipe::OkStatus();
     }
+#endif  // MEDIAPIPE_ANDROID
+
     // Return, no need for GPU delegate below.
     return ::mediapipe::OkStatus();
   }
-#endif  // ANDROID
 
 #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
   // Configure and create the delegate.
@@ -568,7 +599,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       TFLITE_GL_OBJECT_TYPE_FASTEST;
   options.compile_options.dynamic_batch_enabled = 0;
   options.compile_options.inline_parameters = 1;
-  if (!delegate_) delegate_ = TfLiteGpuDelegateCreate(&options);
+  if (!delegate_)
+    delegate_ = TfLiteDelegatePtr(TfLiteGpuDelegateCreate(&options),
+                                  &TfLiteGpuDelegateDelete);
 
   if (gpu_input_) {
     // Get input image sizes.
@@ -586,7 +619,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
           ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
               gpu_data_in_[i]->elements, &gpu_data_in_[i]->buffer));
       RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
-                       delegate_, gpu_data_in_[i]->buffer.id(),
+                       delegate_.get(), gpu_data_in_[i]->buffer.id(),
                        interpreter_->inputs()[i]),
                    kTfLiteOk);
     }
@@ -609,15 +642,16 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
     for (int i = 0; i < gpu_data_out_.size(); ++i) {
       RET_CHECK_CALL(CreateReadWriteShaderStorageBuffer<float>(
           gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer));
-      RET_CHECK_EQ(
+      RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
-          TfLiteGpuDelegateBindBufferToTensor(
+                       delegate_.get(), gpu_data_out_[i]->buffer.id(),
-              delegate_, gpu_data_out_[i]->buffer.id(), output_indices[i]),
+                       output_indices[i]),
                    kTfLiteOk);
     }
   }
 
   // Must call this last.
-  RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
+  RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
+               kTfLiteOk);
 #endif  // OpenGL
 
 #if defined(MEDIAPIPE_IOS)
@@ -626,7 +660,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
   TFLGpuDelegateOptions options;
   options.allow_precision_loss = true;
   options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive;
-  if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options);
+  if (!delegate_)
+    delegate_ = TfLiteDelegatePtr(TFLGpuDelegateCreate(&options),
+                                  &TFLGpuDelegateDelete);
   id<MTLDevice> device = gpu_helper_.mtlDevice;
 
   if (gpu_input_) {
@@ -678,9 +714,11 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       gpu_data_in_[i]->buffer =
           [device newBufferWithLength:gpu_data_in_[i]->elements * kHalfSize
                               options:MTLResourceStorageModeShared];
-      RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
+      RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
-      RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
+                   kTfLiteOk);
-                       delegate_, input_indices[i], gpu_data_in_[i]->buffer),
+      RET_CHECK_EQ(
+          TFLGpuDelegateBindMetalBufferToTensor(
+              delegate_.get(), input_indices[i], gpu_data_in_[i]->buffer),
           true);
     }
   }
@@ -725,8 +763,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
       gpu_data_out_[i]->buffer =
           [device newBufferWithLength:gpu_data_out_[i]->elements * kHalfSize
                               options:MTLResourceStorageModeShared];
-      RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
+      RET_CHECK_EQ(
-                       delegate_, output_indices[i], gpu_data_out_[i]->buffer),
+          TFLGpuDelegateBindMetalBufferToTensor(
+              delegate_.get(), output_indices[i], gpu_data_out_[i]->buffer),
           true);
     }
 

mediapipe/calculators/tflite/tflite_inference_calculator.proto

@@ -27,7 +27,7 @@ import "mediapipe/framework/calculator.proto";
 //   options {
 //     [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
 //       model_path: "model.tflite"
-//       use_gpu: true
+//       delegate { gpu {} }
 //     }
 //   }
 // }
@@ -37,6 +37,22 @@ message TfLiteInferenceCalculatorOptions {
     optional TfLiteInferenceCalculatorOptions ext = 233867213;
   }
 
+  message Delegate {
+    // Default inference provided by tflite.
+    message TfLite {}
+    // Delegate to run GPU inference depending on the device.
+    // (Can use OpenGl, OpenCl, Metal depending on the device.)
+    message Gpu {}
+    // Android only.
+    message Nnapi {}
+
+    oneof delegate {
+      TfLite tflite = 1;
+      Gpu gpu = 2;
+      Nnapi nnapi = 3;
+    }
+  }
+
   // Path to the TF Lite model (ex: /path/to/modelname.tflite).
   // On mobile, this is generally just modelname.tflite.
   optional string model_path = 1;
@@ -44,10 +60,22 @@ message TfLiteInferenceCalculatorOptions {
   // Whether the TF Lite GPU or CPU backend should be used. Effective only when
   // input tensors are on CPU. For input tensors on GPU, GPU backend is always
   // used.
-  optional bool use_gpu = 2 [default = false];
+  // DEPRECATED: configure "delegate" instead.
+  optional bool use_gpu = 2 [deprecated = true, default = false];
 
   // Android only. When true, an NNAPI delegate will be used for inference.
   // If NNAPI is not available, then the default CPU delegate will be used
   // automatically.
-  optional bool use_nnapi = 3 [default = false];
+  // DEPRECATED: configure "delegate" instead.
+  optional bool use_nnapi = 3 [deprecated = true, default = false];
+
+  // The number of threads available to the interpreter. Effective only when
+  // input tensors are on CPU and 'use_gpu' is false.
+  optional int32 cpu_num_thread = 4 [default = -1];
+
+  // TfLite delegate to run inference.
+  // NOTE: calculator is free to choose delegate if not specified explicitly.
+  // NOTE: use_gpu/use_nnapi are ignored if specified. (Delegate takes
+  // precedence over use_* deprecated options.)
+  optional Delegate delegate = 5;
 }

mediapipe/calculators/tflite/tflite_inference_calculator_test.cc

@@ -16,6 +16,8 @@
 #include <string>
 #include <vector>
 
+#include "absl/strings/str_replace.h"
+#include "absl/strings/string_view.h"
 #include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
@@ -39,13 +41,7 @@ namespace mediapipe {
 
 using ::tflite::Interpreter;
 
-class TfLiteInferenceCalculatorTest : public ::testing::Test {
+void DoSmokeTest(absl::string_view delegate) {
- protected:
-  std::unique_ptr<CalculatorRunner> runner_ = nullptr;
-};
-
-// Tests a simple add model that adds an input tensor to itself.
-TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
   const int width = 8;
   const int height = 8;
   const int channels = 3;
@@ -73,10 +69,7 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
   auto input_vec = absl::make_unique<std::vector<TfLiteTensor>>();
   input_vec->emplace_back(*tensor);
 
-  // Prepare single calculator graph to and wait for packets.
+  std::string graph_proto = R"(
-  CalculatorGraphConfig graph_config =
-      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
-          R"(
     input_stream: "tensor_in"
     node {
       calculator: "TfLiteInferenceCalculator"
@@ -84,12 +77,16 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
       output_stream: "TENSORS:tensor_out"
       options {
         [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
-                  use_gpu: false
           model_path: "mediapipe/calculators/tflite/testdata/add.bin"
+          $delegate
         }
       }
     }
-          )");
+  )";
+  ASSERT_EQ(absl::StrReplaceAll({{"$delegate", delegate}}, &graph_proto), 1);
+  // Prepare single calculator graph to and wait for packets.
+  CalculatorGraphConfig graph_config =
+      ::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(graph_proto);
   std::vector<Packet> output_packets;
   tool::AddVectorSink("tensor_out", &graph_config, &output_packets);
   CalculatorGraph graph(graph_config);
@@ -120,4 +117,10 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
   MP_ASSERT_OK(graph.WaitUntilDone());
 }
 
+// Tests a simple add model that adds an input tensor to itself.
+TEST(TfLiteInferenceCalculatorTest, SmokeTest) {
+  DoSmokeTest(/*delegate=*/"");
+  DoSmokeTest(/*delegate=*/"delegate { tflite {} }");
+}
+
 }  // namespace mediapipe

mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc

@@ -28,6 +28,21 @@ namespace mediapipe {
 //  TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first
 //            tensor will be used. The size of the values must be
 //            (num_dimension x num_landmarks).
+//
+//  FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
+//  not. Overrides corresponding side packet and/or field in the calculator
+//  options.
+//
+//  FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
+//  Overrides corresponding side packet and/or field in the calculator options.
+//
+// Input side packet:
+//   FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
+//   not. Overrides the corresponding field in the calculator options.
+//
+//   FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
+//   Overrides the corresponding field in the calculator options.
+//
 // Output:
 //  LANDMARKS(optional) - Result MediaPipe landmarks.
 //  NORM_LANDMARKS(optional) - Result MediaPipe normalized landmarks.
@@ -61,6 +76,8 @@ class TfLiteTensorsToLandmarksCalculator : public CalculatorBase {
  private:
   ::mediapipe::Status LoadOptions(CalculatorContext* cc);
   int num_landmarks_ = 0;
+  bool flip_vertically_ = false;
+  bool flip_horizontally_ = false;
 
   ::mediapipe::TfLiteTensorsToLandmarksCalculatorOptions options_;
 };
@@ -75,6 +92,22 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
     cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
   }
 
+  if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) {
+    cc->Inputs().Tag("FLIP_HORIZONTALLY").Set<bool>();
+  }
+
+  if (cc->Inputs().HasTag("FLIP_VERTICALLY")) {
+    cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
+  }
+
+  if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
+    cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set<bool>();
+  }
+
+  if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
+    cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set<bool>();
+  }
+
   if (cc->Outputs().HasTag("LANDMARKS")) {
     cc->Outputs().Tag("LANDMARKS").Set<LandmarkList>();
   }
@@ -98,17 +131,40 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
         << "Must provide input with/height for getting normalized landmarks.";
   }
   if (cc->Outputs().HasTag("LANDMARKS") &&
-      (options_.flip_vertically() || options_.flip_horizontally())) {
+      (options_.flip_vertically() || options_.flip_horizontally() ||
+       cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY") ||
+       cc->InputSidePackets().HasTag("FLIP_VERTICALLY"))) {
     RET_CHECK(options_.has_input_image_height() &&
               options_.has_input_image_width())
         << "Must provide input with/height for using flip_vertically option "
            "when outputing landmarks in absolute coordinates.";
   }
+
+  flip_horizontally_ =
+      cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")
+          ? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>()
+          : options_.flip_horizontally();
+
+  flip_horizontally_ =
+      cc->InputSidePackets().HasTag("FLIP_VERTICALLY")
+          ? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>()
+          : options_.flip_vertically();
+
   return ::mediapipe::OkStatus();
 }
 
 ::mediapipe::Status TfLiteTensorsToLandmarksCalculator::Process(
     CalculatorContext* cc) {
+  // Override values if specified so.
+  if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") &&
+      !cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) {
+    flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get<bool>();
+  }
+  if (cc->Inputs().HasTag("FLIP_VERTICALLY") &&
+      !cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
+    flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
+  }
+
   if (cc->Inputs().Tag("TENSORS").IsEmpty()) {
     return ::mediapipe::OkStatus();
   }
@@ -133,13 +189,13 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
     const int offset = ld * num_dimensions;
     Landmark* landmark = output_landmarks.add_landmark();
 
-    if (options_.flip_horizontally()) {
+    if (flip_horizontally_) {
       landmark->set_x(options_.input_image_width() - raw_landmarks[offset]);
     } else {
       landmark->set_x(raw_landmarks[offset]);
     }
     if (num_dimensions > 1) {
-      if (options_.flip_vertically()) {
+      if (flip_vertically_) {
         landmark->set_y(options_.input_image_height() -
                         raw_landmarks[offset + 1]);
       } else {

mediapipe/calculators/util/BUILD

@@ -437,6 +437,7 @@ cc_library(
         "//mediapipe/framework/formats:rect_cc_proto",
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
+        "@com_google_absl//absl/types:optional",
     ],
     alwayslink = 1,
 )
@@ -928,6 +929,19 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "local_file_pattern_contents_calculator",
+    srcs = ["local_file_pattern_contents_calculator.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/port:file_helpers",
+        "//mediapipe/framework/port:ret_check",
+        "//mediapipe/framework/port:status",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "filter_collection_calculator",
     srcs = ["filter_collection_calculator.cc"],

mediapipe/calculators/util/annotation_overlay_calculator.cc

@@ -39,13 +39,13 @@ namespace mediapipe {
 
 namespace {
 
-constexpr char kInputFrameTag[] = "INPUT_FRAME";
+constexpr char kInputFrameTag[] = "IMAGE";
-constexpr char kOutputFrameTag[] = "OUTPUT_FRAME";
+constexpr char kOutputFrameTag[] = "IMAGE";
 
 constexpr char kInputVectorTag[] = "VECTOR";
 
-constexpr char kInputFrameTagGpu[] = "INPUT_FRAME_GPU";
+constexpr char kInputFrameTagGpu[] = "IMAGE_GPU";
-constexpr char kOutputFrameTagGpu[] = "OUTPUT_FRAME_GPU";
+constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU";
 
 enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
 
@@ -61,7 +61,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
 // A calculator for rendering data on images.
 //
 // Inputs:
-//  1. INPUT_FRAME or INPUT_FRAME_GPU (optional): An ImageFrame (or GpuBuffer)
+//  1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer)
 //     containing the input image.
 //     If output is CPU, and input isn't provided, the renderer creates a
 //     blank canvas with the width, height and color provided in the options.
@@ -73,7 +73,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
 //     input vector items. These input streams are tagged with "VECTOR".
 //
 // Output:
-//  1. OUTPUT_FRAME or OUTPUT_FRAME_GPU: A rendered ImageFrame (or GpuBuffer).
+//  1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer).
 //
 // For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The
 // output format is the same as input except for GRAY8 where the output is in
@@ -87,13 +87,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
 // Example config (CPU):
 // node {
 //   calculator: "AnnotationOverlayCalculator"
-//   input_stream: "INPUT_FRAME:image_frames"
+//   input_stream: "IMAGE:image_frames"
 //   input_stream: "render_data_1"
 //   input_stream: "render_data_2"
 //   input_stream: "render_data_3"
 //   input_stream: "VECTOR:0:render_data_vec_0"
 //   input_stream: "VECTOR:1:render_data_vec_1"
-//   output_stream: "OUTPUT_FRAME:decorated_frames"
+//   output_stream: "IMAGE:decorated_frames"
 //   options {
 //     [mediapipe.AnnotationOverlayCalculatorOptions.ext] {
 //     }
@@ -103,13 +103,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
 // Example config (GPU):
 // node {
 //   calculator: "AnnotationOverlayCalculator"
-//   input_stream: "INPUT_FRAME_GPU:image_frames"
+//   input_stream: "IMAGE_GPU:image_frames"
 //   input_stream: "render_data_1"
 //   input_stream: "render_data_2"
 //   input_stream: "render_data_3"
 //   input_stream: "VECTOR:0:render_data_vec_0"
 //   input_stream: "VECTOR:1:render_data_vec_1"
-//   output_stream: "OUTPUT_FRAME_GPU:decorated_frames"
+//   output_stream: "IMAGE_GPU:decorated_frames"
 //   options {
 //     [mediapipe.AnnotationOverlayCalculatorOptions.ext] {
 //     }

mediapipe/calculators/util/detections_to_rects_calculator.cc

@@ -39,7 +39,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
 }  // namespace
 
 ::mediapipe::Status DetectionsToRectsCalculator::DetectionToRect(
-    const Detection& detection, Rect* rect) {
+    const Detection& detection, const DetectionSpec& detection_spec,
+    Rect* rect) {
   const LocationData location_data = detection.location_data();
   RET_CHECK(location_data.format() == LocationData::BOUNDING_BOX)
       << "Only Detection with formats of BOUNDING_BOX can be converted to Rect";
@@ -52,7 +53,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
 }
 
 ::mediapipe::Status DetectionsToRectsCalculator::DetectionToNormalizedRect(
-    const Detection& detection, NormalizedRect* rect) {
+    const Detection& detection, const DetectionSpec& detection_spec,
+    NormalizedRect* rect) {
   const LocationData location_data = detection.location_data();
   RET_CHECK(location_data.format() == LocationData::RELATIVE_BOUNDING_BOX)
       << "Only Detection with formats of RELATIVE_BOUNDING_BOX can be "
@@ -174,27 +176,31 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
     }
   }
 
-  std::pair<int, int> image_size;
+  // Get dynamic calculator options (e.g. `image_size`).
-  if (rotate_) {
+  const DetectionSpec detection_spec = GetDetectionSpec(cc);
-    RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty());
-    image_size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
-  }
 
   if (cc->Outputs().HasTag(kRectTag)) {
     auto output_rect = absl::make_unique<Rect>();
-    MP_RETURN_IF_ERROR(DetectionToRect(detections[0], output_rect.get()));
+    MP_RETURN_IF_ERROR(
+        DetectionToRect(detections[0], detection_spec, output_rect.get()));
     if (rotate_) {
-      output_rect->set_rotation(ComputeRotation(detections[0], image_size));
+      float rotation;
+      MP_RETURN_IF_ERROR(
+          ComputeRotation(detections[0], detection_spec, &rotation));
+      output_rect->set_rotation(rotation);
     }
     cc->Outputs().Tag(kRectTag).Add(output_rect.release(),
                                     cc->InputTimestamp());
   }
   if (cc->Outputs().HasTag(kNormRectTag)) {
     auto output_rect = absl::make_unique<NormalizedRect>();
-    MP_RETURN_IF_ERROR(
+    MP_RETURN_IF_ERROR(DetectionToNormalizedRect(detections[0], detection_spec,
-        DetectionToNormalizedRect(detections[0], output_rect.get()));
+                                                 output_rect.get()));
     if (rotate_) {
-      output_rect->set_rotation(ComputeRotation(detections[0], image_size));
+      float rotation;
+      MP_RETURN_IF_ERROR(
+          ComputeRotation(detections[0], detection_spec, &rotation));
+      output_rect->set_rotation(rotation);
     }
     cc->Outputs()
         .Tag(kNormRectTag)
@@ -203,11 +209,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
   if (cc->Outputs().HasTag(kRectsTag)) {
     auto output_rects = absl::make_unique<std::vector<Rect>>(detections.size());
     for (int i = 0; i < detections.size(); ++i) {
-      MP_RETURN_IF_ERROR(
+      MP_RETURN_IF_ERROR(DetectionToRect(detections[i], detection_spec,
-          DetectionToRect(detections[i], &(output_rects->at(i))));
+                                         &(output_rects->at(i))));
       if (rotate_) {
-        output_rects->at(i).set_rotation(
+        float rotation;
-            ComputeRotation(detections[i], image_size));
+        MP_RETURN_IF_ERROR(
+            ComputeRotation(detections[i], detection_spec, &rotation));
+        output_rects->at(i).set_rotation(rotation);
       }
     }
     cc->Outputs().Tag(kRectsTag).Add(output_rects.release(),
@@ -217,11 +225,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
     auto output_rects =
         absl::make_unique<std::vector<NormalizedRect>>(detections.size());
     for (int i = 0; i < detections.size(); ++i) {
-      MP_RETURN_IF_ERROR(
+      MP_RETURN_IF_ERROR(DetectionToNormalizedRect(
-          DetectionToNormalizedRect(detections[i], &(output_rects->at(i))));
+          detections[i], detection_spec, &(output_rects->at(i))));
       if (rotate_) {
-        output_rects->at(i).set_rotation(
+        float rotation;
-            ComputeRotation(detections[i], image_size));
+        MP_RETURN_IF_ERROR(
+            ComputeRotation(detections[i], detection_spec, &rotation));
+        output_rects->at(i).set_rotation(rotation);
       }
     }
     cc->Outputs()
@@ -232,21 +242,35 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
   return ::mediapipe::OkStatus();
 }
 
-float DetectionsToRectsCalculator::ComputeRotation(
+::mediapipe::Status DetectionsToRectsCalculator::ComputeRotation(
-    const Detection& detection, const std::pair<int, int> image_size) {
+    const Detection& detection, const DetectionSpec& detection_spec,
+    float* rotation) {
   const auto& location_data = detection.location_data();
+  const auto& image_size = detection_spec.image_size;
+  RET_CHECK(image_size) << "Image size is required to calculate rotation";
+
   const float x0 = location_data.relative_keypoints(start_keypoint_index_).x() *
-                   image_size.first;
+                   image_size->first;
   const float y0 = location_data.relative_keypoints(start_keypoint_index_).y() *
-                   image_size.second;
+                   image_size->second;
   const float x1 = location_data.relative_keypoints(end_keypoint_index_).x() *
-                   image_size.first;
+                   image_size->first;
   const float y1 = location_data.relative_keypoints(end_keypoint_index_).y() *
-                   image_size.second;
+                   image_size->second;
 
-  float rotation = target_angle_ - std::atan2(-(y1 - y0), x1 - x0);
+  *rotation = NormalizeRadians(target_angle_ - std::atan2(-(y1 - y0), x1 - x0));
 
-  return NormalizeRadians(rotation);
+  return ::mediapipe::OkStatus();
+}
+
+DetectionSpec DetectionsToRectsCalculator::GetDetectionSpec(
+    const CalculatorContext* cc) {
+  absl::optional<std::pair<int, int>> image_size;
+  if (cc->Inputs().HasTag(kImageSizeTag)) {
+    image_size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
+  }
+
+  return {image_size};
 }
 
 REGISTER_CALCULATOR(DetectionsToRectsCalculator);

mediapipe/calculators/util/detections_to_rects_calculator.h

@@ -16,6 +16,7 @@
 
 #include <cmath>
 
+#include "absl/types/optional.h"
 #include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_options.pb.h"
@@ -27,6 +28,13 @@
 
 namespace mediapipe {
 
+// Dynamic options passed as calculator `input_stream` that can be used for
+// calculation of rectangle or rotation for given detection. Does not include
+// static calculator options which are available via private fields.
+struct DetectionSpec {
+  absl::optional<std::pair<int, int>> image_size;
+};
+
 // A calculator that converts Detection proto to Rect proto.
 //
 // Detection is the format for encoding one or more detections in an image.
@@ -81,13 +89,16 @@ class DetectionsToRectsCalculator : public CalculatorBase {
   ::mediapipe::Status Process(CalculatorContext* cc) override;
 
  protected:
-  virtual float ComputeRotation(const ::mediapipe::Detection& detection,
-                                const std::pair<int, int> image_size);
   virtual ::mediapipe::Status DetectionToRect(
-      const ::mediapipe::Detection& detection, ::mediapipe::Rect* rect);
+      const ::mediapipe::Detection& detection,
+      const DetectionSpec& detection_spec, ::mediapipe::Rect* rect);
   virtual ::mediapipe::Status DetectionToNormalizedRect(
       const ::mediapipe::Detection& detection,
-      ::mediapipe::NormalizedRect* rect);
+      const DetectionSpec& detection_spec, ::mediapipe::NormalizedRect* rect);
+  virtual ::mediapipe::Status ComputeRotation(
+      const ::mediapipe::Detection& detection,
+      const DetectionSpec& detection_spec, float* rotation);
+  virtual DetectionSpec GetDetectionSpec(const CalculatorContext* cc);
 
   static inline float NormalizeRadians(float angle) {
     return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI));

mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc

@@ -12,20 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Copyright 2019 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 #include <cmath>
 #include <vector>
 
@@ -67,6 +53,15 @@ constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING";
 //   input_stream: "LETTERBOX_PADDING:letterbox_padding"
 //   output_stream: "LANDMARKS:adjusted_landmarks"
 // }
+//
+// node {
+//   calculator: "LandmarkLetterboxRemovalCalculator"
+//   input_stream: "LANDMARKS:0:landmarks_0"
+//   input_stream: "LANDMARKS:1:landmarks_1"
+//   input_stream: "LETTERBOX_PADDING:letterbox_padding"
+//   output_stream: "LANDMARKS:0:adjusted_landmarks_0"
+//   output_stream: "LANDMARKS:1:adjusted_landmarks_1"
+// }
 class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -74,10 +69,20 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
               cc->Inputs().HasTag(kLetterboxPaddingTag))
         << "Missing one or more input streams.";
 
-    cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
+    RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag),
+                 cc->Outputs().NumEntries(kLandmarksTag))
+        << "Same number of input and output landmarks is required.";
+
+    for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag);
+         id != cc->Inputs().EndId(kLandmarksTag); ++id) {
+      cc->Inputs().Get(id).Set<NormalizedLandmarkList>();
+    }
     cc->Inputs().Tag(kLetterboxPaddingTag).Set<std::array<float, 4>>();
 
-    cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
+    for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag);
+         id != cc->Outputs().EndId(kLandmarksTag); ++id) {
+      cc->Outputs().Get(id).Set<NormalizedLandmarkList>();
+    }
 
     return ::mediapipe::OkStatus();
   }
@@ -89,21 +94,28 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Process(CalculatorContext* cc) override {
-    // Only process if there's input landmarks.
+    if (cc->Inputs().Tag(kLetterboxPaddingTag).IsEmpty()) {
-    if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) {
       return ::mediapipe::OkStatus();
     }
-
-    const NormalizedLandmarkList& input_landmarks =
-        cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
     const auto& letterbox_padding =
         cc->Inputs().Tag(kLetterboxPaddingTag).Get<std::array<float, 4>>();
-
     const float left = letterbox_padding[0];
     const float top = letterbox_padding[1];
     const float left_and_right = letterbox_padding[0] + letterbox_padding[2];
     const float top_and_bottom = letterbox_padding[1] + letterbox_padding[3];
 
+    CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag);
+    CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag);
+    // Number of inputs and outpus is the same according to the contract.
+    for (; input_id != cc->Inputs().EndId(kLandmarksTag);
+         ++input_id, ++output_id) {
+      const auto& input_packet = cc->Inputs().Get(input_id);
+      if (input_packet.IsEmpty()) {
+        continue;
+      }
+
+      const NormalizedLandmarkList& input_landmarks =
+          input_packet.Get<NormalizedLandmarkList>();
       NormalizedLandmarkList output_landmarks;
       for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
         const NormalizedLandmark& landmark = input_landmarks.landmark(i);
@@ -117,10 +129,10 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
         new_landmark->set_z(landmark.z());
       }
 
-    cc->Outputs()
+      cc->Outputs().Get(output_id).AddPacket(
-        .Tag(kLandmarksTag)
+          MakePacket<NormalizedLandmarkList>(output_landmarks)
-        .AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
               .At(cc->InputTimestamp()));
+    }
     return ::mediapipe::OkStatus();
   }
 };

mediapipe/calculators/util/landmark_projection_calculator.cc

@@ -12,20 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Copyright 2019 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
 #include <cmath>
 #include <vector>
 
@@ -63,6 +49,15 @@ constexpr char kRectTag[] = "NORM_RECT";
 //   input_stream: "NORM_RECT:rect"
 //   output_stream: "NORM_LANDMARKS:projected_landmarks"
 // }
+//
+// node {
+//   calculator: "LandmarkProjectionCalculator"
+//   input_stream: "NORM_LANDMARKS:0:landmarks_0"
+//   input_stream: "NORM_LANDMARKS:1:landmarks_1"
+//   input_stream: "NORM_RECT:rect"
+//   output_stream: "NORM_LANDMARKS:0:projected_landmarks_0"
+//   output_stream: "NORM_LANDMARKS:1:projected_landmarks_1"
+// }
 class LandmarkProjectionCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@@ -70,10 +65,20 @@ class LandmarkProjectionCalculator : public CalculatorBase {
               cc->Inputs().HasTag(kRectTag))
         << "Missing one or more input streams.";
 
-    cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
+    RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag),
+                 cc->Outputs().NumEntries(kLandmarksTag))
+        << "Same number of input and output landmarks is required.";
+
+    for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag);
+         id != cc->Inputs().EndId(kLandmarksTag); ++id) {
+      cc->Inputs().Get(id).Set<NormalizedLandmarkList>();
+    }
     cc->Inputs().Tag(kRectTag).Set<NormalizedRect>();
 
-    cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
+    for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag);
+         id != cc->Outputs().EndId(kLandmarksTag); ++id) {
+      cc->Outputs().Get(id).Set<NormalizedLandmarkList>();
+    }
 
     return ::mediapipe::OkStatus();
   }
@@ -85,17 +90,25 @@ class LandmarkProjectionCalculator : public CalculatorBase {
   }
 
   ::mediapipe::Status Process(CalculatorContext* cc) override {
-    const auto& options =
+    if (cc->Inputs().Tag(kRectTag).IsEmpty()) {
-        cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>();
-    // Only process if there's input landmarks.
-    if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) {
       return ::mediapipe::OkStatus();
     }
-
-    const NormalizedLandmarkList& input_landmarks =
-        cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
     const auto& input_rect = cc->Inputs().Tag(kRectTag).Get<NormalizedRect>();
 
+    const auto& options =
+        cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>();
+
+    CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag);
+    CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag);
+    // Number of inputs and outpus is the same according to the contract.
+    for (; input_id != cc->Inputs().EndId(kLandmarksTag);
+         ++input_id, ++output_id) {
+      const auto& input_packet = cc->Inputs().Get(input_id);
+      if (input_packet.IsEmpty()) {
+        continue;
+      }
+
+      const auto& input_landmarks = input_packet.Get<NormalizedLandmarkList>();
       NormalizedLandmarkList output_landmarks;
       for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
         const NormalizedLandmark& landmark = input_landmarks.landmark(i);
@@ -103,7 +116,8 @@ class LandmarkProjectionCalculator : public CalculatorBase {
 
         const float x = landmark.x() - 0.5f;
         const float y = landmark.y() - 0.5f;
-      const float angle = options.ignore_rotation() ? 0 : input_rect.rotation();
+        const float angle =
+            options.ignore_rotation() ? 0 : input_rect.rotation();
         float new_x = std::cos(angle) * x - std::sin(angle) * y;
         float new_y = std::sin(angle) * x + std::cos(angle) * y;
 
@@ -116,10 +130,10 @@ class LandmarkProjectionCalculator : public CalculatorBase {
         new_landmark->set_z(landmark.z());
       }
 
-    cc->Outputs()
+      cc->Outputs().Get(output_id).AddPacket(
-        .Tag(kLandmarksTag)
+          MakePacket<NormalizedLandmarkList>(output_landmarks)
-        .AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
               .At(cc->InputTimestamp()));
+    }
     return ::mediapipe::OkStatus();
   }
 };

mediapipe/calculators/util/local_file_pattern_contents_calculator.cc

@@ -0,0 +1,75 @@
+// Copyright 2019 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <memory>
+#include <string>
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/port/file_helpers.h"
+#include "mediapipe/framework/port/status.h"
+
+namespace mediapipe {
+// The calculator takes the path to local directory and desired file suffix to
+// mach as input side packets, and outputs the contents of those files that
+// match the pattern. Those matched files will be sent sequentially through the
+// output stream with incremental timestamp difference by 1.
+//
+// Example config:
+// node {
+//   calculator: "LocalFilePatternContentsCalculator"
+//   input_side_packet: "FILE_DIRECTORY:file_directory"
+//   input_side_packet: "FILE_SUFFIX:file_suffix"
+//   output_stream: "CONTENTS:contents"
+// }
+class LocalFilePatternContentsCalculator : public CalculatorBase {
+ public:
+  static ::mediapipe::Status GetContract(CalculatorContract* cc) {
+    cc->InputSidePackets().Tag("FILE_DIRECTORY").Set<std::string>();
+    cc->InputSidePackets().Tag("FILE_SUFFIX").Set<std::string>();
+    cc->Outputs().Tag("CONTENTS").Set<std::string>();
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Open(CalculatorContext* cc) override {
+    MP_RETURN_IF_ERROR(::mediapipe::file::MatchFileTypeInDirectory(
+        cc->InputSidePackets().Tag("FILE_DIRECTORY").Get<std::string>(),
+        cc->InputSidePackets().Tag("FILE_SUFFIX").Get<std::string>(),
+        &filenames_));
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext* cc) override {
+    if (current_output_ < filenames_.size()) {
+      auto contents = absl::make_unique<std::string>();
+      LOG(INFO) << filenames_[current_output_];
+      MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
+          filenames_[current_output_], contents.get()));
+      ++current_output_;
+      cc->Outputs()
+          .Tag("CONTENTS")
+          .Add(contents.release(), Timestamp(current_output_));
+    } else {
+      return tool::StatusStop();
+    }
+    return ::mediapipe::OkStatus();
+  }
+
+ private:
+  std::vector<std::string> filenames_;
+  int current_output_ = 0;
+};
+
+REGISTER_CALCULATOR(LocalFilePatternContentsCalculator);
+
+}  // namespace mediapipe

mediapipe/calculators/util/rect_to_render_data_calculator.cc

@@ -45,15 +45,17 @@ RenderAnnotation::Rectangle* NewRect(
 void SetRect(bool normalized, double xmin, double ymin, double width,
              double height, double rotation,
              RenderAnnotation::Rectangle* rect) {
+  if (rotation == 0.0) {
     if (xmin + width < 0.0 || ymin + height < 0.0) return;
     if (normalized) {
       if (xmin > 1.0 || ymin > 1.0) return;
     }
+  }
   rect->set_normalized(normalized);
-  rect->set_left(normalized ? std::max(xmin, 0.0) : xmin);
+  rect->set_left(xmin);
-  rect->set_top(normalized ? std::max(ymin, 0.0) : ymin);
+  rect->set_top(ymin);
-  rect->set_right(normalized ? std::min(xmin + width, 1.0) : xmin + width);
+  rect->set_right(xmin + width);
-  rect->set_bottom(normalized ? std::min(ymin + height, 1.0) : ymin + height);
+  rect->set_bottom(ymin + height);
   rect->set_rotation(rotation);
 }
 

mediapipe/calculators/video/BUILD

@@ -368,6 +368,7 @@ cc_test(
 cc_test(
     name = "tvl1_optical_flow_calculator_test",
     srcs = ["tvl1_optical_flow_calculator_test.cc"],
+    linkstatic = 1,
     deps = [
         ":tvl1_optical_flow_calculator",
         "//mediapipe/framework:calculator_framework",

mediapipe/docs/face_detection_desktop.md

@@ -259,9 +259,9 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:throttled_input_video"
+  input_stream: "IMAGE:throttled_input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }
 ```
 

mediapipe/docs/face_detection_mobile_cpu.md

@@ -229,9 +229,9 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_video_cpu"
+  input_stream: "IMAGE:input_video_cpu"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video_cpu"
+  output_stream: "IMAGE:output_video_cpu"
 }
 
 # Transfers the annotated image from CPU back to GPU memory, to be sent out of

mediapipe/docs/face_detection_mobile_gpu.md

@@ -221,8 +221,8 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:throttled_input_video"
+  input_stream: "IMAGE_GPU:throttled_input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_video"
+  output_stream: "IMAGE_GPU:output_video"
 }
 ```

mediapipe/docs/hand_detection_mobile_gpu.md

@@ -136,10 +136,10 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:throttled_input_video"
+  input_stream: "IMAGE_GPU:throttled_input_video"
   input_stream: "detection_render_data"
   input_stream: "rect_render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_video"
+  output_stream: "IMAGE_GPU:output_video"
 }
 ```
 

mediapipe/docs/hand_tracking_mobile_gpu.md

@@ -716,10 +716,10 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:input_image"
+  input_stream: "IMAGE_GPU:input_image"
   input_stream: "detection_render_data"
   input_stream: "landmark_render_data"
   input_stream: "rect_render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_image"
+  output_stream: "IMAGE_GPU:output_image"
 }
 ```

mediapipe/docs/install.md

@@ -40,7 +40,7 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (version between 0.24.1 and 1.2.1).
+2.  Install Bazel (version between 1.0.0 and 1.2.1).
 
     Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
@@ -152,7 +152,7 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-2.  Install Bazel (version between 0.24.1 and 1.2.1).
+2.  Install Bazel (version between 1.0.0 and 1.2.1).
 
     Follow the official
     [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
@@ -241,7 +241,7 @@ To build and run iOS apps:
     $ cd mediapipe
     ```
 
-3.  Install Bazel (version between 0.24.1 and 1.1.0).
+3.  Install Bazel (version between 1.0.0 and 1.1.0).
 
     Option 1. Use package manager tool to install Bazel 1.1.0
 
@@ -389,18 +389,18 @@ a video file as input.
     username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
     ```
 
-5.  Install Bazel (version between 0.24.1 and 1.2.1).
+5.  Install Bazel (version between 1.0.0 and 1.2.1).
 
     ```bash
     username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
-    https://storage.googleapis.com/bazel/0.27.0/release/bazel-0.27.0-installer-linux-x86_64.sh && \
+    https://storage.googleapis.com/bazel/1.0.0/release/bazel-1.0.0-installer-linux-x86_64.sh && \
-    sudo mkdir -p /usr/local/bazel/0.27.0 && \
+    sudo mkdir -p /usr/local/bazel/1.0.0 && \
-    chmod 755 bazel-0.27.0-installer-linux-x86_64.sh && \
+    chmod 755 bazel-1.0.0-installer-linux-x86_64.sh && \
-    sudo ./bazel-0.27.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/0.27.0 && \
+    sudo ./bazel-1.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/1.0.0 && \
-    source /usr/local/bazel/0.27.0/lib/bazel/bin/bazel-complete.bash
+    source /usr/local/bazel/1.0.0/lib/bazel/bin/bazel-complete.bash
 
-    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/0.27.0/lib/bazel/bin/bazel version && \
+    username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/1.0.0/lib/bazel/bin/bazel version && \
-    alias bazel='/usr/local/bazel/0.27.0/lib/bazel/bin/bazel'
+    alias bazel='/usr/local/bazel/1.0.0/lib/bazel/bin/bazel'
     ```
 
 6.  Checkout MediaPipe repository.

mediapipe/docs/multi_hand_tracking_mobile_gpu.md

@@ -745,11 +745,11 @@ node {
 # a vector of RenderData objects and draws each of them on the input frame.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:input_image"
+  input_stream: "IMAGE_GPU:input_image"
   input_stream: "detection_render_data"
   input_stream: "multi_hand_rects_render_data"
   input_stream: "multi_palm_rects_render_data"
   input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_image"
+  output_stream: "IMAGE_GPU:output_image"
 }
 ```

mediapipe/docs/object_detection_desktop.md

@@ -163,9 +163,9 @@ node {
 # the graph.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_video"
+  input_stream: "IMAGE:input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }
 
 # Encodes the annotated images into a video file, adopting properties specified
@@ -396,9 +396,9 @@ node {
 # the graph.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_video"
+  input_stream: "IMAGE:input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }
 
 # Encodes the annotated images into a video file, adopting properties specified

mediapipe/docs/object_detection_mobile_cpu.md

@@ -230,9 +230,9 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:throttled_input_video_cpu"
+  input_stream: "IMAGE:throttled_input_video_cpu"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video_cpu"
+  output_stream: "IMAGE:output_video_cpu"
 }
 
 # Transfers the annotated image from CPU back to GPU memory, to be sent out of

mediapipe/docs/object_detection_mobile_gpu.md

@@ -212,8 +212,8 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:throttled_input_video"
+  input_stream: "IMAGE_GPU:throttled_input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_video"
+  output_stream: "IMAGE_GPU:output_video"
 }
 ```

mediapipe/docs/object_tracking_mobile_gpu.md

@@ -467,9 +467,9 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:input_image"
+  input_stream: "IMAGE_GPU:input_image"
   input_stream: "detections_render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_image"
+  output_stream: "IMAGE_GPU:output_image"
 }
 
 ```
@@ -484,7 +484,8 @@ CPU.
 To build and run the app:
 
 ```bash
-bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu
+bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu \
+  --define MEDIAPIPE_DISABLE_GPU=1
 
 bazel-bin/mediapipe/examples/desktop/object_tracking/object_tracking_cpu \
   --calculator_graph_config_file=mediapipe/graphs/tracking/object_detection_tracking_desktop_live.pbtxt

mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt

@@ -182,8 +182,8 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:throttled_input_video"
+  input_stream: "IMAGE:throttled_input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }
 

mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt

@@ -173,7 +173,7 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:throttled_input_video"
+  input_stream: "IMAGE:throttled_input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }

mediapipe/examples/desktop/BUILD

@@ -53,7 +53,7 @@ cc_library(
 
 # Linux only.
 # Must have a GPU with EGL support:
-# ex: sudo aptitude install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
+# ex: sudo apt-get install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
 # (or similar nvidia/amd equivalent)
 cc_library(
     name = "demo_run_graph_main_gpu",

mediapipe/examples/desktop/media_sequence/charades_dataset.py

@@ -68,7 +68,7 @@ import zipfile
 from absl import app
 from absl import flags
 from absl import logging
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 from mediapipe.util.sequence import media_sequence as ms
 
 

mediapipe/examples/desktop/media_sequence/demo_dataset.py

@@ -62,7 +62,7 @@ import urllib
 from absl import app
 from absl import flags
 from absl import logging
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 from mediapipe.util.sequence import media_sequence as ms
 

mediapipe/examples/desktop/media_sequence/kinetics_dataset.py

@@ -78,7 +78,7 @@ import urllib
 from absl import app
 from absl import flags
 from absl import logging
-import tensorflow as tf
+import tensorflow.compat.v1 as tf
 
 from mediapipe.util.sequence import media_sequence as ms
 

mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py

@@ -21,6 +21,7 @@ import sys
 
 from absl import app
 from absl import flags
+import six
 import tensorflow.compat.v1 as tf
 from mediapipe.util.sequence import media_sequence as ms
 
@@ -54,7 +55,7 @@ def main(argv):
   ms.set_clip_end_timestamp(
       flags.FLAGS.clip_end_time_sec * SECONDS_TO_MICROSECONDS, metadata)
   with open('/tmp/mediapipe/metadata.pb', 'wb') as writer:
-    writer.write(metadata.SerializeToString())
+    writer.write(six.ensure_binary(metadata.SerializeToString()))
 
 
 if __name__ == '__main__':

mediapipe/framework/calculator_graph.cc

@@ -1301,7 +1301,7 @@ void PrintTimingToInfo(const std::string& label, int64 timer_value) {
                    "%02lld days, %02lld:%02lld:%02lld.%03lld (total seconds: "
                    "%lld.%06lld)",
                    days, hours, minutes, seconds, milliseconds, total_seconds,
-                   timer_value % 1000000ll);
+                   timer_value % int64{1000000});
 }
 
 bool MetricElementComparator(const std::pair<std::string, int64>& e1,

mediapipe/framework/deps/BUILD

@@ -251,6 +251,7 @@ cc_library(
         "//mediapipe/framework/port:logging",
         "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings",
     ],
 )

mediapipe/framework/deps/file_helpers.cc

@@ -105,6 +105,30 @@ namespace file {
   return ::mediapipe::OkStatus();
 }
 
+::mediapipe::Status MatchFileTypeInDirectory(
+    const std::string& directory, const std::string& file_suffix,
+    std::vector<std::string>* results) {
+  DIR* dir = opendir(directory.c_str());
+  CHECK(dir);
+  // Iterates through the direcotry.
+  while (true) {
+    struct dirent* dir_ent = readdir(dir);
+    if (dir_ent == nullptr) {
+      break;
+    }
+    if (std::string(dir_ent->d_name) == "." ||
+        std::string(dir_ent->d_name) == "..") {
+      continue;
+    }
+
+    if (absl::EndsWith(std::string(dir_ent->d_name), file_suffix)) {
+      results->push_back(JoinPath(directory, std::string(dir_ent->d_name)));
+    }
+  }
+  closedir(dir);
+  return ::mediapipe::OkStatus();
+}
+
 ::mediapipe::Status Exists(absl::string_view file_name) {
   struct stat buffer;
   int status;

mediapipe/framework/deps/file_helpers.h

@@ -30,6 +30,10 @@ namespace file {
     const std::string& parent_directory, const std::string& file_name,
     std::vector<std::string>* results);
 
+::mediapipe::Status MatchFileTypeInDirectory(const std::string& directory,
+                                             const std::string& file_suffix,
+                                             std::vector<std::string>* results);
+
 ::mediapipe::Status Exists(absl::string_view file_name);
 
 }  // namespace file

mediapipe/framework/deps/status.cc

@@ -18,103 +18,6 @@
 
 namespace mediapipe {
 
-Status::Status(::mediapipe::StatusCode code, absl::string_view msg) {
-  state_ = std::unique_ptr<State>(new State);
-  state_->code = code;
-  state_->msg = std::string(msg);
-}
-
-void Status::Update(const Status& new_status) {
-  if (ok()) {
-    *this = new_status;
-  }
-}
-
-void Status::SlowCopyFrom(const State* src) {
-  if (src == nullptr) {
-    state_ = nullptr;
-  } else {
-    state_ = std::unique_ptr<State>(new State(*src));
-  }
-}
-
-const std::string& Status::empty_string() {
-  static std::string* empty = new std::string;
-  return *empty;
-}
-
-std::string Status::ToString() const {
-  if (state_ == nullptr) {
-    return "OK";
-  } else {
-    char tmp[30];
-    const char* type;
-    switch (code()) {
-      case ::mediapipe::StatusCode::kCancelled:
-        type = "Cancelled";
-        break;
-      case ::mediapipe::StatusCode::kUnknown:
-        type = "Unknown";
-        break;
-      case ::mediapipe::StatusCode::kInvalidArgument:
-        type = "Invalid argument";
-        break;
-      case ::mediapipe::StatusCode::kDeadlineExceeded:
-        type = "Deadline exceeded";
-        break;
-      case ::mediapipe::StatusCode::kNotFound:
-        type = "Not found";
-        break;
-      case ::mediapipe::StatusCode::kAlreadyExists:
-        type = "Already exists";
-        break;
-      case ::mediapipe::StatusCode::kPermissionDenied:
-        type = "Permission denied";
-        break;
-      case ::mediapipe::StatusCode::kUnauthenticated:
-        type = "Unauthenticated";
-        break;
-      case ::mediapipe::StatusCode::kResourceExhausted:
-        type = "Resource exhausted";
-        break;
-      case ::mediapipe::StatusCode::kFailedPrecondition:
-        type = "Failed precondition";
-        break;
-      case ::mediapipe::StatusCode::kAborted:
-        type = "Aborted";
-        break;
-      case ::mediapipe::StatusCode::kOutOfRange:
-        type = "Out of range";
-        break;
-      case ::mediapipe::StatusCode::kUnimplemented:
-        type = "Unimplemented";
-        break;
-      case ::mediapipe::StatusCode::kInternal:
-        type = "Internal";
-        break;
-      case ::mediapipe::StatusCode::kUnavailable:
-        type = "Unavailable";
-        break;
-      case ::mediapipe::StatusCode::kDataLoss:
-        type = "Data loss";
-        break;
-      default:
-        snprintf(tmp, sizeof(tmp), "Unknown code(%d)",
-                 static_cast<int>(code()));
-        type = tmp;
-        break;
-    }
-    std::string result(type);
-    result += ": ";
-    result += state_->msg;
-    return result;
-  }
-}
-
-void Status::IgnoreError() const {
-  // no-op
-}
-
 std::ostream& operator<<(std::ostream& os, const Status& x) {
   os << x.ToString();
   return os;

mediapipe/framework/deps/status.h

@@ -20,126 +20,16 @@
 #include <memory>
 #include <string>
 
+#include "absl/status/status.h"
 #include "absl/strings/string_view.h"
 #include "mediapipe/framework/port/logging.h"
 
 namespace mediapipe {
 
-enum class StatusCode {
+using Status = absl::Status;
-  kOk = 0,
+using StatusCode = absl::StatusCode;
-  kCancelled = 1,
-  kUnknown = 2,
-  kInvalidArgument = 3,
-  kDeadlineExceeded = 4,
-  kNotFound = 5,
-  kAlreadyExists = 6,
-  kPermissionDenied = 7,
-  kResourceExhausted = 8,
-  kFailedPrecondition = 9,
-  kAborted = 10,
-  kOutOfRange = 11,
-  kUnimplemented = 12,
-  kInternal = 13,
-  kUnavailable = 14,
-  kDataLoss = 15,
-  kUnauthenticated = 16,
-  kDoNotUseReservedForFutureExpansionUseDefaultInSwitchInstead_ = 20
-};
 
-#if defined(__clang__)
+inline ::mediapipe::Status OkStatus() { return absl::OkStatus(); }
-// Only clang supports warn_unused_result as a type annotation.
-class ABSL_MUST_USE_RESULT Status;
-#endif
-
-// Denotes success or failure of a call in MediaPipe.
-class Status {
- public:
-  // Creates a success status.
-  Status() {}
-
-  // Creates a status with the specified error code and msg as a
-  // human-readable std::string containing more detailed information.
-  Status(::mediapipe::StatusCode code, absl::string_view msg);
-
-  // Copies the specified status.
-  Status(const Status& s);
-  void operator=(const Status& s);
-
-  // Returns true iff the status indicates success.
-  bool ok() const {
-    return (state_ == NULL) || (state_->code == ::mediapipe::StatusCode::kOk);
-  }
-
-  ::mediapipe::StatusCode code() const {
-    return ok() ? ::mediapipe::StatusCode::kOk : state_->code;
-  }
-
-  const std::string& error_message() const {
-    return ok() ? empty_string() : state_->msg;
-  }
-
-  absl::string_view message() const {
-    return absl::string_view(error_message());
-  }
-
-  bool operator==(const Status& x) const;
-  bool operator!=(const Status& x) const;
-
-  // If `ok()`, stores `new_status` into `*this`.  If `!ok()`,
-  // preserves the current status, but may augment with additional
-  // information about `new_status`.
-  //
-  // Convenient way of keeping track of the first error encountered.
-  // Instead of:
-  //   `if (overall_status.ok()) overall_status = new_status`
-  // Use:
-  //   `overall_status.Update(new_status);`
-  void Update(const Status& new_status);
-
-  // Returns a std::string representation of this status suitable for
-  // printing. Returns the std::string `"OK"` for success.
-  std::string ToString() const;
-
-  // Ignores any errors. This method does nothing except potentially suppress
-  // complaints from any tools that are checking that errors are not dropped on
-  // the floor.
-  void IgnoreError() const;
-
- private:
-  static const std::string& empty_string();
-  struct State {
-    ::mediapipe::StatusCode code;
-    std::string msg;
-  };
-  // OK status has a `NULL` state_.  Otherwise, `state_` points to
-  // a `State` structure containing the error code and message(s)
-  std::unique_ptr<State> state_;
-
-  void SlowCopyFrom(const State* src);
-};
-
-inline Status::Status(const Status& s)
-    : state_((s.state_ == NULL) ? NULL : new State(*s.state_)) {}
-
-inline void Status::operator=(const Status& s) {
-  // The following condition catches both aliasing (when this == &s),
-  // and the common case where both s and *this are ok.
-  if (state_ != s.state_) {
-    SlowCopyFrom(s.state_.get());
-  }
-}
-
-inline bool Status::operator==(const Status& x) const {
-  return (this->state_ == x.state_) || (ToString() == x.ToString());
-}
-
-inline bool Status::operator!=(const Status& x) const { return !(*this == x); }
-
-inline Status OkStatus() { return Status(); }
-
-std::ostream& operator<<(std::ostream& os, const Status& x);
-
-typedef std::function<void(const Status&)> StatusCallback;
 
 extern std::string* MediaPipeCheckOpHelperOutOfLine(
     const ::mediapipe::Status& v, const char* msg);

mediapipe/framework/deps/status_builder.cc

@@ -72,12 +72,12 @@ StatusBuilder::operator Status() && {
   std::string message;
   if (join_style_ == MessageJoinStyle::kAnnotate) {
     if (!status_.ok()) {
-      message = absl::StrCat(status_.error_message(), "; ", stream_->str());
+      message = absl::StrCat(status_.message(), "; ", stream_->str());
     }
   } else {
     message = join_style_ == MessageJoinStyle::kPrepend
-                  ? absl::StrCat(stream_->str(), status_.error_message())
+                  ? absl::StrCat(stream_->str(), status_.message())
-                  : absl::StrCat(status_.error_message(), stream_->str());
+                  : absl::StrCat(status_.message(), stream_->str());
   }
   return Status(status_.code(), message);
 }

mediapipe/framework/deps/status_builder_test.cc

@@ -27,7 +27,7 @@ TEST(StatusBuilder, AnnotateMode) {
       << "annotated message2";
   ASSERT_FALSE(status.ok());
   EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kNotFound);
-  EXPECT_EQ(status.error_message(),
+  EXPECT_EQ(status.message(),
             "original message; annotated message1 annotated message2");
 }
 
@@ -42,7 +42,7 @@ TEST(StatusBuilder, PrependMode) {
       << "prepended message2 ";
   ASSERT_FALSE(status.ok());
   EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kInvalidArgument);
-  EXPECT_EQ(status.error_message(),
+  EXPECT_EQ(status.message(),
             "prepended message1 prepended message2 original message");
 }
 
@@ -56,8 +56,7 @@ TEST(StatusBuilder, AppendMode) {
       << " extra message2";
   ASSERT_FALSE(status.ok());
   EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kInternal);
-  EXPECT_EQ(status.error_message(),
+  EXPECT_EQ(status.message(), "original message extra message1 extra message2");
-            "original message extra message1 extra message2");
 }
 
 TEST(StatusBuilder, NoLoggingMode) {
@@ -69,7 +68,7 @@ TEST(StatusBuilder, NoLoggingMode) {
       << " extra message";
   ASSERT_FALSE(status.ok());
   EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kUnavailable);
-  EXPECT_EQ(status.error_message(), "original message");
+  EXPECT_EQ(status.message(), "original message");
 }
 
 }  // namespace mediapipe

mediapipe/framework/deps/status_test.cc

@@ -21,7 +21,7 @@ namespace mediapipe {
 
 TEST(Status, OK) {
   EXPECT_EQ(OkStatus().code(), ::mediapipe::StatusCode::kOk);
-  EXPECT_EQ(OkStatus().error_message(), "");
+  EXPECT_EQ(OkStatus().message(), "");
   MP_EXPECT_OK(OkStatus());
   MP_ASSERT_OK(OkStatus());
   EXPECT_EQ(OkStatus(), Status());
@@ -38,7 +38,7 @@ TEST(Status, Set) {
   Status status;
   status = Status(::mediapipe::StatusCode::kCancelled, "Error message");
   EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kCancelled);
-  EXPECT_EQ(status.error_message(), "Error message");
+  EXPECT_EQ(status.message(), "Error message");
 }
 
 TEST(Status, Copy) {

mediapipe/framework/deps/statusor_test.cc

@@ -158,12 +158,12 @@ TEST(StatusOr, TestMoveWithValuesAndErrors) {
   // Overwrite the value in status_or with an error.
   status_or = std::move(error1);
   ASSERT_FALSE(status_or.ok());
-  EXPECT_EQ("error1", status_or.status().error_message());
+  EXPECT_EQ("error1", status_or.status().message());
 
   // Overwrite the error in status_or with another error.
   status_or = std::move(error2);
   ASSERT_FALSE(status_or.ok());
-  EXPECT_EQ("error2", status_or.status().error_message());
+  EXPECT_EQ("error2", status_or.status().message());
 
   // Overwrite the error with a value.
   status_or = std::move(value2);
@@ -191,12 +191,12 @@ TEST(StatusOr, TestCopyWithValuesAndErrors) {
   // Overwrite the value in status_or with an error.
   status_or = error1;
   ASSERT_FALSE(status_or.ok());
-  EXPECT_EQ("error1", status_or.status().error_message());
+  EXPECT_EQ("error1", status_or.status().message());
 
   // Overwrite the error in status_or with another error.
   status_or = error2;
   ASSERT_FALSE(status_or.ok());
-  EXPECT_EQ("error2", status_or.status().error_message());
+  EXPECT_EQ("error2", status_or.status().message());
 
   // Overwrite the error with a value.
   status_or = value2;
@@ -205,8 +205,8 @@ TEST(StatusOr, TestCopyWithValuesAndErrors) {
 
   // Verify original values unchanged.
   EXPECT_EQ(std::string(1000, '1'), value1.ValueOrDie());
-  EXPECT_EQ("error1", error1.status().error_message());
+  EXPECT_EQ("error1", error1.status().message());
-  EXPECT_EQ("error2", error2.status().error_message());
+  EXPECT_EQ("error2", error2.status().message());
   EXPECT_EQ(std::string(1000, '2'), value2.ValueOrDie());
 }
 

mediapipe/framework/encode_binary_proto.bzl

@@ -36,6 +36,11 @@ def _canonicalize_proto_path_oss(all_protos, genfile_path):
     for s in all_protos.to_list():
         if s.path.startswith(genfile_path):
             repo_name, _, file_name = s.path[len(genfile_path + "/external/"):].partition("/")
+
+            # handle virtual imports
+            if file_name.startswith("_virtual_imports"):
+                repo_name = repo_name + "/" + "/".join(file_name.split("/", 2)[:2])
+                file_name = file_name.split("/", 2)[-1]
             proto_paths.append(genfile_path + "/external/" + repo_name)
             proto_file_names.append(file_name)
         else:

mediapipe/framework/formats/BUILD

@@ -268,7 +268,7 @@ java_lite_proto_library(
     visibility = [
         "//mediapipe:__subpackages__",
     ],
-    deps = [":landmark_proto"],
+    deps = [":rect_proto"],
 )
 
 proto_library(

mediapipe/framework/formats/rect.proto

@@ -16,6 +16,9 @@ syntax = "proto2";
 
 package mediapipe;
 
+option java_package = "com.google.mediapipe.formats.proto";
+option java_outer_classname = "RectProto";
+
 // A rectangle with rotation in image coordinates.
 message Rect {
   // Location of the center of the rectangle in image coordinates.
@@ -27,7 +30,7 @@ message Rect {
   required int32 height = 3;
   required int32 width = 4;
 
-  // Rotation angle is counter-clockwise in radian.
+  // Rotation angle is clockwise in radians.
   optional float rotation = 5 [default = 0.0];
 
   // Optional unique id to help associate different Rects to each other.
@@ -46,7 +49,7 @@ message NormalizedRect {
   required float height = 3;
   required float width = 4;
 
-  // Rotation angle is counter-clockwise in radian.
+  // Rotation angle is clockwise in radians.
   optional float rotation = 5 [default = 0.0];
 
   // Optional unique id to help associate different NormalizedRects to each

mediapipe/framework/graph_validation_test.cc

@@ -325,10 +325,8 @@ class OptionalSideInputTestCalculator : public CalculatorBase {
  public:
   static ::mediapipe::Status GetContract(CalculatorContract* cc) {
     cc->InputSidePackets().Tag("SIDEINPUT").Set<std::string>().Optional();
-    if (!cc->Outputs().HasTag("OUTPUT")) {
+    cc->Inputs().Tag("SELECT").Set<int>().Optional();
-      return ::mediapipe::InvalidArgumentError(
+    cc->Inputs().Tag("ENABLE").Set<bool>().Optional();
-          "Expected std::string as output.");
-    }
     cc->Outputs().Tag("OUTPUT").Set<std::string>();
     return ::mediapipe::OkStatus();
   }
@@ -394,5 +392,64 @@ TEST(GraphValidationTest, OptionalInputNotProvidedForSubgraphCalculator) {
   MP_EXPECT_OK(graph_1.WaitUntilDone());
 }
 
+TEST(GraphValidationTest, MultipleOptionalInputsForSubgraph) {
+  // A subgraph defining one optional side-packet and two optional inputs.
+  auto config_1 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    type: "PassThroughGraph"
+    input_side_packet: "INPUT:input_0"
+    input_stream: "SELECT:select"
+    input_stream: "ENABLE:enable"
+    output_stream: "OUTPUT:output_0"
+    node {
+      calculator: "OptionalSideInputTestCalculator"
+      input_side_packet: "SIDEINPUT:input_0"  # std::string
+      input_stream: "SELECT:select"
+      input_stream: "ENABLE:enable"
+      output_stream: "OUTPUT:output_0"  # std::string
+    }
+  )");
+
+  // An enclosing graph that specifies just one optional input.
+  auto config_2 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+    input_side_packet: "INPUT:foo_in"
+    input_stream: "SELECT:foo_select"
+    output_stream: "OUTPUT:foo_out"
+    node {
+      calculator: "PassThroughGraph"
+      input_stream: "SELECT:foo_select"
+      output_stream: "OUTPUT:foo_out"  # std::string
+    }
+  )");
+
+  GraphValidation validation_1;
+  MP_ASSERT_OK(validation_1.Validate({config_1, config_2}, {}));
+  CalculatorGraph graph_1;
+  MP_ASSERT_OK(graph_1.Initialize({config_1, config_2}, {}));
+  EXPECT_THAT(
+      graph_1.Config(),
+
+      // The expanded graph includes only the specified input, "SELECT".
+      // Without the fix to RemoveIgnoredStreams(), the expanded graph
+      // includes the wrong input.
+      EqualsProto(::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
+        input_side_packet: "INPUT:foo_in"
+        input_stream: "SELECT:foo_select"
+        output_stream: "OUTPUT:foo_out"
+        node {
+          calculator: "OptionalSideInputTestCalculator"
+          name: "passthroughgraph__OptionalSideInputTestCalculator"
+          input_stream: "SELECT:foo_select"
+          output_stream: "OUTPUT:foo_out"
+        }
+        executor {}
+      )")));
+
+  std::map<std::string, Packet> side_packets;
+  side_packets.insert({"foo_in", mediapipe::Adopt(new std::string("input"))});
+  MP_EXPECT_OK(graph_1.StartRun(side_packets));
+  MP_EXPECT_OK(graph_1.CloseAllPacketSources());
+  MP_EXPECT_OK(graph_1.WaitUntilDone());
+}
+
 }  // namespace
 }  // namespace mediapipe

mediapipe/framework/packet.cc

@@ -40,6 +40,13 @@ Packet Create(HolderBase* holder, Timestamp timestamp) {
   return result;
 }
 
+Packet Create(std::shared_ptr<HolderBase> holder, Timestamp timestamp) {
+  Packet result;
+  result.holder_ = std::move(holder);
+  result.timestamp_ = timestamp;
+  return result;
+}
+
 const HolderBase* GetHolder(const Packet& packet) {
   return packet.holder_.get();
 }

mediapipe/framework/packet.h

@@ -48,7 +48,9 @@ class HolderBase;
 
 Packet Create(HolderBase* holder);
 Packet Create(HolderBase* holder, Timestamp timestamp);
+Packet Create(std::shared_ptr<HolderBase> holder, Timestamp timestamp);
 const HolderBase* GetHolder(const Packet& packet);
+const std::shared_ptr<HolderBase>& GetHolderShared(const Packet& packet);
 }  // namespace packet_internal
 
 // A generic container class which can hold data of any type.  The type of
@@ -201,8 +203,14 @@ class Packet {
   friend Packet packet_internal::Create(packet_internal::HolderBase* holder);
   friend Packet packet_internal::Create(packet_internal::HolderBase* holder,
                                         class Timestamp timestamp);
+  friend Packet packet_internal::Create(
+      std::shared_ptr<packet_internal::HolderBase> holder,
+      class Timestamp timestamp);
   friend const packet_internal::HolderBase* packet_internal::GetHolder(
       const Packet& packet);
+  friend const std::shared_ptr<packet_internal::HolderBase>&
+  packet_internal::GetHolderShared(const Packet& packet);
+
   std::shared_ptr<packet_internal::HolderBase> holder_;
   class Timestamp timestamp_;
 };
@@ -713,6 +721,15 @@ inline bool operator!=(const Packet& p1, const Packet& p2) {
   return !(p1 == p2);
 }
 
+namespace packet_internal {
+
+inline const std::shared_ptr<HolderBase>& GetHolderShared(
+    const Packet& packet) {
+  return packet.holder_;
+}
+
+}  // namespace packet_internal
+
 }  // namespace mediapipe
 
 #endif  // MEDIAPIPE_FRAMEWORK_PACKET_H_

mediapipe/framework/port.h

@@ -50,4 +50,34 @@
 #define MEDIAPIPE_DISABLE_GL_COMPUTE
 #endif
 
+// Compile time target platform definitions.
+// Example: #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
+#define MEDIAPIPE_OPENGL_ES_UNSUPPORTED 0
+#define MEDIAPIPE_OPENGL_ES_20 200
+#define MEDIAPIPE_OPENGL_ES_31 310
+
+#if defined(MEDIAPIPE_DISABLE_GPU)
+#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
+#define MEDIAPIPE_METAL_ENABLED 0
+#else
+#if defined(MEDIAPIPE_ANDROID)
+#if defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
+#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_20
+#else
+#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_31
+#endif
+#define MEDIAPIPE_METAL_ENABLED 0
+#elif defined(MEDIAPIPE_IOS)
+#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_20
+#define MEDIAPIPE_METAL_ENABLED 1
+#elif defined(MEDIAPIPE_OSX)
+#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
+#define MEDIAPIPE_METAL_ENABLED 1
+#else
+// GPU is not supported on Linux yet.
+#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
+#define MEDIAPIPE_METAL_ENABLED 0
+#endif
+#endif
+
 #endif  // MEDIAPIPE_FRAMEWORK_PORT_H_

mediapipe/framework/port/BUILD

@@ -351,6 +351,7 @@ cc_library(
         ":source_location",
         "//mediapipe/framework:port",
         "//mediapipe/framework/deps:status",
+        "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings",
     ],
 )

mediapipe/framework/profiler/graph_tracer_test.cc

@@ -1267,9 +1267,9 @@ TEST_F(GraphTracerE2ETest, GpuTracing) {
         output_stream: "annotated_buffer"
         node {
           calculator: "AnnotationOverlayCalculator"
-          input_stream: "INPUT_FRAME:input_buffer"
+          input_stream: "IMAGE:input_buffer"
           input_stream: "render_data"
-          output_stream: "OUTPUT_FRAME:annotated_buffer"
+          output_stream: "IMAGE:annotated_buffer"
         }
         profiler_config {
           trace_enabled: true

mediapipe/framework/profiler/profiler_resource_util_apple.cc

@@ -28,7 +28,7 @@ StatusOr<std::string> GetDefaultTraceLogDirectory() {
 
   // Note: "createDirectoryAtURL:..." method doesn't successfully create
   // the directory, hence this code uses "createDirectoryAtPath:..".
-  NSString* ns_documents_directory = [documents_directory_url absoluteString];
+  NSString* ns_documents_directory = [documents_directory_url path];
   NSError* error;
   BOOL success = [[NSFileManager defaultManager]
             createDirectoryAtPath:ns_documents_directory

mediapipe/framework/profiler/testdata/BUILD

@@ -0,0 +1,24 @@
+load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
+
+package(default_visibility = ["//mediapipe/framework/profiler:__subpackages__"])
+
+licenses(["notice"])
+
+# Compile any proto data into wire format for use in our tests.
+[encode_binary_proto(
+    name = graph.split("/")[-1].rsplit(".", 1)[0],
+    input = graph,
+    message_type = "mediapipe.GraphProfile",
+    output = graph.rsplit(".", 1)[0] + ".binarypb",
+    deps = ["//mediapipe/framework:calculator_profile_proto"],
+) for graph in glob(["profile_*.pbtxt"])]
+
+filegroup(
+    name = "mediapipe_profile_graphs",
+    srcs = [binarypb.rsplit(".", 1)[0] + ".binarypb" for binarypb in glob(["profile_*.pbtxt"])],
+)
+
+filegroup(
+    name = "pbtxt_files",
+    srcs = glob(["*.pbtxt"]),
+)

mediapipe/framework/tool/BUILD

@@ -177,7 +177,7 @@ cc_library(
     deps = [
         "//mediapipe/framework:packet",
         "//mediapipe/framework/port:statusor",
-        "@org_tensorflow//tensorflow/core:protos_all",
+        "@org_tensorflow//tensorflow/core:protos_all_cc",
     ],
 )
 

mediapipe/framework/tool/status_util_test.cc

@@ -52,8 +52,10 @@ TEST(StatusTest, CombinedStatus) {
   errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument,
                       "error_with_that_string");
   status = tool::CombinedStatus(prefix_error_message, errors);
-  EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[0].error_message()));
+  EXPECT_THAT(status.ToString(),
-  EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[1].error_message()));
+              testing::HasSubstr(std::string(errors[0].message())));
+  EXPECT_THAT(status.ToString(),
+              testing::HasSubstr(std::string(errors[1].message())));
   EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message));
   EXPECT_EQ(::mediapipe::StatusCode::kInvalidArgument, status.code());
 
@@ -63,8 +65,10 @@ TEST(StatusTest, CombinedStatus) {
   errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument,
                       "error_with_that_string");
   status = tool::CombinedStatus(prefix_error_message, errors);
-  EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[0].error_message()));
+  EXPECT_THAT(status.ToString(),
-  EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[1].error_message()));
+              testing::HasSubstr(std::string(errors[0].message())));
+  EXPECT_THAT(status.ToString(),
+              testing::HasSubstr(std::string(errors[1].message())));
   EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message));
   EXPECT_EQ(::mediapipe::StatusCode::kUnknown, status.code());
   errors.clear();
@@ -72,7 +76,8 @@ TEST(StatusTest, CombinedStatus) {
   errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument,
                       "error_with_that_string");
   status = tool::CombinedStatus(prefix_error_message, errors);
-  EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[1].error_message()));
+  EXPECT_THAT(status.ToString(),
+              testing::HasSubstr(std::string(errors[1].message())));
   EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message));
   EXPECT_EQ(::mediapipe::StatusCode::kInvalidArgument, status.code());
 

mediapipe/framework/tool/subgraph_expansion.cc

@@ -76,10 +76,11 @@ namespace tool {
 ::mediapipe::Status RemoveIgnoredStreams(
     proto_ns::RepeatedPtrField<ProtoString>* streams,
     const std::set<std::string>& missing_streams) {
-  ASSIGN_OR_RETURN(auto src_map, tool::TagMap::Create(*streams));
-  std::vector<std::string> src_names = src_map->Names();
   for (int i = streams->size() - 1; i >= 0; --i) {
-    if (missing_streams.count(src_names[i]) > 0) {
+    std::string tag, name;
+    int index;
+    MP_RETURN_IF_ERROR(ParseTagIndexName(streams->Get(i), &tag, &index, &name));
+    if (missing_streams.count(name) > 0) {
       streams->DeleteSubrange(i, 1);
     }
   }

mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt

@@ -177,8 +177,8 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:throttled_input_video"
+  input_stream: "IMAGE:throttled_input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }
 

mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt

@@ -188,9 +188,9 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_video_cpu"
+  input_stream: "IMAGE:input_video_cpu"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video_cpu"
+  output_stream: "IMAGE:output_video_cpu"
 }
 
 # Transfers the annotated image from CPU back to GPU memory, to be sent out of

mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt

@@ -178,7 +178,7 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:throttled_input_video"
+  input_stream: "IMAGE_GPU:throttled_input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_video"
+  output_stream: "IMAGE_GPU:output_video"
 }

mediapipe/graphs/hand_tracking/BUILD

@@ -21,6 +21,10 @@ licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:public"])
 
+exports_files(glob([
+    "*.pbtxt",
+]))
+
 cc_library(
     name = "desktop_offline_calculators",
     deps = [

mediapipe/graphs/hand_tracking/hand_detection_desktop.pbtxt

@@ -41,9 +41,9 @@ node {
 # the graph.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_video"
+  input_stream: "IMAGE:input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }
 
 # Encodes the annotated images into a video file, adopting properties specified

mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt

@@ -32,7 +32,7 @@ node {
 # the graph.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_video"
+  input_stream: "IMAGE:input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }

mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt

@@ -66,8 +66,8 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:throttled_input_video"
+  input_stream: "IMAGE_GPU:throttled_input_video"
   input_stream: "detection_render_data"
   input_stream: "rect_render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_video"
+  output_stream: "IMAGE_GPU:output_video"
 }

mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_cpu.pbtxt

@@ -14,6 +14,11 @@ node {
   input_stream: "IMAGE:input_video"
   input_stream: "NORM_RECT:hand_rect"
   output_stream: "IMAGE:hand_image"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
+      border_mode: BORDER_REPLICATE
+    }
+  }
 }
 
 # Transforms the input image on CPU to a 256x256 image. To scale the input
@@ -33,13 +38,9 @@ node: {
   }
 }
 
-# Converts the transformed input image on GPU into an image tensor stored in
+# Converts the transformed input image on CPU into an image tensor stored in
-# tflite::gpu::GlBuffer. The zero_center option is set to true to normalize the
+# TfliteTensor. The zero_center option is set to false to normalize the
-# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f]. The flip_vertically
+# pixel values to [0.f, 1.f].
-# option is set to true to account for the descrepancy between the
-# representation of the input image (origin at the bottom-left corner, the
-# OpenGL convention) and what the model used in this graph is expecting (origin
-# at the top-left corner).
 node {
   calculator: "TfLiteConverterCalculator"
   input_stream: "IMAGE:transformed_input_video"

mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt

@@ -14,6 +14,11 @@ node {
   input_stream: "IMAGE_GPU:input_video"
   input_stream: "NORM_RECT:hand_rect"
   output_stream: "IMAGE_GPU:hand_image"
+  node_options: {
+    [type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
+      border_mode: BORDER_REPLICATE
+    }
+  }
 }
 
 # Transforms the input image on GPU to a 256x256 image. To scale the input

mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_cpu.pbtxt

@@ -135,10 +135,10 @@ node {
 # a vector of RenderData objects and draws each of them on the input frame.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_image"
+  input_stream: "IMAGE:input_image"
   input_stream: "detection_render_data"
   input_stream: "multi_hand_rects_render_data"
   input_stream: "multi_palm_rects_render_data"
   input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
-  output_stream: "OUTPUT_FRAME:output_image"
+  output_stream: "IMAGE:output_image"
 }

mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt

@@ -135,10 +135,10 @@ node {
 # a vector of RenderData objects and draws each of them on the input frame.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:input_image"
+  input_stream: "IMAGE_GPU:input_image"
   input_stream: "detection_render_data"
   input_stream: "multi_hand_rects_render_data"
   input_stream: "multi_palm_rects_render_data"
   input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_image"
+  output_stream: "IMAGE_GPU:output_image"
 }

mediapipe/graphs/hand_tracking/subgraphs/renderer_cpu.pbtxt

@@ -94,9 +94,9 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_image"
+  input_stream: "IMAGE:input_image"
   input_stream: "detection_render_data"
   input_stream: "landmark_render_data"
   input_stream: "rect_render_data"
-  output_stream: "OUTPUT_FRAME:output_image"
+  output_stream: "IMAGE:output_image"
 }

mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt

@@ -94,9 +94,9 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:input_image"
+  input_stream: "IMAGE_GPU:input_image"
   input_stream: "detection_render_data"
   input_stream: "landmark_render_data"
   input_stream: "rect_render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_image"
+  output_stream: "IMAGE_GPU:output_image"
 }

mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt

@@ -168,7 +168,7 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:throttled_input_video"
+  input_stream: "IMAGE:throttled_input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }

mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt

@@ -109,9 +109,9 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_video"
+  input_stream: "IMAGE:input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }
 
 # Encodes the annotated images into a video file, adopting properties specified

mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt

@@ -159,9 +159,9 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_video"
+  input_stream: "IMAGE:input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }
 
 # Encodes the annotated images into a video file, adopting properties specified

mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt

@@ -179,9 +179,9 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:throttled_input_video_cpu"
+  input_stream: "IMAGE:throttled_input_video_cpu"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME:output_video_cpu"
+  output_stream: "IMAGE:output_video_cpu"
 }
 
 # Transfers the annotated image from CPU back to GPU memory, to be sent out of

mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt

@@ -169,7 +169,7 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:throttled_input_video"
+  input_stream: "IMAGE_GPU:throttled_input_video"
   input_stream: "render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_video"
+  output_stream: "IMAGE_GPU:output_video"
 }

mediapipe/graphs/tracking/subgraphs/renderer_cpu.pbtxt

@@ -23,7 +23,7 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_image"
+  input_stream: "IMAGE:input_image"
   input_stream: "detections_render_data"
-  output_stream: "OUTPUT_FRAME:output_image"
+  output_stream: "IMAGE:output_image"
 }

mediapipe/graphs/tracking/subgraphs/renderer_gpu.pbtxt

@@ -23,7 +23,7 @@ node {
 # Draws annotations and overlays them on top of the input images.
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME_GPU:input_image"
+  input_stream: "IMAGE_GPU:input_image"
   input_stream: "detections_render_data"
-  output_stream: "OUTPUT_FRAME_GPU:output_image"
+  output_stream: "IMAGE_GPU:output_image"
 }

mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt

@@ -158,9 +158,9 @@ node {
 
 node {
   calculator: "AnnotationOverlayCalculator"
-  input_stream: "INPUT_FRAME:input_video"
+  input_stream: "IMAGE:input_video"
   input_stream: "synchronized_render_data"
-  output_stream: "OUTPUT_FRAME:output_video"
+  output_stream: "IMAGE:output_video"
 }
 
 node {