cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 6f964e58d874e47fb6207aa97d060a4cd6428527
This commit is contained in:
MediaPipe Team 2020-02-28 20:44:27 -08:00 committed by jqtang
parent de4fbc10e6
commit 252a5713c7
134 changed files with 16665 additions and 874 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
WORKSPACE
View File

@ -10,15 +10,15 @@ http_archive(
sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e",
)
load("@bazel_skylib//lib:versions.bzl", "versions")
versions.check(minimum_bazel_version = "0.24.1",
versions.check(minimum_bazel_version = "1.0.0",
maximum_bazel_version = "1.2.1")
# ABSL cpp library lts_2019_08_08.
# ABSL cpp library lts_2020_02_25
http_archive(
name = "com_google_absl",
urls = [
"https://github.com/abseil/abseil-cpp/archive/20190808.tar.gz",
"https://github.com/abseil/abseil-cpp/archive/20200225.tar.gz",
],
# Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved.
patches = [
@ -27,8 +27,8 @@ http_archive(
patch_args = [
"-p1",
],
strip_prefix = "abseil-cpp-20190808",
sha256 = "8100085dada279bf3ee00cd064d43b5f55e5d913be0dfe2906f06f8f28d5b37e"
strip_prefix = "abseil-cpp-20200225",
sha256 = "728a813291bdec2aa46eab8356ace9f75ac2ed9dfe2df5ab603c4e6c09f1c353"
)
http_archive(
@ -117,18 +117,19 @@ http_archive(
],
)
# 2019-11-21
_TENSORFLOW_GIT_COMMIT = "f482488b481a799ca07e7e2d153cf47b8e91a60c"
_TENSORFLOW_SHA256= "8d9118c2ce186c7e1403f04b96982fe72c184060c7f7a93e30a28dca358694f0"
# 2020-02-12
# The last commit before TensorFlow switched to Bazel 2.0
_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328"
_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13"
http_archive(
name = "org_tensorflow",
urls = [
"https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
"https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT,
],
# Patch https://github.com/tensorflow/tensorflow/commit/e3a7bdbebb99352351a19e2e403136166aa52934
# A compatibility patch
patches = [
"@//third_party:org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff"
"@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff"
],
patch_args = [
"-p1",

16
mediapipe/calculators/core/BUILD
View File

@ -610,6 +610,22 @@ cc_library(
alwayslink = 1,
)
cc_test(
name = "side_packet_to_stream_calculator_test",
srcs = ["side_packet_to_stream_calculator_test.cc"],
deps = [
":side_packet_to_stream_calculator",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "immediate_mux_calculator_test",
srcs = ["immediate_mux_calculator_test.cc"],

116
mediapipe/calculators/core/side_packet_to_stream_calculator.cc
View File

@ -28,55 +28,133 @@ using mediapipe::PacketTypeSet;
using mediapipe::Timestamp;
namespace {
constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
constexpr char kTagAtZero[] = "AT_ZERO";
constexpr char kTagAtTick[] = "AT_TICK";
constexpr char kTagTick[] = "TICK";
static std::map<std::string, Timestamp>* kTimestampMap = []() {
auto* res = new std::map<std::string, Timestamp>();
res->emplace("AT_PRESTREAM", Timestamp::PreStream());
res->emplace("AT_POSTSTREAM", Timestamp::PostStream());
res->emplace("AT_ZERO", Timestamp(0));
res->emplace(kTagAtPreStream, Timestamp::PreStream());
res->emplace(kTagAtPostStream, Timestamp::PostStream());
res->emplace(kTagAtZero, Timestamp(0));
res->emplace(kTagAtTick, Timestamp::Unset());
return res;
}();
template <typename CC>
std::string GetOutputTag(const CC& cc) {
// Single output tag only is required by contract.
return *cc.Outputs().GetTags().begin();
}
} // namespace
// Outputs the single input_side_packet at the timestamp specified in the
// output_stream tag. Valid tags are AT_PRESTREAM, AT_POSTSTREAM and AT_ZERO.
// Outputs side packet(s) in corresponding output stream(s) with a particular
// timestamp, depending on the tag used to define output stream(s). (One tag can
// be used only.)
//
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK and
// corresponding timestamps are Timestamp::PreStream(), Timestamp::PostStream(),
// Timestamp(0) and timestamp of a packet received in TICK input.
//
// Examples:
// node {
// calculator: "SidePacketToStreamCalculator"
// input_side_packet: "side_packet"
// output_stream: "AT_PRESTREAM:packet"
// }
//
// node {
// calculator: "SidePacketToStreamCalculator"
// input_stream: "TICK:tick"
// input_side_packet: "side_packet"
// output_stream: "AT_TICK:packet"
// }
class SidePacketToStreamCalculator : public CalculatorBase {
public:
SidePacketToStreamCalculator() = default;
~SidePacketToStreamCalculator() override = default;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
private:
bool is_tick_processing_ = false;
std::string output_tag_;
};
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
::mediapipe::Status SidePacketToStreamCalculator::GetContract(
CalculatorContract* cc) {
cc->InputSidePackets().Index(0).SetAny();
const auto& tags = cc->Outputs().GetTags();
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK tags is "
"allowed and required to specify output stream(s).";
RET_CHECK(
(cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) ||
(!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick)))
<< "Either both of TICK and AT_TICK should be used or none of them.";
const std::string output_tag = GetOutputTag(*cc);
const int num_entries = cc->Outputs().NumEntries(output_tag);
RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries())
<< "Same number of input side packets and output streams is required.";
for (int i = 0; i < num_entries; ++i) {
cc->InputSidePackets().Index(i).SetAny();
cc->Outputs()
.Get(output_tag, i)
.SetSameAs(cc->InputSidePackets().Index(i).GetSameAs());
}
std::set<std::string> tags = cc->Outputs().GetTags();
RET_CHECK_EQ(tags.size(), 1);
if (cc->Inputs().HasTag(kTagTick)) {
cc->Inputs().Tag(kTagTick).SetAny();
}
RET_CHECK_EQ(kTimestampMap->count(*tags.begin()), 1);
cc->Outputs().Tag(*tags.begin()).SetAny();
return ::mediapipe::OkStatus();
}
::mediapipe::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
output_tag_ = GetOutputTag(*cc);
if (cc->Inputs().HasTag(kTagTick)) {
is_tick_processing_ = true;
// Set offset, so output timestamp bounds are updated in response to TICK
// timestamp bound update.
cc->SetOffset(TimestampDiff(0));
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status SidePacketToStreamCalculator::Process(
CalculatorContext* cc) {
return mediapipe::tool::StatusStop();
if (is_tick_processing_) {
// TICK input is guaranteed to be non-empty, as it's the only input stream
// for this calculator.
const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
cc->Outputs()
.Get(output_tag_, i)
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
}
::mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
std::set<std::string> tags = cc->Outputs().GetTags();
RET_CHECK_EQ(tags.size(), 1);
const std::string& tag = *tags.begin();
RET_CHECK_EQ(kTimestampMap->count(tag), 1);
cc->Outputs().Tag(tag).AddPacket(
cc->InputSidePackets().Index(0).At(kTimestampMap->at(tag)));
return ::mediapipe::OkStatus();
}
return ::mediapipe::tool::StatusStop();
}
::mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
if (!cc->Outputs().HasTag(kTagAtTick)) {
const auto& timestamp = kTimestampMap->at(output_tag_);
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
cc->Outputs()
.Get(output_tag_, i)
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
}
}
return ::mediapipe::OkStatus();
}

275
mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc Normal file
View File

@ -0,0 +1,275 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <string>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/strings/match.h"
#include "absl/strings/str_replace.h"
#include "absl/strings/string_view.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/tool/options_util.h"
namespace mediapipe {
namespace {
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "AT_TICK:packet"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Either both of TICK and AT_TICK should be used or none of them.");
}
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "DOES_NOT_EXIST:packet"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(absl::StrContains, status.message(),
"Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK "
"tags is allowed and required to specify output stream(s).");
}
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_TICK:packet0"
output_stream: "AT_PRE_STREAM:packet1"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(absl::StrContains, status.message(),
"Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK "
"tags is allowed and required to specify output stream(s).");
}
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet0"
output_stream: "AT_PRESTREAM:0:packet0"
output_stream: "AT_PRESTREAM:1:packet1"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Same number of input side packets and output streams is required.");
}
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_PRESTREAM:packet0"
}
)");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_PRED2(
absl::StrContains, status.message(),
"Same number of input side packets and output streams is required.");
}
void DoTestNonAtTickOutputTag(absl::string_view tag,
Timestamp expected_timestamp) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(absl::StrReplaceAll(
R"(
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "$tag:packet"
}
)",
{{"$tag", tag}}));
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value = 10;
std::vector<Packet> output_packets;
MP_ASSERT_OK(graph.ObserveOutputStream(
"packet", [&output_packets](const Packet& packet) {
output_packets.push_back(packet);
return ::mediapipe::OkStatus();
}));
MP_ASSERT_OK(
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
MP_ASSERT_OK(graph.WaitForObservedOutput());
ASSERT_FALSE(output_packets.empty());
EXPECT_EQ(expected_timestamp, output_packets.back().Timestamp());
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
}
TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) {
DoTestNonAtTickOutputTag("AT_PRESTREAM", Timestamp::PreStream());
DoTestNonAtTickOutputTag("AT_POSTSTREAM", Timestamp::PostStream());
DoTestNonAtTickOutputTag("AT_ZERO", Timestamp(0));
}
TEST(SidePacketToStreamCalculator, AtTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet"
output_stream: "AT_TICK:packet"
}
)");
std::vector<Packet> output_packets;
tool::AddVectorSink("packet", &graph_config, &output_packets);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value = 20;
MP_ASSERT_OK(
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
auto tick_and_verify = [&graph, &output_packets,
expected_value](int at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick",
MakePacket<int>(/*doesn't matter*/ 1).At(Timestamp(at_timestamp))));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_FALSE(output_packets.empty());
EXPECT_EQ(Timestamp(at_timestamp), output_packets.back().Timestamp());
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
};
tick_and_verify(/*at_timestamp=*/0);
tick_and_verify(/*at_timestamp=*/1);
tick_and_verify(/*at_timestamp=*/128);
tick_and_verify(/*at_timestamp=*/1024);
tick_and_verify(/*at_timestamp=*/1025);
}
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
input_stream: "tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "packet0"
output_stream: "packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_TICK:0:packet0"
output_stream: "AT_TICK:1:packet1"
}
)");
std::vector<Packet> output_packets0;
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
std::vector<Packet> output_packets1;
tool::AddVectorSink("packet1", &graph_config, &output_packets1);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value0 = 20;
const int expected_value1 = 128;
MP_ASSERT_OK(
graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
{"side_packet1", MakePacket<int>(expected_value1)}}));
auto tick_and_verify = [&graph, &output_packets0, &output_packets1,
expected_value0, expected_value1](int at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick",
MakePacket<int>(/*doesn't matter*/ 1).At(Timestamp(at_timestamp))));
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_FALSE(output_packets0.empty());
ASSERT_FALSE(output_packets1.empty());
EXPECT_EQ(Timestamp(at_timestamp), output_packets0.back().Timestamp());
EXPECT_EQ(expected_value0, output_packets0.back().Get<int>());
EXPECT_EQ(Timestamp(at_timestamp), output_packets1.back().Timestamp());
EXPECT_EQ(expected_value1, output_packets1.back().Get<int>());
};
tick_and_verify(/*at_timestamp=*/0);
tick_and_verify(/*at_timestamp=*/1);
tick_and_verify(/*at_timestamp=*/128);
tick_and_verify(/*at_timestamp=*/1024);
tick_and_verify(/*at_timestamp=*/1025);
}
} // namespace
} // namespace mediapipe

4
mediapipe/calculators/image/BUILD
View File

@ -346,9 +346,7 @@ cc_library(
],
"//conditions:default": [],
}),
visibility = [
"//visibility:public",
],
visibility = ["//visibility:public"],
deps = [
":image_cropping_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",

5
mediapipe/calculators/image/color_convert_calculator.cc
View File

@ -75,6 +75,11 @@ class ColorConvertCalculator : public CalculatorBase {
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return ::mediapipe::OkStatus();
}
private:
// Wrangles the appropriate inputs and outputs to perform the color
// conversion. The ImageFrame on input_tag is converted using the

62
mediapipe/calculators/image/image_cropping_calculator.cc
View File

@ -119,6 +119,13 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
#endif // !MEDIAPIPE_DISABLE_GPU
}
// Validate border mode.
if (use_gpu_) {
MP_RETURN_IF_ERROR(ValidateBorderModeForGPU(cc));
} else {
MP_RETURN_IF_ERROR(ValidateBorderModeForCPU(cc));
}
return ::mediapipe::OkStatus();
}
@ -162,6 +169,32 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
return ::mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForCPU(
CalculatorContext* cc) {
int border_mode;
return GetBorderModeForOpenCV(cc, &border_mode);
}
::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForGPU(
CalculatorContext* cc) {
mediapipe::ImageCroppingCalculatorOptions options =
cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
switch (options.border_mode()) {
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU "
<< "implementation and will fall back into BORDER_REPLICATE";
break;
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
break;
default:
RET_CHECK_FAIL() << "Unsupported border mode for GPU: "
<< options.border_mode();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) {
if (cc->Inputs().Tag(kImageTag).IsEmpty()) {
return ::mediapipe::OkStatus();
@ -172,6 +205,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
auto [target_width, target_height, rect_center_x, rect_center_y, rotation] =
GetCropSpecs(cc, input_img.Width(), input_img.Height());
// Get border mode and value for OpenCV.
int border_mode;
MP_RETURN_IF_ERROR(GetBorderModeForOpenCV(cc, &border_mode));
const cv::RotatedRect min_rect(cv::Point2f(rect_center_x, rect_center_y),
cv::Size2f(target_width, target_height),
rotation * 180.f / M_PI);
@ -191,7 +228,9 @@ REGISTER_CALCULATOR(ImageCroppingCalculator);
cv::getPerspectiveTransform(src_points, dst_points);
cv::Mat cropped_image;
cv::warpPerspective(input_mat, cropped_image, projection_matrix,
cv::Size(min_rect.size.width, min_rect.size.height));
cv::Size(min_rect.size.width, min_rect.size.height),
/* flags = */ 0,
/* borderMode = */ border_mode);
std::unique_ptr<ImageFrame> output_frame(new ImageFrame(
input_img.Format(), cropped_image.cols, cropped_image.rows));
@ -453,6 +492,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
rotation = options.rotation();
}
}
return {
.width = crop_width,
.height = crop_height,
@ -462,4 +502,24 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc,
};
}
::mediapipe::Status ImageCroppingCalculator::GetBorderModeForOpenCV(
CalculatorContext* cc, int* border_mode) {
mediapipe::ImageCroppingCalculatorOptions options =
cc->Options<mediapipe::ImageCroppingCalculatorOptions>();
switch (options.border_mode()) {
case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO:
*border_mode = cv::BORDER_CONSTANT;
break;
case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE:
*border_mode = cv::BORDER_REPLICATE;
break;
default:
RET_CHECK_FAIL() << "Unsupported border mode for CPU: "
<< options.border_mode();
}
return ::mediapipe::OkStatus();
}
} // namespace mediapipe

5
mediapipe/calculators/image/image_cropping_calculator.h
View File

@ -36,6 +36,7 @@
// Note: input_stream values take precedence over options defined in the graph.
//
namespace mediapipe {
struct RectSpec {
int width;
int height;
@ -63,12 +64,16 @@ class ImageCroppingCalculator : public CalculatorBase {
int src_height);
private:
::mediapipe::Status ValidateBorderModeForCPU(CalculatorContext* cc);
::mediapipe::Status ValidateBorderModeForGPU(CalculatorContext* cc);
::mediapipe::Status RenderCpu(CalculatorContext* cc);
::mediapipe::Status RenderGpu(CalculatorContext* cc);
::mediapipe::Status InitGpu(CalculatorContext* cc);
void GlRender();
void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height,
int* dst_width, int* dst_height);
::mediapipe::Status GetBorderModeForOpenCV(CalculatorContext* cc,
int* border_mode);
mediapipe::ImageCroppingCalculatorOptions options_;

11
mediapipe/calculators/image/image_cropping_calculator.proto
View File

@ -40,4 +40,15 @@ message ImageCroppingCalculatorOptions {
// The (0, 0) point is at the (top, left) corner.
optional float norm_center_x = 6 [default = 0];
optional float norm_center_y = 7 [default = 0];
enum BorderMode {
// First unspecified value is required by the guideline. See details here:
// https://developers.google.com/protocol-buffers/docs/style#enums
BORDER_UNSPECIFIED = 0;
BORDER_ZERO = 1;
BORDER_REPLICATE = 2;
}
// Specifies behaviour for crops that go beyond image borders.
optional BorderMode border_mode = 8 [default = BORDER_ZERO];
}

52
mediapipe/calculators/image/image_cropping_calculator_test.cc
View File

@ -56,10 +56,10 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto cc =
CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(),
tool::CreateTagMap({}).ValueOrDie());
RectSpec expectRect = {
@ -69,8 +69,8 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) {
.center_y = 50,
.rotation = 0.3,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST
@ -96,10 +96,10 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto cc =
CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(),
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(),
tool::CreateTagMap({}).ValueOrDie());
RectSpec expectRect = {
.width = 50,
@ -108,8 +108,8 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) {
.center_y = 50,
.rotation = 0.3,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST
@ -138,16 +138,16 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto inputTags = tool::CreateTagMap({
"HEIGHT:0:crop_height",
"WIDTH:0:crop_width",
})
.ValueOrDie();
auto cc = CalculatorContext(&calculator_state, inputTags,
tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc.Inputs();
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc->Inputs();
inputs.Tag(kHeightTag).Value() = MakePacket<int>(1);
inputs.Tag(kWidthTag).Value() = MakePacket<int>(1);
RectSpec expectRect = {
@ -157,8 +157,8 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) {
.center_y = 50,
.rotation = 0.3,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST
@ -186,15 +186,15 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
}
)");
auto calculator_state =
CalculatorState("Node", 0, "Calculator", calculator_node, nullptr);
auto calculator_state = absl::make_unique<CalculatorState>(
"Node", 0, "Calculator", calculator_node, nullptr);
auto inputTags = tool::CreateTagMap({
"RECT:0:rect",
})
.ValueOrDie();
auto cc = CalculatorContext(&calculator_state, inputTags,
tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc.Inputs();
auto cc = absl::make_unique<CalculatorContext>(
calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie());
auto& inputs = cc->Inputs();
mediapipe::Rect rect = ParseTextProtoOrDie<mediapipe::Rect>(
R"(
width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5
@ -207,8 +207,8 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) {
.center_y = 40,
.rotation = 0.5,
};
EXPECT_EQ(
ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height),
EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width,
input_height),
expectRect);
} // TEST

86
mediapipe/calculators/image/image_transformation_calculator.cc
View File

@ -104,6 +104,14 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
// to be a multiple of 90 degrees. If provided, it overrides the
// ROTATION_DEGREES input side packet.
//
// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not. If
// provided, it overrides the FLIP_HORIZONTALLY input side packet and/or
// corresponding field in the calculator options.
//
// FLIP_VERTICALLY (optional): Whether to flip image vertically or not. If
// provided, it overrides the FLIP_VERTICALLY input side packet and/or
// corresponding field in the calculator options.
//
// Output:
// One of the following two tags:
// IMAGE - ImageFrame representing the output image.
@ -129,6 +137,12 @@ mediapipe::ScaleMode_Mode ParseScaleMode(
// degrees. It has to be a multiple of 90 degrees. It overrides the
// corresponding field in the calculator options.
//
// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not.
// It overrides the corresponding field in the calculator options.
//
// FLIP_VERTICALLY (optional): Whether to flip image vertically or not.
// It overrides the corresponding field in the calculator options.
//
// Calculator options (see image_transformation_calculator.proto):
// output_width, output_height - (optional) Desired scaled image size.
// rotation_mode - (optional) Rotation in multiples of 90 degrees.
@ -167,6 +181,8 @@ class ImageTransformationCalculator : public CalculatorBase {
int output_height_ = 0;
mediapipe::RotationMode_Mode rotation_;
mediapipe::ScaleMode_Mode scale_mode_;
bool flip_horizontally_ = false;
bool flip_vertically_ = false;
bool use_gpu_ = false;
#if !defined(MEDIAPIPE_DISABLE_GPU)
@ -203,6 +219,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
if (cc->Inputs().HasTag("ROTATION_DEGREES")) {
cc->Inputs().Tag("ROTATION_DEGREES").Set<int>();
}
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) {
cc->Inputs().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY")) {
cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("OUTPUT_DIMENSIONS")) {
cc->InputSidePackets().Tag("OUTPUT_DIMENSIONS").Set<DimensionsPacketType>();
@ -210,6 +232,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
cc->InputSidePackets().Tag("ROTATION_DEGREES").Set<int>();
}
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->Outputs().HasTag("LETTERBOX_PADDING")) {
cc->Outputs().Tag("LETTERBOX_PADDING").Set<std::array<float, 4>>();
@ -245,6 +273,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
output_width_ = options_.output_width();
output_height_ = options_.output_height();
}
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
rotation_ = DegreesToRotationMode(
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
@ -252,6 +281,20 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
rotation_ = options_.rotation_mode();
}
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
flip_horizontally_ =
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>();
} else {
flip_horizontally_ = options_.flip_horizontally();
}
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
flip_vertically_ =
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>();
} else {
flip_vertically_ = options_.flip_vertically();
}
scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE);
if (use_gpu_) {
@ -268,12 +311,37 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
::mediapipe::Status ImageTransformationCalculator::Process(
CalculatorContext* cc) {
// Override values if specified so.
if (cc->Inputs().HasTag("ROTATION_DEGREES") &&
!cc->Inputs().Tag("ROTATION_DEGREES").IsEmpty()) {
rotation_ =
DegreesToRotationMode(cc->Inputs().Tag("ROTATION_DEGREES").Get<int>());
}
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") &&
!cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) {
flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY") &&
!cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
}
if (use_gpu_) {
#if !defined(MEDIAPIPE_DISABLE_GPU)
if (cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) {
// Image is missing, hence no way to produce output image. (Timestamp
// bound will be updated automatically.)
return ::mediapipe::OkStatus();
}
return helper_.RunInGlContext(
[this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); });
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
if (cc->Inputs().Tag("IMAGE").IsEmpty()) {
// Image is missing, hence no way to produce output image. (Timestamp
// bound will be updated automatically.)
return ::mediapipe::OkStatus();
}
return RenderCpu(cc);
}
return ::mediapipe::OkStatus();
@ -360,11 +428,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
.Add(padding.release(), cc->InputTimestamp());
}
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
rotation_ = DegreesToRotationMode(
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
}
cv::Mat rotated_mat;
const int angle = RotationModeToDegrees(rotation_);
cv::Point2f src_center(scaled_mat.cols / 2.0, scaled_mat.rows / 2.0);
@ -372,11 +435,9 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
cv::warpAffine(scaled_mat, rotated_mat, rotation_mat, scaled_mat.size());
cv::Mat flipped_mat;
if (options_.flip_horizontally() || options_.flip_vertically()) {
if (flip_horizontally_ || flip_vertically_) {
const int flip_code =
options_.flip_horizontally() && options_.flip_vertically()
? -1
: options_.flip_horizontally();
flip_horizontally_ && flip_vertically_ ? -1 : flip_horizontally_;
cv::flip(rotated_mat, flipped_mat, flip_code);
} else {
flipped_mat = rotated_mat;
@ -450,11 +511,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
}
RET_CHECK(renderer) << "Unsupported input texture type";
if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) {
rotation_ = DegreesToRotationMode(
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
}
mediapipe::FrameScaleMode scale_mode = mediapipe::FrameScaleModeFromProto(
scale_mode_, mediapipe::FrameScaleMode::kStretch);
mediapipe::FrameRotation rotation =
@ -469,7 +525,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
MP_RETURN_IF_ERROR(renderer->GlRender(
src1.width(), src1.height(), dst.width(), dst.height(), scale_mode,
rotation, options_.flip_horizontally(), options_.flip_vertically(),
rotation, flip_horizontally_, flip_vertically_,
/*flip_texture=*/false));
glActiveTexture(GL_TEXTURE1);

16
mediapipe/calculators/image/scale_image_calculator.cc
View File

@ -264,7 +264,7 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
options_.target_width(), //
options_.target_height(), //
options_.preserve_aspect_ratio(), //
options_.scale_to_multiple_of_two(), //
options_.scale_to_multiple_of(), //
&output_width_, &output_height_));
MP_RETURN_IF_ERROR(FindInterpolationAlgorithm(options_.algorithm(),
&interpolation_algorithm_));
@ -361,17 +361,21 @@ ScaleImageCalculator::~ScaleImageCalculator() {}
output_format_ = input_format_;
}
const bool is_positive_and_even =
(options_.scale_to_multiple_of() >= 1) &&
(options_.scale_to_multiple_of() % 2 == 0);
if (output_format_ == ImageFormat::YCBCR420P) {
RET_CHECK(options_.scale_to_multiple_of_two())
RET_CHECK(is_positive_and_even)
<< "ScaleImageCalculator always outputs width and height that are "
"divisible by 2 when output format is YCbCr420P. To scale to "
"width and height of odd numbers, the output format must be SRGB.";
} else if (options_.preserve_aspect_ratio()) {
RET_CHECK(options_.scale_to_multiple_of_two())
RET_CHECK(options_.scale_to_multiple_of() == 2)
<< "ScaleImageCalculator always outputs width and height that are "
"divisible by 2 when perserving aspect ratio. To scale to width "
"and height of odd numbers, please set "
"preserve_aspect_ratio to false.";
"divisible by 2 when preserving aspect ratio. If you'd like to "
"set scale_to_multiple_of to something other than 2, please "
"set preserve_aspect_ratio to false.";
}
if (input_width_ > 0 && input_height_ > 0 &&

24
mediapipe/calculators/image/scale_image_calculator.proto
View File

@ -11,9 +11,10 @@ import "mediapipe/framework/formats/image_format.proto";
// 2) Scale and convert the image to fit inside target_width x target_height
// using the specified scaling algorithm. (maintaining the aspect
// ratio if preserve_aspect_ratio is true).
// The output width and height will be divisible by 2. It is possible to output
// width and height that are odd number when the output format is SRGB and not
// perserving the aspect ratio. See scale_to_multiple_of_two option for details.
// The output width and height will be divisible by 2, by default. It is
// possible to output width and height that are odd numbers when the output
// format is SRGB and the aspect ratio is left unpreserved. See
// scale_to_multiple_of for details.
message ScaleImageCalculatorOptions {
extend CalculatorOptions {
optional ScaleImageCalculatorOptions ext = 66237115;
@ -23,7 +24,7 @@ message ScaleImageCalculatorOptions {
// depending on the other options below. If unset, use the same width
// or height as the input. If only one is set then determine the other
// from the aspect ratio (after cropping). The output width and height
// will be divisible by 2.
// will be divisible by 2, by default.
optional int32 target_width = 1;
optional int32 target_height = 2;
@ -31,7 +32,8 @@ message ScaleImageCalculatorOptions {
// fits inside the box represented by target_width and target_height.
// Otherwise it is scaled to fit target_width and target_height
// completely. In any case, the aspect ratio that is preserved is
// that after cropping to the minimum/maximum aspect ratio.
// that after cropping to the minimum/maximum aspect ratio. Additionally, if
// true, the output width and height will be divisible by 2.
optional bool preserve_aspect_ratio = 3 [default = true];
// If ratio is positive, crop the image to this minimum and maximum
@ -95,11 +97,13 @@ message ScaleImageCalculatorOptions {
// SRGB or YCBCR420P.
optional ImageFormat.Format input_format = 12;
// If true, the output width and height will be divisible by 2. Otherwise it
// will use the exact specified output width and height, which is only
// supported when the output format is SRGB and preserve_aspect_ratio option
// is set to false.
optional bool scale_to_multiple_of_two = 13 [default = true];
// If set to 2, the target width and height will be rounded-down
// to the nearest even number. If set to any positive value other than 2,
// preserve_aspect_ratio must be false and the target width and height will be
// rounded-down to multiples of the given value. If set to any value less than
// 1, it will be treated like 1.
// NOTE: If set to an odd number, the output format must be SRGB.
optional int32 scale_to_multiple_of = 13 [default = 2];
// If true, assume the input YUV is BT.709 (this is the HDTV standard, so most
// content is likely using it). If false use the previous assumption of BT.601

22
mediapipe/calculators/image/scale_image_utils.cc
View File

@ -93,12 +93,22 @@ double ParseRational(const std::string& rational) {
int target_width, //
int target_height, //
bool preserve_aspect_ratio, //
bool scale_to_multiple_of_two, //
int scale_to_multiple_of, //
int* output_width,
int* output_height) {
CHECK(output_width);
CHECK(output_height);
if (preserve_aspect_ratio) {
RET_CHECK(scale_to_multiple_of == 2)
<< "FindOutputDimensions always outputs width and height that are "
"divisible by 2 when preserving aspect ratio. If you'd like to "
"set scale_to_multiple_of to something other than 2, please "
"set preserve_aspect_ratio to false.";
}
if (scale_to_multiple_of < 1) scale_to_multiple_of = 1;
if (!preserve_aspect_ratio || (target_width <= 0 && target_height <= 0)) {
if (target_width <= 0) {
target_width = input_width;
@ -106,13 +116,13 @@ double ParseRational(const std::string& rational) {
if (target_height <= 0) {
target_height = input_height;
}
if (scale_to_multiple_of_two) {
*output_width = (target_width / 2) * 2;
*output_height = (target_height / 2) * 2;
} else {
target_width -= target_width % scale_to_multiple_of;
target_height -= target_height % scale_to_multiple_of;
*output_width = target_width;
*output_height = target_height;
}
return ::mediapipe::OkStatus();
}

16
mediapipe/calculators/image/scale_image_utils.h
View File

@ -35,17 +35,19 @@ namespace scale_image {
int* col_start, int* row_start);
// Given an input width and height, a target width and height, whether to
// preserve the aspect ratio, and whether to round down to a multiple of 2,
// determine the output width and height. If target_width or target_height is
// non-positive, then they will be set to the input_width and input_height
// respectively. The output_width and output_height will be reduced as necessary
// to preserve_aspect_ratio and to scale_to_multipe_of_two if these options are
// specified.
// preserve the aspect ratio, and whether to round-down to the multiple of a
// given number nearest to the targets, determine the output width and height.
// If target_width or target_height is non-positive, then they will be set to
// the input_width and input_height respectively. If scale_to_multiple_of is
// less than 1, it will be treated like 1. The output_width and
// output_height will be reduced as necessary to preserve_aspect_ratio if the
// option is specified. If preserving the aspect ratio is desired, you must set
// scale_to_multiple_of to 2.
::mediapipe::Status FindOutputDimensions(int input_width, int input_height, //
int target_width,
int target_height, //
bool preserve_aspect_ratio, //
bool scale_to_multiple_of_two, //
int scale_to_multiple_of, //
int* output_width, int* output_height);
} // namespace scale_image

86
mediapipe/calculators/image/scale_image_utils_test.cc
View File

@ -79,49 +79,49 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) {
int output_width;
int output_height;
// Not scale.
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, true, &output_width,
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(200, output_width);
EXPECT_EQ(100, output_height);
// Not scale with odd input size.
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, false,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, 1, &output_width,
&output_height));
EXPECT_EQ(201, output_width);
EXPECT_EQ(101, output_height);
// Scale down by 1/2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale up, doubling dimensions.
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, 2, &output_width,
&output_height));
EXPECT_EQ(400, output_width);
EXPECT_EQ(200, output_height);
// Fits a 2:1 image into a 150 x 150 box. Output dimensions are always
// visible by 2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, 2, &output_width,
&output_height));
EXPECT_EQ(150, output_width);
EXPECT_EQ(74, output_height);
// Fits a 2:1 image into a 400 x 50 box.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale to multiple number with odd targe size.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale to multiple number with odd targe size.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, false,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(50, output_height);
// Scale to odd size.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, false,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, 1, &output_width,
&output_height));
EXPECT_EQ(151, output_width);
EXPECT_EQ(101, output_height);
}
@ -131,22 +131,62 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsNoAspectRatio) {
int output_width;
int output_height;
// Scale width only.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 2, &output_width,
&output_height));
EXPECT_EQ(100, output_width);
EXPECT_EQ(100, output_height);
// Scale height only.
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, 2, &output_width,
&output_height));
EXPECT_EQ(200, output_width);
EXPECT_EQ(200, output_height);
// Scale both dimensions.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, true,
&output_width, &output_height));
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 2, &output_width,
&output_height));
EXPECT_EQ(150, output_width);
EXPECT_EQ(200, output_height);
}
// Tests scale_to_multiple_of.
TEST(ScaleImageUtilsTest, FindOutputDimensionsDownScaleToMultipleOf) {
int output_width;
int output_height;
// Set no targets, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(100, 100, -1, -1, false, 8, &output_width,
&output_height));
EXPECT_EQ(96, output_width);
EXPECT_EQ(96, output_height);
// Set width target, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 8, &output_width,
&output_height));
EXPECT_EQ(96, output_width);
EXPECT_EQ(96, output_height);
// Set height target, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, 201, false, 8, &output_width,
&output_height));
EXPECT_EQ(200, output_width);
EXPECT_EQ(200, output_height);
// Set both targets, downscale to a multiple of 8.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 8, &output_width,
&output_height));
EXPECT_EQ(144, output_width);
EXPECT_EQ(200, output_height);
// Doesn't throw error if keep aspect is true and downscale multiple is 2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 200, true, 2, &output_width,
&output_height));
EXPECT_EQ(400, output_width);
EXPECT_EQ(200, output_height);
// Throws error if keep aspect is true, but downscale multiple is not 2.
ASSERT_THAT(FindOutputDimensions(200, 100, 400, 200, true, 4, &output_width,
&output_height),
testing::Not(testing::status::IsOk()));
// Downscaling to multiple ignored if multiple is less than 2.
MP_ASSERT_OK(FindOutputDimensions(200, 100, 401, 201, false, 1, &output_width,
&output_height));
EXPECT_EQ(401, output_width);
EXPECT_EQ(201, output_height);
}
} // namespace
} // namespace scale_image
} // namespace mediapipe

52
mediapipe/calculators/tensorflow/BUILD
View File

@ -138,7 +138,7 @@ mediapipe_cc_proto_library(
srcs = ["image_frame_to_tensor_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":image_frame_to_tensor_calculator_proto"],
@ -173,7 +173,7 @@ mediapipe_cc_proto_library(
srcs = ["pack_media_sequence_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":pack_media_sequence_calculator_proto"],
@ -192,7 +192,7 @@ mediapipe_cc_proto_library(
srcs = ["tensorflow_session_from_frozen_graph_generator.proto"],
cc_deps = [
"//mediapipe/framework:packet_generator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":tensorflow_session_from_frozen_graph_generator_proto"],
@ -203,7 +203,7 @@ mediapipe_cc_proto_library(
srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":tensorflow_session_from_frozen_graph_calculator_proto"],
@ -277,7 +277,7 @@ mediapipe_cc_proto_library(
srcs = ["vector_int_to_tensor_calculator_options.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//visibility:public"],
deps = [":vector_int_to_tensor_calculator_options_proto"],
@ -408,7 +408,7 @@ cc_library(
"//mediapipe/util/sequence:media_sequence",
"//mediapipe/util/sequence:media_sequence_util",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@ -423,7 +423,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@ -654,7 +654,7 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:lib",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@ -695,7 +695,7 @@ cc_library(
"//mediapipe/util:audio_decoder_cc_proto",
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@ -737,7 +737,7 @@ cc_library(
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:packet",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
alwayslink = 1,
)
@ -745,6 +745,7 @@ cc_library(
cc_test(
name = "graph_tensors_packet_generator_test",
srcs = ["graph_tensors_packet_generator_test.cc"],
linkstatic = 1,
deps = [
":graph_tensors_packet_generator",
":graph_tensors_packet_generator_cc_proto",
@ -761,6 +762,7 @@ cc_test(
name = "image_frame_to_tensor_calculator_test",
size = "small",
srcs = ["image_frame_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":image_frame_to_tensor_calculator",
"//mediapipe/framework:calculator_framework",
@ -777,6 +779,7 @@ cc_test(
name = "matrix_to_tensor_calculator_test",
size = "small",
srcs = ["matrix_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":matrix_to_tensor_calculator",
":matrix_to_tensor_calculator_options_cc_proto",
@ -793,6 +796,7 @@ cc_test(
name = "lapped_tensor_buffer_calculator_test",
size = "small",
srcs = ["lapped_tensor_buffer_calculator_test.cc"],
linkstatic = 1,
deps = [
":lapped_tensor_buffer_calculator",
":lapped_tensor_buffer_calculator_cc_proto",
@ -801,7 +805,7 @@ cc_test(
"//mediapipe/framework/port:gtest_main",
"@com_google_absl//absl/memory",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@ -840,7 +844,7 @@ cc_test(
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@ -867,7 +871,7 @@ cc_test(
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:testlib",
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
"@org_tensorflow//tensorflow/core/kernels:math",
@ -897,7 +901,7 @@ cc_test(
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:testlib",
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
"@org_tensorflow//tensorflow/core/kernels:math",
@ -956,6 +960,7 @@ cc_test(
cc_test(
name = "tensor_squeeze_dimensions_calculator_test",
srcs = ["tensor_squeeze_dimensions_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_squeeze_dimensions_calculator",
":tensor_squeeze_dimensions_calculator_cc_proto",
@ -963,7 +968,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@ -971,6 +976,7 @@ cc_test(
name = "tensor_to_image_frame_calculator_test",
size = "small",
srcs = ["tensor_to_image_frame_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_to_image_frame_calculator",
":tensor_to_image_frame_calculator_cc_proto",
@ -979,7 +985,7 @@ cc_test(
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@ -987,6 +993,7 @@ cc_test(
name = "tensor_to_matrix_calculator_test",
size = "small",
srcs = ["tensor_to_matrix_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_to_matrix_calculator",
":tensor_to_matrix_calculator_cc_proto",
@ -996,13 +1003,14 @@ cc_test(
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
cc_test(
name = "tensor_to_vector_float_calculator_test",
srcs = ["tensor_to_vector_float_calculator_test.cc"],
linkstatic = 1,
deps = [
":tensor_to_vector_float_calculator",
":tensor_to_vector_float_calculator_options_cc_proto",
@ -1010,7 +1018,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
@ -1030,13 +1038,14 @@ cc_test(
"//mediapipe/util/sequence:media_sequence",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
cc_test(
name = "vector_int_to_tensor_calculator_test",
srcs = ["vector_int_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":vector_int_to_tensor_calculator",
":vector_int_to_tensor_calculator_options_cc_proto",
@ -1044,13 +1053,14 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)
cc_test(
name = "vector_float_to_tensor_calculator_test",
srcs = ["vector_float_to_tensor_calculator_test.cc"],
linkstatic = 1,
deps = [
":vector_float_to_tensor_calculator",
":vector_float_to_tensor_calculator_options_cc_proto",
@ -1058,7 +1068,7 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)

4
mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc
View File

@ -17,7 +17,7 @@
#if !defined(__ANDROID__)
#include "mediapipe/framework/port/file_helpers.h"
#endif
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@ -63,7 +63,7 @@ const std::string MaybeConvertSignatureToTag(
output.resize(name.length());
std::transform(name.begin(), name.end(), output.begin(),
[](unsigned char c) { return std::toupper(c); });
output = absl::Substitute(output, "/", "_");
output = absl::StrReplaceAll(output, {{"/", "_"}});
return output;
} else {
return name;

2
mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"

4
mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc
View File

@ -17,7 +17,7 @@
#if !defined(__ANDROID__)
#include "mediapipe/framework/port/file_helpers.h"
#endif
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
#include "mediapipe/framework/deps/file_path.h"
@ -65,7 +65,7 @@ const std::string MaybeConvertSignatureToTag(
output.resize(name.length());
std::transform(name.begin(), name.end(), output.begin(),
[](unsigned char c) { return std::toupper(c); });
output = absl::Substitute(output, "/", "_");
output = absl::StrReplaceAll(output, {{"/", "_"}});
return output;
} else {
return name;

2
mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc
View File

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/strings/substitute.h"
#include "absl/strings/str_replace.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h"
#include "mediapipe/framework/calculator_framework.h"

1
mediapipe/calculators/tflite/BUILD
View File

@ -485,6 +485,7 @@ cc_test(
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/lite:framework",
"@org_tensorflow//tensorflow/lite/kernels:builtin_ops",
],

99
mediapipe/calculators/tflite/tflite_inference_calculator.cc
View File

@ -148,7 +148,7 @@ struct GPUData {
// options: {
// [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
// model_path: "modelname.tflite"
// use_gpu: true
// delegate { gpu {} }
// }
// }
// }
@ -163,6 +163,9 @@ struct GPUData {
//
class TfLiteInferenceCalculator : public CalculatorBase {
public:
using TfLiteDelegatePtr =
std::unique_ptr<TfLiteDelegate, std::function<void(TfLiteDelegate*)>>;
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
@ -176,7 +179,7 @@ class TfLiteInferenceCalculator : public CalculatorBase {
std::unique_ptr<tflite::Interpreter> interpreter_;
std::unique_ptr<tflite::FlatBufferModel> model_;
TfLiteDelegate* delegate_ = nullptr;
TfLiteDelegatePtr delegate_;
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
mediapipe::GlCalculatorHelper gpu_helper_;
@ -212,12 +215,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
RET_CHECK(cc->Outputs().HasTag("TENSORS") ^
cc->Outputs().HasTag("TENSORS_GPU"));
bool use_gpu = false;
const auto& options =
cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
bool use_gpu =
options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
if (cc->Inputs().HasTag("TENSORS"))
cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
if (cc->Inputs().HasTag("TENSORS_GPU")) {
RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
<< "GPU input is compatible with GPU delegate only.";
cc->Inputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
use_gpu |= true;
}
@ -227,6 +236,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
cc->Outputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
#if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__)
if (cc->Outputs().HasTag("TENSORS_GPU")) {
RET_CHECK(!options.has_delegate() || options.delegate().has_gpu())
<< "GPU output is compatible with GPU delegate only.";
cc->Outputs().Tag("TENSORS_GPU").Set<std::vector<GpuTensor>>();
use_gpu |= true;
}
@ -238,10 +250,6 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
.Set<tflite::ops::builtin::BuiltinOpResolver>();
}
const auto& options =
cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>();
use_gpu |= options.use_gpu();
if (use_gpu) {
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc));
@ -454,7 +462,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
if (gpu_inference_) {
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status {
TfLiteGpuDelegateDelete(delegate_);
delegate_ = nullptr;
for (int i = 0; i < gpu_data_in_.size(); ++i) {
gpu_data_in_[i].reset();
}
@ -464,7 +472,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
return ::mediapipe::OkStatus();
}));
#elif defined(MEDIAPIPE_IOS)
TFLGpuDelegateDelete(delegate_);
delegate_ = nullptr;
for (int i = 0; i < gpu_data_in_.size(); ++i) {
gpu_data_in_[i].reset();
}
@ -472,9 +480,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
gpu_data_out_[i].reset();
}
#endif
}
} else {
delegate_ = nullptr;
}
}
#if defined(MEDIAPIPE_EDGE_TPU)
edgetpu_context_.reset();
#endif
@ -501,7 +510,8 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
}
// Get execution modes.
gpu_inference_ = options.use_gpu();
gpu_inference_ =
options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu();
return ::mediapipe::OkStatus();
}
@ -526,8 +536,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
RET_CHECK(interpreter_);
#if defined(__EMSCRIPTEN__)
#if defined(__EMSCRIPTEN__) || defined(MEDIAPIPE_EDGE_TPU)
interpreter_->SetNumThreads(1);
#else
interpreter_->SetNumThreads(
cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
.cpu_num_thread());
#endif // __EMSCRIPTEN__
if (gpu_output_) {
@ -545,20 +559,37 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
::mediapipe::Status TfLiteInferenceCalculator::LoadDelegate(
CalculatorContext* cc) {
#if defined(MEDIAPIPE_ANDROID)
const auto& calculator_opts =
cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>();
if (calculator_opts.has_delegate() &&
calculator_opts.delegate().has_tflite()) {
// Default tflite inference requeqsted - no need to modify graph.
return ::mediapipe::OkStatus();
}
if (!gpu_inference_) {
if (cc->Options<mediapipe::TfLiteInferenceCalculatorOptions>()
.use_nnapi()) {
#if defined(MEDIAPIPE_ANDROID)
const bool nnapi_requested = calculator_opts.has_delegate()
? calculator_opts.delegate().has_nnapi()
: calculator_opts.use_nnapi();
if (nnapi_requested) {
// Attempt to use NNAPI.
// If not supported, the default CPU delegate will be created and used.
interpreter_->SetAllowFp16PrecisionForFp32(1);
delegate_ = tflite::NnApiDelegate();
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
delegate_ =
TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) {
// No need to free according to tflite::NnApiDelegate()
// documentation.
});
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
kTfLiteOk);
return ::mediapipe::OkStatus();
}
#endif // MEDIAPIPE_ANDROID
// Return, no need for GPU delegate below.
return ::mediapipe::OkStatus();
}
#endif // ANDROID
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
// Configure and create the delegate.
@ -568,7 +599,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
TFLITE_GL_OBJECT_TYPE_FASTEST;
options.compile_options.dynamic_batch_enabled = 0;
options.compile_options.inline_parameters = 1;
if (!delegate_) delegate_ = TfLiteGpuDelegateCreate(&options);
if (!delegate_)
delegate_ = TfLiteDelegatePtr(TfLiteGpuDelegateCreate(&options),
&TfLiteGpuDelegateDelete);
if (gpu_input_) {
// Get input image sizes.
@ -586,7 +619,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer<float>(
gpu_data_in_[i]->elements, &gpu_data_in_[i]->buffer));
RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
delegate_, gpu_data_in_[i]->buffer.id(),
delegate_.get(), gpu_data_in_[i]->buffer.id(),
interpreter_->inputs()[i]),
kTfLiteOk);
}
@ -609,15 +642,16 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
for (int i = 0; i < gpu_data_out_.size(); ++i) {
RET_CHECK_CALL(CreateReadWriteShaderStorageBuffer<float>(
gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer));
RET_CHECK_EQ(
TfLiteGpuDelegateBindBufferToTensor(
delegate_, gpu_data_out_[i]->buffer.id(), output_indices[i]),
RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor(
delegate_.get(), gpu_data_out_[i]->buffer.id(),
output_indices[i]),
kTfLiteOk);
}
}
// Must call this last.
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
kTfLiteOk);
#endif // OpenGL
#if defined(MEDIAPIPE_IOS)
@ -626,7 +660,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
TFLGpuDelegateOptions options;
options.allow_precision_loss = true;
options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive;
if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options);
if (!delegate_)
delegate_ = TfLiteDelegatePtr(TFLGpuDelegateCreate(&options),
&TFLGpuDelegateDelete);
id<MTLDevice> device = gpu_helper_.mtlDevice;
if (gpu_input_) {
@ -678,9 +714,11 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
gpu_data_in_[i]->buffer =
[device newBufferWithLength:gpu_data_in_[i]->elements * kHalfSize
options:MTLResourceStorageModeShared];
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk);
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
delegate_, input_indices[i], gpu_data_in_[i]->buffer),
RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()),
kTfLiteOk);
RET_CHECK_EQ(
TFLGpuDelegateBindMetalBufferToTensor(
delegate_.get(), input_indices[i], gpu_data_in_[i]->buffer),
true);
}
}
@ -725,8 +763,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
gpu_data_out_[i]->buffer =
[device newBufferWithLength:gpu_data_out_[i]->elements * kHalfSize
options:MTLResourceStorageModeShared];
RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor(
delegate_, output_indices[i], gpu_data_out_[i]->buffer),
RET_CHECK_EQ(
TFLGpuDelegateBindMetalBufferToTensor(
delegate_.get(), output_indices[i], gpu_data_out_[i]->buffer),
true);
}

34
mediapipe/calculators/tflite/tflite_inference_calculator.proto
View File

@ -27,7 +27,7 @@ import "mediapipe/framework/calculator.proto";
// options {
// [mediapipe.TfLiteInferenceCalculatorOptions.ext] {
// model_path: "model.tflite"
// use_gpu: true
// delegate { gpu {} }
// }
// }
// }
@ -37,6 +37,22 @@ message TfLiteInferenceCalculatorOptions {
optional TfLiteInferenceCalculatorOptions ext = 233867213;
}
message Delegate {
// Default inference provided by tflite.
message TfLite {}
// Delegate to run GPU inference depending on the device.
// (Can use OpenGl, OpenCl, Metal depending on the device.)
message Gpu {}
// Android only.
message Nnapi {}
oneof delegate {
TfLite tflite = 1;
Gpu gpu = 2;
Nnapi nnapi = 3;
}
}
// Path to the TF Lite model (ex: /path/to/modelname.tflite).
// On mobile, this is generally just modelname.tflite.
optional string model_path = 1;
@ -44,10 +60,22 @@ message TfLiteInferenceCalculatorOptions {
// Whether the TF Lite GPU or CPU backend should be used. Effective only when
// input tensors are on CPU. For input tensors on GPU, GPU backend is always
// used.
optional bool use_gpu = 2 [default = false];
// DEPRECATED: configure "delegate" instead.
optional bool use_gpu = 2 [deprecated = true, default = false];
// Android only. When true, an NNAPI delegate will be used for inference.
// If NNAPI is not available, then the default CPU delegate will be used
// automatically.
optional bool use_nnapi = 3 [default = false];
// DEPRECATED: configure "delegate" instead.
optional bool use_nnapi = 3 [deprecated = true, default = false];
// The number of threads available to the interpreter. Effective only when
// input tensors are on CPU and 'use_gpu' is false.
optional int32 cpu_num_thread = 4 [default = -1];
// TfLite delegate to run inference.
// NOTE: calculator is free to choose delegate if not specified explicitly.
// NOTE: use_gpu/use_nnapi are ignored if specified. (Delegate takes
// precedence over use_* deprecated options.)
optional Delegate delegate = 5;
}

29
mediapipe/calculators/tflite/tflite_inference_calculator_test.cc
View File

@ -16,6 +16,8 @@
#include <string>
#include <vector>
#include "absl/strings/str_replace.h"
#include "absl/strings/string_view.h"
#include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
@ -39,13 +41,7 @@ namespace mediapipe {
using ::tflite::Interpreter;
class TfLiteInferenceCalculatorTest : public ::testing::Test {
protected:
std::unique_ptr<CalculatorRunner> runner_ = nullptr;
};
// Tests a simple add model that adds an input tensor to itself.
TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
void DoSmokeTest(absl::string_view delegate) {
const int width = 8;
const int height = 8;
const int channels = 3;
@ -73,10 +69,7 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
auto input_vec = absl::make_unique<std::vector<TfLiteTensor>>();
input_vec->emplace_back(*tensor);
// Prepare single calculator graph to and wait for packets.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"(
std::string graph_proto = R"(
input_stream: "tensor_in"
node {
calculator: "TfLiteInferenceCalculator"
@ -84,12 +77,16 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
output_stream: "TENSORS:tensor_out"
options {
[mediapipe.TfLiteInferenceCalculatorOptions.ext] {
use_gpu: false
model_path: "mediapipe/calculators/tflite/testdata/add.bin"
$delegate
}
}
}
)");
)";
ASSERT_EQ(absl::StrReplaceAll({{"$delegate", delegate}}, &graph_proto), 1);
// Prepare single calculator graph to and wait for packets.
CalculatorGraphConfig graph_config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(graph_proto);
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor_out", &graph_config, &output_packets);
CalculatorGraph graph(graph_config);
@ -120,4 +117,10 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) {
MP_ASSERT_OK(graph.WaitUntilDone());
}
// Tests a simple add model that adds an input tensor to itself.
TEST(TfLiteInferenceCalculatorTest, SmokeTest) {
DoSmokeTest(/*delegate=*/"");
DoSmokeTest(/*delegate=*/"delegate { tflite {} }");
}
} // namespace mediapipe

62
mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc
View File

@ -28,6 +28,21 @@ namespace mediapipe {
// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first
// tensor will be used. The size of the values must be
// (num_dimension x num_landmarks).
//
// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
// not. Overrides corresponding side packet and/or field in the calculator
// options.
//
// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
// Overrides corresponding side packet and/or field in the calculator options.
//
// Input side packet:
// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or
// not. Overrides the corresponding field in the calculator options.
//
// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not.
// Overrides the corresponding field in the calculator options.
//
// Output:
// LANDMARKS(optional) - Result MediaPipe landmarks.
// NORM_LANDMARKS(optional) - Result MediaPipe normalized landmarks.
@ -61,6 +76,8 @@ class TfLiteTensorsToLandmarksCalculator : public CalculatorBase {
private:
::mediapipe::Status LoadOptions(CalculatorContext* cc);
int num_landmarks_ = 0;
bool flip_vertically_ = false;
bool flip_horizontally_ = false;
::mediapipe::TfLiteTensorsToLandmarksCalculatorOptions options_;
};
@ -75,6 +92,22 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
}
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) {
cc->Inputs().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY")) {
cc->Inputs().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) {
cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set<bool>();
}
if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) {
cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set<bool>();
}
if (cc->Outputs().HasTag("LANDMARKS")) {
cc->Outputs().Tag("LANDMARKS").Set<LandmarkList>();
}
@ -98,17 +131,40 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
<< "Must provide input with/height for getting normalized landmarks.";
}
if (cc->Outputs().HasTag("LANDMARKS") &&
(options_.flip_vertically() || options_.flip_horizontally())) {
(options_.flip_vertically() || options_.flip_horizontally() ||
cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY") ||
cc->InputSidePackets().HasTag("FLIP_VERTICALLY"))) {
RET_CHECK(options_.has_input_image_height() &&
options_.has_input_image_width())
<< "Must provide input with/height for using flip_vertically option "
"when outputing landmarks in absolute coordinates.";
}
flip_horizontally_ =
cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")
? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get<bool>()
: options_.flip_horizontally();
flip_horizontally_ =
cc->InputSidePackets().HasTag("FLIP_VERTICALLY")
? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get<bool>()
: options_.flip_vertically();
return ::mediapipe::OkStatus();
}
::mediapipe::Status TfLiteTensorsToLandmarksCalculator::Process(
CalculatorContext* cc) {
// Override values if specified so.
if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") &&
!cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) {
flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get<bool>();
}
if (cc->Inputs().HasTag("FLIP_VERTICALLY") &&
!cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) {
flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get<bool>();
}
if (cc->Inputs().Tag("TENSORS").IsEmpty()) {
return ::mediapipe::OkStatus();
}
@ -133,13 +189,13 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
const int offset = ld * num_dimensions;
Landmark* landmark = output_landmarks.add_landmark();
if (options_.flip_horizontally()) {
if (flip_horizontally_) {
landmark->set_x(options_.input_image_width() - raw_landmarks[offset]);
} else {
landmark->set_x(raw_landmarks[offset]);
}
if (num_dimensions > 1) {
if (options_.flip_vertically()) {
if (flip_vertically_) {
landmark->set_y(options_.input_image_height() -
raw_landmarks[offset + 1]);
} else {

14
mediapipe/calculators/util/BUILD
View File

@ -437,6 +437,7 @@ cc_library(
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/types:optional",
],
alwayslink = 1,
)
@ -928,6 +929,19 @@ cc_library(
alwayslink = 1,
)
cc_library(
name = "local_file_pattern_contents_calculator",
srcs = ["local_file_pattern_contents_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)
cc_library(
name = "filter_collection_calculator",
srcs = ["filter_collection_calculator.cc"],

20
mediapipe/calculators/util/annotation_overlay_calculator.cc
View File

@ -39,13 +39,13 @@ namespace mediapipe {
namespace {
constexpr char kInputFrameTag[] = "INPUT_FRAME";
constexpr char kOutputFrameTag[] = "OUTPUT_FRAME";
constexpr char kInputFrameTag[] = "IMAGE";
constexpr char kOutputFrameTag[] = "IMAGE";
constexpr char kInputVectorTag[] = "VECTOR";
constexpr char kInputFrameTagGpu[] = "INPUT_FRAME_GPU";
constexpr char kOutputFrameTagGpu[] = "OUTPUT_FRAME_GPU";
constexpr char kInputFrameTagGpu[] = "IMAGE_GPU";
constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU";
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
@ -61,7 +61,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// A calculator for rendering data on images.
//
// Inputs:
// 1. INPUT_FRAME or INPUT_FRAME_GPU (optional): An ImageFrame (or GpuBuffer)
// 1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer)
// containing the input image.
// If output is CPU, and input isn't provided, the renderer creates a
// blank canvas with the width, height and color provided in the options.
@ -73,7 +73,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// input vector items. These input streams are tagged with "VECTOR".
//
// Output:
// 1. OUTPUT_FRAME or OUTPUT_FRAME_GPU: A rendered ImageFrame (or GpuBuffer).
// 1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer).
//
// For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The
// output format is the same as input except for GRAY8 where the output is in
@ -87,13 +87,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// Example config (CPU):
// node {
// calculator: "AnnotationOverlayCalculator"
// input_stream: "INPUT_FRAME:image_frames"
// input_stream: "IMAGE:image_frames"
// input_stream: "render_data_1"
// input_stream: "render_data_2"
// input_stream: "render_data_3"
// input_stream: "VECTOR:0:render_data_vec_0"
// input_stream: "VECTOR:1:render_data_vec_1"
// output_stream: "OUTPUT_FRAME:decorated_frames"
// output_stream: "IMAGE:decorated_frames"
// options {
// [mediapipe.AnnotationOverlayCalculatorOptions.ext] {
// }
@ -103,13 +103,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102};
// Example config (GPU):
// node {
// calculator: "AnnotationOverlayCalculator"
// input_stream: "INPUT_FRAME_GPU:image_frames"
// input_stream: "IMAGE_GPU:image_frames"
// input_stream: "render_data_1"
// input_stream: "render_data_2"
// input_stream: "render_data_3"
// input_stream: "VECTOR:0:render_data_vec_0"
// input_stream: "VECTOR:1:render_data_vec_1"
// output_stream: "OUTPUT_FRAME_GPU:decorated_frames"
// output_stream: "IMAGE_GPU:decorated_frames"
// options {
// [mediapipe.AnnotationOverlayCalculatorOptions.ext] {
// }

80
mediapipe/calculators/util/detections_to_rects_calculator.cc
View File

@ -39,7 +39,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
} // namespace
::mediapipe::Status DetectionsToRectsCalculator::DetectionToRect(
const Detection& detection, Rect* rect) {
const Detection& detection, const DetectionSpec& detection_spec,
Rect* rect) {
const LocationData location_data = detection.location_data();
RET_CHECK(location_data.format() == LocationData::BOUNDING_BOX)
<< "Only Detection with formats of BOUNDING_BOX can be converted to Rect";
@ -52,7 +53,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
}
::mediapipe::Status DetectionsToRectsCalculator::DetectionToNormalizedRect(
const Detection& detection, NormalizedRect* rect) {
const Detection& detection, const DetectionSpec& detection_spec,
NormalizedRect* rect) {
const LocationData location_data = detection.location_data();
RET_CHECK(location_data.format() == LocationData::RELATIVE_BOUNDING_BOX)
<< "Only Detection with formats of RELATIVE_BOUNDING_BOX can be "
@ -174,27 +176,31 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
}
}
std::pair<int, int> image_size;
if (rotate_) {
RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty());
image_size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
}
// Get dynamic calculator options (e.g. `image_size`).
const DetectionSpec detection_spec = GetDetectionSpec(cc);
if (cc->Outputs().HasTag(kRectTag)) {
auto output_rect = absl::make_unique<Rect>();
MP_RETURN_IF_ERROR(DetectionToRect(detections[0], output_rect.get()));
MP_RETURN_IF_ERROR(
DetectionToRect(detections[0], detection_spec, output_rect.get()));
if (rotate_) {
output_rect->set_rotation(ComputeRotation(detections[0], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[0], detection_spec, &rotation));
output_rect->set_rotation(rotation);
}
cc->Outputs().Tag(kRectTag).Add(output_rect.release(),
cc->InputTimestamp());
}
if (cc->Outputs().HasTag(kNormRectTag)) {
auto output_rect = absl::make_unique<NormalizedRect>();
MP_RETURN_IF_ERROR(
DetectionToNormalizedRect(detections[0], output_rect.get()));
MP_RETURN_IF_ERROR(DetectionToNormalizedRect(detections[0], detection_spec,
output_rect.get()));
if (rotate_) {
output_rect->set_rotation(ComputeRotation(detections[0], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[0], detection_spec, &rotation));
output_rect->set_rotation(rotation);
}
cc->Outputs()
.Tag(kNormRectTag)
@ -203,11 +209,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
if (cc->Outputs().HasTag(kRectsTag)) {
auto output_rects = absl::make_unique<std::vector<Rect>>(detections.size());
for (int i = 0; i < detections.size(); ++i) {
MP_RETURN_IF_ERROR(
DetectionToRect(detections[i], &(output_rects->at(i))));
MP_RETURN_IF_ERROR(DetectionToRect(detections[i], detection_spec,
&(output_rects->at(i))));
if (rotate_) {
output_rects->at(i).set_rotation(
ComputeRotation(detections[i], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[i], detection_spec, &rotation));
output_rects->at(i).set_rotation(rotation);
}
}
cc->Outputs().Tag(kRectsTag).Add(output_rects.release(),
@ -217,11 +225,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
auto output_rects =
absl::make_unique<std::vector<NormalizedRect>>(detections.size());
for (int i = 0; i < detections.size(); ++i) {
MP_RETURN_IF_ERROR(
DetectionToNormalizedRect(detections[i], &(output_rects->at(i))));
MP_RETURN_IF_ERROR(DetectionToNormalizedRect(
detections[i], detection_spec, &(output_rects->at(i))));
if (rotate_) {
output_rects->at(i).set_rotation(
ComputeRotation(detections[i], image_size));
float rotation;
MP_RETURN_IF_ERROR(
ComputeRotation(detections[i], detection_spec, &rotation));
output_rects->at(i).set_rotation(rotation);
}
}
cc->Outputs()
@ -232,21 +242,35 @@ constexpr char kNormRectsTag[] = "NORM_RECTS";
return ::mediapipe::OkStatus();
}
float DetectionsToRectsCalculator::ComputeRotation(
const Detection& detection, const std::pair<int, int> image_size) {
::mediapipe::Status DetectionsToRectsCalculator::ComputeRotation(
const Detection& detection, const DetectionSpec& detection_spec,
float* rotation) {
const auto& location_data = detection.location_data();
const auto& image_size = detection_spec.image_size;
RET_CHECK(image_size) << "Image size is required to calculate rotation";
const float x0 = location_data.relative_keypoints(start_keypoint_index_).x() *
image_size.first;
image_size->first;
const float y0 = location_data.relative_keypoints(start_keypoint_index_).y() *
image_size.second;
image_size->second;
const float x1 = location_data.relative_keypoints(end_keypoint_index_).x() *
image_size.first;
image_size->first;
const float y1 = location_data.relative_keypoints(end_keypoint_index_).y() *
image_size.second;
image_size->second;
float rotation = target_angle_ - std::atan2(-(y1 - y0), x1 - x0);
*rotation = NormalizeRadians(target_angle_ - std::atan2(-(y1 - y0), x1 - x0));
return NormalizeRadians(rotation);
return ::mediapipe::OkStatus();
}
DetectionSpec DetectionsToRectsCalculator::GetDetectionSpec(
const CalculatorContext* cc) {
absl::optional<std::pair<int, int>> image_size;
if (cc->Inputs().HasTag(kImageSizeTag)) {
image_size = cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
}
return {image_size};
}
REGISTER_CALCULATOR(DetectionsToRectsCalculator);

19
mediapipe/calculators/util/detections_to_rects_calculator.h
View File

@ -16,6 +16,7 @@
#include <cmath>
#include "absl/types/optional.h"
#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
@ -27,6 +28,13 @@
namespace mediapipe {
// Dynamic options passed as calculator `input_stream` that can be used for
// calculation of rectangle or rotation for given detection. Does not include
// static calculator options which are available via private fields.
struct DetectionSpec {
absl::optional<std::pair<int, int>> image_size;
};
// A calculator that converts Detection proto to Rect proto.
//
// Detection is the format for encoding one or more detections in an image.
@ -81,13 +89,16 @@ class DetectionsToRectsCalculator : public CalculatorBase {
::mediapipe::Status Process(CalculatorContext* cc) override;
protected:
virtual float ComputeRotation(const ::mediapipe::Detection& detection,
const std::pair<int, int> image_size);
virtual ::mediapipe::Status DetectionToRect(
const ::mediapipe::Detection& detection, ::mediapipe::Rect* rect);
const ::mediapipe::Detection& detection,
const DetectionSpec& detection_spec, ::mediapipe::Rect* rect);
virtual ::mediapipe::Status DetectionToNormalizedRect(
const ::mediapipe::Detection& detection,
::mediapipe::NormalizedRect* rect);
const DetectionSpec& detection_spec, ::mediapipe::NormalizedRect* rect);
virtual ::mediapipe::Status ComputeRotation(
const ::mediapipe::Detection& detection,
const DetectionSpec& detection_spec, float* rotation);
virtual DetectionSpec GetDetectionSpec(const CalculatorContext* cc);
static inline float NormalizeRadians(float angle) {
return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI));

62
mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc
View File

@ -12,20 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include <vector>
@ -67,6 +53,15 @@ constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING";
// input_stream: "LETTERBOX_PADDING:letterbox_padding"
// output_stream: "LANDMARKS:adjusted_landmarks"
// }
//
// node {
// calculator: "LandmarkLetterboxRemovalCalculator"
// input_stream: "LANDMARKS:0:landmarks_0"
// input_stream: "LANDMARKS:1:landmarks_1"
// input_stream: "LETTERBOX_PADDING:letterbox_padding"
// output_stream: "LANDMARKS:0:adjusted_landmarks_0"
// output_stream: "LANDMARKS:1:adjusted_landmarks_1"
// }
class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@ -74,10 +69,20 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
cc->Inputs().HasTag(kLetterboxPaddingTag))
<< "Missing one or more input streams.";
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag),
cc->Outputs().NumEntries(kLandmarksTag))
<< "Same number of input and output landmarks is required.";
for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag);
id != cc->Inputs().EndId(kLandmarksTag); ++id) {
cc->Inputs().Get(id).Set<NormalizedLandmarkList>();
}
cc->Inputs().Tag(kLetterboxPaddingTag).Set<std::array<float, 4>>();
cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag);
id != cc->Outputs().EndId(kLandmarksTag); ++id) {
cc->Outputs().Get(id).Set<NormalizedLandmarkList>();
}
return ::mediapipe::OkStatus();
}
@ -89,21 +94,28 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
}
::mediapipe::Status Process(CalculatorContext* cc) override {
// Only process if there's input landmarks.
if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) {
if (cc->Inputs().Tag(kLetterboxPaddingTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
const NormalizedLandmarkList& input_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
const auto& letterbox_padding =
cc->Inputs().Tag(kLetterboxPaddingTag).Get<std::array<float, 4>>();
const float left = letterbox_padding[0];
const float top = letterbox_padding[1];
const float left_and_right = letterbox_padding[0] + letterbox_padding[2];
const float top_and_bottom = letterbox_padding[1] + letterbox_padding[3];
CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag);
CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag);
// Number of inputs and outpus is the same according to the contract.
for (; input_id != cc->Inputs().EndId(kLandmarksTag);
++input_id, ++output_id) {
const auto& input_packet = cc->Inputs().Get(input_id);
if (input_packet.IsEmpty()) {
continue;
}
const NormalizedLandmarkList& input_landmarks =
input_packet.Get<NormalizedLandmarkList>();
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
@ -117,10 +129,10 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
new_landmark->set_z(landmark.z());
}
cc->Outputs()
.Tag(kLandmarksTag)
.AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
cc->Outputs().Get(output_id).AddPacket(
MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
}
return ::mediapipe::OkStatus();
}
};

68
mediapipe/calculators/util/landmark_projection_calculator.cc
View File

@ -12,20 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include <vector>
@ -63,6 +49,15 @@ constexpr char kRectTag[] = "NORM_RECT";
// input_stream: "NORM_RECT:rect"
// output_stream: "NORM_LANDMARKS:projected_landmarks"
// }
//
// node {
// calculator: "LandmarkProjectionCalculator"
// input_stream: "NORM_LANDMARKS:0:landmarks_0"
// input_stream: "NORM_LANDMARKS:1:landmarks_1"
// input_stream: "NORM_RECT:rect"
// output_stream: "NORM_LANDMARKS:0:projected_landmarks_0"
// output_stream: "NORM_LANDMARKS:1:projected_landmarks_1"
// }
class LandmarkProjectionCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
@ -70,10 +65,20 @@ class LandmarkProjectionCalculator : public CalculatorBase {
cc->Inputs().HasTag(kRectTag))
<< "Missing one or more input streams.";
cc->Inputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag),
cc->Outputs().NumEntries(kLandmarksTag))
<< "Same number of input and output landmarks is required.";
for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag);
id != cc->Inputs().EndId(kLandmarksTag); ++id) {
cc->Inputs().Get(id).Set<NormalizedLandmarkList>();
}
cc->Inputs().Tag(kRectTag).Set<NormalizedRect>();
cc->Outputs().Tag(kLandmarksTag).Set<NormalizedLandmarkList>();
for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag);
id != cc->Outputs().EndId(kLandmarksTag); ++id) {
cc->Outputs().Get(id).Set<NormalizedLandmarkList>();
}
return ::mediapipe::OkStatus();
}
@ -85,17 +90,25 @@ class LandmarkProjectionCalculator : public CalculatorBase {
}
::mediapipe::Status Process(CalculatorContext* cc) override {
const auto& options =
cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>();
// Only process if there's input landmarks.
if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) {
if (cc->Inputs().Tag(kRectTag).IsEmpty()) {
return ::mediapipe::OkStatus();
}
const NormalizedLandmarkList& input_landmarks =
cc->Inputs().Tag(kLandmarksTag).Get<NormalizedLandmarkList>();
const auto& input_rect = cc->Inputs().Tag(kRectTag).Get<NormalizedRect>();
const auto& options =
cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>();
CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag);
CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag);
// Number of inputs and outpus is the same according to the contract.
for (; input_id != cc->Inputs().EndId(kLandmarksTag);
++input_id, ++output_id) {
const auto& input_packet = cc->Inputs().Get(input_id);
if (input_packet.IsEmpty()) {
continue;
}
const auto& input_landmarks = input_packet.Get<NormalizedLandmarkList>();
NormalizedLandmarkList output_landmarks;
for (int i = 0; i < input_landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = input_landmarks.landmark(i);
@ -103,7 +116,8 @@ class LandmarkProjectionCalculator : public CalculatorBase {
const float x = landmark.x() - 0.5f;
const float y = landmark.y() - 0.5f;
const float angle = options.ignore_rotation() ? 0 : input_rect.rotation();
const float angle =
options.ignore_rotation() ? 0 : input_rect.rotation();
float new_x = std::cos(angle) * x - std::sin(angle) * y;
float new_y = std::sin(angle) * x + std::cos(angle) * y;
@ -116,10 +130,10 @@ class LandmarkProjectionCalculator : public CalculatorBase {
new_landmark->set_z(landmark.z());
}
cc->Outputs()
.Tag(kLandmarksTag)
.AddPacket(MakePacket<NormalizedLandmarkList>(output_landmarks)
cc->Outputs().Get(output_id).AddPacket(
MakePacket<NormalizedLandmarkList>(output_landmarks)
.At(cc->InputTimestamp()));
}
return ::mediapipe::OkStatus();
}
};

75
mediapipe/calculators/util/local_file_pattern_contents_calculator.cc Normal file
View File

@ -0,0 +1,75 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
// The calculator takes the path to local directory and desired file suffix to
// mach as input side packets, and outputs the contents of those files that
// match the pattern. Those matched files will be sent sequentially through the
// output stream with incremental timestamp difference by 1.
//
// Example config:
// node {
// calculator: "LocalFilePatternContentsCalculator"
// input_side_packet: "FILE_DIRECTORY:file_directory"
// input_side_packet: "FILE_SUFFIX:file_suffix"
// output_stream: "CONTENTS:contents"
// }
class LocalFilePatternContentsCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->InputSidePackets().Tag("FILE_DIRECTORY").Set<std::string>();
cc->InputSidePackets().Tag("FILE_SUFFIX").Set<std::string>();
cc->Outputs().Tag("CONTENTS").Set<std::string>();
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
MP_RETURN_IF_ERROR(::mediapipe::file::MatchFileTypeInDirectory(
cc->InputSidePackets().Tag("FILE_DIRECTORY").Get<std::string>(),
cc->InputSidePackets().Tag("FILE_SUFFIX").Get<std::string>(),
&filenames_));
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
if (current_output_ < filenames_.size()) {
auto contents = absl::make_unique<std::string>();
LOG(INFO) << filenames_[current_output_];
MP_RETURN_IF_ERROR(mediapipe::file::GetContents(
filenames_[current_output_], contents.get()));
++current_output_;
cc->Outputs()
.Tag("CONTENTS")
.Add(contents.release(), Timestamp(current_output_));
} else {
return tool::StatusStop();
}
return ::mediapipe::OkStatus();
}
private:
std::vector<std::string> filenames_;
int current_output_ = 0;
};
REGISTER_CALCULATOR(LocalFilePatternContentsCalculator);
} // namespace mediapipe

10
mediapipe/calculators/util/rect_to_render_data_calculator.cc
View File

@ -45,15 +45,17 @@ RenderAnnotation::Rectangle* NewRect(
void SetRect(bool normalized, double xmin, double ymin, double width,
double height, double rotation,
RenderAnnotation::Rectangle* rect) {
if (rotation == 0.0) {
if (xmin + width < 0.0 || ymin + height < 0.0) return;
if (normalized) {
if (xmin > 1.0 || ymin > 1.0) return;
}
}
rect->set_normalized(normalized);
rect->set_left(normalized ? std::max(xmin, 0.0) : xmin);
rect->set_top(normalized ? std::max(ymin, 0.0) : ymin);
rect->set_right(normalized ? std::min(xmin + width, 1.0) : xmin + width);
rect->set_bottom(normalized ? std::min(ymin + height, 1.0) : ymin + height);
rect->set_left(xmin);
rect->set_top(ymin);
rect->set_right(xmin + width);
rect->set_bottom(ymin + height);
rect->set_rotation(rotation);
}

1
mediapipe/calculators/video/BUILD
View File

@ -368,6 +368,7 @@ cc_test(
cc_test(
name = "tvl1_optical_flow_calculator_test",
srcs = ["tvl1_optical_flow_calculator_test.cc"],
linkstatic = 1,
deps = [
":tvl1_optical_flow_calculator",
"//mediapipe/framework:calculator_framework",

4
mediapipe/docs/face_detection_desktop.md
View File

@ -259,9 +259,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video"
input_stream: "IMAGE:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
```

4
mediapipe/docs/face_detection_mobile_cpu.md
View File

@ -229,9 +229,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video_cpu"
input_stream: "IMAGE:input_video_cpu"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video_cpu"
output_stream: "IMAGE:output_video_cpu"
}
# Transfers the annotated image from CPU back to GPU memory, to be sent out of

4
mediapipe/docs/face_detection_mobile_gpu.md
View File

@ -221,8 +221,8 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
input_stream: "IMAGE_GPU:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME_GPU:output_video"
output_stream: "IMAGE_GPU:output_video"
}
```

4
mediapipe/docs/hand_detection_mobile_gpu.md
View File

@ -136,10 +136,10 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
input_stream: "IMAGE_GPU:throttled_input_video"
input_stream: "detection_render_data"
input_stream: "rect_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_video"
output_stream: "IMAGE_GPU:output_video"
}
```

4
mediapipe/docs/hand_tracking_mobile_gpu.md
View File

@ -716,10 +716,10 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:input_image"
input_stream: "IMAGE_GPU:input_image"
input_stream: "detection_render_data"
input_stream: "landmark_render_data"
input_stream: "rect_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_image"
output_stream: "IMAGE_GPU:output_image"
}
```

22
mediapipe/docs/install.md
View File

@ -40,7 +40,7 @@ To build and run iOS apps:
$ cd mediapipe
```
2. Install Bazel (version between 0.24.1 and 1.2.1).
2. Install Bazel (version between 1.0.0 and 1.2.1).
Follow the official
[Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
@ -152,7 +152,7 @@ To build and run iOS apps:
$ cd mediapipe
```
2. Install Bazel (version between 0.24.1 and 1.2.1).
2. Install Bazel (version between 1.0.0 and 1.2.1).
Follow the official
[Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html)
@ -241,7 +241,7 @@ To build and run iOS apps:
$ cd mediapipe
```
3. Install Bazel (version between 0.24.1 and 1.1.0).
3. Install Bazel (version between 1.0.0 and 1.1.0).
Option 1. Use package manager tool to install Bazel 1.1.0
@ -389,18 +389,18 @@ a video file as input.
username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk
```
5. Install Bazel (version between 0.24.1 and 1.2.1).
5. Install Bazel (version between 1.0.0 and 1.2.1).
```bash
username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \
https://storage.googleapis.com/bazel/0.27.0/release/bazel-0.27.0-installer-linux-x86_64.sh && \
sudo mkdir -p /usr/local/bazel/0.27.0 && \
chmod 755 bazel-0.27.0-installer-linux-x86_64.sh && \
sudo ./bazel-0.27.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/0.27.0 && \
source /usr/local/bazel/0.27.0/lib/bazel/bin/bazel-complete.bash
https://storage.googleapis.com/bazel/1.0.0/release/bazel-1.0.0-installer-linux-x86_64.sh && \
sudo mkdir -p /usr/local/bazel/1.0.0 && \
chmod 755 bazel-1.0.0-installer-linux-x86_64.sh && \
sudo ./bazel-1.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/1.0.0 && \
source /usr/local/bazel/1.0.0/lib/bazel/bin/bazel-complete.bash
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/0.27.0/lib/bazel/bin/bazel version && \
alias bazel='/usr/local/bazel/0.27.0/lib/bazel/bin/bazel'
username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/1.0.0/lib/bazel/bin/bazel version && \
alias bazel='/usr/local/bazel/1.0.0/lib/bazel/bin/bazel'
```
6. Checkout MediaPipe repository.

4
mediapipe/docs/multi_hand_tracking_mobile_gpu.md
View File

@ -745,11 +745,11 @@ node {
# a vector of RenderData objects and draws each of them on the input frame.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:input_image"
input_stream: "IMAGE_GPU:input_image"
input_stream: "detection_render_data"
input_stream: "multi_hand_rects_render_data"
input_stream: "multi_palm_rects_render_data"
input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_image"
output_stream: "IMAGE_GPU:output_image"
}
```

8
mediapipe/docs/object_detection_desktop.md
View File

@ -163,9 +163,9 @@ node {
# the graph.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video"
input_stream: "IMAGE:input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
# Encodes the annotated images into a video file, adopting properties specified
@ -396,9 +396,9 @@ node {
# the graph.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video"
input_stream: "IMAGE:input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
# Encodes the annotated images into a video file, adopting properties specified

4
mediapipe/docs/object_detection_mobile_cpu.md
View File

@ -230,9 +230,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video_cpu"
input_stream: "IMAGE:throttled_input_video_cpu"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video_cpu"
output_stream: "IMAGE:output_video_cpu"
}
# Transfers the annotated image from CPU back to GPU memory, to be sent out of

4
mediapipe/docs/object_detection_mobile_gpu.md
View File

@ -212,8 +212,8 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
input_stream: "IMAGE_GPU:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME_GPU:output_video"
output_stream: "IMAGE_GPU:output_video"
}
```

7
mediapipe/docs/object_tracking_mobile_gpu.md
View File

@ -467,9 +467,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:input_image"
input_stream: "IMAGE_GPU:input_image"
input_stream: "detections_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_image"
output_stream: "IMAGE_GPU:output_image"
}
```
@ -484,7 +484,8 @@ CPU.
To build and run the app:
```bash
bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu
bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu \
--define MEDIAPIPE_DISABLE_GPU=1
bazel-bin/mediapipe/examples/desktop/object_tracking/object_tracking_cpu \
--calculator_graph_config_file=mediapipe/graphs/tracking/object_detection_tracking_desktop_live.pbtxt

4
mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt
View File

@ -182,8 +182,8 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video"
input_stream: "IMAGE:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}

4
mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt
View File

@ -173,7 +173,7 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video"
input_stream: "IMAGE:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}

2
mediapipe/examples/desktop/BUILD
View File

@ -53,7 +53,7 @@ cc_library(
# Linux only.
# Must have a GPU with EGL support:
# ex: sudo aptitude install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
# ex: sudo apt-get install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev
# (or similar nvidia/amd equivalent)
cc_library(
name = "demo_run_graph_main_gpu",

2
mediapipe/examples/desktop/media_sequence/charades_dataset.py
View File

@ -68,7 +68,7 @@ import zipfile
from absl import app
from absl import flags
from absl import logging
import tensorflow as tf
import tensorflow.compat.v1 as tf
from mediapipe.util.sequence import media_sequence as ms

2
mediapipe/examples/desktop/media_sequence/demo_dataset.py
View File

@ -62,7 +62,7 @@ import urllib
from absl import app
from absl import flags
from absl import logging
import tensorflow as tf
import tensorflow.compat.v1 as tf
from mediapipe.util.sequence import media_sequence as ms

2
mediapipe/examples/desktop/media_sequence/kinetics_dataset.py
View File

@ -78,7 +78,7 @@ import urllib
from absl import app
from absl import flags
from absl import logging
import tensorflow as tf
import tensorflow.compat.v1 as tf
from mediapipe.util.sequence import media_sequence as ms

3
mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py
View File

@ -21,6 +21,7 @@ import sys
from absl import app
from absl import flags
import six
import tensorflow.compat.v1 as tf
from mediapipe.util.sequence import media_sequence as ms
@ -54,7 +55,7 @@ def main(argv):
ms.set_clip_end_timestamp(
flags.FLAGS.clip_end_time_sec * SECONDS_TO_MICROSECONDS, metadata)
with open('/tmp/mediapipe/metadata.pb', 'wb') as writer:
writer.write(metadata.SerializeToString())
writer.write(six.ensure_binary(metadata.SerializeToString()))
if __name__ == '__main__':

2
mediapipe/framework/calculator_graph.cc
View File

@ -1301,7 +1301,7 @@ void PrintTimingToInfo(const std::string& label, int64 timer_value) {
"%02lld days, %02lld:%02lld:%02lld.%03lld (total seconds: "
"%lld.%06lld)",
days, hours, minutes, seconds, milliseconds, total_seconds,
timer_value % 1000000ll);
timer_value % int64{1000000});
}
bool MetricElementComparator(const std::pair<std::string, int64>& e1,

1
mediapipe/framework/deps/BUILD
View File

@ -251,6 +251,7 @@ cc_library(
"//mediapipe/framework/port:logging",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
],
)

24
mediapipe/framework/deps/file_helpers.cc
View File

@ -105,6 +105,30 @@ namespace file {
return ::mediapipe::OkStatus();
}
::mediapipe::Status MatchFileTypeInDirectory(
const std::string& directory, const std::string& file_suffix,
std::vector<std::string>* results) {
DIR* dir = opendir(directory.c_str());
CHECK(dir);
// Iterates through the direcotry.
while (true) {
struct dirent* dir_ent = readdir(dir);
if (dir_ent == nullptr) {
break;
}
if (std::string(dir_ent->d_name) == "." ||
std::string(dir_ent->d_name) == "..") {
continue;
}
if (absl::EndsWith(std::string(dir_ent->d_name), file_suffix)) {
results->push_back(JoinPath(directory, std::string(dir_ent->d_name)));
}
}
closedir(dir);
return ::mediapipe::OkStatus();
}
::mediapipe::Status Exists(absl::string_view file_name) {
struct stat buffer;
int status;

4
mediapipe/framework/deps/file_helpers.h
View File

@ -30,6 +30,10 @@ namespace file {
const std::string& parent_directory, const std::string& file_name,
std::vector<std::string>* results);
::mediapipe::Status MatchFileTypeInDirectory(const std::string& directory,
const std::string& file_suffix,
std::vector<std::string>* results);
::mediapipe::Status Exists(absl::string_view file_name);
} // namespace file

97
mediapipe/framework/deps/status.cc
View File

@ -18,103 +18,6 @@
namespace mediapipe {
Status::Status(::mediapipe::StatusCode code, absl::string_view msg) {
state_ = std::unique_ptr<State>(new State);
state_->code = code;
state_->msg = std::string(msg);
}
void Status::Update(const Status& new_status) {
if (ok()) {
*this = new_status;
}
}
void Status::SlowCopyFrom(const State* src) {
if (src == nullptr) {
state_ = nullptr;
} else {
state_ = std::unique_ptr<State>(new State(*src));
}
}
const std::string& Status::empty_string() {
static std::string* empty = new std::string;
return *empty;
}
std::string Status::ToString() const {
if (state_ == nullptr) {
return "OK";
} else {
char tmp[30];
const char* type;
switch (code()) {
case ::mediapipe::StatusCode::kCancelled:
type = "Cancelled";
break;
case ::mediapipe::StatusCode::kUnknown:
type = "Unknown";
break;
case ::mediapipe::StatusCode::kInvalidArgument:
type = "Invalid argument";
break;
case ::mediapipe::StatusCode::kDeadlineExceeded:
type = "Deadline exceeded";
break;
case ::mediapipe::StatusCode::kNotFound:
type = "Not found";
break;
case ::mediapipe::StatusCode::kAlreadyExists:
type = "Already exists";
break;
case ::mediapipe::StatusCode::kPermissionDenied:
type = "Permission denied";
break;
case ::mediapipe::StatusCode::kUnauthenticated:
type = "Unauthenticated";
break;
case ::mediapipe::StatusCode::kResourceExhausted:
type = "Resource exhausted";
break;
case ::mediapipe::StatusCode::kFailedPrecondition:
type = "Failed precondition";
break;
case ::mediapipe::StatusCode::kAborted:
type = "Aborted";
break;
case ::mediapipe::StatusCode::kOutOfRange:
type = "Out of range";
break;
case ::mediapipe::StatusCode::kUnimplemented:
type = "Unimplemented";
break;
case ::mediapipe::StatusCode::kInternal:
type = "Internal";
break;
case ::mediapipe::StatusCode::kUnavailable:
type = "Unavailable";
break;
case ::mediapipe::StatusCode::kDataLoss:
type = "Data loss";
break;
default:
snprintf(tmp, sizeof(tmp), "Unknown code(%d)",
static_cast<int>(code()));
type = tmp;
break;
}
std::string result(type);
result += ": ";
result += state_->msg;
return result;
}
}
void Status::IgnoreError() const {
// no-op
}
std::ostream& operator<<(std::ostream& os, const Status& x) {
os << x.ToString();
return os;

118
mediapipe/framework/deps/status.h
View File

@ -20,126 +20,16 @@
#include <memory>
#include <string>
#include "absl/status/status.h"
#include "absl/strings/string_view.h"
#include "mediapipe/framework/port/logging.h"
namespace mediapipe {
enum class StatusCode {
kOk = 0,
kCancelled = 1,
kUnknown = 2,
kInvalidArgument = 3,
kDeadlineExceeded = 4,
kNotFound = 5,
kAlreadyExists = 6,
kPermissionDenied = 7,
kResourceExhausted = 8,
kFailedPrecondition = 9,
kAborted = 10,
kOutOfRange = 11,
kUnimplemented = 12,
kInternal = 13,
kUnavailable = 14,
kDataLoss = 15,
kUnauthenticated = 16,
kDoNotUseReservedForFutureExpansionUseDefaultInSwitchInstead_ = 20
};
using Status = absl::Status;
using StatusCode = absl::StatusCode;
#if defined(__clang__)
// Only clang supports warn_unused_result as a type annotation.
class ABSL_MUST_USE_RESULT Status;
#endif
// Denotes success or failure of a call in MediaPipe.
class Status {
public:
// Creates a success status.
Status() {}
// Creates a status with the specified error code and msg as a
// human-readable std::string containing more detailed information.
Status(::mediapipe::StatusCode code, absl::string_view msg);
// Copies the specified status.
Status(const Status& s);
void operator=(const Status& s);
// Returns true iff the status indicates success.
bool ok() const {
return (state_ == NULL) || (state_->code == ::mediapipe::StatusCode::kOk);
}
::mediapipe::StatusCode code() const {
return ok() ? ::mediapipe::StatusCode::kOk : state_->code;
}
const std::string& error_message() const {
return ok() ? empty_string() : state_->msg;
}
absl::string_view message() const {
return absl::string_view(error_message());
}
bool operator==(const Status& x) const;
bool operator!=(const Status& x) const;
// If `ok()`, stores `new_status` into `*this`. If `!ok()`,
// preserves the current status, but may augment with additional
// information about `new_status`.
//
// Convenient way of keeping track of the first error encountered.
// Instead of:
// `if (overall_status.ok()) overall_status = new_status`
// Use:
// `overall_status.Update(new_status);`
void Update(const Status& new_status);
// Returns a std::string representation of this status suitable for
// printing. Returns the std::string `"OK"` for success.
std::string ToString() const;
// Ignores any errors. This method does nothing except potentially suppress
// complaints from any tools that are checking that errors are not dropped on
// the floor.
void IgnoreError() const;
private:
static const std::string& empty_string();
struct State {
::mediapipe::StatusCode code;
std::string msg;
};
// OK status has a `NULL` state_. Otherwise, `state_` points to
// a `State` structure containing the error code and message(s)
std::unique_ptr<State> state_;
void SlowCopyFrom(const State* src);
};
inline Status::Status(const Status& s)
: state_((s.state_ == NULL) ? NULL : new State(*s.state_)) {}
inline void Status::operator=(const Status& s) {
// The following condition catches both aliasing (when this == &s),
// and the common case where both s and *this are ok.
if (state_ != s.state_) {
SlowCopyFrom(s.state_.get());
}
}
inline bool Status::operator==(const Status& x) const {
return (this->state_ == x.state_) || (ToString() == x.ToString());
}
inline bool Status::operator!=(const Status& x) const { return !(*this == x); }
inline Status OkStatus() { return Status(); }
std::ostream& operator<<(std::ostream& os, const Status& x);
typedef std::function<void(const Status&)> StatusCallback;
inline ::mediapipe::Status OkStatus() { return absl::OkStatus(); }
extern std::string* MediaPipeCheckOpHelperOutOfLine(
const ::mediapipe::Status& v, const char* msg);

6
mediapipe/framework/deps/status_builder.cc
View File

@ -72,12 +72,12 @@ StatusBuilder::operator Status() && {
std::string message;
if (join_style_ == MessageJoinStyle::kAnnotate) {
if (!status_.ok()) {
message = absl::StrCat(status_.error_message(), "; ", stream_->str());
message = absl::StrCat(status_.message(), "; ", stream_->str());
}
} else {
message = join_style_ == MessageJoinStyle::kPrepend
? absl::StrCat(stream_->str(), status_.error_message())
: absl::StrCat(status_.error_message(), stream_->str());
? absl::StrCat(stream_->str(), status_.message())
: absl::StrCat(status_.message(), stream_->str());
}
return Status(status_.code(), message);
}

9
mediapipe/framework/deps/status_builder_test.cc
View File

@ -27,7 +27,7 @@ TEST(StatusBuilder, AnnotateMode) {
<< "annotated message2";
ASSERT_FALSE(status.ok());
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kNotFound);
EXPECT_EQ(status.error_message(),
EXPECT_EQ(status.message(),
"original message; annotated message1 annotated message2");
}
@ -42,7 +42,7 @@ TEST(StatusBuilder, PrependMode) {
<< "prepended message2 ";
ASSERT_FALSE(status.ok());
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kInvalidArgument);
EXPECT_EQ(status.error_message(),
EXPECT_EQ(status.message(),
"prepended message1 prepended message2 original message");
}
@ -56,8 +56,7 @@ TEST(StatusBuilder, AppendMode) {
<< " extra message2";
ASSERT_FALSE(status.ok());
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kInternal);
EXPECT_EQ(status.error_message(),
"original message extra message1 extra message2");
EXPECT_EQ(status.message(), "original message extra message1 extra message2");
}
TEST(StatusBuilder, NoLoggingMode) {
@ -69,7 +68,7 @@ TEST(StatusBuilder, NoLoggingMode) {
<< " extra message";
ASSERT_FALSE(status.ok());
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kUnavailable);
EXPECT_EQ(status.error_message(), "original message");
EXPECT_EQ(status.message(), "original message");
}
} // namespace mediapipe

4
mediapipe/framework/deps/status_test.cc
View File

@ -21,7 +21,7 @@ namespace mediapipe {
TEST(Status, OK) {
EXPECT_EQ(OkStatus().code(), ::mediapipe::StatusCode::kOk);
EXPECT_EQ(OkStatus().error_message(), "");
EXPECT_EQ(OkStatus().message(), "");
MP_EXPECT_OK(OkStatus());
MP_ASSERT_OK(OkStatus());
EXPECT_EQ(OkStatus(), Status());
@ -38,7 +38,7 @@ TEST(Status, Set) {
Status status;
status = Status(::mediapipe::StatusCode::kCancelled, "Error message");
EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kCancelled);
EXPECT_EQ(status.error_message(), "Error message");
EXPECT_EQ(status.message(), "Error message");
}
TEST(Status, Copy) {

12
mediapipe/framework/deps/statusor_test.cc
View File

@ -158,12 +158,12 @@ TEST(StatusOr, TestMoveWithValuesAndErrors) {
// Overwrite the value in status_or with an error.
status_or = std::move(error1);
ASSERT_FALSE(status_or.ok());
EXPECT_EQ("error1", status_or.status().error_message());
EXPECT_EQ("error1", status_or.status().message());
// Overwrite the error in status_or with another error.
status_or = std::move(error2);
ASSERT_FALSE(status_or.ok());
EXPECT_EQ("error2", status_or.status().error_message());
EXPECT_EQ("error2", status_or.status().message());
// Overwrite the error with a value.
status_or = std::move(value2);
@ -191,12 +191,12 @@ TEST(StatusOr, TestCopyWithValuesAndErrors) {
// Overwrite the value in status_or with an error.
status_or = error1;
ASSERT_FALSE(status_or.ok());
EXPECT_EQ("error1", status_or.status().error_message());
EXPECT_EQ("error1", status_or.status().message());
// Overwrite the error in status_or with another error.
status_or = error2;
ASSERT_FALSE(status_or.ok());
EXPECT_EQ("error2", status_or.status().error_message());
EXPECT_EQ("error2", status_or.status().message());
// Overwrite the error with a value.
status_or = value2;
@ -205,8 +205,8 @@ TEST(StatusOr, TestCopyWithValuesAndErrors) {
// Verify original values unchanged.
EXPECT_EQ(std::string(1000, '1'), value1.ValueOrDie());
EXPECT_EQ("error1", error1.status().error_message());
EXPECT_EQ("error2", error2.status().error_message());
EXPECT_EQ("error1", error1.status().message());
EXPECT_EQ("error2", error2.status().message());
EXPECT_EQ(std::string(1000, '2'), value2.ValueOrDie());
}

5
mediapipe/framework/encode_binary_proto.bzl
View File

@ -36,6 +36,11 @@ def _canonicalize_proto_path_oss(all_protos, genfile_path):
for s in all_protos.to_list():
if s.path.startswith(genfile_path):
repo_name, _, file_name = s.path[len(genfile_path + "/external/"):].partition("/")
# handle virtual imports
if file_name.startswith("_virtual_imports"):
repo_name = repo_name + "/" + "/".join(file_name.split("/", 2)[:2])
file_name = file_name.split("/", 2)[-1]
proto_paths.append(genfile_path + "/external/" + repo_name)
proto_file_names.append(file_name)
else:

2
mediapipe/framework/formats/BUILD
View File

@ -268,7 +268,7 @@ java_lite_proto_library(
visibility = [
"//mediapipe:__subpackages__",
],
deps = [":landmark_proto"],
deps = [":rect_proto"],
)
proto_library(

7
mediapipe/framework/formats/rect.proto
View File

@ -16,6 +16,9 @@ syntax = "proto2";
package mediapipe;
option java_package = "com.google.mediapipe.formats.proto";
option java_outer_classname = "RectProto";
// A rectangle with rotation in image coordinates.
message Rect {
// Location of the center of the rectangle in image coordinates.
@ -27,7 +30,7 @@ message Rect {
required int32 height = 3;
required int32 width = 4;
// Rotation angle is counter-clockwise in radian.
// Rotation angle is clockwise in radians.
optional float rotation = 5 [default = 0.0];
// Optional unique id to help associate different Rects to each other.
@ -46,7 +49,7 @@ message NormalizedRect {
required float height = 3;
required float width = 4;
// Rotation angle is counter-clockwise in radian.
// Rotation angle is clockwise in radians.
optional float rotation = 5 [default = 0.0];
// Optional unique id to help associate different NormalizedRects to each

65
mediapipe/framework/graph_validation_test.cc
View File

@ -325,10 +325,8 @@ class OptionalSideInputTestCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->InputSidePackets().Tag("SIDEINPUT").Set<std::string>().Optional();
if (!cc->Outputs().HasTag("OUTPUT")) {
return ::mediapipe::InvalidArgumentError(
"Expected std::string as output.");
}
cc->Inputs().Tag("SELECT").Set<int>().Optional();
cc->Inputs().Tag("ENABLE").Set<bool>().Optional();
cc->Outputs().Tag("OUTPUT").Set<std::string>();
return ::mediapipe::OkStatus();
}
@ -394,5 +392,64 @@ TEST(GraphValidationTest, OptionalInputNotProvidedForSubgraphCalculator) {
MP_EXPECT_OK(graph_1.WaitUntilDone());
}
TEST(GraphValidationTest, MultipleOptionalInputsForSubgraph) {
// A subgraph defining one optional side-packet and two optional inputs.
auto config_1 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
type: "PassThroughGraph"
input_side_packet: "INPUT:input_0"
input_stream: "SELECT:select"
input_stream: "ENABLE:enable"
output_stream: "OUTPUT:output_0"
node {
calculator: "OptionalSideInputTestCalculator"
input_side_packet: "SIDEINPUT:input_0" # std::string
input_stream: "SELECT:select"
input_stream: "ENABLE:enable"
output_stream: "OUTPUT:output_0" # std::string
}
)");
// An enclosing graph that specifies just one optional input.
auto config_2 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
input_side_packet: "INPUT:foo_in"
input_stream: "SELECT:foo_select"
output_stream: "OUTPUT:foo_out"
node {
calculator: "PassThroughGraph"
input_stream: "SELECT:foo_select"
output_stream: "OUTPUT:foo_out" # std::string
}
)");
GraphValidation validation_1;
MP_ASSERT_OK(validation_1.Validate({config_1, config_2}, {}));
CalculatorGraph graph_1;
MP_ASSERT_OK(graph_1.Initialize({config_1, config_2}, {}));
EXPECT_THAT(
graph_1.Config(),
// The expanded graph includes only the specified input, "SELECT".
// Without the fix to RemoveIgnoredStreams(), the expanded graph
// includes the wrong input.
EqualsProto(::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
input_side_packet: "INPUT:foo_in"
input_stream: "SELECT:foo_select"
output_stream: "OUTPUT:foo_out"
node {
calculator: "OptionalSideInputTestCalculator"
name: "passthroughgraph__OptionalSideInputTestCalculator"
input_stream: "SELECT:foo_select"
output_stream: "OUTPUT:foo_out"
}
executor {}
)")));
std::map<std::string, Packet> side_packets;
side_packets.insert({"foo_in", mediapipe::Adopt(new std::string("input"))});
MP_EXPECT_OK(graph_1.StartRun(side_packets));
MP_EXPECT_OK(graph_1.CloseAllPacketSources());
MP_EXPECT_OK(graph_1.WaitUntilDone());
}
} // namespace
} // namespace mediapipe

7
mediapipe/framework/packet.cc
View File

@ -40,6 +40,13 @@ Packet Create(HolderBase* holder, Timestamp timestamp) {
return result;
}
Packet Create(std::shared_ptr<HolderBase> holder, Timestamp timestamp) {
Packet result;
result.holder_ = std::move(holder);
result.timestamp_ = timestamp;
return result;
}
const HolderBase* GetHolder(const Packet& packet) {
return packet.holder_.get();
}

17
mediapipe/framework/packet.h
View File

@ -48,7 +48,9 @@ class HolderBase;
Packet Create(HolderBase* holder);
Packet Create(HolderBase* holder, Timestamp timestamp);
Packet Create(std::shared_ptr<HolderBase> holder, Timestamp timestamp);
const HolderBase* GetHolder(const Packet& packet);
const std::shared_ptr<HolderBase>& GetHolderShared(const Packet& packet);
} // namespace packet_internal
// A generic container class which can hold data of any type. The type of
@ -201,8 +203,14 @@ class Packet {
friend Packet packet_internal::Create(packet_internal::HolderBase* holder);
friend Packet packet_internal::Create(packet_internal::HolderBase* holder,
class Timestamp timestamp);
friend Packet packet_internal::Create(
std::shared_ptr<packet_internal::HolderBase> holder,
class Timestamp timestamp);
friend const packet_internal::HolderBase* packet_internal::GetHolder(
const Packet& packet);
friend const std::shared_ptr<packet_internal::HolderBase>&
packet_internal::GetHolderShared(const Packet& packet);
std::shared_ptr<packet_internal::HolderBase> holder_;
class Timestamp timestamp_;
};
@ -713,6 +721,15 @@ inline bool operator!=(const Packet& p1, const Packet& p2) {
return !(p1 == p2);
}
namespace packet_internal {
inline const std::shared_ptr<HolderBase>& GetHolderShared(
const Packet& packet) {
return packet.holder_;
}
} // namespace packet_internal
} // namespace mediapipe
#endif // MEDIAPIPE_FRAMEWORK_PACKET_H_

30
mediapipe/framework/port.h
View File

@ -50,4 +50,34 @@
#define MEDIAPIPE_DISABLE_GL_COMPUTE
#endif
// Compile time target platform definitions.
// Example: #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
#define MEDIAPIPE_OPENGL_ES_UNSUPPORTED 0
#define MEDIAPIPE_OPENGL_ES_20 200
#define MEDIAPIPE_OPENGL_ES_31 310
#if defined(MEDIAPIPE_DISABLE_GPU)
#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
#define MEDIAPIPE_METAL_ENABLED 0
#else
#if defined(MEDIAPIPE_ANDROID)
#if defined(MEDIAPIPE_DISABLE_GL_COMPUTE)
#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_20
#else
#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_31
#endif
#define MEDIAPIPE_METAL_ENABLED 0
#elif defined(MEDIAPIPE_IOS)
#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_20
#define MEDIAPIPE_METAL_ENABLED 1
#elif defined(MEDIAPIPE_OSX)
#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
#define MEDIAPIPE_METAL_ENABLED 1
#else
// GPU is not supported on Linux yet.
#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED
#define MEDIAPIPE_METAL_ENABLED 0
#endif
#endif
#endif // MEDIAPIPE_FRAMEWORK_PORT_H_

1
mediapipe/framework/port/BUILD
View File

@ -351,6 +351,7 @@ cc_library(
":source_location",
"//mediapipe/framework:port",
"//mediapipe/framework/deps:status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
],
)

4
mediapipe/framework/profiler/graph_tracer_test.cc
View File

@ -1267,9 +1267,9 @@ TEST_F(GraphTracerE2ETest, GpuTracing) {
output_stream: "annotated_buffer"
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_buffer"
input_stream: "IMAGE:input_buffer"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:annotated_buffer"
output_stream: "IMAGE:annotated_buffer"
}
profiler_config {
trace_enabled: true

2
mediapipe/framework/profiler/profiler_resource_util_apple.cc
View File

@ -28,7 +28,7 @@ StatusOr<std::string> GetDefaultTraceLogDirectory() {
// Note: "createDirectoryAtURL:..." method doesn't successfully create
// the directory, hence this code uses "createDirectoryAtPath:..".
NSString* ns_documents_directory = [documents_directory_url absoluteString];
NSString* ns_documents_directory = [documents_directory_url path];
NSError* error;
BOOL success = [[NSFileManager defaultManager]
createDirectoryAtPath:ns_documents_directory

24
mediapipe/framework/profiler/testdata/BUILD vendored Normal file
View File

@ -0,0 +1,24 @@
load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
package(default_visibility = ["//mediapipe/framework/profiler:__subpackages__"])
licenses(["notice"])
# Compile any proto data into wire format for use in our tests.
[encode_binary_proto(
name = graph.split("/")[-1].rsplit(".", 1)[0],
input = graph,
message_type = "mediapipe.GraphProfile",
output = graph.rsplit(".", 1)[0] + ".binarypb",
deps = ["//mediapipe/framework:calculator_profile_proto"],
) for graph in glob(["profile_*.pbtxt"])]
filegroup(
name = "mediapipe_profile_graphs",
srcs = [binarypb.rsplit(".", 1)[0] + ".binarypb" for binarypb in glob(["profile_*.pbtxt"])],
)
filegroup(
name = "pbtxt_files",
srcs = glob(["*.pbtxt"]),
)

8814
mediapipe/framework/profiler/testdata/profile_opencv_0.pbtxt vendored Normal file
View File

File diff suppressed because it is too large Load Diff

4957
mediapipe/framework/profiler/testdata/profile_opencv_1.pbtxt vendored Normal file
View File

File diff suppressed because it is too large Load Diff

2
mediapipe/framework/tool/BUILD
View File

@ -177,7 +177,7 @@ cc_library(
deps = [
"//mediapipe/framework:packet",
"//mediapipe/framework/port:statusor",
"@org_tensorflow//tensorflow/core:protos_all",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
)

15
mediapipe/framework/tool/status_util_test.cc
View File

@ -52,8 +52,10 @@ TEST(StatusTest, CombinedStatus) {
errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument,
"error_with_that_string");
status = tool::CombinedStatus(prefix_error_message, errors);
EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[0].error_message()));
EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[1].error_message()));
EXPECT_THAT(status.ToString(),
testing::HasSubstr(std::string(errors[0].message())));
EXPECT_THAT(status.ToString(),
testing::HasSubstr(std::string(errors[1].message())));
EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message));
EXPECT_EQ(::mediapipe::StatusCode::kInvalidArgument, status.code());
@ -63,8 +65,10 @@ TEST(StatusTest, CombinedStatus) {
errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument,
"error_with_that_string");
status = tool::CombinedStatus(prefix_error_message, errors);
EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[0].error_message()));
EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[1].error_message()));
EXPECT_THAT(status.ToString(),
testing::HasSubstr(std::string(errors[0].message())));
EXPECT_THAT(status.ToString(),
testing::HasSubstr(std::string(errors[1].message())));
EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message));
EXPECT_EQ(::mediapipe::StatusCode::kUnknown, status.code());
errors.clear();
@ -72,7 +76,8 @@ TEST(StatusTest, CombinedStatus) {
errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument,
"error_with_that_string");
status = tool::CombinedStatus(prefix_error_message, errors);
EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[1].error_message()));
EXPECT_THAT(status.ToString(),
testing::HasSubstr(std::string(errors[1].message())));
EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message));
EXPECT_EQ(::mediapipe::StatusCode::kInvalidArgument, status.code());

7
mediapipe/framework/tool/subgraph_expansion.cc
View File

@ -76,10 +76,11 @@ namespace tool {
::mediapipe::Status RemoveIgnoredStreams(
proto_ns::RepeatedPtrField<ProtoString>* streams,
const std::set<std::string>& missing_streams) {
ASSIGN_OR_RETURN(auto src_map, tool::TagMap::Create(*streams));
std::vector<std::string> src_names = src_map->Names();
for (int i = streams->size() - 1; i >= 0; --i) {
if (missing_streams.count(src_names[i]) > 0) {
std::string tag, name;
int index;
MP_RETURN_IF_ERROR(ParseTagIndexName(streams->Get(i), &tag, &index, &name));
if (missing_streams.count(name) > 0) {
streams->DeleteSubrange(i, 1);
}
}

4
mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt
View File

@ -177,8 +177,8 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video"
input_stream: "IMAGE:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}

4
mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt
View File

@ -188,9 +188,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video_cpu"
input_stream: "IMAGE:input_video_cpu"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video_cpu"
output_stream: "IMAGE:output_video_cpu"
}
# Transfers the annotated image from CPU back to GPU memory, to be sent out of

4
mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt
View File

@ -178,7 +178,7 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
input_stream: "IMAGE_GPU:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME_GPU:output_video"
output_stream: "IMAGE_GPU:output_video"
}

4
mediapipe/graphs/hand_tracking/BUILD
View File

@ -21,6 +21,10 @@ licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:public"])
exports_files(glob([
"*.pbtxt",
]))
cc_library(
name = "desktop_offline_calculators",
deps = [

4
mediapipe/graphs/hand_tracking/hand_detection_desktop.pbtxt
View File

@ -41,9 +41,9 @@ node {
# the graph.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video"
input_stream: "IMAGE:input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
# Encodes the annotated images into a video file, adopting properties specified

4
mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt
View File

@ -32,7 +32,7 @@ node {
# the graph.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video"
input_stream: "IMAGE:input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}

4
mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt
View File

@ -66,8 +66,8 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
input_stream: "IMAGE_GPU:throttled_input_video"
input_stream: "detection_render_data"
input_stream: "rect_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_video"
output_stream: "IMAGE_GPU:output_video"
}

15
mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_cpu.pbtxt
View File

@ -14,6 +14,11 @@ node {
input_stream: "IMAGE:input_video"
input_stream: "NORM_RECT:hand_rect"
output_stream: "IMAGE:hand_image"
node_options: {
[type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
border_mode: BORDER_REPLICATE
}
}
}
# Transforms the input image on CPU to a 256x256 image. To scale the input
@ -33,13 +38,9 @@ node: {
}
}
# Converts the transformed input image on GPU into an image tensor stored in
# tflite::gpu::GlBuffer. The zero_center option is set to true to normalize the
# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f]. The flip_vertically
# option is set to true to account for the descrepancy between the
# representation of the input image (origin at the bottom-left corner, the
# OpenGL convention) and what the model used in this graph is expecting (origin
# at the top-left corner).
# Converts the transformed input image on CPU into an image tensor stored in
# TfliteTensor. The zero_center option is set to false to normalize the
# pixel values to [0.f, 1.f].
node {
calculator: "TfLiteConverterCalculator"
input_stream: "IMAGE:transformed_input_video"

5
mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt
View File

@ -14,6 +14,11 @@ node {
input_stream: "IMAGE_GPU:input_video"
input_stream: "NORM_RECT:hand_rect"
output_stream: "IMAGE_GPU:hand_image"
node_options: {
[type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] {
border_mode: BORDER_REPLICATE
}
}
}
# Transforms the input image on GPU to a 256x256 image. To scale the input

4
mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_cpu.pbtxt
View File

@ -135,10 +135,10 @@ node {
# a vector of RenderData objects and draws each of them on the input frame.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_image"
input_stream: "IMAGE:input_image"
input_stream: "detection_render_data"
input_stream: "multi_hand_rects_render_data"
input_stream: "multi_palm_rects_render_data"
input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
output_stream: "OUTPUT_FRAME:output_image"
output_stream: "IMAGE:output_image"
}

4
mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt
View File

@ -135,10 +135,10 @@ node {
# a vector of RenderData objects and draws each of them on the input frame.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:input_image"
input_stream: "IMAGE_GPU:input_image"
input_stream: "detection_render_data"
input_stream: "multi_hand_rects_render_data"
input_stream: "multi_palm_rects_render_data"
input_stream: "VECTOR:0:multi_hand_landmarks_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_image"
output_stream: "IMAGE_GPU:output_image"
}

4
mediapipe/graphs/hand_tracking/subgraphs/renderer_cpu.pbtxt
View File

@ -94,9 +94,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_image"
input_stream: "IMAGE:input_image"
input_stream: "detection_render_data"
input_stream: "landmark_render_data"
input_stream: "rect_render_data"
output_stream: "OUTPUT_FRAME:output_image"
output_stream: "IMAGE:output_image"
}

4
mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt
View File

@ -94,9 +94,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:input_image"
input_stream: "IMAGE_GPU:input_image"
input_stream: "detection_render_data"
input_stream: "landmark_render_data"
input_stream: "rect_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_image"
output_stream: "IMAGE_GPU:output_image"
}

4
mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt
View File

@ -168,7 +168,7 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video"
input_stream: "IMAGE:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}

4
mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt
View File

@ -109,9 +109,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video"
input_stream: "IMAGE:input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
# Encodes the annotated images into a video file, adopting properties specified

4
mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt
View File

@ -159,9 +159,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video"
input_stream: "IMAGE:input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
# Encodes the annotated images into a video file, adopting properties specified

4
mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt
View File

@ -179,9 +179,9 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:throttled_input_video_cpu"
input_stream: "IMAGE:throttled_input_video_cpu"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME:output_video_cpu"
output_stream: "IMAGE:output_video_cpu"
}
# Transfers the annotated image from CPU back to GPU memory, to be sent out of

4
mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt
View File

@ -169,7 +169,7 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:throttled_input_video"
input_stream: "IMAGE_GPU:throttled_input_video"
input_stream: "render_data"
output_stream: "OUTPUT_FRAME_GPU:output_video"
output_stream: "IMAGE_GPU:output_video"
}

4
mediapipe/graphs/tracking/subgraphs/renderer_cpu.pbtxt
View File

@ -23,7 +23,7 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_image"
input_stream: "IMAGE:input_image"
input_stream: "detections_render_data"
output_stream: "OUTPUT_FRAME:output_image"
output_stream: "IMAGE:output_image"
}

4
mediapipe/graphs/tracking/subgraphs/renderer_gpu.pbtxt
View File

@ -23,7 +23,7 @@ node {
# Draws annotations and overlays them on top of the input images.
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME_GPU:input_image"
input_stream: "IMAGE_GPU:input_image"
input_stream: "detections_render_data"
output_stream: "OUTPUT_FRAME_GPU:output_image"
output_stream: "IMAGE_GPU:output_image"
}

4
mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt
View File

@ -158,9 +158,9 @@ node {
node {
calculator: "AnnotationOverlayCalculator"
input_stream: "INPUT_FRAME:input_video"
input_stream: "IMAGE:input_video"
input_stream: "synchronized_render_data"
output_stream: "OUTPUT_FRAME:output_video"
output_stream: "IMAGE:output_video"
}
node {

Some files were not shown because too many files have changed in this diff Show More