Compare commits

..

No commits in common. "master" and "mrschmidt/gpu" have entirely different histories.

407 changed files with 2583 additions and 31577 deletions

View File

@ -80,7 +80,7 @@ message SpectrogramCalculatorOptions {
// If use_local_timestamp is true, the output packet's timestamp is based on
// the last sample of the packet and it's inferred from the latest input
// packet's timestamp. If false, the output packet's timestamp is based on
// the cumulative timestamping, which is inferred from the initial input
// the cumulative timestamping, which is inferred from the intial input
// timestamp and the cumulative number of samples.
optional bool use_local_timestamp = 8 [default = false];
}

View File

@ -66,7 +66,7 @@ message TimeSeriesFramerCalculatorOptions {
// If use_local_timestamp is true, the output packet's timestamp is based on
// the last sample of the packet and it's inferred from the latest input
// packet's timestamp. If false, the output packet's timestamp is based on
// the cumulative timestamping, which is inferred from the initial input
// the cumulative timestamping, which is inferred from the intial input
// timestamp and the cumulative number of samples.
optional bool use_local_timestamp = 6 [default = false];
}

View File

@ -727,7 +727,6 @@ cc_library(
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/status",
],
alwayslink = 1,
)
@ -743,7 +742,6 @@ cc_test(
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:options_util",
"//mediapipe/util:packet_test_util",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/strings",
],

View File

@ -71,7 +71,7 @@ TEST_F(PacketSequencerCalculatorTest, IsRegistered) {
CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator"));
}
// Shows how control packets receive timestamps before and after frame packets
// Shows how control packets recieve timestamps before and after frame packets
// have arrived.
TEST_F(PacketSequencerCalculatorTest, ChannelEarly) {
CalculatorGraphConfig::Node node_config = BuildNodeConfig();

View File

@ -17,7 +17,6 @@
#include <set>
#include <string>
#include "absl/status/status.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/ret_check.h"
@ -33,7 +32,6 @@ namespace {
constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
constexpr char kTagAtZero[] = "AT_ZERO";
constexpr char kTagAtFirstTick[] = "AT_FIRST_TICK";
constexpr char kTagAtTick[] = "AT_TICK";
constexpr char kTagTick[] = "TICK";
constexpr char kTagAtTimestamp[] = "AT_TIMESTAMP";
@ -45,7 +43,6 @@ static std::map<std::string, Timestamp>* kTimestampMap = []() {
res->emplace(kTagAtPostStream, Timestamp::PostStream());
res->emplace(kTagAtZero, Timestamp(0));
res->emplace(kTagAtTick, Timestamp::Unset());
res->emplace(kTagAtFirstTick, Timestamp::Unset());
res->emplace(kTagAtTimestamp, Timestamp::Unset());
return res;
}();
@ -62,8 +59,8 @@ std::string GetOutputTag(const CC& cc) {
// timestamp, depending on the tag used to define output stream(s). (One tag can
// be used only.)
//
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_FIRST_TICK,
// AT_TIMESTAMP and corresponding timestamps are Timestamp::PreStream(),
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_TIMESTAMP
// and corresponding timestamps are Timestamp::PreStream(),
// Timestamp::PostStream(), Timestamp(0), timestamp of a packet received in TICK
// input, and timestamp received from a side input.
//
@ -99,7 +96,6 @@ class SidePacketToStreamCalculator : public CalculatorBase {
private:
bool is_tick_processing_ = false;
bool close_on_first_tick_ = false;
std::string output_tag_;
};
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
@ -107,16 +103,13 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator);
absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
const auto& tags = cc->Outputs().GetTags();
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, "
"AT_FIRST_TICK and AT_TIMESTAMP tags is allowed and required to "
"specify output stream(s).";
const bool has_tick_output =
cc->Outputs().HasTag(kTagAtTick) || cc->Outputs().HasTag(kTagAtFirstTick);
const bool has_tick_input = cc->Inputs().HasTag(kTagTick);
RET_CHECK((has_tick_output && has_tick_input) ||
(!has_tick_output && !has_tick_input))
<< "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
"should be used or none of them.";
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
"AT_TIMESTAMP tags is allowed and required to specify output "
"stream(s).";
RET_CHECK(
(cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) ||
(!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick)))
<< "Either both of TICK and AT_TICK should be used or none of them.";
RET_CHECK((cc->Outputs().HasTag(kTagAtTimestamp) &&
cc->InputSidePackets().HasTag(kTagSideInputTimestamp)) ||
(!cc->Outputs().HasTag(kTagAtTimestamp) &&
@ -155,17 +148,11 @@ absl::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
// timestamp bound update.
cc->SetOffset(TimestampDiff(0));
}
if (output_tag_ == kTagAtFirstTick) {
close_on_first_tick_ = true;
}
return absl::OkStatus();
}
absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
if (is_tick_processing_) {
if (cc->Outputs().Get(output_tag_, 0).IsClosed()) {
return absl::OkStatus();
}
// TICK input is guaranteed to be non-empty, as it's the only input stream
// for this calculator.
const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
@ -173,9 +160,6 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
cc->Outputs()
.Get(output_tag_, i)
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
if (close_on_first_tick_) {
cc->Outputs().Get(output_tag_, i).Close();
}
}
return absl::OkStatus();
@ -186,7 +170,6 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
if (!cc->Outputs().HasTag(kTagAtTick) &&
!cc->Outputs().HasTag(kTagAtFirstTick) &&
!cc->Outputs().HasTag(kTagAtTimestamp)) {
const auto& timestamp = kTimestampMap->at(output_tag_);
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {

View File

@ -27,17 +27,13 @@
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/tool/options_util.h"
#include "mediapipe/util/packet_test_util.h"
namespace mediapipe {
namespace {
using ::testing::ElementsAre;
using ::testing::Eq;
using ::testing::HasSubstr;
using ::testing::IsEmpty;
using testing::HasSubstr;
TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTick) {
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -56,35 +52,10 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTick) {
EXPECT_THAT(
status.message(),
HasSubstr(
"Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
"should be used or none of them."));
"Either both of TICK and AT_TICK should be used or none of them."));
}
TEST(SidePacketToStreamCalculator,
WrongConfigWithMissingTickForFirstTickProcessing) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_side_packet: "side_packet"
output_stream: "AT_FIRST_TICK:packet"
}
)pb");
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_THAT(
status.message(),
HasSubstr(
"Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
"should be used or none of them."));
}
TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTimestampSideInput) {
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -105,7 +76,7 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTimestampSideInput) {
"or none of them."));
}
TEST(SidePacketToStreamCalculator, WrongConfigWithNonExistentTag) {
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -121,13 +92,14 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithNonExistentTag) {
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.message(),
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
"AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
"allowed and required to specify output stream(s)."));
EXPECT_THAT(
status.message(),
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
"AT_TIMESTAMP tags is allowed and required to specify output "
"stream(s)."));
}
TEST(SidePacketToStreamCalculator, WrongConfigWithMixedTags) {
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -145,13 +117,14 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithMixedTags) {
CalculatorGraph graph;
auto status = graph.Initialize(graph_config);
EXPECT_FALSE(status.ok());
EXPECT_THAT(status.message(),
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
"AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
"allowed and required to specify output stream(s)."));
EXPECT_THAT(
status.message(),
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
"AT_TIMESTAMP tags is allowed and required to specify output "
"stream(s)."));
}
TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughSidePackets) {
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -173,7 +146,7 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughSidePackets) {
"Same number of input side packets and output streams is required."));
}
TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughOutputStreams) {
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -275,50 +248,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
tick_and_verify(/*at_timestamp=*/1025);
}
TEST(SidePacketToStreamCalculator, AtFirstTick) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "tick"
input_side_packet: "side_packet"
output_stream: "packet"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet"
output_stream: "AT_FIRST_TICK:packet"
}
)pb");
std::vector<Packet> output_packets;
tool::AddVectorSink("packet", &graph_config, &output_packets);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value = 20;
const Timestamp kTestTimestamp(1234);
MP_ASSERT_OK(
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
auto insert_tick = [&graph](Timestamp at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
MP_ASSERT_OK(graph.WaitUntilIdle());
};
insert_tick(kTestTimestamp);
EXPECT_THAT(output_packets,
ElementsAre(PacketContainsTimestampAndPayload<int>(
Eq(kTestTimestamp), Eq(expected_value))));
output_packets.clear();
// Should not result in an additional output.
insert_tick(kTestTimestamp + 1);
EXPECT_THAT(output_packets, IsEmpty());
}
TEST(SidePacketToStreamCalculator, AtTickWithMultipleSidePackets) {
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
@ -372,62 +302,6 @@ TEST(SidePacketToStreamCalculator, AtTickWithMultipleSidePackets) {
tick_and_verify(/*at_timestamp=*/1025);
}
TEST(SidePacketToStreamCalculator, AtFirstTickWithMultipleSidePackets) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "packet0"
output_stream: "packet1"
node {
calculator: "SidePacketToStreamCalculator"
input_stream: "TICK:tick"
input_side_packet: "side_packet0"
input_side_packet: "side_packet1"
output_stream: "AT_FIRST_TICK:0:packet0"
output_stream: "AT_FIRST_TICK:1:packet1"
}
)pb");
std::vector<Packet> output_packets0;
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
std::vector<Packet> output_packets1;
tool::AddVectorSink("packet1", &graph_config, &output_packets1);
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
const int expected_value0 = 20;
const int expected_value1 = 128;
const Timestamp kTestTimestamp(1234);
MP_ASSERT_OK(
graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
{"side_packet1", MakePacket<int>(expected_value1)}}));
auto insert_tick = [&graph](Timestamp at_timestamp) {
MP_ASSERT_OK(graph.AddPacketToInputStream(
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
MP_ASSERT_OK(graph.WaitUntilIdle());
};
insert_tick(kTestTimestamp);
EXPECT_THAT(output_packets0,
ElementsAre(PacketContainsTimestampAndPayload<int>(
Eq(kTestTimestamp), Eq(expected_value0))));
EXPECT_THAT(output_packets1,
ElementsAre(PacketContainsTimestampAndPayload<int>(
Eq(kTestTimestamp), Eq(expected_value1))));
output_packets0.clear();
output_packets1.clear();
// Should not result in an additional output.
insert_tick(kTestTimestamp + 1);
EXPECT_THAT(output_packets0, IsEmpty());
EXPECT_THAT(output_packets1, IsEmpty());
}
TEST(SidePacketToStreamCalculator, AtTimestamp) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
@ -460,7 +334,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
}
TEST(SidePacketToStreamCalculator, AtTimestampWithMultipleOutputs) {
TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(

View File

@ -174,7 +174,7 @@ TEST(ValueOrDefaultCalculatorTest, DefaultAndValues) {
ElementsAre(kDefaultValue, 1, 2, kDefaultValue, 3, kDefaultValue));
}
TEST(ValueOrDefaultCalculatorTest, TimestampsMismatch) {
TEST(ValueOrDefaultCalculatorTest, TimestampsMissmatch) {
// Check that when we provide the inputs not on time - we don't get them.
ValueOrDefaultRunner runner;
const std::vector<int64_t> ticks = {1, 2, 5, 8, 12, 33, 231};

View File

@ -301,11 +301,9 @@ cc_test(
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
"//mediapipe/gpu:multi_pool",
"//third_party:opencv",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest_main",
],

View File

@ -59,7 +59,7 @@ class OpenCvRunner
const ImageFrame& input, const std::array<float, 16>& matrix,
const AffineTransformation::Size& size,
AffineTransformation::BorderMode border_mode) override {
// OpenCV warpAffine works in absolute coordinates, so the transform (which
// OpenCV warpAffine works in absolute coordinates, so the transfom (which
// accepts and produces relative coordinates) should be adjusted to first
// normalize coordinates and then scale them.
// clang-format off

View File

@ -65,7 +65,7 @@ class ImageCloneCalculator : public Node {
}
#else
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*request_gpu_as_optional=*/true));
cc, /*requesst_gpu_as_optional=*/true));
#endif // MEDIAPIPE_DISABLE_GPU
return absl::OkStatus();
}

View File

@ -24,7 +24,7 @@ message ImageCroppingCalculatorOptions {
}
// Output texture buffer dimensions. The values defined in the options will be
// overridden by the WIDTH and HEIGHT input streams if they exist.
// overriden by the WIDTH and HEIGHT input streams if they exist.
optional int32 width = 1;
optional int32 height = 2;

View File

@ -77,7 +77,7 @@ absl::StatusOr<double> ComputeFocalLengthInPixels(int image_width,
return focal_length_pixels;
}
absl::StatusOr<ImageFileProperties> GetImageFileProperties(
absl::StatusOr<ImageFileProperties> GetImageFileProperites(
const std::string& image_bytes) {
easyexif::EXIFInfo result;
int code = result.parseFrom(image_bytes);
@ -151,7 +151,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
if (cc->InputSidePackets().NumEntries() == 1) {
const std::string& image_bytes =
cc->InputSidePackets().Index(0).Get<std::string>();
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
read_properties_ = true;
}
@ -169,7 +169,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
return absl::OkStatus();
}
const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
read_properties_ = true;
}
if (read_properties_) {

View File

@ -656,15 +656,6 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
input.format());
gpu_helper_.BindFramebuffer(dst);
if (scale_mode_ == mediapipe::ScaleMode::FIT) {
// In kFit scale mode, the rendered quad does not fill the whole
// framebuffer, so clear it beforehand.
glClearColor(padding_color_[0] / 255.0f, padding_color_[1] / 255.0f,
padding_color_[2] / 255.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
}
glActiveTexture(GL_TEXTURE1);
glBindTexture(src1.target(), src1.name());

View File

@ -46,14 +46,13 @@ message ImageTransformationCalculatorOptions {
optional bool flip_horizontally = 5 [default = false];
// Scale mode.
optional ScaleMode.Mode scale_mode = 6;
// Padding type. This option is only used when the scale mode is FIT. If set
// to true (default), a constant border is added with color specified by
// padding_color. If set to false, a border is added by replicating edge
// pixels (only supported for CPU).
// Padding type. This option is only used when the scale mode is FIT.
// Default is to use BORDER_CONSTANT. If set to false, it will use
// BORDER_REPLICATE instead.
optional bool constant_padding = 7 [default = true];
// The color for the padding. This option is only used when the scale mode is
// FIT. Default is black.
// FIT. Default is black. This is for CPU only.
optional Color padding_color = 8;
// Interpolation method to use. Note that on CPU when LINEAR is specified,

View File

@ -1,11 +1,9 @@
#include <algorithm>
#include <string>
#include <utility>
#include <vector>
#include "absl/container/flat_hash_set.h"
#include "absl/flags/flag.h"
#include "absl/log/absl_check.h"
#include "absl/strings/substitute.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
@ -18,14 +16,10 @@
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/gpu/multi_pool.h"
#include "testing/base/public/gmock.h"
#include "testing/base/public/googletest.h"
#include "testing/base/public/gunit.h"
#include "third_party/OpenCV/core.hpp" // IWYU pragma: keep
#include "third_party/OpenCV/core/base.hpp"
#include "third_party/OpenCV/core/mat.hpp"
#include "third_party/OpenCV/core/types.hpp"
namespace mediapipe {
@ -82,12 +76,11 @@ TEST(ImageTransformationCalculatorTest, NearestNeighborResizing) {
->Tag("OUTPUT_DIMENSIONS")
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
ABSL_QCHECK_OK(runner.Run());
MP_ASSERT_OK(runner.Run());
const auto& outputs = runner.Outputs();
ABSL_QCHECK_EQ(outputs.NumEntries(), 1);
ASSERT_EQ(outputs.NumEntries(), 1);
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
ABSL_QCHECK_EQ(packets.size(), 1);
ASSERT_EQ(packets.size(), 1);
const auto& result = packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, result.Width());
ASSERT_EQ(output_dim.second, result.Height());
@ -144,12 +137,11 @@ TEST(ImageTransformationCalculatorTest,
->Tag("OUTPUT_DIMENSIONS")
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
ABSL_QCHECK_OK(runner.Run());
MP_ASSERT_OK(runner.Run());
const auto& outputs = runner.Outputs();
ABSL_QCHECK_EQ(outputs.NumEntries(), 1);
ASSERT_EQ(outputs.NumEntries(), 1);
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
ABSL_QCHECK_EQ(packets.size(), 1);
ASSERT_EQ(packets.size(), 1);
const auto& result = packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, result.Width());
ASSERT_EQ(output_dim.second, result.Height());
@ -215,17 +207,17 @@ TEST(ImageTransformationCalculatorTest, NearestNeighborResizingGpu) {
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
CalculatorGraph graph(graph_config);
ABSL_QCHECK_OK(graph.StartRun({}));
MP_ASSERT_OK(graph.StartRun({}));
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image",
MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0))));
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
MP_ASSERT_OK(graph.AddPacketToInputStream(
"image_size",
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
ABSL_QCHECK_OK(graph.WaitUntilIdle());
ABSL_QCHECK_EQ(output_image_packets.size(), 1);
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_THAT(output_image_packets, testing::SizeIs(1));
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, output_image.Width());
@ -295,16 +287,16 @@ TEST(ImageTransformationCalculatorTest,
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
CalculatorGraph graph(graph_config);
ABSL_QCHECK_OK(graph.StartRun({}));
MP_ASSERT_OK(graph.StartRun({}));
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image", input_image_packet.At(Timestamp(0))));
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
MP_ASSERT_OK(graph.AddPacketToInputStream(
"image_size",
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
ABSL_QCHECK_OK(graph.WaitUntilIdle());
ABSL_QCHECK_EQ(output_image_packets.size(), 1);
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_THAT(output_image_packets, testing::SizeIs(1));
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
ASSERT_EQ(output_dim.first, output_image.Width());
@ -319,112 +311,5 @@ TEST(ImageTransformationCalculatorTest,
}
}
TEST(ImageTransformationCalculatorTest, FitScalingClearsBackground) {
// Regression test for not clearing the background in FIT scaling mode.
// First scale an all-red (=r) image from 8x4 to 8x4, so it's a plain copy:
// rrrrrrrr
// rrrrrrrr
// rrrrrrrr
// rrrrrrrr
// Then scale an all-blue image from 4x4 to 8x4 in FIT mode. This should
// introduce dark yellow (=y) letterboxes left and right due to padding_color:
// yybbbbyy
// yybbbbyy
// yybbbbyy
// yybbbbyy
// We make sure that the all-red buffer gets reused. Without clearing the
// background, the blue (=b) image will have red letterboxes:
// rrbbbbrr
// rrbbbbrr
// rrbbbbrr
// rrbbbbrr
constexpr int kSmall = 4, kLarge = 8;
ImageFrame input_image_red(ImageFormat::SRGBA, kLarge, kSmall);
cv::Mat input_image_red_mat = formats::MatView(&input_image_red);
input_image_red_mat = cv::Scalar(255, 0, 0, 255);
ImageFrame input_image_blue(ImageFormat::SRGBA, kSmall, kSmall);
cv::Mat input_image_blue_mat = formats::MatView(&input_image_blue);
input_image_blue_mat = cv::Scalar(0, 0, 255, 255);
Packet input_image_red_packet =
MakePacket<ImageFrame>(std::move(input_image_red));
Packet input_image_blue_packet =
MakePacket<ImageFrame>(std::move(input_image_blue));
CalculatorGraphConfig graph_config =
ParseTextProtoOrDie<CalculatorGraphConfig>(absl::Substitute(
R"pb(
input_stream: "input_image"
output_stream: "output_image"
node {
calculator: "ImageFrameToGpuBufferCalculator"
input_stream: "input_image"
output_stream: "input_image_gpu"
}
node {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:input_image_gpu"
output_stream: "IMAGE_GPU:output_image_gpu"
options: {
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
scale_mode: FIT
output_width: $0,
output_height: $1,
padding_color: { red: 128, green: 128, blue: 0 }
}
}
}
node {
calculator: "GpuBufferToImageFrameCalculator"
input_stream: "output_image_gpu"
output_stream: "output_image"
})pb",
kLarge, kSmall));
std::vector<Packet> output_image_packets;
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
CalculatorGraph graph(graph_config);
ABSL_QCHECK_OK(graph.StartRun({}));
// Send the red image multiple times to cause the GPU pool to actually use
// a pool.
int num_red_packets =
std::max(kDefaultMultiPoolOptions.min_requests_before_pool, 1);
for (int n = 0; n < num_red_packets; ++n) {
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
"input_image", input_image_red_packet.At(Timestamp(n))));
}
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
"input_image", input_image_blue_packet.At(Timestamp(num_red_packets))));
ABSL_QCHECK_OK(graph.WaitUntilIdle());
ABSL_QCHECK_EQ(output_image_packets.size(), num_red_packets + 1);
const auto& output_image_red = output_image_packets[0].Get<ImageFrame>();
const auto& output_image_blue =
output_image_packets[num_red_packets].Get<ImageFrame>();
ABSL_QCHECK_EQ(output_image_red.Width(), kLarge);
ABSL_QCHECK_EQ(output_image_red.Height(), kSmall);
ABSL_QCHECK_EQ(output_image_blue.Width(), kLarge);
ABSL_QCHECK_EQ(output_image_blue.Height(), kSmall);
cv::Mat output_image_blue_mat = formats::MatView(&output_image_blue);
ImageFrame expected_image_blue(ImageFormat::SRGBA, kLarge, kSmall);
cv::Mat expected_image_blue_mat = formats::MatView(&expected_image_blue);
expected_image_blue_mat = cv::Scalar(128, 128, 0, 255);
cv::Rect rect((kLarge - kSmall) / 2, 0, kSmall, kSmall);
cv::rectangle(expected_image_blue_mat, rect, cv::Scalar(0, 0, 255, 255),
cv::FILLED);
EXPECT_EQ(cv::sum(cv::sum(output_image_blue_mat != expected_image_blue_mat)),
cv::Scalar(0));
}
} // namespace
} // namespace mediapipe

View File

@ -118,7 +118,7 @@ absl::Status SegmentationSmoothingCalculator::GetContract(
#if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*request_gpu_as_optional=*/true));
cc, /*requesst_gpu_as_optional=*/true));
#endif // !MEDIAPIPE_DISABLE_GPU
return absl::OkStatus();

View File

@ -206,7 +206,7 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
if constexpr (std::is_same_v<InterfaceT, WarpAffineCalculatorGpu> ||
std::is_same_v<InterfaceT, WarpAffineCalculator>) {
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
cc, /*request_gpu_as_optional=*/true));
cc, /*requesst_gpu_as_optional=*/true));
}
return absl::OkStatus();
}

View File

@ -284,7 +284,7 @@ std::array<float, 16> GetMatrix(cv::Mat input, mediapipe::NormalizedRect roi,
.IgnoreError();
mediapipe::GetRotatedSubRectToRectTransformMatrix(
roi_absolute, input.cols, input.rows,
/*flip_horizontally=*/false, &transform_mat);
/*flip_horizontaly=*/false, &transform_mat);
return transform_mat;
}

View File

@ -49,7 +49,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
// The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka
// YV21) format (as per the `fourcc()` property). This covers the most commonly
// used YUV image formats used on mobile devices. Other formats are not
// supported and will result in an `InvalidArgumentError`.
// supported and wil result in an `InvalidArgumentError`.
class YUVToImageCalculator : public Node {
public:
static constexpr Input<YUVImage> kInput{"YUV_IMAGE"};

View File

@ -657,7 +657,6 @@ cc_library(
}),
deps = [
":tensor_converter_calculator_cc_proto",
":tensor_converter_cpu",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port",
"//mediapipe/framework/formats:image_frame",
@ -666,7 +665,6 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_buffer_format",
"//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/util:resource_util",
@ -676,17 +674,10 @@ cc_library(
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"tensor_converter_calculator_gpu_deps",
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:shader_util",
],
"//conditions:default": ["tensor_converter_calculator_gpu_deps"],
}) + select({
"//mediapipe:apple": [
"//third_party/apple_frameworks:MetalKit",
@ -696,35 +687,6 @@ cc_library(
alwayslink = 1,
)
cc_library(
name = "tensor_converter_cpu",
srcs = ["tensor_converter_cpu.cc"],
hdrs = ["tensor_converter_cpu.h"],
deps = [
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
],
)
cc_test(
name = "tensor_converter_cpu_test",
srcs = ["tensor_converter_cpu_test.cc"],
deps = [
":tensor_converter_cpu",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status_matchers",
"//mediapipe/util:image_test_utils",
],
)
cc_library(
name = "tensor_converter_calculator_gpu_deps",
visibility = ["//visibility:private"],
@ -1452,8 +1414,6 @@ cc_library(
}),
deps = [
":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_converter",
":tensors_to_segmentation_utils",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:port",
@ -1461,11 +1421,9 @@ cc_library(
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gpu_origin_cc_proto",
"//mediapipe/util:resource_util",
"@com_google_absl//absl/status",
"@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span",
@ -1476,7 +1434,6 @@ cc_library(
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gl_simple_shaders",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/gpu:gpu_buffer_format",
"//mediapipe/gpu:shader_util",
],
}) + selects.with_or({
@ -1496,96 +1453,19 @@ cc_library(
}) + select({
"//mediapipe/framework/port:disable_opencv": [],
"//conditions:default": [
":tensors_to_segmentation_converter_opencv",
"//mediapipe/framework/formats:image_opencv",
"//mediapipe/framework/port:opencv_imgproc",
],
}),
alwayslink = 1,
)
cc_library(
name = "tensors_to_segmentation_utils",
srcs = ["tensors_to_segmentation_utils.cc"],
hdrs = ["tensors_to_segmentation_utils.h"],
deps = [
"//mediapipe/framework:port",
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/status:statusor",
],
)
cc_test(
name = "tensors_to_segmentation_utils_test",
srcs = ["tensors_to_segmentation_utils_test.cc"],
deps = [
":tensors_to_segmentation_utils",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:status_matchers",
"@com_google_absl//absl/status:statusor",
],
)
cc_library(
name = "tensors_to_segmentation_converter",
hdrs = ["tensors_to_segmentation_converter.h"],
deps = [
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:tensor",
"@com_google_absl//absl/status:statusor",
],
)
cc_library(
name = "tensors_to_segmentation_converter_opencv",
srcs = ["tensors_to_segmentation_converter_opencv.cc"],
hdrs = ["tensors_to_segmentation_converter_opencv.h"],
deps = [
":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_converter",
":tensors_to_segmentation_utils",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_opencv",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
],
)
cc_library(
name = "tensors_to_segmentation_calculator_test_utils",
testonly = 1,
srcs = ["tensors_to_segmentation_calculator_test_utils.cc"],
hdrs = ["tensors_to_segmentation_calculator_test_utils.h"],
deps = [
":tensors_to_segmentation_calculator_cc_proto",
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
],
)
cc_test(
name = "tensors_to_segmentation_calculator_test_utils_test",
srcs = ["tensors_to_segmentation_calculator_test_utils_test.cc"],
deps = [
":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_calculator_test_utils",
"//mediapipe/framework/port:gtest_main",
],
)
cc_test(
name = "tensors_to_segmentation_calculator_test",
srcs = ["tensors_to_segmentation_calculator_test.cc"],
deps = [
":tensors_to_segmentation_calculator",
":tensors_to_segmentation_calculator_cc_proto",
":tensors_to_segmentation_calculator_test_utils",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:packet",
@ -1596,6 +1476,10 @@ cc_test(
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/log",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/strings",
],
)

View File

@ -109,7 +109,7 @@ bool IsValidFftSize(int size) {
// Non-streaming mode: when "stream_mode" is set to false in the calculator
// options, the calculators treats the packets in the input audio stream as
// a batch of unrelated audio buffers. In each Process() call, the input
// buffer will be first resampled, and framed as fixed-sized, possibly
// buffer will be frist resampled, and framed as fixed-sized, possibly
// overlapping tensors. The last tensor produced by a Process() invocation
// will be zero-padding if the remaining samples are insufficient. As the
// calculator treats the input packets as unrelated, all samples will be
@ -159,7 +159,7 @@ class AudioToTensorCalculator : public Node {
public:
static constexpr Input<Matrix> kAudioIn{"AUDIO"};
// TODO: Removes this optional input stream when the "AUDIO" stream
// uses the new mediapipe audio data containers that carry audio metadata,
// uses the new mediapipe audio data containers that carry audio metatdata,
// such as sample rate.
static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"};
static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};

View File

@ -37,7 +37,7 @@ message AudioToTensorCalculatorOptions {
// will be converted into tensors.
optional double target_sample_rate = 4;
// Whether to treat the input audio stream as a continuous stream or a batch
// Whether to treat the input audio stream as a continous stream or a batch
// of unrelated audio buffers.
optional bool stream_mode = 5 [default = true];

View File

@ -206,7 +206,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
} else if (image_channels == 1) {
return ImageFormat::GRAY8;
}
ABSL_CHECK(false) << "Unsupported input image channels: " << image_channels;
ABSL_CHECK(false) << "Unsupported input image channles: " << image_channels;
}
Packet MakeImageFramePacket(cv::Mat input) {

View File

@ -14,7 +14,6 @@
#include <cstdint>
#include <string>
#include <utility>
#include <vector>
#include "absl/log/absl_check.h"
@ -22,22 +21,17 @@
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_format.h"
#include "absl/strings/substitute.h"
#include "absl/types/optional.h"
#include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_origin.pb.h"
#if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/gpu_buffer.h"
#if MEDIAPIPE_METAL_ENABLED
#import <CoreVideo/CoreVideo.h>
@ -100,13 +94,16 @@ absl::StatusOr<bool> ShouldFlipVertically(
}
}
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
RowMajorMatrixXf;
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
ColMajorMatrixXf;
constexpr char kImageFrameTag[] = "IMAGE";
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
constexpr char kTensorsTag[] = "TENSORS";
constexpr char kMatrixTag[] = "MATRIX";
constexpr std::pair<float, float> kDefaultOutputRange = {0.0f, 1.0f};
} // namespace
namespace mediapipe {
@ -159,6 +156,10 @@ class TensorConverterCalculator : public CalculatorBase {
private:
absl::Status InitGpu(CalculatorContext* cc);
absl::Status LoadOptions(CalculatorContext* cc, bool use_gpu);
template <class T>
absl::Status NormalizeImage(const ImageFrame& image_frame,
bool flip_vertically, float* tensor_ptr);
absl::Status CopyMatrixToTensor(const Matrix& matrix, float* tensor_ptr);
absl::Status ProcessCPU(CalculatorContext* cc);
absl::Status ProcessGPU(CalculatorContext* cc);
@ -278,21 +279,46 @@ absl::Status TensorConverterCalculator::ProcessCPU(CalculatorContext* cc) {
}
const auto& image_frame =
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
MP_ASSIGN_OR_RETURN(Tensor output,
ConvertImageFrameToTensorOnCpu(
image_frame,
output_range_.has_value() ? output_range_.value()
: kDefaultOutputRange,
flip_vertically_, max_num_channels_));
output_tensors->emplace_back(std::move(output));
const int height = image_frame.Height();
const int width = image_frame.Width();
const int channels = image_frame.NumberOfChannels();
const int channels_preserved = std::min(channels, max_num_channels_);
const mediapipe::ImageFormat::Format format = image_frame.Format();
if (!(format == mediapipe::ImageFormat::SRGBA ||
format == mediapipe::ImageFormat::SRGB ||
format == mediapipe::ImageFormat::GRAY8 ||
format == mediapipe::ImageFormat::VEC32F1))
RET_CHECK_FAIL() << "Unsupported CPU input format.";
output_tensors->emplace_back(
Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels_preserved});
auto cpu_view = output_tensors->back().GetCpuWriteView();
// Copy image data into tensor.
if (image_frame.ByteDepth() == 1) {
MP_RETURN_IF_ERROR(NormalizeImage<uint8_t>(image_frame, flip_vertically_,
cpu_view.buffer<float>()));
} else if (image_frame.ByteDepth() == 4) {
MP_RETURN_IF_ERROR(NormalizeImage<float>(image_frame, flip_vertically_,
cpu_view.buffer<float>()));
} else {
return absl::InternalError(
"Only byte-based (8 bit) and float (32 bit) images supported.");
}
} else if (cc->Inputs().HasTag(kMatrixTag)) {
if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) {
return absl::OkStatus();
}
const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get<Matrix>();
MP_ASSIGN_OR_RETURN(Tensor output,
ConvertMatrixToTensorOnCpu(matrix, row_major_matrix_));
output_tensors->emplace_back(std::move(output));
const int height = matrix.rows();
const int width = matrix.cols();
const int channels = 1;
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels});
MP_RETURN_IF_ERROR(CopyMatrixToTensor(
matrix, output_tensors->back().GetCpuWriteView().buffer<float>()));
} else {
return absl::OkStatus();
}
@ -643,4 +669,67 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc,
return absl::OkStatus();
}
template <class T>
absl::Status TensorConverterCalculator::NormalizeImage(
const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) {
const int height = image_frame.Height();
const int width = image_frame.Width();
const int channels = image_frame.NumberOfChannels();
const int channels_preserved = std::min(channels, max_num_channels_);
const int channels_ignored = channels - channels_preserved;
if (output_range_.has_value()) {
// If the output float range is set and we are not using custom
// normalization, normalize the pixel values from [0, 255] to the specified
// output range.
RET_CHECK_NE(output_range_->first, output_range_->second);
const float scale = (output_range_->second - output_range_->first) / 255.0f;
const float bias = output_range_->first;
for (int i = 0; i < height; ++i) {
const T* image_ptr = reinterpret_cast<const T*>(
image_frame.PixelData() +
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
for (int j = 0; j < width; ++j) {
for (int c = 0; c < channels_preserved; ++c) {
*tensor_ptr++ = *image_ptr++ * scale + bias;
}
image_ptr += channels_ignored;
}
}
} else {
// [0,1], scale only (bias == 0)
// Verified that there are no precision issues with 1.0f / 255.0f expression
const float scale = 1.0f / 255.0f;
for (int i = 0; i < height; ++i) {
const T* image_ptr = reinterpret_cast<const T*>(
image_frame.PixelData() +
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
for (int j = 0; j < width; ++j) {
for (int c = 0; c < channels_preserved; ++c) {
*tensor_ptr++ = *image_ptr++ * scale;
}
image_ptr += channels_ignored;
}
}
}
return absl::OkStatus();
}
absl::Status TensorConverterCalculator::CopyMatrixToTensor(const Matrix& matrix,
float* tensor_ptr) {
if (row_major_matrix_) {
auto matrix_map =
Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
} else {
auto matrix_map =
Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
}
return absl::OkStatus();
}
} // namespace mediapipe

View File

@ -321,61 +321,6 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) {
}
}
TEST_F(TensorConverterCalculatorTest,
ShouldConvertImageWithDefaultOutputRange) {
CalculatorGraph graph;
CalculatorGraphConfig graph_config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
R"pb(
input_stream: "input_image"
node {
calculator: "TensorConverterCalculator"
input_stream: "IMAGE:input_image"
output_stream: "TENSORS:tensor"
options {
[mediapipe.TensorConverterCalculatorOptions.ext] {
zero_center: false
}
}
}
)pb");
std::vector<Packet> output_packets;
tool::AddVectorSink("tensor", &graph_config, &output_packets);
// Run the graph.
MP_ASSERT_OK(graph.Initialize(graph_config));
MP_ASSERT_OK(graph.StartRun({}));
auto input_image = std::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
cv::Mat mat = mediapipe::formats::MatView(input_image.get());
mat.at<uint8_t>(0, 0) = 200;
MP_ASSERT_OK(graph.AddPacketToInputStream(
"input_image", Adopt(input_image.release()).At(Timestamp(0))));
// Wait until the calculator finishes processing.
MP_ASSERT_OK(graph.WaitUntilIdle());
ASSERT_EQ(output_packets.size(), 1);
// Get and process results.
const std::vector<Tensor>& tensor_vec =
output_packets[0].Get<std::vector<Tensor>>();
ASSERT_EQ(tensor_vec.size(), 1);
const Tensor* tensor = &tensor_vec[0];
// Calculate the expected normalized value:
float expected_value = 200.0 / 255.0;
EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32);
auto view = tensor->GetCpuReadView();
float actual_value = *view.buffer<float>();
EXPECT_FLOAT_EQ(actual_value, expected_value);
// Fully close graph at end, otherwise calculator+tensors are destroyed
// after calling WaitUntilDone().
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
MP_ASSERT_OK(graph.WaitUntilDone());
}
TEST_F(TensorConverterCalculatorTest, FlipVertically) {
CalculatorGraph graph;
CalculatorGraphConfig graph_config =

View File

@ -1,145 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
#include <algorithm>
#include <cstdint>
#include <utility>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status_macros.h"
namespace mediapipe {
namespace {
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
RowMajorMatrixXf;
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
ColMajorMatrixXf;
template <class T>
absl::Status NormalizeImage(const ImageFrame& image_frame, bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr) {
const int height = image_frame.Height();
const int width = image_frame.Width();
const int channels = image_frame.NumberOfChannels();
const int channels_preserved = std::min(channels, max_num_channels);
const int channels_ignored = channels - channels_preserved;
RET_CHECK_NE(output_range.first, output_range.second);
const float scale = (output_range.second - output_range.first) / 255.0f;
const float bias = output_range.first;
for (int i = 0; i < height; ++i) {
const T* image_ptr = reinterpret_cast<const T*>(
image_frame.PixelData() +
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
for (int j = 0; j < width; ++j) {
for (int c = 0; c < channels_preserved; ++c) {
*tensor_ptr++ = *image_ptr++ * scale + bias;
}
image_ptr += channels_ignored;
}
}
return absl::OkStatus();
}
} // namespace
absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr) {
return NormalizeImage<uint8_t>(image_frame, flip_vertically, output_range,
max_num_channels, tensor_ptr);
}
absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr) {
return NormalizeImage<float>(image_frame, flip_vertically, output_range,
max_num_channels, tensor_ptr);
}
absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
float* tensor_ptr) {
if (is_row_major_matrix) {
auto matrix_map =
Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
} else {
auto matrix_map =
Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
matrix_map = matrix;
}
return absl::OkStatus();
}
absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
const ImageFrame& image_frame, const std::pair<float, float>& output_range,
bool flip_vertically, int max_num_channels) {
const int height = image_frame.Height();
const int width = image_frame.Width();
const int channels = image_frame.NumberOfChannels();
const int channels_preserved = std::min(channels, max_num_channels);
const mediapipe::ImageFormat::Format format = image_frame.Format();
if (!(format == mediapipe::ImageFormat::SRGBA ||
format == mediapipe::ImageFormat::SRGB ||
format == mediapipe::ImageFormat::GRAY8 ||
format == mediapipe::ImageFormat::VEC32F1))
RET_CHECK_FAIL() << "Unsupported CPU input format.";
Tensor output_tensor(Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels_preserved});
auto cpu_view = output_tensor.GetCpuWriteView();
// Copy image data into tensor.
if (image_frame.ByteDepth() == 1) {
MP_RETURN_IF_ERROR(NormalizeUInt8Image(image_frame, flip_vertically,
output_range, max_num_channels,
cpu_view.buffer<float>()));
} else if (image_frame.ByteDepth() == 4) {
MP_RETURN_IF_ERROR(NormalizeFloatImage(image_frame, flip_vertically,
output_range, max_num_channels,
cpu_view.buffer<float>()));
} else {
return absl::InternalError(
"Only byte-based (8 bit) and float (32 bit) images supported.");
}
return output_tensor;
}
absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
bool row_major_matrix) {
const int height = matrix.rows();
const int width = matrix.cols();
const int channels = 1;
Tensor output_tensor(Tensor::ElementType::kFloat32,
Tensor::Shape{1, height, width, channels});
MP_RETURN_IF_ERROR(
CopyMatrixToTensor(matrix, row_major_matrix,
output_tensor.GetCpuWriteView().buffer<float>()));
return output_tensor;
}
} // namespace mediapipe

View File

@ -1,61 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
#include <utility>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h"
namespace mediapipe {
// Converts an ImageFrame to a vector of Tensors.
// @flip_vertically enables to flip the image during conversion.
// @max_num_channels can be used to reserve extra channels in the output
// tensors.
// Returns output Tensor.
absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
const ImageFrame& image_frame, const std::pair<float, float>& output_range,
bool flip_vertically, int max_num_channels);
// Converts a Matrix to a vector of Tensors.
// @row_major_matrix defines the ordering in the input matrix.
// @max_num_channels can be used to reserve extra channels in the output
// tensors.
// Returns output Tensor.
absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
bool row_major_matrix);
// For testing only below.
absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr);
absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
bool flip_vertically,
const std::pair<float, float>& output_range,
int max_num_channels, float* tensor_ptr);
absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
float* tensor_ptr);
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_

View File

@ -1,175 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
#include <cstdint>
#include <utility>
#include <vector>
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/util/image_test_utils.h"
namespace mediapipe {
namespace {
Matrix CreateTestMatrix(int num_rows, int num_columns) {
Matrix matrix(num_rows, num_columns);
for (int r = 0; r < num_rows; ++r) {
for (int c = 0; c < num_columns; ++c) {
matrix(r, c) = r * num_columns + c;
}
}
return matrix;
}
TEST(TensorConverterCpuTest, ShouldCopyMatrixInRowMajorFormatToTensor) {
auto test_matrix = CreateTestMatrix(/* num_rows=*/3, /*num_columns=*/4);
std::vector<float> tensor_data(test_matrix.size(), 0.0f);
MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/true,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
const int row = i / test_matrix.cols();
const int column = i % test_matrix.cols();
EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
}
}
TEST(TensorConverterCpuTest, ShouldCopyMatrixInColumnMajorFormatToTensor) {
auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
std::vector<float> tensor_data(test_matrix.size(), 0.0f);
MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/false,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
const int row = i % test_matrix.rows();
const int column = i / test_matrix.rows();
EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithDefaultRange) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false,
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
EXPECT_FLOAT_EQ(
tensor_data[i],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithSpecifiedRange) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
const auto range = std::make_pair(2.0f, 3.0f);
MP_EXPECT_OK(
NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, range,
/*num_tensor_channels=*/1, tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
EXPECT_FLOAT_EQ(tensor_data[i],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) /
255.0f * (range.second - range.first) +
range.first);
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageFlipped) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/true,
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
const int x = i % grey8_image_frame.Width();
const int y = i / grey8_image_frame.Width();
const int flipped_y = grey8_image_frame.Height() - y - 1;
const int index = flipped_y * grey8_image_frame.Width() + x;
EXPECT_FLOAT_EQ(
tensor_data[index],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
}
}
TEST(TensorConverterCpuTest, ShouldNormalizeFloatImageWithDefaultRange) {
auto float_image_frame =
CreateTestFloat32ImageFrame(/*width=*/3, /*height=*/4);
std::vector<float> tensor_data(
float_image_frame.Width() * float_image_frame.Height(), 0.0f);
MP_EXPECT_OK(NormalizeFloatImage(float_image_frame, /*flip_vertically=*/false,
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
tensor_data.data()));
for (int i = 0; i < tensor_data.size(); ++i) {
EXPECT_FLOAT_EQ(tensor_data[i], reinterpret_cast<const float*>(
float_image_frame.PixelData())[i] /
255.0f);
}
}
TEST(TensorConverterCpuTest, ConvertImageFrameToTensorOnCpu) {
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
MP_ASSERT_OK_AND_ASSIGN(Tensor output, ConvertImageFrameToTensorOnCpu(
grey8_image_frame, {0.0f, 1.0f},
/*flip_vertically=*/false,
/*max_num_channels=*/1));
const auto cpu_read_view = output.GetCpuReadView();
const float* tensor_ptr = cpu_read_view.buffer<float>();
for (int i = 0; i < grey8_image_frame.Width() * grey8_image_frame.Height();
++i) {
EXPECT_FLOAT_EQ(
tensor_ptr[i],
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0);
}
}
TEST(TensorConverterCpuTest, ConvertMatrixToTensorOnCpu) {
auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
MP_ASSERT_OK_AND_ASSIGN(
Tensor output, ConvertMatrixToTensorOnCpu(test_matrix,
/*row_major_matrix=*/false));
const auto cpu_read_view = output.GetCpuReadView();
const float* tensor_ptr = cpu_read_view.buffer<float>();
for (int i = 0; i < test_matrix.size(); ++i) {
EXPECT_FLOAT_EQ(tensor_ptr[i], test_matrix.data()[i]);
}
}
} // namespace
} // namespace mediapipe

View File

@ -879,6 +879,13 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
if (max_results_ > 0 && output_detections->size() == max_results_) {
break;
}
if (options_.has_min_score_thresh() &&
detection_scores[i] < options_.min_score_thresh()) {
continue;
}
if (!IsClassIndexAllowed(detection_classes[i])) {
continue;
}
const int box_offset = i * num_coords_;
Detection detection = ConvertToDetection(
/*box_ymin=*/detection_boxes[box_offset + box_indices_[0]],
@ -888,11 +895,6 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
absl::MakeConstSpan(detection_scores + i, classes_per_detection_),
absl::MakeConstSpan(detection_classes + i, classes_per_detection_),
options_.flip_vertically());
// if all the scores and classes are filtered out, we skip the empty
// detection.
if (detection.score().empty()) {
continue;
}
const auto& bbox = detection.location_data().relative_bounding_box();
if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) ||
std::isnan(bbox.height())) {
@ -930,10 +932,6 @@ Detection TensorsToDetectionsCalculator::ConvertToDetection(
if (!IsClassIndexAllowed(class_ids[i])) {
continue;
}
if (options_.has_min_score_thresh() &&
scores[i] < options_.min_score_thresh()) {
continue;
}
detection.add_score(scores[i]);
detection.add_label_id(class_ids[i]);
}

View File

@ -124,7 +124,7 @@ absl::Status TensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
kFlipVertically(cc).IsConnected())) {
RET_CHECK(options_.has_input_image_height() &&
options_.has_input_image_width())
<< "Must provide input width/height for using flipping when outputting "
<< "Must provide input width/height for using flipping when outputing "
"landmarks in absolute coordinates.";
}
return absl::OkStatus();

View File

@ -12,35 +12,32 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/types/span.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/gpu/gpu_origin.pb.h"
#include "mediapipe/util/resource_util.h"
#include "tensorflow/lite/interpreter.h"
#if !MEDIAPIPE_DISABLE_GPU
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gl_simple_shaders.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/shader_util.h"
#endif // !MEDIAPIPE_DISABLE_GPU
#if !MEDIAPIPE_DISABLE_OPENCV
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
#include "mediapipe/framework/formats/image_opencv.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#endif // !MEDIAPIPE_DISABLE_OPENCV
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
@ -65,9 +62,37 @@ namespace {
constexpr int kWorkgroupSize = 8; // Block size for GPU shader.
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
// Commonly used to compute the number of blocks to launch in a kernel.
int NumGroups(const int size, const int group_size) { // NOLINT
return (size + group_size - 1) / group_size;
}
bool CanUseGpu() {
#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
// TODO: Configure GPU usage policy in individual calculators.
constexpr bool kAllowGpuProcessing = true;
return kAllowGpuProcessing;
#else
return false;
#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
}
constexpr char kTensorsTag[] = "TENSORS";
constexpr char kOutputSizeTag[] = "OUTPUT_SIZE";
constexpr char kMaskTag[] = "MASK";
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
const std::vector<int>& dims) {
if (dims.size() == 3) {
return std::make_tuple(dims[0], dims[1], dims[2]);
} else if (dims.size() == 4) {
// BHWC format check B == 1
RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap";
return std::make_tuple(dims[1], dims[2], dims[3]);
} else {
RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
}
}
} // namespace
namespace mediapipe {
@ -131,28 +156,19 @@ class TensorsToSegmentationCalculator : public CalculatorBase {
private:
absl::Status LoadOptions(CalculatorContext* cc);
absl::Status InitGpu(CalculatorContext* cc);
absl::Status ProcessGpu(CalculatorContext* cc,
const std::vector<Tensor>& input_tensors,
std::tuple<int, int, int> hwc, int output_width,
int output_height);
absl::Status ProcessGpu(CalculatorContext* cc);
absl::Status ProcessCpu(CalculatorContext* cc);
void GlRender();
bool DoesGpuTextureStartAtBottom() {
return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
}
absl::Status InitConverterIfNecessary() {
#if !MEDIAPIPE_DISABLE_OPENCV
if (!cpu_converter_) {
MP_ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(options_));
}
#else
RET_CHECK_FAIL() << "OpenCV processing disabled.";
#endif // !MEDIAPIPE_DISABLE_OPENCV
return absl::OkStatus();
}
mediapipe::TensorsToSegmentationCalculatorOptions options_;
std::unique_ptr<TensorsToSegmentationConverter> cpu_converter_;
#if !MEDIAPIPE_DISABLE_OPENCV
template <class T>
absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat);
#endif // !MEDIAPIPE_DISABLE_OPENCV
::mediapipe::TensorsToSegmentationCalculatorOptions options_;
#if !MEDIAPIPE_DISABLE_GPU
mediapipe::GlCalculatorHelper gpu_helper_;
@ -245,7 +261,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
MP_ASSIGN_OR_RETURN(auto hwc,
GetHwcFromDims(input_tensors[0].shape().dims));
int tensor_channels = std::get<2>(hwc);
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
switch (options_.activation()) {
case Options::NONE:
RET_CHECK_EQ(tensor_channels, 1);
@ -259,17 +275,6 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
}
}
// Get dimensions.
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc;
int output_width = tensor_width, output_height = tensor_height;
if (cc->Inputs().HasTag(kOutputSizeTag)) {
const auto& size =
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
output_width = size.first;
output_height = size.second;
}
if (use_gpu) {
#if !MEDIAPIPE_DISABLE_GPU
if (!gpu_initialized_) {
@ -281,25 +286,16 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
#endif // !MEDIAPIPE_DISABLE_GPU
#if !MEDIAPIPE_DISABLE_GPU
MP_RETURN_IF_ERROR(
gpu_helper_.RunInGlContext([this, cc, &input_tensors, output_width,
output_height, hwc]() -> absl::Status {
MP_RETURN_IF_ERROR(
ProcessGpu(cc, input_tensors, hwc, output_width, output_height));
return absl::OkStatus();
}));
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status {
MP_RETURN_IF_ERROR(ProcessGpu(cc));
return absl::OkStatus();
}));
#else
RET_CHECK_FAIL() << "GPU processing disabled.";
#endif // !MEDIAPIPE_DISABLE_GPU
} else {
#if !MEDIAPIPE_DISABLE_OPENCV
// Lazily initialize converter.
MP_RETURN_IF_ERROR(InitConverterIfNecessary());
MP_ASSIGN_OR_RETURN(
std::unique_ptr<Image> output_mask,
cpu_converter_->Convert(input_tensors, output_width, output_height));
cc->Outputs().Tag(kMaskTag).Add(output_mask.release(),
cc->InputTimestamp());
MP_RETURN_IF_ERROR(ProcessCpu(cc));
#else
RET_CHECK_FAIL() << "OpenCV processing disabled.";
#endif // !MEDIAPIPE_DISABLE_OPENCV
@ -333,15 +329,132 @@ absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) {
return absl::OkStatus();
}
absl::Status TensorsToSegmentationCalculator::ProcessCpu(
CalculatorContext* cc) {
#if !MEDIAPIPE_DISABLE_OPENCV
// Get input streams, and dimensions.
const auto& input_tensors =
cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc;
int output_width = tensor_width, output_height = tensor_height;
if (cc->Inputs().HasTag(kOutputSizeTag)) {
const auto& size =
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
output_width = size.first;
output_height = size.second;
}
// Create initial working mask.
cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
// Wrap input tensor.
auto raw_input_tensor = &input_tensors[0];
auto raw_input_view = raw_input_tensor->GetCpuReadView();
const float* raw_input_data = raw_input_view.buffer<float>();
cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
CV_MAKETYPE(CV_32F, tensor_channels),
const_cast<float*>(raw_input_data));
// Process mask tensor and apply activation function.
if (tensor_channels == 2) {
MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
} else if (tensor_channels == 1) {
RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
options_.activation()); // Requires 2 channels.
if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
options_.activation()) // Pass-through optimization.
tensor_mat.copyTo(small_mask_mat);
else
MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
} else {
RET_CHECK_FAIL() << "Unsupported number of tensor channels "
<< tensor_channels;
}
// Send out image as CPU packet.
std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
ImageFormat::VEC32F1, output_width, output_height);
std::unique_ptr<Image> output_mask = absl::make_unique<Image>(mask_frame);
auto output_mat = formats::MatView(output_mask.get());
// Upsample small mask into output.
cv::resize(small_mask_mat, *output_mat,
cv::Size(output_width, output_height));
cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp());
#endif // !MEDIAPIPE_DISABLE_OPENCV
return absl::OkStatus();
}
#if !MEDIAPIPE_DISABLE_OPENCV
template <class T>
absl::Status TensorsToSegmentationCalculator::ApplyActivation(
cv::Mat& tensor_mat, cv::Mat* small_mask_mat) {
// Configure activation function.
const int output_layer_index = options_.output_layer_index();
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
const auto activation_fn = [&](const cv::Vec2f& mask_value) {
float new_mask_value = 0;
// TODO consider moving switch out of the loop,
// and also avoid float/Vec2f casting.
switch (options_.activation()) {
case Options::NONE: {
new_mask_value = mask_value[0];
break;
}
case Options::SIGMOID: {
const float pixel0 = mask_value[0];
new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
break;
}
case Options::SOFTMAX: {
const float pixel0 = mask_value[0];
const float pixel1 = mask_value[1];
const float max_pixel = std::max(pixel0, pixel1);
const float min_pixel = std::min(pixel0, pixel1);
const float softmax_denom =
/*exp(max_pixel - max_pixel)=*/1.0f +
std::exp(min_pixel - max_pixel);
new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
softmax_denom;
break;
}
}
return new_mask_value;
};
// Process mask tensor.
for (int i = 0; i < tensor_mat.rows; ++i) {
for (int j = 0; j < tensor_mat.cols; ++j) {
const T& input_pix = tensor_mat.at<T>(i, j);
const float mask_value = activation_fn(input_pix);
small_mask_mat->at<float>(i, j) = mask_value;
}
}
return absl::OkStatus();
}
#endif // !MEDIAPIPE_DISABLE_OPENCV
// Steps:
// 1. receive tensor
// 2. process segmentation tensor into small mask
// 3. upsample small mask into output mask to be same size as input image
absl::Status TensorsToSegmentationCalculator::ProcessGpu(
CalculatorContext* cc, const std::vector<Tensor>& input_tensors,
std::tuple<int, int, int> hwc, int output_width, int output_height) {
CalculatorContext* cc) {
#if !MEDIAPIPE_DISABLE_GPU
// Get input streams, and dimensions.
const auto& input_tensors =
cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc;
int output_width = tensor_width, output_height = tensor_height;
if (cc->Inputs().HasTag(kOutputSizeTag)) {
const auto& size =
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
output_width = size.first;
output_height = size.second;
}
// Create initial working mask texture.
#if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31)
@ -519,7 +632,7 @@ void TensorsToSegmentationCalculator::GlRender() {
absl::Status TensorsToSegmentationCalculator::LoadOptions(
CalculatorContext* cc) {
// Get calculator options specified in the graph.
options_ = cc->Options<mediapipe::TensorsToSegmentationCalculatorOptions>();
options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>();
return absl::OkStatus();
}
@ -713,7 +826,7 @@ void main() {
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
// Shader defines.
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
const std::string output_layer_index =
"\n#define OUTPUT_LAYER_INDEX int(" +
std::to_string(options_.output_layer_index()) + ")";

View File

@ -17,8 +17,10 @@
#include <utility>
#include <vector>
#include "absl/log/absl_log.h"
#include "absl/log/log.h"
#include "absl/strings/substitute.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/image.h"
@ -28,6 +30,7 @@
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/timestamp.h"
@ -37,17 +40,58 @@ namespace {
using ::testing::SizeIs;
using ::testing::TestWithParam;
using Options = mediapipe::TensorsToSegmentationCalculatorOptions;
namespace test_utils = ::mediapipe::tensors_to_segmentation_utils;
using TensorsToSegmentationCalculatorTest =
TestWithParam<test_utils::FormattingTestCase>;
std::string ActivationTypeToString(Options::Activation activation) {
switch (activation) {
case Options::NONE:
return "NONE";
case Options::SIGMOID:
return "SIGMOID";
case Options::SOFTMAX:
return "SOFTMAX";
default:
ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
return "UNKNOWN";
}
}
struct FormattingTestCase {
std::string test_name;
std::vector<float> inputs;
std::vector<float> expected_outputs;
Options::Activation activation;
int rows = 1;
int cols = 1;
int rows_new = 1;
int cols_new = 1;
int channels = 1;
double max_abs_diff = 1e-7;
};
using TensorsToSegmentationCalculatorTest = TestWithParam<FormattingTestCase>;
TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
const auto& [test_name, inputs, expected_outputs, activation, rows, cols,
rows_new, cols_new, channels, max_abs_diff] = GetParam();
auto graph_config =
test_utils::CreateGraphConfigForTest(/*test_gpu=*/false, activation);
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(absl::Substitute(
R"pb(
input_stream: "tensors"
input_stream: "size"
node {
calculator: "TensorsToSegmentationCalculator"
input_stream: "TENSORS:tensors"
input_stream: "OUTPUT_SIZE:size"
output_stream: "MASK:image_as_mask"
options: {
[mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
activation: $0
}
}
}
)pb",
ActivationTypeToString(activation)));
std::vector<Packet> output_packets;
tool::AddVectorSink("image_as_mask", &graph_config, &output_packets);
@ -107,7 +151,7 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
INSTANTIATE_TEST_SUITE_P(
TensorsToSegmentationCalculatorTests, TensorsToSegmentationCalculatorTest,
testing::ValuesIn<test_utils::FormattingTestCase>({
testing::ValuesIn<FormattingTestCase>({
{.test_name = "NoActivationAndNoOutputResize",
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
12.0, 13.0, 14.0, 15.0, 16.0},

View File

@ -1,111 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
#include <string>
#include <vector>
#include "absl/log/absl_log.h"
#include "absl/strings/substitute.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/port/parse_text_proto.h"
namespace mediapipe {
namespace tensors_to_segmentation_utils {
std::string ActivationTypeToString(
const TensorsToSegmentationCalculatorOptions::Activation& activation) {
switch (activation) {
case TensorsToSegmentationCalculatorOptions::NONE:
return "NONE";
case TensorsToSegmentationCalculatorOptions::SIGMOID:
return "SIGMOID";
case TensorsToSegmentationCalculatorOptions::SOFTMAX:
return "SOFTMAX";
}
ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
return "UNKNOWN";
}
std::vector<unsigned char> ArrayFloatToUnsignedChar(
const std::vector<float>& array) {
std::vector<unsigned char> result;
result.reserve(array.size());
for (int i = 0; i < array.size(); ++i) {
result.push_back(static_cast<unsigned char>(array[i]));
}
return result;
}
std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values) {
std::vector<float> result;
result.reserve(values.size() * 4);
for (const float& value : values) {
result.push_back(value);
result.push_back(0);
result.push_back(0);
result.push_back(value);
}
return result;
}
// For GPU tests, the input tensor needs to be moved to GPU, using
// TensorViewRequestor. After calculation, the output needs to be moved back
// to CPU, using ToImageCalculator. The output is an ImageFrame.
mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
bool test_gpu,
const TensorsToSegmentationCalculatorOptions::Activation& activation) {
std::string pre_process = R"pb(
node {
calculator: "mediapipe.aimatter.TensorViewRequestor"
input_stream: "TENSORS:tensors"
output_stream: "TENSORS:tensors_gpu"
options {
[mediapipe.aimatter.TensorViewRequestorOptions.ext] { gpu {} }
}
}
)pb";
std::string post_process = R"pb(
node {
calculator: "FromImageCalculator"
input_stream: "IMAGE:image_as_mask_gpu"
output_stream: "IMAGE_CPU:image_as_mask"
}
)pb";
return mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
absl::Substitute(
R"pb(
input_stream: "tensors"
input_stream: "size" $0
node {
calculator: "TensorsToSegmentationCalculator"
input_stream: "TENSORS:tensors$1"
input_stream: "OUTPUT_SIZE:size"
output_stream: "MASK:image_as_mask$2"
options: {
[mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
activation: $3
gpu_origin: TOP_LEFT
}
}
} $4
)pb",
test_gpu ? pre_process : "", test_gpu ? "_gpu" : "",
test_gpu ? "_gpu" : "", ActivationTypeToString(activation),
test_gpu ? post_process : ""));
}
} // namespace tensors_to_segmentation_utils
} // namespace mediapipe

View File

@ -1,57 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
namespace mediapipe {
namespace tensors_to_segmentation_utils {
std::string ActivationTypeToString(
const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
activation);
std::vector<unsigned char> ArrayFloatToUnsignedChar(
const std::vector<float>& array);
std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values);
mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
bool test_gpu,
const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
activation);
struct FormattingTestCase {
std::string test_name;
std::vector<float> inputs;
std::vector<float> expected_outputs;
mediapipe::TensorsToSegmentationCalculatorOptions::Activation activation;
int rows = 1;
int cols = 1;
int rows_new = 1;
int cols_new = 1;
int channels = 1;
double max_abs_diff = 1e-7;
};
} // namespace tensors_to_segmentation_utils
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_

View File

@ -1,50 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
#include <vector>
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/framework/port/gtest.h"
namespace mediapipe::tensors_to_segmentation_utils {
namespace {
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
ActivationTypeToStringWorksCorrectly) {
EXPECT_EQ(ActivationTypeToString(Options::NONE), "NONE");
EXPECT_EQ(ActivationTypeToString(Options::SIGMOID), "SIGMOID");
EXPECT_EQ(ActivationTypeToString(Options::SOFTMAX), "SOFTMAX");
}
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
ArrayFloatToUnsignedCharWorksCorrectly) {
std::vector<float> input = {1.0, 2.0, 3.0};
std::vector<unsigned char> expected = {1, 2, 3};
EXPECT_EQ(ArrayFloatToUnsignedChar(input), expected);
}
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
MakeRedAlphaMatrixWorksCorrectly) {
std::vector<float> input = {1.0, 2.0, 3.0};
std::vector<float> expected = {1.0, 0.0, 0.0, 1.0, 2.0, 0.0,
0.0, 2.0, 3.0, 0.0, 0.0, 3.0};
EXPECT_EQ(MakeRedAlphaMatrix(input), expected);
}
} // namespace
} // namespace mediapipe::tensors_to_segmentation_utils

View File

@ -1,43 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
#include <memory>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/tensor.h"
namespace mediapipe {
class TensorsToSegmentationConverter {
public:
virtual ~TensorsToSegmentationConverter() = default;
// Converts tensors to image mask.
// Returns a unique pointer containing the converted image.
// @input_tensors contains the tensors needed to be processed.
// @output_width/height describes output dimensions to reshape the output mask
// into.
virtual absl::StatusOr<std::unique_ptr<Image>> Convert(
const std::vector<Tensor>& input_tensors, int output_width,
int output_height) = 0;
};
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_

View File

@ -1,157 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
#include <algorithm>
#include <cmath>
#include <memory>
#include <vector>
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_opencv.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/opencv_core_inc.h"
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status_macros.h"
namespace mediapipe {
namespace {
class OpenCvProcessor : public TensorsToSegmentationConverter {
public:
absl::Status Init(const TensorsToSegmentationCalculatorOptions& options) {
options_ = options;
return absl::OkStatus();
}
absl::StatusOr<std::unique_ptr<Image>> Convert(
const std::vector<Tensor>& input_tensors, int output_width,
int output_height) override;
private:
template <class T>
absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat);
TensorsToSegmentationCalculatorOptions options_;
};
absl::StatusOr<std::unique_ptr<Image>> OpenCvProcessor::Convert(
const std::vector<Tensor>& input_tensors, int output_width,
int output_height) {
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
auto [tensor_height, tensor_width, tensor_channels] = hwc;
// Create initial working mask.
cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
// Wrap input tensor.
auto raw_input_tensor = &input_tensors[0];
auto raw_input_view = raw_input_tensor->GetCpuReadView();
const float* raw_input_data = raw_input_view.buffer<float>();
cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
CV_MAKETYPE(CV_32F, tensor_channels),
const_cast<float*>(raw_input_data));
// Process mask tensor and apply activation function.
if (tensor_channels == 2) {
MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
} else if (tensor_channels == 1) {
RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
options_.activation()); // Requires 2 channels.
if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
options_.activation()) // Pass-through optimization.
tensor_mat.copyTo(small_mask_mat);
else
MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
} else {
RET_CHECK_FAIL() << "Unsupported number of tensor channels "
<< tensor_channels;
}
// Send out image as CPU packet.
std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
ImageFormat::VEC32F1, output_width, output_height);
auto output_mask = std::make_unique<Image>(mask_frame);
auto output_mat = formats::MatView(output_mask.get());
// Upsample small mask into output.
cv::resize(small_mask_mat, *output_mat,
cv::Size(output_width, output_height));
return output_mask;
}
template <class T>
absl::Status OpenCvProcessor::ApplyActivation(cv::Mat& tensor_mat,
cv::Mat* small_mask_mat) {
// Configure activation function.
const int output_layer_index = options_.output_layer_index();
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
const auto activation_fn = [&](const cv::Vec2f& mask_value) {
float new_mask_value = 0;
// TODO consider moving switch out of the loop,
// and also avoid float/Vec2f casting.
switch (options_.activation()) {
case Options::NONE: {
new_mask_value = mask_value[0];
break;
}
case Options::SIGMOID: {
const float pixel0 = mask_value[0];
new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
break;
}
case Options::SOFTMAX: {
const float pixel0 = mask_value[0];
const float pixel1 = mask_value[1];
const float max_pixel = std::max(pixel0, pixel1);
const float min_pixel = std::min(pixel0, pixel1);
const float softmax_denom =
/*exp(max_pixel - max_pixel)=*/1.0f +
std::exp(min_pixel - max_pixel);
new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
softmax_denom;
break;
}
}
return new_mask_value;
};
// Process mask tensor.
for (int i = 0; i < tensor_mat.rows; ++i) {
for (int j = 0; j < tensor_mat.cols; ++j) {
const T& input_pix = tensor_mat.at<T>(i, j);
const float mask_value = activation_fn(input_pix);
small_mask_mat->at<float>(i, j) = mask_value;
}
}
return absl::OkStatus();
}
} // namespace
absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
CreateOpenCvConverter(const TensorsToSegmentationCalculatorOptions& options) {
auto converter = std::make_unique<OpenCvProcessor>();
MP_RETURN_IF_ERROR(converter->Init(options));
return converter;
}
} // namespace mediapipe

View File

@ -1,31 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
#include <memory>
#include "absl/status/statusor.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
namespace mediapipe {
// Creates OpenCV tensors-to-segmentation converter.
absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
CreateOpenCvConverter(
const mediapipe::TensorsToSegmentationCalculatorOptions& options);
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_

View File

@ -1,52 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include <tuple>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/framework/port.h"
#include "mediapipe/framework/port/ret_check.h"
namespace mediapipe {
int NumGroups(int size, int group_size) {
return (size + group_size - 1) / group_size;
}
bool CanUseGpu() {
#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
// TODO: Configure GPU usage policy in individual calculators.
constexpr bool kAllowGpuProcessing = true;
return kAllowGpuProcessing;
#else
return false;
#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
}
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
const std::vector<int>& dims) {
if (dims.size() == 3) {
return std::make_tuple(dims[0], dims[1], dims[2]);
} else if (dims.size() == 4) {
// BHWC format check B == 1
RET_CHECK_EQ(dims[0], 1) << "Expected batch to be 1 for BHWC heatmap";
return std::make_tuple(dims[1], dims[2], dims[3]);
} else {
RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
}
}
} // namespace mediapipe

View File

@ -1,34 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
#include <tuple>
#include <vector>
#include "absl/status/statusor.h"
namespace mediapipe {
// Commonly used to compute the number of blocks to launch in a kernel.
int NumGroups(const int size, const int group_size); // NOLINT
bool CanUseGpu();
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
const std::vector<int>& dims);
} // namespace mediapipe
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_

View File

@ -1,63 +0,0 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
#include <tuple>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/status_matchers.h"
namespace mediapipe {
namespace {
using ::testing::HasSubstr;
TEST(TensorsToSegmentationUtilsTest, NumGroupsWorksProperly) {
EXPECT_EQ(NumGroups(13, 4), 4);
EXPECT_EQ(NumGroups(4, 13), 1);
}
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsWorksProperly) {
std::vector<int> dims_3 = {2, 3, 4};
absl::StatusOr<std::tuple<int, int, int>> result_1 = GetHwcFromDims(dims_3);
MP_ASSERT_OK(result_1);
EXPECT_EQ(result_1.value(), (std::make_tuple(2, 3, 4)));
std::vector<int> dims_4 = {1, 3, 4, 5};
absl::StatusOr<std::tuple<int, int, int>> result_2 = GetHwcFromDims(dims_4);
MP_ASSERT_OK(result_2);
EXPECT_EQ(result_2.value(), (std::make_tuple(3, 4, 5)));
}
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsBatchCheckFail) {
std::vector<int> dims_4 = {2, 3, 4, 5};
absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_4);
EXPECT_FALSE(result.ok());
EXPECT_THAT(result.status().message(),
HasSubstr("Expected batch to be 1 for BHWC heatmap"));
}
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsInvalidShape) {
std::vector<int> dims_5 = {1, 2, 3, 4, 5};
absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_5);
EXPECT_FALSE(result.ok());
EXPECT_THAT(result.status().message(),
HasSubstr("Invalid shape for segmentation tensor"));
}
} // namespace
} // namespace mediapipe

View File

@ -79,7 +79,7 @@ namespace mpms = mediapipe::mediasequence;
// and label and label_id are optional but at least one of them should be set.
// "IMAGE_${NAME}", "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store
// prefixed versions of each stream, which allows for multiple image streams to
// be included. However, the default names are supported by more tools.
// be included. However, the default names are suppored by more tools.
//
// Example config:
// node {

View File

@ -67,8 +67,8 @@ absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet,
// -- 1-D or 2-D Tensor
// Output:
// -- Matrix with the same values as the Tensor
// If input tensor is 1 dimensional, the output Matrix is of (1xn) shape.
// If input tensor is 2 dimensional (batched), the output Matrix is (mxn) shape.
// If input tensor is 1 dimensional, the ouput Matrix is of (1xn) shape.
// If input tensor is 2 dimensional (batched), the ouput Matrix is (mxn) shape.
//
// Example Config
// node: {

View File

@ -15,9 +15,9 @@
// Calculator converts from one-dimensional Tensor of DT_FLOAT to vector<float>
// OR from (batched) two-dimensional Tensor of DT_FLOAT to vector<vector<float>.
#include <cstdint>
#include <memory>
#include "absl/base/integral_types.h"
#include "mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/status.h"

View File

@ -111,8 +111,8 @@ class InferenceState {
// input_side_packet.
//
// The input and output streams are TensorFlow tensors labeled by tags. The tags
// for the streams are matched to feeds and fetches in a TensorFlow session
// using a named_signature.generic_signature in the ModelManifest. The
// for the streams are matched to feeds and fetchs in a TensorFlow session using
// a named_signature.generic_signature in the ModelManifest. The
// generic_signature is used as key-value pairs between the MediaPipe tag and
// the TensorFlow tensor. The signature_name in the options proto determines
// which named_signature is used. The keys in the generic_signature must be
@ -128,7 +128,7 @@ class InferenceState {
// addition. Once batch_size inputs have been provided, the batch will be run
// and the output tensors sent out on the output streams with timestamps
// corresponding to the input stream packets. Setting the batch_size to 1
// completely disables batching, but is independent of add_batch_dim_to_tensors.
// completely disables batching, but is indepdent of add_batch_dim_to_tensors.
//
// The TensorFlowInferenceCalculator also support feeding states recurrently for
// RNNs and LSTMs. Simply set the recurrent_tag_pair options to define the

View File

@ -42,7 +42,7 @@ message TensorFlowInferenceCalculatorOptions {
// If the 0th dimension is the batch dimension, then the tensors are
// concatenated on that dimension. If the 0th is a data dimension, then a 0th
// dimension is added before concatenating. If added, the extra dimension is
// removed before outputting the tensor. Examples of each case: If you want
// removed before outputing the tensor. Examples of each case: If you want
// to batch spectra of audio over time for an LSTM, a time-frequency
// representation has a 0th dimension as the batch dimension. If you want to
// batch frames of video that are [width, height, channels], the batch

View File

@ -1,2 +1,2 @@
The model files add.bin, add_quantized.bin
(and corresponding metadata json files) come from tensorflow/lite/testdata/
(and corresponding metatada json files) come from tensorflow/lite/testdata/

View File

@ -95,7 +95,7 @@ struct GPUData {
// into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer
// or MTLBuffer.
//
// This calculator is designed to be used with the TfLiteInferenceCalculator,
// This calculator is designed to be used with the TfLiteInferenceCalcualtor,
// as a pre-processing step for calculator inputs.
//
// IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1],

View File

@ -31,7 +31,7 @@ message TfLiteConverterCalculatorOptions {
// Custom settings to override the internal scaling factors `div` and `sub`.
// Both values must be set to non-negative values. Will only take effect on
// CPU AND when |use_custom_normalization| is set to true. When these custom
// values take effect, the |zero_center| setting above will be overridden, and
// values take effect, the |zero_center| setting above will be overriden, and
// the normalized_value will be calculated as:
// normalized_value = input / custom_div - custom_sub.
optional bool use_custom_normalization = 6 [default = false];

View File

@ -25,7 +25,7 @@ message TfLiteTensorsToClassificationCalculatorOptions {
optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463;
}
// Score threshold for preserving the class.
// Score threshold for perserving the class.
optional float min_score_threshold = 1;
// Number of highest scoring labels to output. If top_k is not positive then
// all labels are used.

View File

@ -116,7 +116,7 @@ void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors,
// tensors can have 2 or 3 tensors. First tensor is the predicted
// raw boxes/keypoints. The size of the values must be (num_boxes
// * num_predicted_values). Second tensor is the score tensor. The
// size of the values must be (num_boxes * num_classes). It's
// size of the valuse must be (num_boxes * num_classes). It's
// optional to pass in a third tensor for anchors (e.g. for SSD
// models) depend on the outputs of the detection model. The size
// of anchor tensor must be (num_boxes * 4).

View File

@ -69,6 +69,6 @@ message TfLiteTensorsToDetectionsCalculatorOptions {
// representation has a bottom-left origin (e.g., in OpenGL).
optional bool flip_vertically = 18 [default = false];
// Score threshold for preserving decoded detections.
// Score threshold for perserving decoded detections.
optional float min_score_thresh = 19;
}

View File

@ -158,7 +158,7 @@ absl::Status TfLiteTensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
RET_CHECK(options_.has_input_image_height() &&
options_.has_input_image_width())
<< "Must provide input width/height for using flip_vertically option "
"when outputting landmarks in absolute coordinates.";
"when outputing landmarks in absolute coordinates.";
}
flip_horizontally_ =

View File

@ -1,7 +1,6 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip
networkTimeout=10000
validateDistributionUrl=true
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists

View File

@ -83,8 +83,10 @@ done
# This is normally unused
# shellcheck disable=SC2034
APP_BASE_NAME=${0##*/}
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD=maximum
@ -131,13 +133,10 @@ location of your Java installation."
fi
else
JAVACMD=java
if ! command -v java >/dev/null 2>&1
then
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
fi
# Increase the maximum file descriptors if we can.
@ -145,7 +144,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
case $MAX_FD in #(
max*)
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC2039,SC3045
# shellcheck disable=SC3045
MAX_FD=$( ulimit -H -n ) ||
warn "Could not query maximum file descriptor limit"
esac
@ -153,7 +152,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
'' | soft) :;; #(
*)
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
# shellcheck disable=SC2039,SC3045
# shellcheck disable=SC3045
ulimit -n "$MAX_FD" ||
warn "Could not set maximum file descriptor limit to $MAX_FD"
esac
@ -198,15 +197,11 @@ if "$cygwin" || "$msys" ; then
done
fi
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Collect all arguments for the java command:
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
# and any embedded shellness will be escaped.
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
# treated as '${Hostname}' itself on the command line.
# Collect all arguments for the java command;
# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
# shell script including quotes and variable substitutions, so put them in
# double quotes to make sure that they get re-expanded; and
# * put everything else in single quotes, so that it's not re-expanded.
set -- \
"-Dorg.gradle.appname=$APP_BASE_NAME" \

View File

@ -43,9 +43,9 @@ class KinematicPathSolver {
initialized_(false),
pixels_per_degree_(pixels_per_degree) {}
// Add an observation (detection) at a position and time.
absl::Status AddObservation(int position, const uint64_t time_us);
absl::Status AddObservation(int position, const uint64 time_us);
// Get the predicted position at a time.
absl::Status UpdatePrediction(const int64_t time_us);
absl::Status UpdatePrediction(const int64 time_us);
// Get the state at a time, as an int.
absl::Status GetState(int* position);
// Get the state at a time, as a float.
@ -63,7 +63,7 @@ class KinematicPathSolver {
bool IsMotionTooSmall(double delta_degs);
// Check if a position measurement will cause the camera to be in motion
// without updating the internal state.
absl::Status PredictMotionState(int position, const uint64_t time_us,
absl::Status PredictMotionState(int position, const uint64 time_us,
bool* state);
// Clear any history buffer of positions that are used when
// filtering_time_window_us is set to a non-zero value.
@ -85,9 +85,9 @@ class KinematicPathSolver {
double current_position_px_;
double prior_position_px_;
double current_velocity_deg_per_s_;
uint64_t current_time_ = 0;
uint64 current_time_ = 0;
// History of observations (second) and their time (first).
std::deque<std::pair<uint64_t, int>> raw_positions_at_time_;
std::deque<std::pair<uint64, int>> raw_positions_at_time_;
// Current target position.
double target_position_px_;
// Defines if the camera is moving to a target (true) or reached a target

View File

@ -67,7 +67,7 @@ class SceneCameraMotionAnalyzer {
const KeyFrameCropOptions& key_frame_crop_options,
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
const int scene_frame_width, const int scene_frame_height,
const std::vector<int64_t>& scene_frame_timestamps,
const std::vector<int64>& scene_frame_timestamps,
const bool has_solid_color_background,
SceneKeyFrameCropSummary* scene_summary,
std::vector<FocusPointFrame>* focus_point_frames,
@ -78,7 +78,7 @@ class SceneCameraMotionAnalyzer {
// crop window in SceneKeyFrameCropSummary in the case of steady motion.
absl::Status DecideCameraMotionType(
const KeyFrameCropOptions& key_frame_crop_options,
const double scene_span_sec, const int64_t end_time_us,
const double scene_span_sec, const int64 end_time_us,
SceneKeyFrameCropSummary* scene_summary,
SceneCameraMotion* scene_camera_motion) const;
@ -87,7 +87,7 @@ class SceneCameraMotionAnalyzer {
absl::Status PopulateFocusPointFrames(
const SceneKeyFrameCropSummary& scene_summary,
const SceneCameraMotion& scene_camera_motion,
const std::vector<int64_t>& scene_frame_timestamps,
const std::vector<int64>& scene_frame_timestamps,
std::vector<FocusPointFrame>* focus_point_frames) const;
private:
@ -118,7 +118,7 @@ class SceneCameraMotionAnalyzer {
absl::Status PopulateFocusPointFramesForTracking(
const SceneKeyFrameCropSummary& scene_summary,
const FocusPointFrameType focus_point_frame_type,
const std::vector<int64_t>& scene_frame_timestamps,
const std::vector<int64>& scene_frame_timestamps,
std::vector<FocusPointFrame>* focus_point_frames) const;
// Decide to use steady motion.
@ -142,7 +142,7 @@ class SceneCameraMotionAnalyzer {
// Last position
SceneCameraMotion last_scene_with_salient_region_;
int64_t time_since_last_salient_region_us_;
int64 time_since_last_salient_region_us_;
// Scene has solid color background.
bool has_solid_color_background_;

View File

@ -62,7 +62,7 @@ class SceneCropper {
// TODO: split this function into two separate functions.
absl::Status CropFrames(
const SceneKeyFrameCropSummary& scene_summary,
const std::vector<int64_t>& scene_timestamps,
const std::vector<int64>& scene_timestamps,
const std::vector<bool>& is_key_frames,
const std::vector<cv::Mat>& scene_frames_or_empty,
const std::vector<FocusPointFrame>& focus_point_frames,
@ -73,7 +73,7 @@ class SceneCropper {
absl::Status ProcessKinematicPathSolver(
const SceneKeyFrameCropSummary& scene_summary,
const std::vector<int64_t>& scene_timestamps,
const std::vector<int64>& scene_timestamps,
const std::vector<bool>& is_key_frames,
const std::vector<FocusPointFrame>& focus_point_frames,
const bool continue_last_scene, std::vector<cv::Mat>* all_xforms);

View File

@ -29,7 +29,7 @@ namespace autoflip {
// Packs detected features and timestamp (ms) into a KeyFrameInfo object. Scales
// features back to the original frame size if features have been detected on a
// different frame size.
absl::Status PackKeyFrameInfo(const int64_t frame_timestamp_ms,
absl::Status PackKeyFrameInfo(const int64 frame_timestamp_ms,
const DetectionSet& detections,
const int original_frame_width,
const int original_frame_height,
@ -71,7 +71,7 @@ absl::Status ComputeSceneStaticBordersSize(
// interpolation functions in Lab space using input timestamps.
absl::Status FindSolidBackgroundColor(
const std::vector<StaticFeatures>& static_features,
const std::vector<int64_t>& static_features_timestamps,
const std::vector<int64>& static_features_timestamps,
const double min_fraction_solid_background_color,
bool* has_solid_background,
PiecewiseLinearFunction* background_color_l_function,

View File

@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
@ -367,7 +368,6 @@ cc_library(
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/log:check",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",

View File

@ -81,13 +81,15 @@ cc_library(
srcs = ["node.cc"],
hdrs = ["node.h"],
deps = [
":const_str",
":contract",
":packet",
":port",
"//mediapipe/framework:calculator_base",
"//mediapipe/framework:calculator_context",
"//mediapipe/framework:calculator_contract",
"//mediapipe/framework:subgraph",
"//mediapipe/framework/deps:registration",
"@com_google_absl//absl/status",
"//mediapipe/framework/deps:no_destructor",
],
)

View File

@ -330,14 +330,6 @@ using MultiSideDestination = MultiPort<SideDestination<T>>;
class NodeBase {
public:
NodeBase() = default;
~NodeBase() = default;
NodeBase(NodeBase&&) = default;
NodeBase& operator=(NodeBase&&) = default;
// Explicitly delete copies to improve error messages.
NodeBase(const NodeBase&) = delete;
NodeBase& operator=(const NodeBase&) = delete;
// TODO: right now access to an indexed port is made directly by
// specifying both a tag and an index. It would be better to represent this
// as a two-step lookup, first getting a multi-port, and then accessing one
@ -593,14 +585,6 @@ class PacketGenerator {
class Graph {
public:
Graph() = default;
~Graph() = default;
Graph(Graph&&) = default;
Graph& operator=(Graph&&) = default;
// Explicitly delete copies to improve error messages.
Graph(const Graph&) = delete;
Graph& operator=(const Graph&) = delete;
void SetType(std::string type) { type_ = std::move(type); }
// Creates a node of a specific type. Should be used for calculators whose

View File

@ -1,15 +1,17 @@
#ifndef MEDIAPIPE_FRAMEWORK_API2_NODE_H_
#define MEDIAPIPE_FRAMEWORK_API2_NODE_H_
#include <memory>
#include <type_traits>
#include <functional>
#include <string>
#include "absl/status/status.h"
#include "mediapipe/framework/api2/const_str.h"
#include "mediapipe/framework/api2/contract.h"
#include "mediapipe/framework/api2/packet.h"
#include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/calculator_base.h"
#include "mediapipe/framework/calculator_context.h"
#include "mediapipe/framework/calculator_contract.h"
#include "mediapipe/framework/deps/registration.h"
#include "mediapipe/framework/deps/no_destructor.h"
#include "mediapipe/framework/subgraph.h"
namespace mediapipe {
@ -176,10 +178,7 @@ class SubgraphImpl : public Subgraph,
absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>)
// This macro is used to register a non-split-contract calculator. Deprecated.
#define MEDIAPIPE_REGISTER_NODE(name) \
MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED( \
mediapipe::CalculatorBaseRegistry, calculator_registration, #name, \
absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<name>>)
#define MEDIAPIPE_REGISTER_NODE(name) REGISTER_CALCULATOR(name)
// This macro is used to define a subgraph that does not use automatic
// registration. Deprecated.

View File

@ -28,7 +28,6 @@
#include "absl/container/flat_hash_set.h"
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/log/check.h"
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/strings/str_cat.h"
@ -891,12 +890,12 @@ absl::Status CalculatorGraph::WaitForObservedOutput() {
}
absl::Status CalculatorGraph::AddPacketToInputStream(
absl::string_view stream_name, const Packet& packet) {
const std::string& stream_name, const Packet& packet) {
return AddPacketToInputStreamInternal(stream_name, packet);
}
absl::Status CalculatorGraph::AddPacketToInputStream(
absl::string_view stream_name, Packet&& packet) {
const std::string& stream_name, Packet&& packet) {
return AddPacketToInputStreamInternal(stream_name, std::move(packet));
}
@ -919,18 +918,14 @@ absl::Status CalculatorGraph::SetInputStreamTimestampBound(
// std::forward will deduce the correct type as we pass along packet.
template <typename T>
absl::Status CalculatorGraph::AddPacketToInputStreamInternal(
absl::string_view stream_name, T&& packet) {
auto stream_it = graph_input_streams_.find(stream_name);
const std::string& stream_name, T&& packet) {
std::unique_ptr<GraphInputStream>* stream =
stream_it == graph_input_streams_.end() ? nullptr : &stream_it->second;
mediapipe::FindOrNull(graph_input_streams_, stream_name);
RET_CHECK(stream).SetNoLogging() << absl::Substitute(
"AddPacketToInputStream called on input stream \"$0\" which is not a "
"graph input stream.",
stream_name);
auto node_id_it = graph_input_stream_node_ids_.find(stream_name);
ABSL_CHECK(node_id_it != graph_input_stream_node_ids_.end())
<< "Map key not found: " << stream_name;
int node_id = node_id_it->second;
int node_id = mediapipe::FindOrDie(graph_input_stream_node_ids_, stream_name);
ABSL_CHECK_GE(node_id, validated_graph_->CalculatorInfos().size());
{
absl::MutexLock lock(&full_input_streams_mutex_);

View File

@ -32,7 +32,6 @@
#include "absl/container/flat_hash_set.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "absl/strings/string_view.h"
#include "absl/synchronization/mutex.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_base.h"
@ -256,7 +255,7 @@ class CalculatorGraph {
// sizes of the queues in the graph. The input stream must have been specified
// in the configuration as a graph level input_stream. On error, nothing is
// added.
absl::Status AddPacketToInputStream(absl::string_view stream_name,
absl::Status AddPacketToInputStream(const std::string& stream_name,
const Packet& packet);
// Same as the l-value version of this function by the same name, but moves
@ -266,7 +265,7 @@ class CalculatorGraph {
// packet may remain valid. In particular, when using the ADD_IF_NOT_FULL
// mode with a full queue, this will return StatusUnavailable and the caller
// may try adding the packet again later.
absl::Status AddPacketToInputStream(absl::string_view stream_name,
absl::Status AddPacketToInputStream(const std::string& stream_name,
Packet&& packet);
// Indicates that input will arrive no earlier than a certain timestamp.
@ -510,7 +509,7 @@ class CalculatorGraph {
// AddPacketToInputStream(Packet&& packet) or
// AddPacketToInputStream(const Packet& packet).
template <typename T>
absl::Status AddPacketToInputStreamInternal(absl::string_view stream_name,
absl::Status AddPacketToInputStreamInternal(const std::string& stream_name,
T&& packet);
// Sets the executor that will run the nodes assigned to the executor

View File

@ -31,14 +31,14 @@
// A StrongInt<T> with a NullStrongIntValidator should compile away to a raw T
// in optimized mode. What this means is that the generated assembly for:
//
// int64_t foo = 123;
// int64_t bar = 456;
// int64_t baz = foo + bar;
// constexpr int64_t fubar = 789;
// int64 foo = 123;
// int64 bar = 456;
// int64 baz = foo + bar;
// constexpr int64 fubar = 789;
//
// ...should be identical to the generated assembly for:
//
// DEFINE_STRONG_INT_TYPE(MyStrongInt, int64_t);
// DEFINE_STRONG_INT_TYPE(MyStrongInt, int64);
// MyStrongInt foo(123);
// MyStrongInt bar(456);
// MyStrongInt baz = foo + bar;
@ -97,7 +97,6 @@
#ifndef MEDIAPIPE_DEPS_STRONG_INT_H_
#define MEDIAPIPE_DEPS_STRONG_INT_H_
#include <cstdint>
#include <iosfwd>
#include <limits>
#include <ostream>
@ -180,11 +179,11 @@ struct NullStrongIntValidator {
}
// Verify lhs << rhs.
template <typename T>
static void ValidateLeftShift(T lhs, int64_t rhs) { /* do nothing */
static void ValidateLeftShift(T lhs, int64 rhs) { /* do nothing */
}
// Verify lhs >> rhs.
template <typename T>
static void ValidateRightShift(T lhs, int64_t rhs) { /* do nothing */
static void ValidateRightShift(T lhs, int64 rhs) { /* do nothing */
}
// Verify lhs & rhs.
template <typename T>
@ -225,8 +224,8 @@ class StrongInt {
//
// Example: Assume you have two StrongInt types.
//
// DEFINE_STRONG_INT_TYPE(Bytes, int64_t);
// DEFINE_STRONG_INT_TYPE(Megabytes, int64_t);
// DEFINE_STRONG_INT_TYPE(Bytes, int64);
// DEFINE_STRONG_INT_TYPE(Megabytes, int64);
//
// If you want to be able to (explicitly) construct an instance of Bytes from
// an instance of Megabytes, simply define a converter function in the same
@ -338,12 +337,12 @@ class StrongInt {
value_ %= arg;
return *this;
}
StrongInt &operator<<=(int64_t arg) { // NOLINT(whitespace/operators)
StrongInt &operator<<=(int64 arg) { // NOLINT(whitespace/operators)
ValidatorType::template ValidateLeftShift<ValueType>(value_, arg);
value_ <<= arg;
return *this;
}
StrongInt &operator>>=(int64_t arg) { // NOLINT(whitespace/operators)
StrongInt &operator>>=(int64 arg) { // NOLINT(whitespace/operators)
ValidatorType::template ValidateRightShift<ValueType>(value_, arg);
value_ >>= arg;
return *this;
@ -379,19 +378,19 @@ std::ostream &operator<<(std::ostream &os,
return os << arg.value();
}
// Provide the << operator, primarily for logging purposes. Specialized for
// int8_t so that an integer and not a character is printed.
// Provide the << operator, primarily for logging purposes. Specialized for int8
// so that an integer and not a character is printed.
template <typename TagType, typename ValidatorType>
std::ostream &operator<<(std::ostream &os,
StrongInt<TagType, int8_t, ValidatorType> arg) {
StrongInt<TagType, int8, ValidatorType> arg) {
return os << static_cast<int>(arg.value());
}
// Provide the << operator, primarily for logging purposes. Specialized for
// uint8_t so that an integer and not a character is printed.
// uint8 so that an integer and not a character is printed.
template <typename TagType, typename ValidatorType>
std::ostream &operator<<(std::ostream &os,
StrongInt<TagType, uint8_t, ValidatorType> arg) {
StrongInt<TagType, uint8, ValidatorType> arg) {
return os << static_cast<unsigned int>(arg.value());
}

View File

@ -76,12 +76,10 @@ def _get_proto_provider(dep):
"""
if ProtoInfo in dep:
return dep[ProtoInfo]
elif hasattr(dep, "proto"):
return dep.proto
else:
fail("cannot happen, rule definition requires .proto" +
" or ProtoInfo")
fail("cannot happen, rule definition requires .proto or ProtoInfo")
def _encode_binary_proto_impl(ctx):
"""Implementation of the encode_binary_proto rule."""
@ -144,10 +142,7 @@ _encode_binary_proto = rule(
cfg = "exec",
),
"deps": attr.label_list(
providers = [
[ProtoInfo],
["proto"],
],
providers = [[ProtoInfo], ["proto"]],
),
"input": attr.label(
mandatory = True,
@ -187,10 +182,7 @@ def _generate_proto_descriptor_set_impl(ctx):
all_protos = depset(transitive = [
_get_proto_provider(dep).transitive_sources
for dep in ctx.attr.deps
if (
ProtoInfo in dep or
hasattr(dep, "proto")
)
if ProtoInfo in dep or hasattr(dep, "proto")
])
descriptor = ctx.outputs.output
@ -221,10 +213,7 @@ generate_proto_descriptor_set = rule(
cfg = "exec",
),
"deps": attr.label_list(
providers = [
[ProtoInfo],
["proto"],
],
providers = [[ProtoInfo], ["proto"]],
),
},
outputs = {"output": "%{name}.proto.bin"},

View File

@ -124,15 +124,6 @@ cc_library(
],
)
cc_library(
name = "ahwb_view",
hdrs = ["ahwb_view.h"],
deps = [
"//mediapipe/framework:port",
"//mediapipe/gpu:gpu_buffer_storage",
],
)
cc_library(
name = "affine_transform",
srcs = ["affine_transform.cc"],
@ -155,27 +146,6 @@ cc_library(
],
)
cc_library(
name = "hardware_buffer",
srcs = ["hardware_buffer_android.cc"],
hdrs = ["hardware_buffer.h"],
linkopts = select({
"//conditions:default": [],
# Option for vendor binaries to avoid linking libandroid.so.
"//mediapipe/framework:android_no_jni": [],
"//mediapipe:android": ["-landroid"],
":android_link_native_window": [
"-lnativewindow", # Provides <android/hardware_buffer.h> to vendor binaries on Android API >= 26.
],
}),
visibility = ["//visibility:private"],
deps = [
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:statusor",
"@com_google_absl//absl/log:absl_check",
],
)
cc_library(
name = "image_frame",
srcs = ["image_frame.cc"],
@ -514,31 +484,28 @@ cc_library(
"//conditions:default": [],
# Option for vendor binaries to avoid linking libandroid.so.
"//mediapipe/framework:android_no_jni": [],
"//mediapipe:android": ["-landroid"],
":android_link_native_window": [
"-lnativewindow", # Provides <android/hardware_buffer.h> to vendor binaries on Android API >= 26.
],
}),
deps = [
"//mediapipe/framework:port",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/synchronization",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:gl_context",
],
"//mediapipe:android": [
":hardware_buffer",
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:gl_context",
],
":android_link_native_window": [
":hardware_buffer",
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:gl_context",
],
}),
"//mediapipe/framework:port",
"@com_google_absl//absl/container:flat_hash_map",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/synchronization",
] + select({
"//mediapipe/gpu:disable_gpu": [],
"//conditions:default": [
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:gl_context",
],
}) +
select({
"//conditions:default": [],
}),
)
cc_test(

View File

@ -1,54 +0,0 @@
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
#define MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
#include "mediapipe/framework/port.h"
#ifdef MEDIAPIPE_GPU_BUFFER_USE_AHWB
#include <android/hardware_buffer.h>
#include "mediapipe/gpu/gpu_buffer_storage.h"
namespace mediapipe {
// Wrapper to facilitate short lived access to Android Hardware Buffer objects.
// Intended use cases:
// - Extracting an AHWB for processing in another library after it's produced by
// MediaPipe.
// - Sending AHWBs to compute devices that are able to map the memory for their
// own usage.
// The AHWB abstractions in GpuBuffer and Tensor are likely more suitable for
// other CPU/GPU uses of AHWBs.
class AhwbView {
public:
explicit AhwbView(AHardwareBuffer* handle) : handle_(handle) {}
// Non-copyable
AhwbView(const AhwbView&) = delete;
AhwbView& operator=(const AhwbView&) = delete;
// Non-movable
AhwbView(AhwbView&&) = delete;
// Only supports synchronous usage. All users of GetHandle must finish
// accessing the buffer before this view object is destroyed to avoid race
// conditions.
// TODO: Support asynchronous usage.
const AHardwareBuffer* GetHandle() const { return handle_; }
private:
const AHardwareBuffer* handle_;
};
namespace internal {
// Makes this class available as a GpuBuffer view.
template <>
class ViewProvider<AhwbView> {
public:
virtual ~ViewProvider() = default;
virtual const AhwbView GetReadView(types<AhwbView>) const = 0;
virtual AhwbView GetWriteView(types<AhwbView>) = 0;
};
} // namespace internal
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_BUFFER_USE_AHWB
#endif // MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_

View File

@ -17,25 +17,10 @@
#include <stdio.h>
#include <utility>
#include "absl/log/absl_log.h"
namespace mediapipe {
DeletingFile::DeletingFile(DeletingFile&& other)
: path_(std::move(other.path_)),
delete_on_destruction_(other.delete_on_destruction_) {
other.delete_on_destruction_ = false;
}
DeletingFile& DeletingFile::operator=(DeletingFile&& other) {
path_ = std::move(other.path_);
delete_on_destruction_ = other.delete_on_destruction_;
other.delete_on_destruction_ = false;
return *this;
}
DeletingFile::DeletingFile(const std::string& path, bool delete_on_destruction)
: path_(path), delete_on_destruction_(delete_on_destruction) {}

View File

@ -28,11 +28,6 @@ class DeletingFile {
DeletingFile(const DeletingFile&) = delete;
DeletingFile& operator=(const DeletingFile&) = delete;
// DeletingFile is movable. The moved-from object remains in valid but
// unspecified state and will not perform any operations on destruction.
DeletingFile(DeletingFile&& other);
DeletingFile& operator=(DeletingFile&& other);
// Provide the path to the file and whether the file should be deleted
// when this object is destroyed.
DeletingFile(const std::string& path, bool delete_on_destruction);

View File

@ -16,11 +16,11 @@ limitations under the License.
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_FRAME_BUFFER_H_
#define MEDIAPIPE_FRAMEWORK_FORMATS_FRAME_BUFFER_H_
#include <cstdint>
#include <vector>
#include "absl/log/absl_check.h"
#include "absl/status/statusor.h"
#include "mediapipe/framework/port/integral_types.h"
namespace mediapipe {
@ -76,13 +76,13 @@ class FrameBuffer {
// Plane encapsulates buffer and stride information.
struct Plane {
Plane(uint8_t* buffer, Stride stride) : buffer_(buffer), stride_(stride) {}
const uint8_t* buffer() const { return buffer_; }
uint8_t* mutable_buffer() { return buffer_; }
Plane(uint8* buffer, Stride stride) : buffer_(buffer), stride_(stride) {}
const uint8* buffer() const { return buffer_; }
uint8* mutable_buffer() { return buffer_; }
Stride stride() const { return stride_; }
private:
uint8_t* buffer_;
uint8* buffer_;
Stride stride_;
};
@ -121,9 +121,9 @@ class FrameBuffer {
// YUV data structure.
struct YuvData {
const uint8_t* y_buffer;
const uint8_t* u_buffer;
const uint8_t* v_buffer;
const uint8* y_buffer;
const uint8* u_buffer;
const uint8* v_buffer;
// Y buffer row stride in bytes.
int y_row_stride;
// U/V buffer row stride in bytes.

View File

@ -1,167 +0,0 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_HARDWARE_BUFFER_H_
#define MEDIAPIPE_FRAMEWORK_FORMATS_HARDWARE_BUFFER_H_
#include <cstdint>
#include <memory>
#include <optional>
#include <utility>
#include "absl/base/attributes.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
typedef struct AHardwareBuffer AHardwareBuffer;
namespace mediapipe {
struct HardwareBufferSpec {
// Buffer pixel formats. See NDK's hardware_buffer.h for descriptions.
enum {
// This must be kept in sync with NDK's hardware_buffer.h
AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM = 0x01,
AHARDWAREBUFFER_FORMAT_R8G8B8_UNORM = 0x03,
AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT = 0x16,
AHARDWAREBUFFER_FORMAT_BLOB = 0x21,
AHARDWAREBUFFER_FORMAT_R8_UNORM = 0x38,
};
// Buffer usage descriptions. See NDK's hardware_buffer.h for descriptions.
enum {
// This must be kept in sync with NDK's hardware_buffer.h
AHARDWAREBUFFER_USAGE_CPU_READ_NEVER = 0x0UL,
AHARDWAREBUFFER_USAGE_CPU_READ_RARELY = 0x2UL,
AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN = 0x3UL,
AHARDWAREBUFFER_USAGE_CPU_WRITE_NEVER = UINT64_C(0) << 4,
AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY = UINT64_C(2) << 4,
AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN = UINT64_C(3) << 4,
AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE = UINT64_C(1) << 8,
AHARDWAREBUFFER_USAGE_GPU_FRAMEBUFFER = UINT64_C(1) << 9,
AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER = UINT64_C(1) << 24,
};
// Hashing required to use HardwareBufferSpec as key in buffer pools. See
// absl::Hash for details.
template <typename H>
friend H AbslHashValue(H h, const HardwareBufferSpec& spec) {
return H::combine(std::move(h), spec.width, spec.height, spec.layers,
spec.format, spec.usage);
}
uint32_t width = 0;
uint32_t height = 0;
uint32_t layers = 0;
uint32_t format = 0;
uint64_t usage = 0;
};
// Equality operators
inline bool operator==(const HardwareBufferSpec& lhs,
const HardwareBufferSpec& rhs) {
return lhs.width == rhs.width && lhs.height == rhs.height &&
lhs.layers == rhs.layers && lhs.format == rhs.format &&
lhs.usage == rhs.usage;
}
inline bool operator!=(const HardwareBufferSpec& lhs,
const HardwareBufferSpec& rhs) {
return !operator==(lhs, rhs);
}
// For internal use only. Thinly wraps the Android NDK AHardwareBuffer.
class HardwareBuffer {
public:
// Constructs a HardwareBuffer instance from a newly allocated Android NDK
// AHardwareBuffer.
static absl::StatusOr<HardwareBuffer> Create(const HardwareBufferSpec& spec);
// Destructs the HardwareBuffer, releasing the AHardwareBuffer.
~HardwareBuffer();
// Support HardwareBuffer moves.
HardwareBuffer(HardwareBuffer&& other);
// Delete assignment and copy constructors.
HardwareBuffer(HardwareBuffer& other) = delete;
HardwareBuffer(const HardwareBuffer& other) = delete;
HardwareBuffer& operator=(const HardwareBuffer&) = delete;
// Returns true if AHWB is supported.
static bool IsSupported();
// Lock the hardware buffer for the given usage flags. fence_file_descriptor
// specifies a fence file descriptor on which to wait before locking the
// buffer. Returns raw memory address if lock is successful, nullptr
// otherwise.
ABSL_MUST_USE_RESULT absl::StatusOr<void*> Lock(
uint64_t usage, std::optional<int> fence_file_descriptor = std::nullopt);
// Unlocks the hardware buffer synchronously. This method blocks until
// unlocking is complete.
absl::Status Unlock();
// Unlocks the hardware buffer asynchronously. It returns a file_descriptor
// which can be used as a fence that is signaled once unlocking is complete.
absl::StatusOr<int> UnlockAsync();
// Returns the underlying raw AHardwareBuffer pointer to be used directly with
// AHardwareBuffer APIs.
AHardwareBuffer* GetAHardwareBuffer() const { return ahw_buffer_; }
// Returns whether this HardwareBuffer contains a valid AHardwareBuffer.
bool IsValid() const { return ahw_buffer_ != nullptr; }
// Returns whether this HardwareBuffer is locked.
bool IsLocked() const { return is_locked_; }
// Releases the AHardwareBuffer.
void Reset();
// Ahwb's are aligned to an implementation specific cacheline size.
uint32_t GetAlignedWidth() const;
// Returns buffer spec.
const HardwareBufferSpec& spec() const { return spec_; }
private:
// Allocates an AHardwareBuffer instance;
static absl::StatusOr<AHardwareBuffer*> AllocateAHardwareBuffer(
const HardwareBufferSpec& spec);
// Constructs a HardwareBuffer instance from an already aquired
// AHardwareBuffer instance and its spec.
HardwareBuffer(const HardwareBufferSpec& spec, AHardwareBuffer* ahwb);
// Unlocks the hardware buffer. If fence_file_descriptor_ptr is not nullptr,
// the function won't block and instead fence_file_descriptor_ptr will be set
// to a file descriptor to become signaled once unlocking is complete.
absl::Status UnlockInternal(int* fence_file_descriptor_ptr);
// Releases ahw_buffer_ AHardwareBuffer instance;
absl::Status ReleaseAHardwareBuffer();
// Buffer spec.
HardwareBufferSpec spec_ = {};
// Android NDK AHardwareBuffer.
AHardwareBuffer* ahw_buffer_ = nullptr;
// Indicates if AHardwareBuffer is locked for reading or writing.
bool is_locked_ = false;
};
} // namespace mediapipe
#endif // MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_BUFFER_H_

View File

@ -1,152 +0,0 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#if !defined(MEDIAPIPE_NO_JNI) && \
(__ANDROID_API__ >= 26 || \
defined(__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__))
#include <android/hardware_buffer.h>
#include <memory>
#include "absl/log/absl_check.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/hardware_buffer.h"
#include "mediapipe/framework/port/ret_check.h"
namespace mediapipe {
HardwareBuffer::HardwareBuffer(HardwareBuffer &&other) {
spec_ = std::exchange(other.spec_, {});
ahw_buffer_ = std::exchange(other.ahw_buffer_, nullptr);
is_locked_ = std::exchange(other.is_locked_, false);
}
HardwareBuffer::HardwareBuffer(const HardwareBufferSpec &spec,
AHardwareBuffer *ahwb)
: spec_(spec), ahw_buffer_(ahwb), is_locked_(false) {}
HardwareBuffer::~HardwareBuffer() { Reset(); }
absl::StatusOr<HardwareBuffer> HardwareBuffer::Create(
const HardwareBufferSpec &spec) {
MP_ASSIGN_OR_RETURN(AHardwareBuffer * ahwb, AllocateAHardwareBuffer(spec));
return HardwareBuffer(spec, ahwb);
}
bool HardwareBuffer::IsSupported() {
if (__builtin_available(android 26, *)) {
return true;
}
return false;
}
absl::StatusOr<AHardwareBuffer *> HardwareBuffer::AllocateAHardwareBuffer(
const HardwareBufferSpec &spec) {
RET_CHECK(IsSupported()) << "AndroidHWBuffers not supported";
AHardwareBuffer *output = nullptr;
int error = 0;
if (__builtin_available(android 26, *)) {
AHardwareBuffer_Desc desc = {
.width = spec.width,
.height = spec.height,
.layers = spec.layers,
.format = spec.format,
.usage = spec.usage,
};
error = AHardwareBuffer_allocate(&desc, &output);
}
RET_CHECK(!error && output != nullptr) << "AHardwareBuffer_allocate failed";
return output;
}
absl::Status HardwareBuffer::ReleaseAHardwareBuffer() {
if (ahw_buffer_ == nullptr) {
return absl::OkStatus();
}
if (is_locked_) {
MP_RETURN_IF_ERROR(Unlock());
}
if (__builtin_available(android 26, *)) {
AHardwareBuffer_release(ahw_buffer_);
}
spec_ = {};
ahw_buffer_ = nullptr;
return absl::OkStatus();
}
absl::StatusOr<void *> HardwareBuffer::Lock(
uint64_t usage, std::optional<int> fence_file_descriptor) {
RET_CHECK(ahw_buffer_ != nullptr) << "Hardware Buffer not allocated";
RET_CHECK(!is_locked_) << "Hardware Buffer already locked";
void *mem = nullptr;
if (__builtin_available(android 26, *)) {
const int error = AHardwareBuffer_lock(
ahw_buffer_, usage,
fence_file_descriptor.has_value() ? *fence_file_descriptor : -1,
nullptr, &mem);
RET_CHECK(error == 0) << "Hardware Buffer lock failed. Error: " << error;
}
is_locked_ = true;
return mem;
}
absl::Status HardwareBuffer::Unlock() {
return UnlockInternal(/*fence_file_descriptor=*/nullptr);
}
absl::StatusOr<int> HardwareBuffer::UnlockAsync() {
int fence_file_descriptor = -1;
MP_RETURN_IF_ERROR(UnlockInternal(&fence_file_descriptor));
return fence_file_descriptor;
}
absl::Status HardwareBuffer::UnlockInternal(int *fence_file_descriptor) {
RET_CHECK(ahw_buffer_ != nullptr) << "Hardware Buffer not allocated";
if (!is_locked_) {
return absl::OkStatus();
}
if (__builtin_available(android 26, *)) {
const int error =
AHardwareBuffer_unlock(ahw_buffer_, fence_file_descriptor);
RET_CHECK(error == 0) << "Hardware Buffer unlock failed. error: " << error;
}
is_locked_ = false;
return absl::OkStatus();
}
uint32_t HardwareBuffer::GetAlignedWidth() const {
if (__builtin_available(android 26, *)) {
ABSL_CHECK(ahw_buffer_ != nullptr) << "Hardware Buffer not allocated";
AHardwareBuffer_Desc desc = {};
AHardwareBuffer_describe(ahw_buffer_, &desc);
ABSL_CHECK_GT(desc.stride, 0);
return desc.stride;
}
return 0;
}
void HardwareBuffer::Reset() {
const auto success = ReleaseAHardwareBuffer();
if (!success.ok()) {
ABSL_LOG(DFATAL) << "Failed to release AHardwareBuffer: " << success;
}
}
} // namespace mediapipe
#endif // !defined(MEDIAPIPE_NO_JNI) && (__ANDROID_API__>= 26 ||
// defined(__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__))

View File

@ -1,131 +0,0 @@
#include "mediapipe/framework/formats/hardware_buffer.h"
#include <android/hardware_buffer.h>
#include <memory>
#include "base/logging.h"
#include "mediapipe/framework/port/status_macros.h"
#include "testing/base/public/gmock.h"
#include "testing/base/public/gunit.h"
namespace mediapipe {
namespace {
HardwareBufferSpec GetTestHardwareBufferSpec(uint32_t size_bytes) {
return {.width = size_bytes,
.height = 1,
.layers = 1,
.format = HardwareBufferSpec::AHARDWAREBUFFER_FORMAT_BLOB,
.usage = HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY |
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN |
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN |
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER};
}
TEST(HardwareBufferTest, ShouldConstructValidAHardwareBuffer) {
MP_ASSERT_OK_AND_ASSIGN(
HardwareBuffer hardware_buffer,
HardwareBuffer::Create(GetTestHardwareBufferSpec(/*size_bytes=*/123)));
EXPECT_NE(hardware_buffer.GetAHardwareBuffer(), nullptr);
EXPECT_TRUE(hardware_buffer.IsValid());
}
TEST(HardwareBufferTest, ShouldResetValidAHardwareBuffer) {
MP_ASSERT_OK_AND_ASSIGN(
HardwareBuffer hardware_buffer,
HardwareBuffer::Create(GetTestHardwareBufferSpec(/*size_bytes=*/123)));
EXPECT_TRUE(hardware_buffer.IsValid());
EXPECT_NE(*hardware_buffer.Lock(
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY),
nullptr);
EXPECT_TRUE(hardware_buffer.IsLocked());
hardware_buffer.Reset();
EXPECT_FALSE(hardware_buffer.IsValid());
EXPECT_FALSE(hardware_buffer.IsLocked());
}
TEST(HardwareBufferTest, ShouldAllocateRequestedBufferSize) {
constexpr int kBufferSize = 123;
const HardwareBufferSpec spec = GetTestHardwareBufferSpec(kBufferSize);
MP_ASSERT_OK_AND_ASSIGN(HardwareBuffer hardware_buffer,
HardwareBuffer::Create(spec));
EXPECT_TRUE(hardware_buffer.IsValid());
if (__builtin_available(android 26, *)) {
AHardwareBuffer_Desc desc;
AHardwareBuffer_describe(hardware_buffer.GetAHardwareBuffer(), &desc);
EXPECT_EQ(desc.width, spec.width);
EXPECT_EQ(desc.height, spec.height);
EXPECT_EQ(desc.layers, spec.layers);
EXPECT_EQ(desc.format, spec.format);
EXPECT_EQ(desc.usage, spec.usage);
}
EXPECT_EQ(hardware_buffer.spec().width, spec.width);
EXPECT_EQ(hardware_buffer.spec().height, spec.height);
EXPECT_EQ(hardware_buffer.spec().layers, spec.layers);
EXPECT_EQ(hardware_buffer.spec().format, spec.format);
EXPECT_EQ(hardware_buffer.spec().usage, spec.usage);
}
TEST(HardwareBufferTest, ShouldSupportMoveConstructor) {
constexpr int kBufferSize = 123;
const auto spec = GetTestHardwareBufferSpec(kBufferSize);
MP_ASSERT_OK_AND_ASSIGN(HardwareBuffer hardware_buffer_a,
HardwareBuffer::Create(spec));
EXPECT_TRUE(hardware_buffer_a.IsValid());
void* const ahardware_buffer_ptr_a = hardware_buffer_a.GetAHardwareBuffer();
EXPECT_NE(ahardware_buffer_ptr_a, nullptr);
EXPECT_FALSE(hardware_buffer_a.IsLocked());
MP_ASSERT_OK_AND_ASSIGN(
void* const hardware_buffer_a_locked_ptr,
hardware_buffer_a.Lock(
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY));
EXPECT_NE(hardware_buffer_a_locked_ptr, nullptr);
EXPECT_TRUE(hardware_buffer_a.IsLocked());
HardwareBuffer hardware_buffer_b(std::move(hardware_buffer_a));
EXPECT_FALSE(hardware_buffer_a.IsValid());
EXPECT_FALSE(hardware_buffer_a.IsLocked());
void* const ahardware_buffer_ptr_b = hardware_buffer_b.GetAHardwareBuffer();
EXPECT_EQ(ahardware_buffer_ptr_a, ahardware_buffer_ptr_b);
EXPECT_TRUE(hardware_buffer_b.IsValid());
EXPECT_TRUE(hardware_buffer_b.IsLocked());
EXPECT_EQ(hardware_buffer_a.spec(), HardwareBufferSpec());
EXPECT_EQ(hardware_buffer_b.spec(), spec);
MP_ASSERT_OK(hardware_buffer_b.Unlock());
}
TEST(HardwareBufferTest, ShouldSupportReadWrite) {
constexpr std::string_view kTestString = "TestString";
constexpr int kBufferSize = kTestString.size();
MP_ASSERT_OK_AND_ASSIGN(
HardwareBuffer hardware_buffer,
HardwareBuffer::Create(GetTestHardwareBufferSpec(kBufferSize)));
// Write test string.
MP_ASSERT_OK_AND_ASSIGN(
void* const write_ptr,
hardware_buffer.Lock(
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY));
memcpy(write_ptr, kTestString.data(), kBufferSize);
MP_ASSERT_OK(hardware_buffer.Unlock());
// Read test string.
MP_ASSERT_OK_AND_ASSIGN(
void* const read_ptr,
hardware_buffer.Lock(
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_READ_RARELY));
EXPECT_EQ(memcmp(read_ptr, kTestString.data(), kBufferSize), 0);
MP_ASSERT_OK(hardware_buffer.Unlock());
}
} // namespace
} // namespace mediapipe

View File

@ -15,7 +15,6 @@
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_H_
#define MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_H_
#include <cstdint>
#include <utility>
#include "absl/synchronization/mutex.h"
@ -208,7 +207,7 @@ inline void Image::UnlockPixels() const {}
// Image buf = ...
// {
// PixelLock lock(&buf);
// uint8_t* buf_ptr = lock.Pixels();
// uint8* buf_ptr = lock.Pixels();
// ... use buf_ptr to access pixel data ...
// ... lock released automatically at end of scope ...
// }
@ -229,7 +228,7 @@ class PixelReadLock {
}
PixelReadLock(const PixelReadLock&) = delete;
const uint8_t* Pixels() const {
const uint8* Pixels() const {
if (frame_) return frame_->PixelData();
return nullptr;
}
@ -255,7 +254,7 @@ class PixelWriteLock {
}
PixelWriteLock(const PixelWriteLock&) = delete;
uint8_t* Pixels() {
uint8* Pixels() {
if (frame_) return frame_->MutablePixelData();
return nullptr;
}

View File

@ -35,13 +35,13 @@
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_H_
#define MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_H_
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/port.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/tool/type_util.h"
#define IMAGE_FRAME_RAW_IMAGE MEDIAPIPE_HAS_RTTI
@ -63,7 +63,7 @@ namespace mediapipe {
// stored with row padding for alignment purposes.
class ImageFrame {
public:
typedef std::function<void(uint8_t*)> Deleter;
typedef std::function<void(uint8*)> Deleter;
// This class offers a few standard delete functions and retains
// compatibility with the previous API.
@ -78,13 +78,13 @@ class ImageFrame {
// Use a default alignment boundary of 16 because Intel SSE2 instructions may
// incur performance penalty when accessing data not aligned on a 16-byte
// boundary. FFmpeg requires at least this level of alignment.
static const uint32_t kDefaultAlignmentBoundary = 16;
static const uint32 kDefaultAlignmentBoundary = 16;
// If the pixel data of an ImageFrame will be passed to an OpenGL function
// such as glTexImage2D() or glReadPixels(), use a four-byte alignment
// boundary because that is the initial value of the OpenGL GL_PACK_ALIGNMENT
// and GL_UNPACK_ALIGNMENT parameters.
static const uint32_t kGlDefaultAlignmentBoundary = 4;
static const uint32 kGlDefaultAlignmentBoundary = 4;
// Returns number of channels for an ImageFormat.
static int NumberOfChannelsForFormat(ImageFormat::Format format);
@ -104,7 +104,7 @@ class ImageFrame {
// must be a power of 2 (the number 1 is valid, and means the data will
// be stored contiguously).
ImageFrame(ImageFormat::Format format, int width, int height,
uint32_t alignment_boundary);
uint32 alignment_boundary);
// Same as above, but use kDefaultAlignmentBoundary for alignment_boundary.
ImageFrame(ImageFormat::Format format, int width, int height);
@ -115,8 +115,8 @@ class ImageFrame {
// width*num_channels*depth. Both width_step and depth are in units
// of bytes.
ImageFrame(ImageFormat::Format format, int width, int height, int width_step,
uint8_t* pixel_data,
Deleter deleter = std::default_delete<uint8_t[]>());
uint8* pixel_data,
Deleter deleter = std::default_delete<uint8[]>());
ImageFrame(ImageFrame&& move_from);
ImageFrame& operator=(ImageFrame&& move_from);
@ -142,7 +142,7 @@ class ImageFrame {
// alignment_boundary. If IsAligned(16) is true then so are
// IsAligned(8), IsAligned(4), IsAligned(2), and IsAligned(1).
// alignment_boundary must be 1 or a power of 2.
bool IsAligned(uint32_t alignment_boundary) const;
bool IsAligned(uint32 alignment_boundary) const;
// Returns the image / video format.
ImageFormat::Format Format() const { return format_; }
@ -167,13 +167,13 @@ class ImageFrame {
// Reset the current image frame and copy the data from image_frame into
// this image frame. The alignment_boundary must be given (and won't
// necessarily match the alignment_boundary of the input image_frame).
void CopyFrom(const ImageFrame& image_frame, uint32_t alignment_boundary);
void CopyFrom(const ImageFrame& image_frame, uint32 alignment_boundary);
// Get a mutable pointer to the underlying image data. The ImageFrame
// retains ownership.
uint8_t* MutablePixelData() { return pixel_data_.get(); }
uint8* MutablePixelData() { return pixel_data_.get(); }
// Get a const pointer to the underlying image data.
const uint8_t* PixelData() const { return pixel_data_.get(); }
const uint8* PixelData() const { return pixel_data_.get(); }
// Returns the total size of the pixel data.
int PixelDataSize() const { return Height() * WidthStep(); }
@ -187,41 +187,41 @@ class ImageFrame {
// ImageFrame takes ownership of pixel_data. See the Constructor
// with the same arguments for details.
void AdoptPixelData(ImageFormat::Format format, int width, int height,
int width_step, uint8_t* pixel_data,
Deleter deleter = std::default_delete<uint8_t[]>());
int width_step, uint8* pixel_data,
Deleter deleter = std::default_delete<uint8[]>());
// Resets the ImageFrame and makes it a copy of the provided pixel
// data, which is assumed to be stored contiguously. The ImageFrame
// will use the given alignment_boundary.
void CopyPixelData(ImageFormat::Format format, int width, int height,
const uint8_t* pixel_data, uint32_t alignment_boundary);
const uint8* pixel_data, uint32 alignment_boundary);
// Resets the ImageFrame and makes it a copy of the provided pixel
// data, with given width_step. The ImageFrame
// will use the given alignment_boundary.
void CopyPixelData(ImageFormat::Format format, int width, int height,
int width_step, const uint8_t* pixel_data,
uint32_t alignment_boundary);
int width_step, const uint8* pixel_data,
uint32 alignment_boundary);
// Allocates a frame of the specified format, width, height, and alignment,
// without clearing any current pixel data. See the constructor with the same
// argument list.
void Reset(ImageFormat::Format format, int width, int height,
uint32_t alignment_boundary);
uint32 alignment_boundary);
// Relinquishes ownership of the pixel data. Notice that the unique_ptr
// uses a non-standard deleter.
std::unique_ptr<uint8_t[], Deleter> Release();
std::unique_ptr<uint8[], Deleter> Release();
// Copy the 8-bit ImageFrame into a contiguous, pre-allocated buffer. Note
// that ImageFrame does not necessarily store its data contiguously (i.e. do
// not use copy_n to move image data).
void CopyToBuffer(uint8_t* buffer, int buffer_size) const;
void CopyToBuffer(uint8* buffer, int buffer_size) const;
// A version of CopyToBuffer for 16-bit pixel data. Note that buffer_size
// stores the number of 16-bit elements in the buffer, not the number of
// bytes.
void CopyToBuffer(uint16_t* buffer, int buffer_size) const;
void CopyToBuffer(uint16* buffer, int buffer_size) const;
// A version of CopyToBuffer for float pixel data. Note that buffer_size
// stores the number of float elements in the buffer, not the number of
@ -233,12 +233,12 @@ class ImageFrame {
private:
// Returns true if alignment_number is 1 or a power of 2.
static bool IsValidAlignmentNumber(uint32_t alignment_boundary);
static bool IsValidAlignmentNumber(uint32 alignment_boundary);
// The internal implementation of copying data from the provided pixel data.
// If width_step is 0, then calculates width_step assuming no padding.
void InternalCopyFrom(int width, int height, int width_step, int channel_size,
const uint8_t* pixel_data);
const uint8* pixel_data);
// The internal implementation of copying data to the provided buffer.
// If width_step is 0, then calculates width_step assuming no padding.
@ -249,7 +249,7 @@ class ImageFrame {
int height_;
int width_step_;
std::unique_ptr<uint8_t[], Deleter> pixel_data_;
std::unique_ptr<uint8[], Deleter> pixel_data_;
};
} // namespace mediapipe

View File

@ -16,7 +16,6 @@
#include <algorithm>
#include <cmath>
#include <cstdint>
#include <memory>
#include "absl/log/absl_check.h"

View File

@ -215,7 +215,7 @@ Location CreateCvMaskLocation(const cv::Mat_<T>& mask) {
return Location(location_data);
}
template Location CreateCvMaskLocation(const cv::Mat_<uint8_t>& mask);
template Location CreateCvMaskLocation(const cv::Mat_<uint8>& mask);
template Location CreateCvMaskLocation(const cv::Mat_<float>& mask);
} // namespace mediapipe

View File

@ -24,9 +24,6 @@
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
#include "mediapipe/gpu/gl_base.h"
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
#ifdef MEDIAPIPE_TENSOR_USE_AHWB
#include "mediapipe/framework/formats/hardware_buffer.h"
#endif // MEDIAPIPE_TENSOR_USE_AHWB
#if MEDIAPIPE_METAL_ENABLED
#import <Metal/Metal.h>
@ -342,14 +339,6 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferReadView() const {
<< "Tensor conversion between different GPU backing formats is not "
"supported yet.";
auto lock(absl::make_unique<absl::MutexLock>(&view_mutex_));
if ((valid_ & kValidOpenGlBuffer) && gl_context_ != nullptr &&
!gl_context_->IsCurrent() && GlContext::IsAnyContextCurrent()) {
ABSL_LOG_FIRST_N(WARNING, 1)
<< "Tensor::GetOpenGlBufferReadView is not executed on the same GL "
"context where GL buffer was created. Note that Tensor has "
"limited synchronization support when sharing OpenGl objects "
"between multiple OpenGL contexts.";
}
AllocateOpenGlBuffer();
if (!(valid_ & kValidOpenGlBuffer)) {
// If the call succeeds then AHWB -> SSBO are synchronized so any usage of
@ -367,13 +356,7 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferReadView() const {
}
return {opengl_buffer_, std::move(lock),
#ifdef MEDIAPIPE_TENSOR_USE_AHWB
// ssbo_read_ is passed to be populated on OpenGlBufferView
// destruction in order to perform delayed resources releasing (see
// tensor_ahwb.cc/DelayedReleaser) only when AHWB is in use.
//
// Not passing for the case when AHWB is not in use to avoid creation
// of unnecessary sync object and memory leak.
use_ahwb_ ? &ssbo_read_ : nullptr
&ssbo_read_
#else
nullptr
#endif // MEDIAPIPE_TENSOR_USE_AHWB
@ -384,14 +367,6 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferWriteView(
uint64_t source_location_hash) const {
auto lock(absl::make_unique<absl::MutexLock>(&view_mutex_));
TrackAhwbUsage(source_location_hash);
if ((valid_ & kValidOpenGlBuffer) && gl_context_ != nullptr &&
!gl_context_->IsCurrent() && GlContext::IsAnyContextCurrent()) {
ABSL_LOG_FIRST_N(WARNING, 1)
<< "Tensor::GetOpenGlBufferWriteView is not executed on the same GL "
"context where GL buffer was created. Note that Tensor has "
"limited synchronization support when sharing OpenGl objects "
"between multiple OpenGL contexts.";
}
AllocateOpenGlBuffer();
valid_ = kValidOpenGlBuffer;
return {opengl_buffer_, std::move(lock), nullptr};
@ -561,8 +536,9 @@ Tensor::CpuReadView Tensor::GetCpuReadView() const {
void* ptr = MapAhwbToCpuRead();
if (ptr) {
valid_ |= kValidCpu;
return {ptr, std::move(lock), [ahwb = ahwb_.get()] {
ABSL_CHECK_OK(ahwb->Unlock()) << "Unlock failed.";
return {ptr, std::move(lock), [ahwb = ahwb_] {
auto error = AHardwareBuffer_unlock(ahwb, nullptr);
ABSL_CHECK(error == 0) << "AHardwareBuffer_unlock " << error;
}};
}
}
@ -644,11 +620,9 @@ Tensor::CpuWriteView Tensor::GetCpuWriteView(
if (__builtin_available(android 26, *)) {
void* ptr = MapAhwbToCpuWrite();
if (ptr) {
return {ptr, std::move(lock),
[ahwb = ahwb_.get(), fence_fd = &fence_fd_] {
auto fence_fd_status = ahwb->UnlockAsync();
ABSL_CHECK_OK(fence_fd_status) << "Unlock failed.";
*fence_fd = fence_fd_status.value();
return {ptr, std::move(lock), [ahwb = ahwb_, fence_fd = &fence_fd_] {
auto error = AHardwareBuffer_unlock(ahwb, fence_fd);
ABSL_CHECK(error == 0) << "AHardwareBuffer_unlock " << error;
}};
}
}

View File

@ -44,8 +44,7 @@
#ifdef MEDIAPIPE_TENSOR_USE_AHWB
#include <EGL/egl.h>
#include <EGL/eglext.h>
#include "mediapipe/framework/formats/hardware_buffer.h"
#include <android/hardware_buffer.h>
#endif // MEDIAPIPE_TENSOR_USE_AHWB
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
#include "mediapipe/gpu/gl_base.h"
@ -196,11 +195,9 @@ class Tensor {
using FinishingFunc = std::function<bool(bool)>;
class AHardwareBufferView : public View {
public:
AHardwareBuffer* handle() const {
return hardware_buffer_->GetAHardwareBuffer();
}
AHardwareBuffer* handle() const { return handle_; }
AHardwareBufferView(AHardwareBufferView&& src) : View(std::move(src)) {
hardware_buffer_ = std::move(src.hardware_buffer_);
handle_ = std::exchange(src.handle_, nullptr);
file_descriptor_ = src.file_descriptor_;
fence_fd_ = std::exchange(src.fence_fd_, nullptr);
ahwb_written_ = std::exchange(src.ahwb_written_, nullptr);
@ -225,17 +222,17 @@ class Tensor {
protected:
friend class Tensor;
AHardwareBufferView(HardwareBuffer* hardware_buffer, int file_descriptor,
AHardwareBufferView(AHardwareBuffer* handle, int file_descriptor,
int* fence_fd, FinishingFunc* ahwb_written,
std::function<void()>* release_callback,
std::unique_ptr<absl::MutexLock>&& lock)
: View(std::move(lock)),
hardware_buffer_(hardware_buffer),
handle_(handle),
file_descriptor_(file_descriptor),
fence_fd_(fence_fd),
ahwb_written_(ahwb_written),
release_callback_(release_callback) {}
HardwareBuffer* hardware_buffer_;
AHardwareBuffer* handle_;
int file_descriptor_;
// The view sets some Tensor's fields. The view is released prior to tensor.
int* fence_fd_;
@ -288,22 +285,18 @@ class Tensor {
class OpenGlBufferView : public View {
public:
GLuint name() const { return name_; }
OpenGlBufferView(OpenGlBufferView&& src) : View(std::move(src)) {
name_ = std::exchange(src.name_, GL_INVALID_INDEX);
ssbo_read_ = std::exchange(src.ssbo_read_, nullptr);
}
~OpenGlBufferView() {
if (ssbo_read_) {
// TODO: update tensor to properly handle cases when
// multiple views were requested multiple sync fence may be needed.
*ssbo_read_ = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
}
}
protected:
friend class Tensor;
OpenGlBufferView(GLuint name, std::unique_ptr<absl::MutexLock>&& lock,
GLsync* ssbo_read)
: View(std::move(lock)), name_(name), ssbo_read_(ssbo_read) {}
@ -391,7 +384,7 @@ class Tensor {
mutable std::unique_ptr<MtlResources> mtl_resources_;
#ifdef MEDIAPIPE_TENSOR_USE_AHWB
mutable std::unique_ptr<HardwareBuffer> ahwb_;
mutable AHardwareBuffer* ahwb_ = nullptr;
// Signals when GPU finished writing into SSBO so AHWB can be used then. Or
// signals when writing into AHWB has been finished so GPU can read from SSBO.
// Sync and FD are bound together.

View File

@ -10,7 +10,7 @@
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "absl/synchronization/mutex.h"
#include "mediapipe/framework/formats/hardware_buffer.h"
#include "mediapipe/framework/port.h"
#include "mediapipe/gpu/gl_base.h"
#endif // MEDIAPIPE_TENSOR_USE_AHWB
@ -97,7 +97,7 @@ class DelayedReleaser {
DelayedReleaser(DelayedReleaser&&) = delete;
DelayedReleaser& operator=(DelayedReleaser&&) = delete;
static void Add(std::unique_ptr<HardwareBuffer> ahwb, GLuint opengl_buffer,
static void Add(AHardwareBuffer* ahwb, GLuint opengl_buffer,
EGLSyncKHR ssbo_sync, GLsync ssbo_read,
Tensor::FinishingFunc&& ahwb_written,
std::shared_ptr<mediapipe::GlContext> gl_context,
@ -115,8 +115,8 @@ class DelayedReleaser {
// Using `new` to access a non-public constructor.
to_release_local.emplace_back(absl::WrapUnique(new DelayedReleaser(
std::move(ahwb), opengl_buffer, ssbo_sync, ssbo_read,
std::move(ahwb_written), gl_context, std::move(callback))));
ahwb, opengl_buffer, ssbo_sync, ssbo_read, std::move(ahwb_written),
gl_context, std::move(callback))));
for (auto it = to_release_local.begin(); it != to_release_local.end();) {
if ((*it)->IsSignaled()) {
it = to_release_local.erase(it);
@ -136,6 +136,9 @@ class DelayedReleaser {
~DelayedReleaser() {
if (release_callback_) release_callback_();
if (__builtin_available(android 26, *)) {
AHardwareBuffer_release(ahwb_);
}
}
bool IsSignaled() {
@ -178,7 +181,7 @@ class DelayedReleaser {
}
protected:
std::unique_ptr<HardwareBuffer> ahwb_;
AHardwareBuffer* ahwb_;
GLuint opengl_buffer_;
// TODO: use wrapper instead.
EGLSyncKHR fence_sync_;
@ -189,12 +192,12 @@ class DelayedReleaser {
std::function<void()> release_callback_;
static inline std::deque<std::unique_ptr<DelayedReleaser>> to_release_;
DelayedReleaser(std::unique_ptr<HardwareBuffer> ahwb, GLuint opengl_buffer,
DelayedReleaser(AHardwareBuffer* ahwb, GLuint opengl_buffer,
EGLSyncKHR fence_sync, GLsync ssbo_read,
Tensor::FinishingFunc&& ahwb_written,
std::shared_ptr<mediapipe::GlContext> gl_context,
std::function<void()>&& callback)
: ahwb_(std::move(ahwb)),
: ahwb_(ahwb),
opengl_buffer_(opengl_buffer),
fence_sync_(fence_sync),
ssbo_read_(ssbo_read),
@ -211,7 +214,7 @@ Tensor::AHardwareBufferView Tensor::GetAHardwareBufferReadView() const {
ABSL_CHECK(!(valid_ & kValidOpenGlTexture2d))
<< "Tensor conversion between OpenGL texture and AHardwareBuffer is not "
"supported.";
bool transfer = ahwb_ == nullptr;
bool transfer = !ahwb_;
ABSL_CHECK(AllocateAHardwareBuffer())
<< "AHardwareBuffer is not supported on the target system.";
valid_ |= kValidAHardwareBuffer;
@ -220,10 +223,12 @@ Tensor::AHardwareBufferView Tensor::GetAHardwareBufferReadView() const {
} else {
if (valid_ & kValidOpenGlBuffer) CreateEglSyncAndFd();
}
return {ahwb_.get(), ssbo_written_,
return {ahwb_,
ssbo_written_,
&fence_fd_, // The FD is created for SSBO -> AHWB synchronization.
&ahwb_written_, // Filled by SetReadingFinishedFunc.
&release_callback_, std::move(lock)};
&release_callback_,
std::move(lock)};
}
void Tensor::CreateEglSyncAndFd() const {
@ -253,11 +258,12 @@ Tensor::AHardwareBufferView Tensor::GetAHardwareBufferWriteView(
ABSL_CHECK(AllocateAHardwareBuffer(size_alignment))
<< "AHardwareBuffer is not supported on the target system.";
valid_ = kValidAHardwareBuffer;
return {ahwb_.get(),
return {ahwb_,
/*ssbo_written=*/-1,
&fence_fd_, // For SetWritingFinishedFD.
&ahwb_written_, // Filled by SetReadingFinishedFunc.
&release_callback_, std::move(lock)};
&fence_fd_, // For SetWritingFinishedFD.
&ahwb_written_,
&release_callback_,
std::move(lock)};
}
bool Tensor::AllocateAHardwareBuffer(int size_alignment) const {
@ -270,43 +276,40 @@ bool Tensor::AllocateAHardwareBuffer(int size_alignment) const {
}
use_ahwb_ = true;
if (ahwb_ == nullptr) {
HardwareBufferSpec spec = {};
if (size_alignment == 0) {
spec.width = bytes();
} else {
// We expect allocations to be page-aligned, implicitly satisfying any
// requirements from Edge TPU. No need to add a check for this,
// since Edge TPU will check for us.
spec.width = AlignedToPowerOf2(bytes(), size_alignment);
if (__builtin_available(android 26, *)) {
if (ahwb_ == nullptr) {
AHardwareBuffer_Desc desc = {};
if (size_alignment == 0) {
desc.width = bytes();
} else {
// We expect allocations to be page-aligned, implicitly satisfying any
// requirements from Edge TPU. No need to add a check for this,
// since Edge TPU will check for us.
desc.width = AlignedToPowerOf2(bytes(), size_alignment);
}
desc.height = 1;
desc.layers = 1;
desc.format = AHARDWAREBUFFER_FORMAT_BLOB;
desc.usage = AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN |
AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN |
AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER;
return AHardwareBuffer_allocate(&desc, &ahwb_) == 0;
}
spec.height = 1;
spec.layers = 1;
spec.format = HardwareBufferSpec::AHARDWAREBUFFER_FORMAT_BLOB;
spec.usage = HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN |
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN |
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER;
auto new_ahwb = HardwareBuffer::Create(spec);
if (!new_ahwb.ok()) {
ABSL_LOG(ERROR) << "Allocation of NDK Hardware Buffer failed: "
<< new_ahwb.status();
return false;
}
ahwb_ = std::make_unique<HardwareBuffer>(std::move(*new_ahwb));
return true;
}
return true;
return false;
}
bool Tensor::AllocateAhwbMapToSsbo() const {
if (__builtin_available(android 26, *)) {
if (AllocateAHardwareBuffer()) {
if (MapAHardwareBufferToGlBuffer(ahwb_->GetAHardwareBuffer(), bytes())
.ok()) {
if (MapAHardwareBufferToGlBuffer(ahwb_, bytes()).ok()) {
glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0);
return true;
}
// Unable to make OpenGL <-> AHWB binding. Use regular SSBO instead.
ahwb_.reset();
AHardwareBuffer_release(ahwb_);
ahwb_ = nullptr;
}
}
return false;
@ -314,11 +317,14 @@ bool Tensor::AllocateAhwbMapToSsbo() const {
// Moves Cpu/Ssbo resource under the Ahwb backed memory.
void Tensor::MoveCpuOrSsboToAhwb() const {
auto dest =
ahwb_->Lock(HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY);
ABSL_CHECK_OK(dest) << "Lock of AHWB failed";
void* dest = nullptr;
if (__builtin_available(android 26, *)) {
auto error = AHardwareBuffer_lock(
ahwb_, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY, -1, nullptr, &dest);
ABSL_CHECK(error == 0) << "AHardwareBuffer_lock " << error;
}
if (valid_ & kValidCpu) {
std::memcpy(*dest, cpu_buffer_, bytes());
std::memcpy(dest, cpu_buffer_, bytes());
// Free CPU memory because next time AHWB is mapped instead.
free(cpu_buffer_);
cpu_buffer_ = nullptr;
@ -328,7 +334,7 @@ void Tensor::MoveCpuOrSsboToAhwb() const {
glBindBuffer(GL_SHADER_STORAGE_BUFFER, opengl_buffer_);
const void* src = glMapBufferRange(GL_SHADER_STORAGE_BUFFER, 0, bytes(),
GL_MAP_READ_BIT);
std::memcpy(*dest, src, bytes());
std::memcpy(dest, src, bytes());
glUnmapBuffer(GL_SHADER_STORAGE_BUFFER);
glDeleteBuffers(1, &opengl_buffer_);
});
@ -341,7 +347,10 @@ void Tensor::MoveCpuOrSsboToAhwb() const {
ABSL_LOG(FATAL) << "Can't convert tensor with mask " << valid_
<< " into AHWB.";
}
ABSL_CHECK_OK(ahwb_->Unlock()) << "Unlock of AHWB failed";
if (__builtin_available(android 26, *)) {
auto error = AHardwareBuffer_unlock(ahwb_, nullptr);
ABSL_CHECK(error == 0) << "AHardwareBuffer_unlock " << error;
}
}
// SSBO is created on top of AHWB. A fence is inserted into the GPU queue before
@ -394,52 +403,59 @@ void Tensor::ReleaseAhwbStuff() {
if (ahwb_) {
if (ssbo_read_ != 0 || fence_sync_ != EGL_NO_SYNC_KHR || ahwb_written_) {
if (ssbo_written_ != -1) close(ssbo_written_);
DelayedReleaser::Add(std::move(ahwb_), opengl_buffer_, fence_sync_,
ssbo_read_, std::move(ahwb_written_), gl_context_,
DelayedReleaser::Add(ahwb_, opengl_buffer_, fence_sync_, ssbo_read_,
std::move(ahwb_written_), gl_context_,
std::move(release_callback_));
opengl_buffer_ = GL_INVALID_INDEX;
} else {
if (release_callback_) release_callback_();
ahwb_.reset();
AHardwareBuffer_release(ahwb_);
}
}
}
}
void* Tensor::MapAhwbToCpuRead() const {
if (ahwb_ != nullptr) {
if (!(valid_ & kValidCpu)) {
if ((valid_ & kValidOpenGlBuffer) && ssbo_written_ == -1) {
// EGLSync is failed. Use another synchronization method.
// TODO: Use tflite::gpu::GlBufferSync and GlActiveSync.
gl_context_->Run([]() { glFinish(); });
} else if (valid_ & kValidAHardwareBuffer) {
ABSL_CHECK(ahwb_written_) << "Ahwb-to-Cpu synchronization requires the "
"completion function to be set";
ABSL_CHECK(ahwb_written_(true))
<< "An error oqcured while waiting for the buffer to be written";
if (__builtin_available(android 26, *)) {
if (ahwb_) {
if (!(valid_ & kValidCpu)) {
if ((valid_ & kValidOpenGlBuffer) && ssbo_written_ == -1) {
// EGLSync is failed. Use another synchronization method.
// TODO: Use tflite::gpu::GlBufferSync and GlActiveSync.
gl_context_->Run([]() { glFinish(); });
} else if (valid_ & kValidAHardwareBuffer) {
ABSL_CHECK(ahwb_written_)
<< "Ahwb-to-Cpu synchronization requires the "
"completion function to be set";
ABSL_CHECK(ahwb_written_(true))
<< "An error oqcured while waiting for the buffer to be written";
}
}
void* ptr;
auto error =
AHardwareBuffer_lock(ahwb_, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN,
ssbo_written_, nullptr, &ptr);
ABSL_CHECK(error == 0) << "AHardwareBuffer_lock " << error;
close(ssbo_written_);
ssbo_written_ = -1;
return ptr;
}
auto ptr =
ahwb_->Lock(HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN,
ssbo_written_);
ABSL_CHECK_OK(ptr) << "Lock of AHWB failed";
close(ssbo_written_);
ssbo_written_ = -1;
return *ptr;
}
return nullptr;
}
void* Tensor::MapAhwbToCpuWrite() const {
if (ahwb_ != nullptr) {
// TODO: If previously acquired view is GPU write view then need
// to be sure that writing is finished. That's a warning: two consequent
// write views should be interleaved with read view.
auto locked_ptr =
ahwb_->Lock(HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN);
ABSL_CHECK_OK(locked_ptr) << "Lock of AHWB failed";
return *locked_ptr;
if (__builtin_available(android 26, *)) {
if (ahwb_) {
// TODO: If previously acquired view is GPU write view then need
// to be sure that writing is finished. That's a warning: two consequent
// write views should be interleaved with read view.
void* ptr;
auto error = AHardwareBuffer_lock(
ahwb_, AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN, -1, nullptr, &ptr);
ABSL_CHECK(error == 0) << "AHardwareBuffer_lock " << error;
return ptr;
}
}
return nullptr;
}

View File

@ -6,7 +6,6 @@
#include <cstdint>
#include "absl/algorithm/container.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/formats/tensor/views/data_types.h"
#include "mediapipe/gpu/gpu_test_base.h"
@ -19,7 +18,7 @@
// Then the test requests the CPU view and compares the values.
// Float32 and Float16 tests are there.
namespace mediapipe {
namespace {
using mediapipe::Float16;
using mediapipe::Tensor;
@ -28,16 +27,6 @@ MATCHER_P(NearWithPrecision, precision, "") {
return std::abs(std::get<0>(arg) - std::get<1>(arg)) < precision;
}
template <typename F = float>
std::vector<F> CreateReferenceData(int num_elements) {
std::vector<F> reference;
reference.resize(num_elements);
for (int i = 0; i < num_elements; i++) {
reference[i] = static_cast<float>(i) / 10.0f;
}
return reference;
}
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
// Utility function to fill the GPU buffer.
@ -121,7 +110,11 @@ TEST_F(TensorAhwbGpuTest, TestGpuToCpuFloat32) {
});
auto ptr = tensor.GetCpuReadView().buffer<float>();
ASSERT_NE(ptr, nullptr);
std::vector<float> reference = CreateReferenceData(num_elements);
std::vector<float> reference;
reference.resize(num_elements);
for (int i = 0; i < num_elements; i++) {
reference[i] = static_cast<float>(i) / 10.0f;
}
EXPECT_THAT(absl::Span<const float>(ptr, num_elements),
testing::Pointwise(testing::FloatEq(), reference));
}
@ -144,7 +137,11 @@ TEST_F(TensorAhwbGpuTest, TestGpuToCpuFloat16) {
});
auto ptr = tensor.GetCpuReadView().buffer<Float16>();
ASSERT_NE(ptr, nullptr);
std::vector<Float16> reference = CreateReferenceData<Float16>(num_elements);
std::vector<Float16> reference;
reference.resize(num_elements);
for (int i = 0; i < num_elements; i++) {
reference[i] = static_cast<float>(i) / 10.0f;
}
// Precision is set to a reasonable value for Float16.
EXPECT_THAT(absl::Span<const Float16>(ptr, num_elements),
testing::Pointwise(NearWithPrecision(0.001), reference));
@ -169,7 +166,11 @@ TEST_F(TensorAhwbGpuTest, TestReplacingCpuByAhwb) {
}
auto ptr = tensor.GetCpuReadView().buffer<float>();
ASSERT_NE(ptr, nullptr);
std::vector<float> reference = CreateReferenceData(num_elements);
std::vector<float> reference;
reference.resize(num_elements);
for (int i = 0; i < num_elements; i++) {
reference[i] = static_cast<float>(i) / 10.0f;
}
EXPECT_THAT(absl::Span<const float>(ptr, num_elements),
testing::Pointwise(testing::FloatEq(), reference));
}
@ -193,107 +194,17 @@ TEST_F(TensorAhwbGpuTest, TestReplacingGpuByAhwb) {
}
auto ptr = tensor.GetCpuReadView().buffer<float>();
ASSERT_NE(ptr, nullptr);
std::vector<float> reference = CreateReferenceData(num_elements);
std::vector<float> reference;
reference.resize(num_elements);
for (int i = 0; i < num_elements; i++) {
reference[i] = static_cast<float>(i) / 10.0f;
}
EXPECT_THAT(absl::Span<const float>(ptr, num_elements),
testing::Pointwise(testing::FloatEq(), reference));
}
std::vector<float> ReadGlBufferView(const Tensor::OpenGlBufferView& view,
int num_elements) {
glBindBuffer(GL_SHADER_STORAGE_BUFFER, view.name());
int bytes = num_elements * sizeof(float);
void* ptr =
glMapBufferRange(GL_SHADER_STORAGE_BUFFER, 0, bytes, GL_MAP_READ_BIT);
ABSL_CHECK(ptr) << "glMapBufferRange failed: " << glGetError();
std::vector<float> data;
data.resize(num_elements);
std::memcpy(data.data(), ptr, bytes);
glUnmapBuffer(GL_SHADER_STORAGE_BUFFER);
return data;
}
TEST_F(TensorAhwbGpuTest, TestGetOpenGlBufferReadViewNoAhwb) {
constexpr size_t kNumElements = 20;
std::vector<float> reference = CreateReferenceData(kNumElements);
Tensor tensor(Tensor::ElementType::kFloat32, Tensor::Shape({kNumElements}));
{
// Populate tensor on CPU and make sure view is destroyed
absl::c_copy(reference, tensor.GetCpuWriteView().buffer<float>());
}
RunInGlContext([&] {
// Triggers conversion to GL buffer.
auto ssbo_view = tensor.GetOpenGlBufferReadView();
ASSERT_NE(ssbo_view.name(), 0);
// ssbo_read_ must NOT be populated, as there's no AHWB associated with
// GL buffer
ASSERT_EQ(ssbo_view.ssbo_read_, nullptr);
std::vector<float> output = ReadGlBufferView(ssbo_view, kNumElements);
EXPECT_THAT(output, testing::Pointwise(testing::FloatEq(), reference));
});
}
TEST_F(TensorAhwbGpuTest, TestGetOpenGlBufferReadViewAhwbFromCpu) {
constexpr size_t kNumElements = 20;
std::vector<float> reference = CreateReferenceData(kNumElements);
Tensor tensor(Tensor::ElementType::kFloat32, Tensor::Shape({kNumElements}));
{
// Populate tensor on CPU and make sure view is destroyed
absl::c_copy(reference, tensor.GetCpuWriteView().buffer<float>());
}
{
// Make tensor to allocate ahwb and make sure view is destroyed.
ASSERT_NE(tensor.GetAHardwareBufferReadView().handle(), nullptr);
}
RunInGlContext([&] {
// Triggers conversion to GL buffer.
auto ssbo_view = tensor.GetOpenGlBufferReadView();
ASSERT_NE(ssbo_view.name(), 0);
// ssbo_read_ must be populated, so during view destruction it's set
// properly for further AHWB destruction
ASSERT_NE(ssbo_view.ssbo_read_, nullptr);
std::vector<float> output = ReadGlBufferView(ssbo_view, kNumElements);
EXPECT_THAT(output, testing::Pointwise(testing::FloatEq(), reference));
});
}
TEST_F(TensorAhwbGpuTest, TestGetOpenGlBufferReadViewAhwbFromGpu) {
constexpr size_t kNumElements = 20;
std::vector<float> reference = CreateReferenceData(kNumElements);
Tensor tensor(Tensor::ElementType::kFloat32, Tensor::Shape({kNumElements}));
{
// Make tensor to allocate ahwb and make sure view is destroyed.
ASSERT_NE(tensor.GetAHardwareBufferWriteView().handle(), nullptr);
}
RunInGlContext([&] {
FillGpuBuffer(tensor.GetOpenGlBufferWriteView().name(),
tensor.shape().num_elements(), tensor.element_type());
});
RunInGlContext([&] {
// Triggers conversion to GL buffer.
auto ssbo_view = tensor.GetOpenGlBufferReadView();
ASSERT_NE(ssbo_view.name(), 0);
// ssbo_read_ must be populated, so during view destruction it's set
// properly for further AHWB destruction
ASSERT_NE(ssbo_view.ssbo_read_, nullptr);
std::vector<float> output = ReadGlBufferView(ssbo_view, kNumElements);
EXPECT_THAT(output, testing::Pointwise(testing::FloatEq(), reference));
});
}
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
} // namespace mediapipe
} // namespace
#endif // !defined(MEDIAPIPE_NO_JNI) && (__ANDROID_API__ >= 26 ||
// defined(__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__))

View File

@ -1,5 +1,3 @@
#include <android/hardware_buffer.h>
#include "mediapipe/framework/formats/tensor.h"
#include "testing/base/public/gmock.h"
#include "testing/base/public/gunit.h"

View File

@ -15,7 +15,6 @@
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_YUV_IMAGE_H_
#define MEDIAPIPE_FRAMEWORK_FORMATS_YUV_IMAGE_H_
#include <cstdint>
#include <functional>
#include <memory>
@ -120,13 +119,13 @@ class YUVImage {
~YUVImage() { Clear(); }
// Convenience constructor
YUVImage(libyuv::FourCC fourcc, //
std::unique_ptr<uint8_t[]> data_location, //
uint8_t* data0, int stride0, //
uint8_t* data1, int stride1, //
uint8_t* data2, int stride2, //
YUVImage(libyuv::FourCC fourcc, //
std::unique_ptr<uint8[]> data_location, //
uint8* data0, int stride0, //
uint8* data1, int stride1, //
uint8* data2, int stride2, //
int width, int height, int bit_depth = 8) {
uint8_t* tmp = data_location.release();
uint8* tmp = data_location.release();
std::function<void()> deallocate = [tmp]() { delete[] tmp; };
Initialize(fourcc, //
deallocate, //
@ -138,14 +137,14 @@ class YUVImage {
// Convenience constructor to construct the YUVImage with data stored
// in three unique_ptrs.
YUVImage(libyuv::FourCC fourcc, //
std::unique_ptr<uint8_t[]> data0, int stride0, //
std::unique_ptr<uint8_t[]> data1, int stride1, //
std::unique_ptr<uint8_t[]> data2, int stride2, //
YUVImage(libyuv::FourCC fourcc, //
std::unique_ptr<uint8[]> data0, int stride0, //
std::unique_ptr<uint8[]> data1, int stride1, //
std::unique_ptr<uint8[]> data2, int stride2, //
int width, int height, int bit_depth = 8) {
uint8_t* tmp0 = data0.release();
uint8_t* tmp1 = data1.release();
uint8_t* tmp2 = data2.release();
uint8* tmp0 = data0.release();
uint8* tmp1 = data1.release();
uint8* tmp2 = data2.release();
std::function<void()> deallocate = [tmp0, tmp1, tmp2]() {
delete[] tmp0;
delete[] tmp1;
@ -178,9 +177,9 @@ class YUVImage {
// pixel format it holds.
void Initialize(libyuv::FourCC fourcc, //
std::function<void()> deallocation_function, //
uint8_t* data0, int stride0, //
uint8_t* data1, int stride1, //
uint8_t* data2, int stride2, //
uint8* data0, int stride0, //
uint8* data1, int stride1, //
uint8* data2, int stride2, //
int width, int height, int bit_depth = 8) {
Clear();
deallocation_function_ = deallocation_function;
@ -215,7 +214,7 @@ class YUVImage {
// Getters.
libyuv::FourCC fourcc() const { return fourcc_; }
const uint8_t* data(int index) const { return data_[index]; }
const uint8* data(int index) const { return data_[index]; }
int stride(int index) const { return stride_[index]; }
int width() const { return width_; }
int height() const { return height_; }
@ -227,7 +226,7 @@ class YUVImage {
// Setters.
void set_fourcc(libyuv::FourCC fourcc) { fourcc_ = fourcc; }
uint8_t* mutable_data(int index) { return data_[index]; }
uint8* mutable_data(int index) { return data_[index]; }
void set_stride(int index, int stride) { stride_[index] = stride; }
void set_width(int width) { width_ = width; }
void set_height(int height) { height_ = height; }
@ -242,7 +241,7 @@ class YUVImage {
std::function<void()> deallocation_function_;
libyuv::FourCC fourcc_ = libyuv::FOURCC_ANY;
uint8_t* data_[kMaxNumPlanes];
uint8* data_[kMaxNumPlanes];
int stride_[kMaxNumPlanes];
int width_ = 0;
int height_ = 0;

View File

@ -50,7 +50,7 @@
// but may or may not still be able to run other OpenGL code.
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) && \
(defined(__APPLE__) || defined(__EMSCRIPTEN__) || MEDIAPIPE_DISABLE_GPU || \
MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
MEDIAPIPE_USING_SWIFTSHADER)
#define MEDIAPIPE_DISABLE_GL_COMPUTE
#endif
@ -104,9 +104,4 @@
#endif
#endif // MEDIAPIPE_HAS_RTTI
// AHardware buffers are only available since Android API 26.
#if (__ANDROID_API__ >= 26)
#define MEDIAPIPE_GPU_BUFFER_USE_AHWB 1
#endif
#endif // MEDIAPIPE_FRAMEWORK_PORT_H_

View File

@ -26,7 +26,7 @@ def replace_suffix(string, old, new):
def mediapipe_ts_library(
name,
srcs = [],
srcs,
visibility = None,
deps = [],
testonly = 0,

View File

@ -13,13 +13,13 @@
# limitations under the License.
#
load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"data_as_c_string",
"mediapipe_binary_graph",
)
load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test")
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
licenses(["notice"])
@ -616,7 +616,6 @@ cc_test(
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/functional:bind_front",
"@com_google_absl//absl/strings",
],
)
@ -857,7 +856,6 @@ cc_library(
mediapipe_cc_test(
name = "switch_demux_calculator_test",
srcs = ["switch_demux_calculator_test.cc"],
requires_full_emulation = False,
deps = [
":container_util",
":switch_demux_calculator",
@ -893,7 +891,6 @@ cc_library(
mediapipe_cc_test(
name = "switch_mux_calculator_test",
srcs = ["switch_mux_calculator_test.cc"],
requires_full_emulation = False,
deps = [
":container_util",
":switch_mux_calculator",

View File

@ -17,7 +17,6 @@
#include <memory>
#include <vector>
#include "absl/functional/bind_front.h"
#include "absl/strings/string_view.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"

View File

@ -196,7 +196,6 @@ cc_library(
":gpu_buffer_format",
"//mediapipe/framework:executor",
"//mediapipe/framework:mediapipe_profiling",
"//mediapipe/framework:port",
"//mediapipe/framework:timestamp",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
@ -210,7 +209,6 @@ cc_library(
"@com_google_absl//absl/memory",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/synchronization",
] + select({
"//conditions:default": [],
@ -513,19 +511,11 @@ cc_library(
],
}),
deps = [
":gl_base_hdr",
":gl_context",
":gl_texture_buffer",
":gl_texture_view",
":gpu_buffer_format",
":gpu_buffer_storage",
":image_frame_view",
"//mediapipe/framework:port",
"//mediapipe/framework/formats:ahwb_view",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:ret_check",
"//third_party/GL:EGL_headers",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/strings:str_format",
],
)
@ -671,8 +661,6 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/base:core_headers",
"@com_google_absl//absl/log:absl_check",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/status",
] + select({
"//conditions:default": [],
"//mediapipe:apple": [
@ -1235,13 +1223,9 @@ mediapipe_cc_test(
],
requires_full_emulation = True,
deps = [
":gl_texture_buffer",
":gl_texture_util",
":gpu_buffer_format",
":gpu_buffer_storage_ahwb",
":gpu_test_base",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/tool:test_util",
],
)

View File

@ -57,8 +57,8 @@ void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) {
// static
absl::Status GlCalculatorHelper::UpdateContract(CalculatorContract* cc,
bool request_gpu_as_optional) {
if (request_gpu_as_optional) {
bool requesst_gpu_as_optional) {
if (requesst_gpu_as_optional) {
cc->UseService(kGpuService).Optional();
} else {
cc->UseService(kGpuService);

View File

@ -68,7 +68,7 @@ class GlCalculatorHelper {
// This method can be called from GetContract to set up the needed GPU
// resources.
static absl::Status UpdateContract(CalculatorContract* cc,
bool request_gpu_as_optional = false);
bool requesst_gpu_as_optional = false);
// This method can be called from FillExpectations to set the correct types
// for the shared GL input side packet(s).

View File

@ -26,9 +26,7 @@
#include "absl/log/absl_log.h"
#include "absl/memory/memory.h"
#include "absl/status/status.h"
#include "absl/strings/str_format.h"
#include "absl/synchronization/mutex.h"
#include "mediapipe/framework/port.h" // IWYU pragma: keep
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_builder.h"
@ -50,17 +48,6 @@
namespace mediapipe {
namespace internal_gl_context {
bool IsOpenGlVersionSameOrAbove(const OpenGlVersion& version,
const OpenGlVersion& expected_version) {
return (version.major == expected_version.major &&
version.minor >= expected_version.minor) ||
version.major > expected_version.major;
}
} // namespace internal_gl_context
static void SetThreadName(const char* name) {
#if defined(__GLIBC_PREREQ)
#define LINUX_STYLE_SETNAME_NP __GLIBC_PREREQ(2, 12)
@ -651,11 +638,6 @@ class GlSyncWrapper {
// TODO: do something if the wait fails?
}
// This method exists only for investigation purposes to distinguish stack
// traces: external vs. internal context.
// TODO: remove after glWaitSync crashes are resolved.
void WaitOnGpuExternalContext() { glWaitSync(sync_, 0, GL_TIMEOUT_IGNORED); }
void WaitOnGpu() {
if (!sync_) return;
// WebGL2 specifies a waitSync call, but since cross-context
@ -663,33 +645,6 @@ class GlSyncWrapper {
// a warning when it's called, so let's just skip the call. See
// b/184637485 for details.
#ifndef __EMSCRIPTEN__
if (!GlContext::IsAnyContextCurrent()) {
// glWaitSync must be called on with some context current. Doing the
// opposite doesn't necessarily result in a crash or GL error. Hence,
// just logging an error and skipping the call.
ABSL_LOG_FIRST_N(ERROR, 1)
<< "An attempt to wait for a sync without any context current.";
return;
}
auto context = GlContext::GetCurrent();
if (context == nullptr) {
// This can happen when WaitOnGpu is invoked on an external context,
// created by other than GlContext::Create means.
WaitOnGpuExternalContext();
return;
}
// GlContext::ShouldUseFenceSync guards creation of sync objects, so this
// CHECK should never fail if clients use MediaPipe APIs in an intended way.
// TODO: remove after glWaitSync crashes are resolved.
ABSL_CHECK(context->ShouldUseFenceSync()) << absl::StrFormat(
"An attempt to wait for a sync when it should not be used. (OpenGL "
"Version "
"%d.%d)",
context->gl_major_version(), context->gl_minor_version());
glWaitSync(sync_, 0, GL_TIMEOUT_IGNORED);
#endif
}
@ -742,13 +697,10 @@ class GlFenceSyncPoint : public GlSyncPoint {
void Wait() override {
if (!sync_) return;
if (GlContext::IsAnyContextCurrent()) {
gl_context_->Run([this] {
// TODO: must this run on the original context??
sync_.Wait();
return;
}
// In case a current GL context is not available, we fall back using the
// captured gl_context_.
gl_context_->Run([this] { sync_.Wait(); });
});
}
void WaitOnGpu() override {
@ -860,25 +812,15 @@ class GlNopSyncPoint : public GlSyncPoint {
#endif
bool GlContext::ShouldUseFenceSync() const {
using internal_gl_context::OpenGlVersion;
#if defined(__EMSCRIPTEN__)
#ifdef __EMSCRIPTEN__
// In Emscripten the glWaitSync function is non-null depending on linkopts,
// but only works in a WebGL2 context.
constexpr OpenGlVersion kMinVersionSyncAvaiable = {.major = 3, .minor = 0};
#elif defined(MEDIAPIPE_MOBILE)
// OpenGL ES, glWaitSync is available since 3.0
constexpr OpenGlVersion kMinVersionSyncAvaiable = {.major = 3, .minor = 0};
// but only works in a WebGL2 context, so fall back to use Finish if it is a
// WebGL1/ES2 context.
// TODO: apply this more generally once b/152794517 is fixed.
return gl_major_version() > 2;
#else
// TODO: specify major/minor version per remaining platforms.
// By default, ignoring major/minor version requirement for backward
// compatibility.
constexpr OpenGlVersion kMinVersionSyncAvaiable = {.major = 0, .minor = 0};
#endif
return SymbolAvailable(&glWaitSync) &&
internal_gl_context::IsOpenGlVersionSameOrAbove(
{.major = gl_major_version(), .minor = gl_minor_version()},
kMinVersionSyncAvaiable);
return SymbolAvailable(&glWaitSync);
#endif // __EMSCRIPTEN__
}
std::shared_ptr<GlSyncPoint> GlContext::CreateSyncToken() {

View File

@ -71,8 +71,6 @@ typedef std::function<void()> GlVoidFunction;
typedef std::function<absl::Status()> GlStatusFunction;
class GlContext;
// TODO: remove after glWaitSync crashes are resolved.
class GlSyncWrapper;
// Generic interface for synchronizing access to a shared resource from a
// different context. This is an abstract class to keep users from
@ -192,7 +190,8 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
// Like Run, but does not wait.
void RunWithoutWaiting(GlVoidFunction gl_func);
// Returns a synchronization token for this GlContext.
// Returns a synchronization token.
// This should not be called outside of the GlContext thread.
std::shared_ptr<GlSyncPoint> CreateSyncToken();
// If another part of the framework calls glFinish, it should call this
@ -331,9 +330,6 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
SyncTokenTypeForTest type);
private:
// TODO: remove after glWaitSync crashes are resolved.
friend GlSyncWrapper;
GlContext();
bool ShouldUseFenceSync() const;
@ -492,18 +488,6 @@ ABSL_DEPRECATED(
const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
int plane);
namespace internal_gl_context {
struct OpenGlVersion {
int major;
int minor;
};
bool IsOpenGlVersionSameOrAbove(const OpenGlVersion& version,
const OpenGlVersion& expected_version);
} // namespace internal_gl_context
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_CONTEXT_H_

View File

@ -32,7 +32,7 @@ namespace mediapipe {
// TODO: Handle webGL "context lost" and "context restored" events.
GlContext::StatusOrGlContext GlContext::Create(std::nullptr_t nullp,
bool create_thread) {
return Create(static_cast<EMSCRIPTEN_WEBGL_CONTEXT_HANDLE>(0), create_thread);
return Create(0, create_thread);
}
GlContext::StatusOrGlContext GlContext::Create(const GlContext& share_context,

View File

@ -14,8 +14,6 @@
#include "mediapipe/gpu/gl_texture_buffer.h"
#include <cstdint>
#include "absl/log/absl_check.h"
#include "absl/log/absl_log.h"
#include "mediapipe/framework/formats/image_frame.h"
@ -133,13 +131,6 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) {
SymbolAvailable(&glTexStorage2D)) {
ABSL_CHECK(data == nullptr) << "unimplemented";
glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
} else if (info.immutable) {
ABSL_CHECK(SymbolAvailable(&glTexStorage2D) &&
context->GetGlVersion() != GlVersion::kGLES2)
<< "Immutable GpuBuffer format requested is not supported in this "
<< "GlContext. Format was " << static_cast<uint32_t>(format_);
ABSL_CHECK(data == nullptr) << "unimplemented";
glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
} else {
glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_,
height_, 0 /* border */, info.gl_format, info.gl_type, data);

View File

@ -19,7 +19,6 @@
#define MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_
#include <atomic>
#include <memory>
#include "absl/memory/memory.h"
#include "mediapipe/framework/formats/image_frame.h"

View File

@ -17,7 +17,7 @@
#include <cstdlib>
#if defined(MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
#if defined(MEDIAPIPE_USING_SWIFTSHADER)
#define MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR 1
#endif

View File

@ -35,10 +35,6 @@ namespace mediapipe {
#endif // GL_HALF_FLOAT_OES
#endif // __EMSCRIPTEN__
#ifndef GL_RGBA8
#define GL_RGBA8 0x8058
#endif // GL_RGBA8
#if !MEDIAPIPE_DISABLE_GPU
#ifdef GL_ES_VERSION_2_0
static void AdaptGlTextureInfoForGLES2(GlTextureInfo* info) {
@ -167,14 +163,6 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
{
{GL_RGBA32F, GL_RGBA, GL_FLOAT, 1},
}},
{GpuBufferFormat::kImmutableRGBAFloat128,
{
{GL_RGBA32F, GL_RGBA, GL_FLOAT, 1, true /* immutable */},
}},
{GpuBufferFormat::kImmutableRGBA32,
{
{GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE, 1, true /* immutable */},
}},
}};
static const auto* gles2_format_info = ([] {
@ -218,7 +206,6 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
switch (format) {
case GpuBufferFormat::kImmutableRGBA32:
case GpuBufferFormat::kBGRA32:
// TODO: verify we are handling order of channels correctly.
return ImageFormat::SRGBA;
@ -234,11 +221,10 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
return ImageFormat::SRGB;
case GpuBufferFormat::kTwoComponentFloat32:
return ImageFormat::VEC32F2;
case GpuBufferFormat::kImmutableRGBAFloat128:
case GpuBufferFormat::kRGBAFloat128:
return ImageFormat::VEC32F4;
case GpuBufferFormat::kRGBA32:
return ImageFormat::SRGBA;
// TODO: this likely maps to ImageFormat::SRGBA
case GpuBufferFormat::kGrayHalf16:
case GpuBufferFormat::kOneComponent8Alpha:
case GpuBufferFormat::kOneComponent8Red:

Some files were not shown because too many files have changed in this diff Show More