Compare commits
No commits in common. "master" and "mrschmidt/gpu" have entirely different histories.
master
...
mrschmidt/
|
@ -80,7 +80,7 @@ message SpectrogramCalculatorOptions {
|
|||
// If use_local_timestamp is true, the output packet's timestamp is based on
|
||||
// the last sample of the packet and it's inferred from the latest input
|
||||
// packet's timestamp. If false, the output packet's timestamp is based on
|
||||
// the cumulative timestamping, which is inferred from the initial input
|
||||
// the cumulative timestamping, which is inferred from the intial input
|
||||
// timestamp and the cumulative number of samples.
|
||||
optional bool use_local_timestamp = 8 [default = false];
|
||||
}
|
||||
|
|
|
@ -66,7 +66,7 @@ message TimeSeriesFramerCalculatorOptions {
|
|||
// If use_local_timestamp is true, the output packet's timestamp is based on
|
||||
// the last sample of the packet and it's inferred from the latest input
|
||||
// packet's timestamp. If false, the output packet's timestamp is based on
|
||||
// the cumulative timestamping, which is inferred from the initial input
|
||||
// the cumulative timestamping, which is inferred from the intial input
|
||||
// timestamp and the cumulative number of samples.
|
||||
optional bool use_local_timestamp = 6 [default = false];
|
||||
}
|
||||
|
|
|
@ -727,7 +727,6 @@ cc_library(
|
|||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/status",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
@ -743,7 +742,6 @@ cc_test(
|
|||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/tool:options_util",
|
||||
"//mediapipe/util:packet_test_util",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
|
|
|
@ -71,7 +71,7 @@ TEST_F(PacketSequencerCalculatorTest, IsRegistered) {
|
|||
CalculatorBaseRegistry::IsRegistered("PacketSequencerCalculator"));
|
||||
}
|
||||
|
||||
// Shows how control packets receive timestamps before and after frame packets
|
||||
// Shows how control packets recieve timestamps before and after frame packets
|
||||
// have arrived.
|
||||
TEST_F(PacketSequencerCalculatorTest, ChannelEarly) {
|
||||
CalculatorGraphConfig::Node node_config = BuildNodeConfig();
|
||||
|
|
|
@ -17,7 +17,6 @@
|
|||
#include <set>
|
||||
#include <string>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/logging.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
@ -33,7 +32,6 @@ namespace {
|
|||
constexpr char kTagAtPreStream[] = "AT_PRESTREAM";
|
||||
constexpr char kTagAtPostStream[] = "AT_POSTSTREAM";
|
||||
constexpr char kTagAtZero[] = "AT_ZERO";
|
||||
constexpr char kTagAtFirstTick[] = "AT_FIRST_TICK";
|
||||
constexpr char kTagAtTick[] = "AT_TICK";
|
||||
constexpr char kTagTick[] = "TICK";
|
||||
constexpr char kTagAtTimestamp[] = "AT_TIMESTAMP";
|
||||
|
@ -45,7 +43,6 @@ static std::map<std::string, Timestamp>* kTimestampMap = []() {
|
|||
res->emplace(kTagAtPostStream, Timestamp::PostStream());
|
||||
res->emplace(kTagAtZero, Timestamp(0));
|
||||
res->emplace(kTagAtTick, Timestamp::Unset());
|
||||
res->emplace(kTagAtFirstTick, Timestamp::Unset());
|
||||
res->emplace(kTagAtTimestamp, Timestamp::Unset());
|
||||
return res;
|
||||
}();
|
||||
|
@ -62,8 +59,8 @@ std::string GetOutputTag(const CC& cc) {
|
|||
// timestamp, depending on the tag used to define output stream(s). (One tag can
|
||||
// be used only.)
|
||||
//
|
||||
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_FIRST_TICK,
|
||||
// AT_TIMESTAMP and corresponding timestamps are Timestamp::PreStream(),
|
||||
// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, AT_TIMESTAMP
|
||||
// and corresponding timestamps are Timestamp::PreStream(),
|
||||
// Timestamp::PostStream(), Timestamp(0), timestamp of a packet received in TICK
|
||||
// input, and timestamp received from a side input.
|
||||
//
|
||||
|
@ -99,7 +96,6 @@ class SidePacketToStreamCalculator : public CalculatorBase {
|
|||
|
||||
private:
|
||||
bool is_tick_processing_ = false;
|
||||
bool close_on_first_tick_ = false;
|
||||
std::string output_tag_;
|
||||
};
|
||||
REGISTER_CALCULATOR(SidePacketToStreamCalculator);
|
||||
|
@ -107,16 +103,13 @@ REGISTER_CALCULATOR(SidePacketToStreamCalculator);
|
|||
absl::Status SidePacketToStreamCalculator::GetContract(CalculatorContract* cc) {
|
||||
const auto& tags = cc->Outputs().GetTags();
|
||||
RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1)
|
||||
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK, "
|
||||
"AT_FIRST_TICK and AT_TIMESTAMP tags is allowed and required to "
|
||||
"specify output stream(s).";
|
||||
const bool has_tick_output =
|
||||
cc->Outputs().HasTag(kTagAtTick) || cc->Outputs().HasTag(kTagAtFirstTick);
|
||||
const bool has_tick_input = cc->Inputs().HasTag(kTagTick);
|
||||
RET_CHECK((has_tick_output && has_tick_input) ||
|
||||
(!has_tick_output && !has_tick_input))
|
||||
<< "Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
|
||||
"should be used or none of them.";
|
||||
<< "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
|
||||
"AT_TIMESTAMP tags is allowed and required to specify output "
|
||||
"stream(s).";
|
||||
RET_CHECK(
|
||||
(cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) ||
|
||||
(!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick)))
|
||||
<< "Either both of TICK and AT_TICK should be used or none of them.";
|
||||
RET_CHECK((cc->Outputs().HasTag(kTagAtTimestamp) &&
|
||||
cc->InputSidePackets().HasTag(kTagSideInputTimestamp)) ||
|
||||
(!cc->Outputs().HasTag(kTagAtTimestamp) &&
|
||||
|
@ -155,17 +148,11 @@ absl::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) {
|
|||
// timestamp bound update.
|
||||
cc->SetOffset(TimestampDiff(0));
|
||||
}
|
||||
if (output_tag_ == kTagAtFirstTick) {
|
||||
close_on_first_tick_ = true;
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
|
||||
if (is_tick_processing_) {
|
||||
if (cc->Outputs().Get(output_tag_, 0).IsClosed()) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
// TICK input is guaranteed to be non-empty, as it's the only input stream
|
||||
// for this calculator.
|
||||
const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp();
|
||||
|
@ -173,9 +160,6 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
|
|||
cc->Outputs()
|
||||
.Get(output_tag_, i)
|
||||
.AddPacket(cc->InputSidePackets().Index(i).At(timestamp));
|
||||
if (close_on_first_tick_) {
|
||||
cc->Outputs().Get(output_tag_, i).Close();
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
|
@ -186,7 +170,6 @@ absl::Status SidePacketToStreamCalculator::Process(CalculatorContext* cc) {
|
|||
|
||||
absl::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) {
|
||||
if (!cc->Outputs().HasTag(kTagAtTick) &&
|
||||
!cc->Outputs().HasTag(kTagAtFirstTick) &&
|
||||
!cc->Outputs().HasTag(kTagAtTimestamp)) {
|
||||
const auto& timestamp = kTimestampMap->at(output_tag_);
|
||||
for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) {
|
||||
|
|
|
@ -27,17 +27,13 @@
|
|||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/tool/options_util.h"
|
||||
#include "mediapipe/util/packet_test_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
using ::testing::ElementsAre;
|
||||
using ::testing::Eq;
|
||||
using ::testing::HasSubstr;
|
||||
using ::testing::IsEmpty;
|
||||
using testing::HasSubstr;
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTick) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -56,35 +52,10 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTick) {
|
|||
EXPECT_THAT(
|
||||
status.message(),
|
||||
HasSubstr(
|
||||
"Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
|
||||
"should be used or none of them."));
|
||||
"Either both of TICK and AT_TICK should be used or none of them."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator,
|
||||
WrongConfigWithMissingTickForFirstTickProcessing) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_FIRST_TICK:packet"
|
||||
}
|
||||
)pb");
|
||||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
HasSubstr(
|
||||
"Either both TICK input and tick (AT_TICK/AT_FIRST_TICK) output "
|
||||
"should be used or none of them."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTimestampSideInput) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MissingTimestampSideInput) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -105,7 +76,7 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithMissingTimestampSideInput) {
|
|||
"or none of them."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithNonExistentTag) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -121,13 +92,14 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithNonExistentTag) {
|
|||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(status.message(),
|
||||
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
|
||||
"AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
|
||||
"allowed and required to specify output stream(s)."));
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
|
||||
"AT_TIMESTAMP tags is allowed and required to specify output "
|
||||
"stream(s)."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithMixedTags) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -145,13 +117,14 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithMixedTags) {
|
|||
CalculatorGraph graph;
|
||||
auto status = graph.Initialize(graph_config);
|
||||
EXPECT_FALSE(status.ok());
|
||||
EXPECT_THAT(status.message(),
|
||||
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, "
|
||||
"AT_TICK, AT_FIRST_TICK and AT_TIMESTAMP tags is "
|
||||
"allowed and required to specify output stream(s)."));
|
||||
EXPECT_THAT(
|
||||
status.message(),
|
||||
HasSubstr("Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO, AT_TICK and "
|
||||
"AT_TIMESTAMP tags is allowed and required to specify output "
|
||||
"stream(s)."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughSidePackets) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -173,7 +146,7 @@ TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughSidePackets) {
|
|||
"Same number of input side packets and output streams is required."));
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, WrongConfigWithNotEnoughOutputStreams) {
|
||||
TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -275,50 +248,7 @@ TEST(SidePacketToStreamCalculator, AtTick) {
|
|||
tick_and_verify(/*at_timestamp=*/1025);
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtFirstTick) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "packet"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_stream: "TICK:tick"
|
||||
input_side_packet: "side_packet"
|
||||
output_stream: "AT_FIRST_TICK:packet"
|
||||
}
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("packet", &graph_config, &output_packets);
|
||||
CalculatorGraph graph;
|
||||
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
const int expected_value = 20;
|
||||
const Timestamp kTestTimestamp(1234);
|
||||
MP_ASSERT_OK(
|
||||
graph.StartRun({{"side_packet", MakePacket<int>(expected_value)}}));
|
||||
|
||||
auto insert_tick = [&graph](Timestamp at_timestamp) {
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
};
|
||||
|
||||
insert_tick(kTestTimestamp);
|
||||
|
||||
EXPECT_THAT(output_packets,
|
||||
ElementsAre(PacketContainsTimestampAndPayload<int>(
|
||||
Eq(kTestTimestamp), Eq(expected_value))));
|
||||
|
||||
output_packets.clear();
|
||||
|
||||
// Should not result in an additional output.
|
||||
insert_tick(kTestTimestamp + 1);
|
||||
EXPECT_THAT(output_packets, IsEmpty());
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtTickWithMultipleSidePackets) {
|
||||
TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
@ -372,62 +302,6 @@ TEST(SidePacketToStreamCalculator, AtTickWithMultipleSidePackets) {
|
|||
tick_and_verify(/*at_timestamp=*/1025);
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtFirstTickWithMultipleSidePackets) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
input_stream: "tick"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
output_stream: "packet0"
|
||||
output_stream: "packet1"
|
||||
node {
|
||||
calculator: "SidePacketToStreamCalculator"
|
||||
input_stream: "TICK:tick"
|
||||
input_side_packet: "side_packet0"
|
||||
input_side_packet: "side_packet1"
|
||||
output_stream: "AT_FIRST_TICK:0:packet0"
|
||||
output_stream: "AT_FIRST_TICK:1:packet1"
|
||||
}
|
||||
)pb");
|
||||
std::vector<Packet> output_packets0;
|
||||
tool::AddVectorSink("packet0", &graph_config, &output_packets0);
|
||||
std::vector<Packet> output_packets1;
|
||||
tool::AddVectorSink("packet1", &graph_config, &output_packets1);
|
||||
CalculatorGraph graph;
|
||||
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
const int expected_value0 = 20;
|
||||
const int expected_value1 = 128;
|
||||
const Timestamp kTestTimestamp(1234);
|
||||
MP_ASSERT_OK(
|
||||
graph.StartRun({{"side_packet0", MakePacket<int>(expected_value0)},
|
||||
{"side_packet1", MakePacket<int>(expected_value1)}}));
|
||||
|
||||
auto insert_tick = [&graph](Timestamp at_timestamp) {
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"tick", MakePacket<int>(/*doesn't matter*/ 1).At(at_timestamp)));
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
};
|
||||
|
||||
insert_tick(kTestTimestamp);
|
||||
|
||||
EXPECT_THAT(output_packets0,
|
||||
ElementsAre(PacketContainsTimestampAndPayload<int>(
|
||||
Eq(kTestTimestamp), Eq(expected_value0))));
|
||||
EXPECT_THAT(output_packets1,
|
||||
ElementsAre(PacketContainsTimestampAndPayload<int>(
|
||||
Eq(kTestTimestamp), Eq(expected_value1))));
|
||||
|
||||
output_packets0.clear();
|
||||
output_packets1.clear();
|
||||
|
||||
// Should not result in an additional output.
|
||||
insert_tick(kTestTimestamp + 1);
|
||||
EXPECT_THAT(output_packets0, IsEmpty());
|
||||
EXPECT_THAT(output_packets1, IsEmpty());
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
|
@ -460,7 +334,7 @@ TEST(SidePacketToStreamCalculator, AtTimestamp) {
|
|||
EXPECT_EQ(expected_value, output_packets.back().Get<int>());
|
||||
}
|
||||
|
||||
TEST(SidePacketToStreamCalculator, AtTimestampWithMultipleOutputs) {
|
||||
TEST(SidePacketToStreamCalculator, AtTimestamp_MultipleOutputs) {
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
|
|
|
@ -174,7 +174,7 @@ TEST(ValueOrDefaultCalculatorTest, DefaultAndValues) {
|
|||
ElementsAre(kDefaultValue, 1, 2, kDefaultValue, 3, kDefaultValue));
|
||||
}
|
||||
|
||||
TEST(ValueOrDefaultCalculatorTest, TimestampsMismatch) {
|
||||
TEST(ValueOrDefaultCalculatorTest, TimestampsMissmatch) {
|
||||
// Check that when we provide the inputs not on time - we don't get them.
|
||||
ValueOrDefaultRunner runner;
|
||||
const std::vector<int64_t> ticks = {1, 2, 5, 8, 12, 33, 231};
|
||||
|
|
|
@ -301,11 +301,9 @@ cc_test(
|
|||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"//mediapipe/gpu:gpu_buffer_to_image_frame_calculator",
|
||||
"//mediapipe/gpu:image_frame_to_gpu_buffer_calculator",
|
||||
"//mediapipe/gpu:multi_pool",
|
||||
"//third_party:opencv",
|
||||
"@com_google_absl//absl/container:flat_hash_set",
|
||||
"@com_google_absl//absl/flags:flag",
|
||||
"@com_google_absl//absl/log:absl_check",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_googletest//:gtest_main",
|
||||
],
|
||||
|
|
|
@ -59,7 +59,7 @@ class OpenCvRunner
|
|||
const ImageFrame& input, const std::array<float, 16>& matrix,
|
||||
const AffineTransformation::Size& size,
|
||||
AffineTransformation::BorderMode border_mode) override {
|
||||
// OpenCV warpAffine works in absolute coordinates, so the transform (which
|
||||
// OpenCV warpAffine works in absolute coordinates, so the transfom (which
|
||||
// accepts and produces relative coordinates) should be adjusted to first
|
||||
// normalize coordinates and then scale them.
|
||||
// clang-format off
|
||||
|
|
|
@ -65,7 +65,7 @@ class ImageCloneCalculator : public Node {
|
|||
}
|
||||
#else
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
|
||||
cc, /*request_gpu_as_optional=*/true));
|
||||
cc, /*requesst_gpu_as_optional=*/true));
|
||||
#endif // MEDIAPIPE_DISABLE_GPU
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
|
|
@ -24,7 +24,7 @@ message ImageCroppingCalculatorOptions {
|
|||
}
|
||||
|
||||
// Output texture buffer dimensions. The values defined in the options will be
|
||||
// overridden by the WIDTH and HEIGHT input streams if they exist.
|
||||
// overriden by the WIDTH and HEIGHT input streams if they exist.
|
||||
optional int32 width = 1;
|
||||
optional int32 height = 2;
|
||||
|
||||
|
|
|
@ -77,7 +77,7 @@ absl::StatusOr<double> ComputeFocalLengthInPixels(int image_width,
|
|||
return focal_length_pixels;
|
||||
}
|
||||
|
||||
absl::StatusOr<ImageFileProperties> GetImageFileProperties(
|
||||
absl::StatusOr<ImageFileProperties> GetImageFileProperites(
|
||||
const std::string& image_bytes) {
|
||||
easyexif::EXIFInfo result;
|
||||
int code = result.parseFrom(image_bytes);
|
||||
|
@ -151,7 +151,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
|
|||
if (cc->InputSidePackets().NumEntries() == 1) {
|
||||
const std::string& image_bytes =
|
||||
cc->InputSidePackets().Index(0).Get<std::string>();
|
||||
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
|
||||
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
||||
read_properties_ = true;
|
||||
}
|
||||
|
||||
|
@ -169,7 +169,7 @@ class ImageFilePropertiesCalculator : public CalculatorBase {
|
|||
return absl::OkStatus();
|
||||
}
|
||||
const std::string& image_bytes = cc->Inputs().Index(0).Get<std::string>();
|
||||
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperties(image_bytes));
|
||||
MP_ASSIGN_OR_RETURN(properties_, GetImageFileProperites(image_bytes));
|
||||
read_properties_ = true;
|
||||
}
|
||||
if (read_properties_) {
|
||||
|
|
|
@ -656,15 +656,6 @@ absl::Status ImageTransformationCalculator::RenderGpu(CalculatorContext* cc) {
|
|||
input.format());
|
||||
|
||||
gpu_helper_.BindFramebuffer(dst);
|
||||
|
||||
if (scale_mode_ == mediapipe::ScaleMode::FIT) {
|
||||
// In kFit scale mode, the rendered quad does not fill the whole
|
||||
// framebuffer, so clear it beforehand.
|
||||
glClearColor(padding_color_[0] / 255.0f, padding_color_[1] / 255.0f,
|
||||
padding_color_[2] / 255.0f, 1.0f);
|
||||
glClear(GL_COLOR_BUFFER_BIT);
|
||||
}
|
||||
|
||||
glActiveTexture(GL_TEXTURE1);
|
||||
glBindTexture(src1.target(), src1.name());
|
||||
|
||||
|
|
|
@ -46,14 +46,13 @@ message ImageTransformationCalculatorOptions {
|
|||
optional bool flip_horizontally = 5 [default = false];
|
||||
// Scale mode.
|
||||
optional ScaleMode.Mode scale_mode = 6;
|
||||
// Padding type. This option is only used when the scale mode is FIT. If set
|
||||
// to true (default), a constant border is added with color specified by
|
||||
// padding_color. If set to false, a border is added by replicating edge
|
||||
// pixels (only supported for CPU).
|
||||
// Padding type. This option is only used when the scale mode is FIT.
|
||||
// Default is to use BORDER_CONSTANT. If set to false, it will use
|
||||
// BORDER_REPLICATE instead.
|
||||
optional bool constant_padding = 7 [default = true];
|
||||
|
||||
// The color for the padding. This option is only used when the scale mode is
|
||||
// FIT. Default is black.
|
||||
// FIT. Default is black. This is for CPU only.
|
||||
optional Color padding_color = 8;
|
||||
|
||||
// Interpolation method to use. Note that on CPU when LINEAR is specified,
|
||||
|
|
|
@ -1,11 +1,9 @@
|
|||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/container/flat_hash_set.h"
|
||||
#include "absl/flags/flag.h"
|
||||
#include "absl/log/absl_check.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/framework/calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
|
@ -18,14 +16,10 @@
|
|||
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h"
|
||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/gpu/multi_pool.h"
|
||||
#include "testing/base/public/gmock.h"
|
||||
#include "testing/base/public/googletest.h"
|
||||
#include "testing/base/public/gunit.h"
|
||||
#include "third_party/OpenCV/core.hpp" // IWYU pragma: keep
|
||||
#include "third_party/OpenCV/core/base.hpp"
|
||||
#include "third_party/OpenCV/core/mat.hpp"
|
||||
#include "third_party/OpenCV/core/types.hpp"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
|
@ -82,12 +76,11 @@ TEST(ImageTransformationCalculatorTest, NearestNeighborResizing) {
|
|||
->Tag("OUTPUT_DIMENSIONS")
|
||||
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
|
||||
|
||||
ABSL_QCHECK_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const auto& outputs = runner.Outputs();
|
||||
ABSL_QCHECK_EQ(outputs.NumEntries(), 1);
|
||||
ASSERT_EQ(outputs.NumEntries(), 1);
|
||||
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
|
||||
ABSL_QCHECK_EQ(packets.size(), 1);
|
||||
|
||||
ASSERT_EQ(packets.size(), 1);
|
||||
const auto& result = packets[0].Get<ImageFrame>();
|
||||
ASSERT_EQ(output_dim.first, result.Width());
|
||||
ASSERT_EQ(output_dim.second, result.Height());
|
||||
|
@ -144,12 +137,11 @@ TEST(ImageTransformationCalculatorTest,
|
|||
->Tag("OUTPUT_DIMENSIONS")
|
||||
.packets.push_back(input_output_dim_packet.At(Timestamp(0)));
|
||||
|
||||
ABSL_QCHECK_OK(runner.Run());
|
||||
MP_ASSERT_OK(runner.Run());
|
||||
const auto& outputs = runner.Outputs();
|
||||
ABSL_QCHECK_EQ(outputs.NumEntries(), 1);
|
||||
ASSERT_EQ(outputs.NumEntries(), 1);
|
||||
const std::vector<Packet>& packets = outputs.Tag("IMAGE").packets;
|
||||
ABSL_QCHECK_EQ(packets.size(), 1);
|
||||
|
||||
ASSERT_EQ(packets.size(), 1);
|
||||
const auto& result = packets[0].Get<ImageFrame>();
|
||||
ASSERT_EQ(output_dim.first, result.Width());
|
||||
ASSERT_EQ(output_dim.second, result.Height());
|
||||
|
@ -215,17 +207,17 @@ TEST(ImageTransformationCalculatorTest, NearestNeighborResizingGpu) {
|
|||
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
|
||||
|
||||
CalculatorGraph graph(graph_config);
|
||||
ABSL_QCHECK_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_image",
|
||||
MakePacket<ImageFrame>(std::move(input_image)).At(Timestamp(0))));
|
||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"image_size",
|
||||
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
|
||||
|
||||
ABSL_QCHECK_OK(graph.WaitUntilIdle());
|
||||
ABSL_QCHECK_EQ(output_image_packets.size(), 1);
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
ASSERT_THAT(output_image_packets, testing::SizeIs(1));
|
||||
|
||||
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
|
||||
ASSERT_EQ(output_dim.first, output_image.Width());
|
||||
|
@ -295,16 +287,16 @@ TEST(ImageTransformationCalculatorTest,
|
|||
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
|
||||
|
||||
CalculatorGraph graph(graph_config);
|
||||
ABSL_QCHECK_OK(graph.StartRun({}));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
|
||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_image", input_image_packet.At(Timestamp(0))));
|
||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"image_size",
|
||||
MakePacket<std::pair<int, int>>(output_dim).At(Timestamp(0))));
|
||||
|
||||
ABSL_QCHECK_OK(graph.WaitUntilIdle());
|
||||
ABSL_QCHECK_EQ(output_image_packets.size(), 1);
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
ASSERT_THAT(output_image_packets, testing::SizeIs(1));
|
||||
|
||||
const auto& output_image = output_image_packets[0].Get<ImageFrame>();
|
||||
ASSERT_EQ(output_dim.first, output_image.Width());
|
||||
|
@ -319,112 +311,5 @@ TEST(ImageTransformationCalculatorTest,
|
|||
}
|
||||
}
|
||||
|
||||
TEST(ImageTransformationCalculatorTest, FitScalingClearsBackground) {
|
||||
// Regression test for not clearing the background in FIT scaling mode.
|
||||
// First scale an all-red (=r) image from 8x4 to 8x4, so it's a plain copy:
|
||||
// rrrrrrrr
|
||||
// rrrrrrrr
|
||||
// rrrrrrrr
|
||||
// rrrrrrrr
|
||||
// Then scale an all-blue image from 4x4 to 8x4 in FIT mode. This should
|
||||
// introduce dark yellow (=y) letterboxes left and right due to padding_color:
|
||||
// yybbbbyy
|
||||
// yybbbbyy
|
||||
// yybbbbyy
|
||||
// yybbbbyy
|
||||
// We make sure that the all-red buffer gets reused. Without clearing the
|
||||
// background, the blue (=b) image will have red letterboxes:
|
||||
// rrbbbbrr
|
||||
// rrbbbbrr
|
||||
// rrbbbbrr
|
||||
// rrbbbbrr
|
||||
|
||||
constexpr int kSmall = 4, kLarge = 8;
|
||||
ImageFrame input_image_red(ImageFormat::SRGBA, kLarge, kSmall);
|
||||
cv::Mat input_image_red_mat = formats::MatView(&input_image_red);
|
||||
input_image_red_mat = cv::Scalar(255, 0, 0, 255);
|
||||
|
||||
ImageFrame input_image_blue(ImageFormat::SRGBA, kSmall, kSmall);
|
||||
cv::Mat input_image_blue_mat = formats::MatView(&input_image_blue);
|
||||
input_image_blue_mat = cv::Scalar(0, 0, 255, 255);
|
||||
|
||||
Packet input_image_red_packet =
|
||||
MakePacket<ImageFrame>(std::move(input_image_red));
|
||||
Packet input_image_blue_packet =
|
||||
MakePacket<ImageFrame>(std::move(input_image_blue));
|
||||
|
||||
CalculatorGraphConfig graph_config =
|
||||
ParseTextProtoOrDie<CalculatorGraphConfig>(absl::Substitute(
|
||||
R"pb(
|
||||
input_stream: "input_image"
|
||||
output_stream: "output_image"
|
||||
|
||||
node {
|
||||
calculator: "ImageFrameToGpuBufferCalculator"
|
||||
input_stream: "input_image"
|
||||
output_stream: "input_image_gpu"
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "ImageTransformationCalculator"
|
||||
input_stream: "IMAGE_GPU:input_image_gpu"
|
||||
output_stream: "IMAGE_GPU:output_image_gpu"
|
||||
options: {
|
||||
[mediapipe.ImageTransformationCalculatorOptions.ext]: {
|
||||
scale_mode: FIT
|
||||
output_width: $0,
|
||||
output_height: $1,
|
||||
padding_color: { red: 128, green: 128, blue: 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
node {
|
||||
calculator: "GpuBufferToImageFrameCalculator"
|
||||
input_stream: "output_image_gpu"
|
||||
output_stream: "output_image"
|
||||
})pb",
|
||||
kLarge, kSmall));
|
||||
|
||||
std::vector<Packet> output_image_packets;
|
||||
tool::AddVectorSink("output_image", &graph_config, &output_image_packets);
|
||||
|
||||
CalculatorGraph graph(graph_config);
|
||||
ABSL_QCHECK_OK(graph.StartRun({}));
|
||||
|
||||
// Send the red image multiple times to cause the GPU pool to actually use
|
||||
// a pool.
|
||||
int num_red_packets =
|
||||
std::max(kDefaultMultiPoolOptions.min_requests_before_pool, 1);
|
||||
for (int n = 0; n < num_red_packets; ++n) {
|
||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
||||
"input_image", input_image_red_packet.At(Timestamp(n))));
|
||||
}
|
||||
ABSL_QCHECK_OK(graph.AddPacketToInputStream(
|
||||
"input_image", input_image_blue_packet.At(Timestamp(num_red_packets))));
|
||||
|
||||
ABSL_QCHECK_OK(graph.WaitUntilIdle());
|
||||
ABSL_QCHECK_EQ(output_image_packets.size(), num_red_packets + 1);
|
||||
|
||||
const auto& output_image_red = output_image_packets[0].Get<ImageFrame>();
|
||||
const auto& output_image_blue =
|
||||
output_image_packets[num_red_packets].Get<ImageFrame>();
|
||||
|
||||
ABSL_QCHECK_EQ(output_image_red.Width(), kLarge);
|
||||
ABSL_QCHECK_EQ(output_image_red.Height(), kSmall);
|
||||
ABSL_QCHECK_EQ(output_image_blue.Width(), kLarge);
|
||||
ABSL_QCHECK_EQ(output_image_blue.Height(), kSmall);
|
||||
|
||||
cv::Mat output_image_blue_mat = formats::MatView(&output_image_blue);
|
||||
ImageFrame expected_image_blue(ImageFormat::SRGBA, kLarge, kSmall);
|
||||
cv::Mat expected_image_blue_mat = formats::MatView(&expected_image_blue);
|
||||
expected_image_blue_mat = cv::Scalar(128, 128, 0, 255);
|
||||
cv::Rect rect((kLarge - kSmall) / 2, 0, kSmall, kSmall);
|
||||
cv::rectangle(expected_image_blue_mat, rect, cv::Scalar(0, 0, 255, 255),
|
||||
cv::FILLED);
|
||||
EXPECT_EQ(cv::sum(cv::sum(output_image_blue_mat != expected_image_blue_mat)),
|
||||
cv::Scalar(0));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
|
|
|
@ -118,7 +118,7 @@ absl::Status SegmentationSmoothingCalculator::GetContract(
|
|||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
|
||||
cc, /*request_gpu_as_optional=*/true));
|
||||
cc, /*requesst_gpu_as_optional=*/true));
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
return absl::OkStatus();
|
||||
|
|
|
@ -206,7 +206,7 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl<InterfaceT> {
|
|||
if constexpr (std::is_same_v<InterfaceT, WarpAffineCalculatorGpu> ||
|
||||
std::is_same_v<InterfaceT, WarpAffineCalculator>) {
|
||||
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(
|
||||
cc, /*request_gpu_as_optional=*/true));
|
||||
cc, /*requesst_gpu_as_optional=*/true));
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
|
|
@ -284,7 +284,7 @@ std::array<float, 16> GetMatrix(cv::Mat input, mediapipe::NormalizedRect roi,
|
|||
.IgnoreError();
|
||||
mediapipe::GetRotatedSubRectToRectTransformMatrix(
|
||||
roi_absolute, input.cols, input.rows,
|
||||
/*flip_horizontally=*/false, &transform_mat);
|
||||
/*flip_horizontaly=*/false, &transform_mat);
|
||||
return transform_mat;
|
||||
}
|
||||
|
||||
|
|
|
@ -49,7 +49,7 @@ std::string FourCCToString(libyuv::FourCC fourcc) {
|
|||
// The input `YUVImage` is expected to be in the NV12, NV21, YV12 or I420 (aka
|
||||
// YV21) format (as per the `fourcc()` property). This covers the most commonly
|
||||
// used YUV image formats used on mobile devices. Other formats are not
|
||||
// supported and will result in an `InvalidArgumentError`.
|
||||
// supported and wil result in an `InvalidArgumentError`.
|
||||
class YUVToImageCalculator : public Node {
|
||||
public:
|
||||
static constexpr Input<YUVImage> kInput{"YUV_IMAGE"};
|
||||
|
|
|
@ -657,7 +657,6 @@ cc_library(
|
|||
}),
|
||||
deps = [
|
||||
":tensor_converter_calculator_cc_proto",
|
||||
":tensor_converter_cpu",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
|
@ -666,7 +665,6 @@ cc_library(
|
|||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:gpu_buffer_format",
|
||||
"//mediapipe/gpu:gpu_origin_cc_proto",
|
||||
"//mediapipe/util:resource_util",
|
||||
|
@ -676,17 +674,10 @@ cc_library(
|
|||
"@com_google_absl//absl/log:check",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_absl//absl/strings:str_format",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": [
|
||||
"tensor_converter_calculator_gpu_deps",
|
||||
"//mediapipe/gpu:gl_base",
|
||||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gl_simple_shaders",
|
||||
"//mediapipe/gpu:shader_util",
|
||||
],
|
||||
"//conditions:default": ["tensor_converter_calculator_gpu_deps"],
|
||||
}) + select({
|
||||
"//mediapipe:apple": [
|
||||
"//third_party/apple_frameworks:MetalKit",
|
||||
|
@ -696,35 +687,6 @@ cc_library(
|
|||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensor_converter_cpu",
|
||||
srcs = ["tensor_converter_cpu.cc"],
|
||||
hdrs = ["tensor_converter_cpu.h"],
|
||||
deps = [
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensor_converter_cpu_test",
|
||||
srcs = ["tensor_converter_cpu_test.cc"],
|
||||
deps = [
|
||||
":tensor_converter_cpu",
|
||||
"//mediapipe/framework/formats:matrix",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:gtest",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status_matchers",
|
||||
"//mediapipe/util:image_test_utils",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensor_converter_calculator_gpu_deps",
|
||||
visibility = ["//visibility:private"],
|
||||
|
@ -1452,8 +1414,6 @@ cc_library(
|
|||
}),
|
||||
deps = [
|
||||
":tensors_to_segmentation_calculator_cc_proto",
|
||||
":tensors_to_segmentation_converter",
|
||||
":tensors_to_segmentation_utils",
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:port",
|
||||
|
@ -1461,11 +1421,9 @@ cc_library(
|
|||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"//mediapipe/gpu:gpu_origin_cc_proto",
|
||||
"//mediapipe/util:resource_util",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/strings",
|
||||
"@com_google_absl//absl/strings:str_format",
|
||||
"@com_google_absl//absl/types:span",
|
||||
|
@ -1476,7 +1434,6 @@ cc_library(
|
|||
"//mediapipe/gpu:gl_calculator_helper",
|
||||
"//mediapipe/gpu:gl_simple_shaders",
|
||||
"//mediapipe/gpu:gpu_buffer",
|
||||
"//mediapipe/gpu:gpu_buffer_format",
|
||||
"//mediapipe/gpu:shader_util",
|
||||
],
|
||||
}) + selects.with_or({
|
||||
|
@ -1496,96 +1453,19 @@ cc_library(
|
|||
}) + select({
|
||||
"//mediapipe/framework/port:disable_opencv": [],
|
||||
"//conditions:default": [
|
||||
":tensors_to_segmentation_converter_opencv",
|
||||
"//mediapipe/framework/formats:image_opencv",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
],
|
||||
}),
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_segmentation_utils",
|
||||
srcs = ["tensors_to_segmentation_utils.cc"],
|
||||
hdrs = ["tensors_to_segmentation_utils.h"],
|
||||
deps = [
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensors_to_segmentation_utils_test",
|
||||
srcs = ["tensors_to_segmentation_utils_test.cc"],
|
||||
deps = [
|
||||
":tensors_to_segmentation_utils",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:status_matchers",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_segmentation_converter",
|
||||
hdrs = ["tensors_to_segmentation_converter.h"],
|
||||
deps = [
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_segmentation_converter_opencv",
|
||||
srcs = ["tensors_to_segmentation_converter_opencv.cc"],
|
||||
hdrs = ["tensors_to_segmentation_converter_opencv.h"],
|
||||
deps = [
|
||||
":tensors_to_segmentation_calculator_cc_proto",
|
||||
":tensors_to_segmentation_converter",
|
||||
":tensors_to_segmentation_utils",
|
||||
"//mediapipe/framework/formats:image",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/formats:image_opencv",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:opencv_core",
|
||||
"//mediapipe/framework/port:opencv_imgproc",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:status",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "tensors_to_segmentation_calculator_test_utils",
|
||||
testonly = 1,
|
||||
srcs = ["tensors_to_segmentation_calculator_test_utils.cc"],
|
||||
hdrs = ["tensors_to_segmentation_calculator_test_utils.h"],
|
||||
deps = [
|
||||
":tensors_to_segmentation_calculator_cc_proto",
|
||||
"//mediapipe/framework:calculator_cc_proto",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/log:absl_log",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensors_to_segmentation_calculator_test_utils_test",
|
||||
srcs = ["tensors_to_segmentation_calculator_test_utils_test.cc"],
|
||||
deps = [
|
||||
":tensors_to_segmentation_calculator_cc_proto",
|
||||
":tensors_to_segmentation_calculator_test_utils",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
],
|
||||
)
|
||||
|
||||
cc_test(
|
||||
name = "tensors_to_segmentation_calculator_test",
|
||||
srcs = ["tensors_to_segmentation_calculator_test.cc"],
|
||||
deps = [
|
||||
":tensors_to_segmentation_calculator",
|
||||
":tensors_to_segmentation_calculator_cc_proto",
|
||||
":tensors_to_segmentation_calculator_test_utils",
|
||||
"//mediapipe/framework:calculator_framework",
|
||||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework:packet",
|
||||
|
@ -1596,6 +1476,10 @@ cc_test(
|
|||
"//mediapipe/framework/formats:rect_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/log",
|
||||
"@com_google_absl//absl/log:absl_log",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
@ -109,7 +109,7 @@ bool IsValidFftSize(int size) {
|
|||
// Non-streaming mode: when "stream_mode" is set to false in the calculator
|
||||
// options, the calculators treats the packets in the input audio stream as
|
||||
// a batch of unrelated audio buffers. In each Process() call, the input
|
||||
// buffer will be first resampled, and framed as fixed-sized, possibly
|
||||
// buffer will be frist resampled, and framed as fixed-sized, possibly
|
||||
// overlapping tensors. The last tensor produced by a Process() invocation
|
||||
// will be zero-padding if the remaining samples are insufficient. As the
|
||||
// calculator treats the input packets as unrelated, all samples will be
|
||||
|
@ -159,7 +159,7 @@ class AudioToTensorCalculator : public Node {
|
|||
public:
|
||||
static constexpr Input<Matrix> kAudioIn{"AUDIO"};
|
||||
// TODO: Removes this optional input stream when the "AUDIO" stream
|
||||
// uses the new mediapipe audio data containers that carry audio metadata,
|
||||
// uses the new mediapipe audio data containers that carry audio metatdata,
|
||||
// such as sample rate.
|
||||
static constexpr Input<double>::Optional kAudioSampleRateIn{"SAMPLE_RATE"};
|
||||
static constexpr Output<std::vector<Tensor>> kTensorsOut{"TENSORS"};
|
||||
|
|
|
@ -37,7 +37,7 @@ message AudioToTensorCalculatorOptions {
|
|||
// will be converted into tensors.
|
||||
optional double target_sample_rate = 4;
|
||||
|
||||
// Whether to treat the input audio stream as a continuous stream or a batch
|
||||
// Whether to treat the input audio stream as a continous stream or a batch
|
||||
// of unrelated audio buffers.
|
||||
optional bool stream_mode = 5 [default = true];
|
||||
|
||||
|
|
|
@ -206,7 +206,7 @@ mediapipe::ImageFormat::Format GetImageFormat(int image_channels) {
|
|||
} else if (image_channels == 1) {
|
||||
return ImageFormat::GRAY8;
|
||||
}
|
||||
ABSL_CHECK(false) << "Unsupported input image channels: " << image_channels;
|
||||
ABSL_CHECK(false) << "Unsupported input image channles: " << image_channels;
|
||||
}
|
||||
|
||||
Packet MakeImageFramePacket(cv::Mat input) {
|
||||
|
|
|
@ -14,7 +14,6 @@
|
|||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/log/absl_check.h"
|
||||
|
@ -22,22 +21,17 @@
|
|||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "absl/strings/str_format.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "absl/types/optional.h"
|
||||
#include "mediapipe/calculators/tensor/tensor_converter_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
#include "mediapipe/gpu/gpu_buffer_format.h"
|
||||
#include "mediapipe/gpu/gpu_origin.pb.h"
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/gpu/gl_base.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
#import <CoreVideo/CoreVideo.h>
|
||||
|
@ -100,13 +94,16 @@ absl::StatusOr<bool> ShouldFlipVertically(
|
|||
}
|
||||
}
|
||||
|
||||
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
|
||||
RowMajorMatrixXf;
|
||||
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
|
||||
ColMajorMatrixXf;
|
||||
|
||||
constexpr char kImageFrameTag[] = "IMAGE";
|
||||
constexpr char kGpuBufferTag[] = "IMAGE_GPU";
|
||||
constexpr char kTensorsTag[] = "TENSORS";
|
||||
constexpr char kMatrixTag[] = "MATRIX";
|
||||
|
||||
constexpr std::pair<float, float> kDefaultOutputRange = {0.0f, 1.0f};
|
||||
|
||||
} // namespace
|
||||
|
||||
namespace mediapipe {
|
||||
|
@ -159,6 +156,10 @@ class TensorConverterCalculator : public CalculatorBase {
|
|||
private:
|
||||
absl::Status InitGpu(CalculatorContext* cc);
|
||||
absl::Status LoadOptions(CalculatorContext* cc, bool use_gpu);
|
||||
template <class T>
|
||||
absl::Status NormalizeImage(const ImageFrame& image_frame,
|
||||
bool flip_vertically, float* tensor_ptr);
|
||||
absl::Status CopyMatrixToTensor(const Matrix& matrix, float* tensor_ptr);
|
||||
absl::Status ProcessCPU(CalculatorContext* cc);
|
||||
absl::Status ProcessGPU(CalculatorContext* cc);
|
||||
|
||||
|
@ -278,21 +279,46 @@ absl::Status TensorConverterCalculator::ProcessCPU(CalculatorContext* cc) {
|
|||
}
|
||||
const auto& image_frame =
|
||||
cc->Inputs().Tag(kImageFrameTag).Get<ImageFrame>();
|
||||
MP_ASSIGN_OR_RETURN(Tensor output,
|
||||
ConvertImageFrameToTensorOnCpu(
|
||||
image_frame,
|
||||
output_range_.has_value() ? output_range_.value()
|
||||
: kDefaultOutputRange,
|
||||
flip_vertically_, max_num_channels_));
|
||||
output_tensors->emplace_back(std::move(output));
|
||||
const int height = image_frame.Height();
|
||||
const int width = image_frame.Width();
|
||||
const int channels = image_frame.NumberOfChannels();
|
||||
const int channels_preserved = std::min(channels, max_num_channels_);
|
||||
const mediapipe::ImageFormat::Format format = image_frame.Format();
|
||||
|
||||
if (!(format == mediapipe::ImageFormat::SRGBA ||
|
||||
format == mediapipe::ImageFormat::SRGB ||
|
||||
format == mediapipe::ImageFormat::GRAY8 ||
|
||||
format == mediapipe::ImageFormat::VEC32F1))
|
||||
RET_CHECK_FAIL() << "Unsupported CPU input format.";
|
||||
|
||||
output_tensors->emplace_back(
|
||||
Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels_preserved});
|
||||
auto cpu_view = output_tensors->back().GetCpuWriteView();
|
||||
|
||||
// Copy image data into tensor.
|
||||
if (image_frame.ByteDepth() == 1) {
|
||||
MP_RETURN_IF_ERROR(NormalizeImage<uint8_t>(image_frame, flip_vertically_,
|
||||
cpu_view.buffer<float>()));
|
||||
} else if (image_frame.ByteDepth() == 4) {
|
||||
MP_RETURN_IF_ERROR(NormalizeImage<float>(image_frame, flip_vertically_,
|
||||
cpu_view.buffer<float>()));
|
||||
} else {
|
||||
return absl::InternalError(
|
||||
"Only byte-based (8 bit) and float (32 bit) images supported.");
|
||||
}
|
||||
} else if (cc->Inputs().HasTag(kMatrixTag)) {
|
||||
if (cc->Inputs().Tag(kMatrixTag).IsEmpty()) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
const auto& matrix = cc->Inputs().Tag(kMatrixTag).Get<Matrix>();
|
||||
MP_ASSIGN_OR_RETURN(Tensor output,
|
||||
ConvertMatrixToTensorOnCpu(matrix, row_major_matrix_));
|
||||
output_tensors->emplace_back(std::move(output));
|
||||
const int height = matrix.rows();
|
||||
const int width = matrix.cols();
|
||||
const int channels = 1;
|
||||
output_tensors->emplace_back(Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels});
|
||||
MP_RETURN_IF_ERROR(CopyMatrixToTensor(
|
||||
matrix, output_tensors->back().GetCpuWriteView().buffer<float>()));
|
||||
} else {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
@ -643,4 +669,67 @@ absl::Status TensorConverterCalculator::LoadOptions(CalculatorContext* cc,
|
|||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
template <class T>
|
||||
absl::Status TensorConverterCalculator::NormalizeImage(
|
||||
const ImageFrame& image_frame, bool flip_vertically, float* tensor_ptr) {
|
||||
const int height = image_frame.Height();
|
||||
const int width = image_frame.Width();
|
||||
const int channels = image_frame.NumberOfChannels();
|
||||
const int channels_preserved = std::min(channels, max_num_channels_);
|
||||
const int channels_ignored = channels - channels_preserved;
|
||||
|
||||
if (output_range_.has_value()) {
|
||||
// If the output float range is set and we are not using custom
|
||||
// normalization, normalize the pixel values from [0, 255] to the specified
|
||||
// output range.
|
||||
RET_CHECK_NE(output_range_->first, output_range_->second);
|
||||
const float scale = (output_range_->second - output_range_->first) / 255.0f;
|
||||
const float bias = output_range_->first;
|
||||
|
||||
for (int i = 0; i < height; ++i) {
|
||||
const T* image_ptr = reinterpret_cast<const T*>(
|
||||
image_frame.PixelData() +
|
||||
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
|
||||
for (int j = 0; j < width; ++j) {
|
||||
for (int c = 0; c < channels_preserved; ++c) {
|
||||
*tensor_ptr++ = *image_ptr++ * scale + bias;
|
||||
}
|
||||
image_ptr += channels_ignored;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// [0,1], scale only (bias == 0)
|
||||
// Verified that there are no precision issues with 1.0f / 255.0f expression
|
||||
const float scale = 1.0f / 255.0f;
|
||||
for (int i = 0; i < height; ++i) {
|
||||
const T* image_ptr = reinterpret_cast<const T*>(
|
||||
image_frame.PixelData() +
|
||||
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
|
||||
for (int j = 0; j < width; ++j) {
|
||||
for (int c = 0; c < channels_preserved; ++c) {
|
||||
*tensor_ptr++ = *image_ptr++ * scale;
|
||||
}
|
||||
image_ptr += channels_ignored;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status TensorConverterCalculator::CopyMatrixToTensor(const Matrix& matrix,
|
||||
float* tensor_ptr) {
|
||||
if (row_major_matrix_) {
|
||||
auto matrix_map =
|
||||
Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
|
||||
matrix_map = matrix;
|
||||
} else {
|
||||
auto matrix_map =
|
||||
Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
|
||||
matrix_map = matrix;
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
|
|
@ -321,61 +321,6 @@ TEST_F(TensorConverterCalculatorTest, SetOutputRange) {
|
|||
}
|
||||
}
|
||||
|
||||
TEST_F(TensorConverterCalculatorTest,
|
||||
ShouldConvertImageWithDefaultOutputRange) {
|
||||
CalculatorGraph graph;
|
||||
CalculatorGraphConfig graph_config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
R"pb(
|
||||
input_stream: "input_image"
|
||||
node {
|
||||
calculator: "TensorConverterCalculator"
|
||||
input_stream: "IMAGE:input_image"
|
||||
output_stream: "TENSORS:tensor"
|
||||
options {
|
||||
[mediapipe.TensorConverterCalculatorOptions.ext] {
|
||||
zero_center: false
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb");
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("tensor", &graph_config, &output_packets);
|
||||
|
||||
// Run the graph.
|
||||
MP_ASSERT_OK(graph.Initialize(graph_config));
|
||||
MP_ASSERT_OK(graph.StartRun({}));
|
||||
auto input_image = std::make_unique<ImageFrame>(ImageFormat::GRAY8, 1, 1);
|
||||
cv::Mat mat = mediapipe::formats::MatView(input_image.get());
|
||||
mat.at<uint8_t>(0, 0) = 200;
|
||||
MP_ASSERT_OK(graph.AddPacketToInputStream(
|
||||
"input_image", Adopt(input_image.release()).At(Timestamp(0))));
|
||||
|
||||
// Wait until the calculator finishes processing.
|
||||
MP_ASSERT_OK(graph.WaitUntilIdle());
|
||||
ASSERT_EQ(output_packets.size(), 1);
|
||||
|
||||
// Get and process results.
|
||||
const std::vector<Tensor>& tensor_vec =
|
||||
output_packets[0].Get<std::vector<Tensor>>();
|
||||
ASSERT_EQ(tensor_vec.size(), 1);
|
||||
|
||||
const Tensor* tensor = &tensor_vec[0];
|
||||
|
||||
// Calculate the expected normalized value:
|
||||
float expected_value = 200.0 / 255.0;
|
||||
|
||||
EXPECT_EQ(tensor->element_type(), Tensor::ElementType::kFloat32);
|
||||
auto view = tensor->GetCpuReadView();
|
||||
float actual_value = *view.buffer<float>();
|
||||
EXPECT_FLOAT_EQ(actual_value, expected_value);
|
||||
|
||||
// Fully close graph at end, otherwise calculator+tensors are destroyed
|
||||
// after calling WaitUntilDone().
|
||||
MP_ASSERT_OK(graph.CloseInputStream("input_image"));
|
||||
MP_ASSERT_OK(graph.WaitUntilDone());
|
||||
}
|
||||
|
||||
TEST_F(TensorConverterCalculatorTest, FlipVertically) {
|
||||
CalculatorGraph graph;
|
||||
CalculatorGraphConfig graph_config =
|
||||
|
|
|
@ -1,145 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>
|
||||
RowMajorMatrixXf;
|
||||
typedef Eigen::Matrix<float, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>
|
||||
ColMajorMatrixXf;
|
||||
|
||||
template <class T>
|
||||
absl::Status NormalizeImage(const ImageFrame& image_frame, bool flip_vertically,
|
||||
const std::pair<float, float>& output_range,
|
||||
int max_num_channels, float* tensor_ptr) {
|
||||
const int height = image_frame.Height();
|
||||
const int width = image_frame.Width();
|
||||
const int channels = image_frame.NumberOfChannels();
|
||||
const int channels_preserved = std::min(channels, max_num_channels);
|
||||
const int channels_ignored = channels - channels_preserved;
|
||||
|
||||
RET_CHECK_NE(output_range.first, output_range.second);
|
||||
const float scale = (output_range.second - output_range.first) / 255.0f;
|
||||
const float bias = output_range.first;
|
||||
|
||||
for (int i = 0; i < height; ++i) {
|
||||
const T* image_ptr = reinterpret_cast<const T*>(
|
||||
image_frame.PixelData() +
|
||||
(flip_vertically ? height - 1 - i : i) * image_frame.WidthStep());
|
||||
for (int j = 0; j < width; ++j) {
|
||||
for (int c = 0; c < channels_preserved; ++c) {
|
||||
*tensor_ptr++ = *image_ptr++ * scale + bias;
|
||||
}
|
||||
image_ptr += channels_ignored;
|
||||
}
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
|
||||
bool flip_vertically,
|
||||
const std::pair<float, float>& output_range,
|
||||
int max_num_channels, float* tensor_ptr) {
|
||||
return NormalizeImage<uint8_t>(image_frame, flip_vertically, output_range,
|
||||
max_num_channels, tensor_ptr);
|
||||
}
|
||||
|
||||
absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
|
||||
bool flip_vertically,
|
||||
const std::pair<float, float>& output_range,
|
||||
int max_num_channels, float* tensor_ptr) {
|
||||
return NormalizeImage<float>(image_frame, flip_vertically, output_range,
|
||||
max_num_channels, tensor_ptr);
|
||||
}
|
||||
|
||||
absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
|
||||
float* tensor_ptr) {
|
||||
if (is_row_major_matrix) {
|
||||
auto matrix_map =
|
||||
Eigen::Map<RowMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
|
||||
matrix_map = matrix;
|
||||
} else {
|
||||
auto matrix_map =
|
||||
Eigen::Map<ColMajorMatrixXf>(tensor_ptr, matrix.rows(), matrix.cols());
|
||||
matrix_map = matrix;
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
|
||||
const ImageFrame& image_frame, const std::pair<float, float>& output_range,
|
||||
bool flip_vertically, int max_num_channels) {
|
||||
const int height = image_frame.Height();
|
||||
const int width = image_frame.Width();
|
||||
const int channels = image_frame.NumberOfChannels();
|
||||
const int channels_preserved = std::min(channels, max_num_channels);
|
||||
const mediapipe::ImageFormat::Format format = image_frame.Format();
|
||||
|
||||
if (!(format == mediapipe::ImageFormat::SRGBA ||
|
||||
format == mediapipe::ImageFormat::SRGB ||
|
||||
format == mediapipe::ImageFormat::GRAY8 ||
|
||||
format == mediapipe::ImageFormat::VEC32F1))
|
||||
RET_CHECK_FAIL() << "Unsupported CPU input format.";
|
||||
|
||||
Tensor output_tensor(Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels_preserved});
|
||||
auto cpu_view = output_tensor.GetCpuWriteView();
|
||||
|
||||
// Copy image data into tensor.
|
||||
if (image_frame.ByteDepth() == 1) {
|
||||
MP_RETURN_IF_ERROR(NormalizeUInt8Image(image_frame, flip_vertically,
|
||||
output_range, max_num_channels,
|
||||
cpu_view.buffer<float>()));
|
||||
} else if (image_frame.ByteDepth() == 4) {
|
||||
MP_RETURN_IF_ERROR(NormalizeFloatImage(image_frame, flip_vertically,
|
||||
output_range, max_num_channels,
|
||||
cpu_view.buffer<float>()));
|
||||
} else {
|
||||
return absl::InternalError(
|
||||
"Only byte-based (8 bit) and float (32 bit) images supported.");
|
||||
}
|
||||
return output_tensor;
|
||||
}
|
||||
|
||||
absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
|
||||
bool row_major_matrix) {
|
||||
const int height = matrix.rows();
|
||||
const int width = matrix.cols();
|
||||
const int channels = 1;
|
||||
Tensor output_tensor(Tensor::ElementType::kFloat32,
|
||||
Tensor::Shape{1, height, width, channels});
|
||||
MP_RETURN_IF_ERROR(
|
||||
CopyMatrixToTensor(matrix, row_major_matrix,
|
||||
output_tensor.GetCpuWriteView().buffer<float>()));
|
||||
return output_tensor;
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
|
@ -1,61 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
|
||||
|
||||
#include <utility>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Converts an ImageFrame to a vector of Tensors.
|
||||
// @flip_vertically enables to flip the image during conversion.
|
||||
// @max_num_channels can be used to reserve extra channels in the output
|
||||
// tensors.
|
||||
// Returns output Tensor.
|
||||
absl::StatusOr<Tensor> ConvertImageFrameToTensorOnCpu(
|
||||
const ImageFrame& image_frame, const std::pair<float, float>& output_range,
|
||||
bool flip_vertically, int max_num_channels);
|
||||
|
||||
// Converts a Matrix to a vector of Tensors.
|
||||
// @row_major_matrix defines the ordering in the input matrix.
|
||||
// @max_num_channels can be used to reserve extra channels in the output
|
||||
// tensors.
|
||||
// Returns output Tensor.
|
||||
absl::StatusOr<Tensor> ConvertMatrixToTensorOnCpu(const Matrix& matrix,
|
||||
bool row_major_matrix);
|
||||
|
||||
// For testing only below.
|
||||
absl::Status NormalizeUInt8Image(const ImageFrame& image_frame,
|
||||
bool flip_vertically,
|
||||
const std::pair<float, float>& output_range,
|
||||
int max_num_channels, float* tensor_ptr);
|
||||
|
||||
absl::Status NormalizeFloatImage(const ImageFrame& image_frame,
|
||||
bool flip_vertically,
|
||||
const std::pair<float, float>& output_range,
|
||||
int max_num_channels, float* tensor_ptr);
|
||||
|
||||
absl::Status CopyMatrixToTensor(const Matrix& matrix, bool is_row_major_matrix,
|
||||
float* tensor_ptr);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSOR_CONVERTER_CPU_H_
|
|
@ -1,175 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensor_converter_cpu.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/framework/formats/matrix.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/util/image_test_utils.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
Matrix CreateTestMatrix(int num_rows, int num_columns) {
|
||||
Matrix matrix(num_rows, num_columns);
|
||||
for (int r = 0; r < num_rows; ++r) {
|
||||
for (int c = 0; c < num_columns; ++c) {
|
||||
matrix(r, c) = r * num_columns + c;
|
||||
}
|
||||
}
|
||||
return matrix;
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldCopyMatrixInRowMajorFormatToTensor) {
|
||||
auto test_matrix = CreateTestMatrix(/* num_rows=*/3, /*num_columns=*/4);
|
||||
std::vector<float> tensor_data(test_matrix.size(), 0.0f);
|
||||
|
||||
MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/true,
|
||||
tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
const int row = i / test_matrix.cols();
|
||||
const int column = i % test_matrix.cols();
|
||||
EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldCopyMatrixInColumnMajorFormatToTensor) {
|
||||
auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
|
||||
std::vector<float> tensor_data(test_matrix.size(), 0.0f);
|
||||
|
||||
MP_EXPECT_OK(CopyMatrixToTensor(test_matrix, /*is_row_major_matrix=*/false,
|
||||
tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
const int row = i % test_matrix.rows();
|
||||
const int column = i / test_matrix.rows();
|
||||
EXPECT_FLOAT_EQ(tensor_data[i], (test_matrix)(row, column));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithDefaultRange) {
|
||||
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
|
||||
std::vector<float> tensor_data(
|
||||
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
|
||||
|
||||
MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false,
|
||||
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
|
||||
tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
EXPECT_FLOAT_EQ(
|
||||
tensor_data[i],
|
||||
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageWithSpecifiedRange) {
|
||||
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
|
||||
std::vector<float> tensor_data(
|
||||
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
|
||||
const auto range = std::make_pair(2.0f, 3.0f);
|
||||
|
||||
MP_EXPECT_OK(
|
||||
NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/false, range,
|
||||
/*num_tensor_channels=*/1, tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
EXPECT_FLOAT_EQ(tensor_data[i],
|
||||
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) /
|
||||
255.0f * (range.second - range.first) +
|
||||
range.first);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldNormalizeGrey8ImageFlipped) {
|
||||
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
|
||||
std::vector<float> tensor_data(
|
||||
grey8_image_frame.Width() * grey8_image_frame.Height(), 0.0f);
|
||||
|
||||
MP_EXPECT_OK(NormalizeUInt8Image(grey8_image_frame, /*flip_vertically=*/true,
|
||||
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
|
||||
tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
const int x = i % grey8_image_frame.Width();
|
||||
const int y = i / grey8_image_frame.Width();
|
||||
const int flipped_y = grey8_image_frame.Height() - y - 1;
|
||||
|
||||
const int index = flipped_y * grey8_image_frame.Width() + x;
|
||||
EXPECT_FLOAT_EQ(
|
||||
tensor_data[index],
|
||||
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0f);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ShouldNormalizeFloatImageWithDefaultRange) {
|
||||
auto float_image_frame =
|
||||
CreateTestFloat32ImageFrame(/*width=*/3, /*height=*/4);
|
||||
std::vector<float> tensor_data(
|
||||
float_image_frame.Width() * float_image_frame.Height(), 0.0f);
|
||||
|
||||
MP_EXPECT_OK(NormalizeFloatImage(float_image_frame, /*flip_vertically=*/false,
|
||||
{0.0f, 1.0f}, /*num_tensor_channels=*/1,
|
||||
tensor_data.data()));
|
||||
|
||||
for (int i = 0; i < tensor_data.size(); ++i) {
|
||||
EXPECT_FLOAT_EQ(tensor_data[i], reinterpret_cast<const float*>(
|
||||
float_image_frame.PixelData())[i] /
|
||||
255.0f);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ConvertImageFrameToTensorOnCpu) {
|
||||
auto grey8_image_frame = CreateTestGrey8ImageFrame(/*width=*/3, /*height=*/4);
|
||||
|
||||
MP_ASSERT_OK_AND_ASSIGN(Tensor output, ConvertImageFrameToTensorOnCpu(
|
||||
grey8_image_frame, {0.0f, 1.0f},
|
||||
/*flip_vertically=*/false,
|
||||
/*max_num_channels=*/1));
|
||||
|
||||
const auto cpu_read_view = output.GetCpuReadView();
|
||||
const float* tensor_ptr = cpu_read_view.buffer<float>();
|
||||
for (int i = 0; i < grey8_image_frame.Width() * grey8_image_frame.Height();
|
||||
++i) {
|
||||
EXPECT_FLOAT_EQ(
|
||||
tensor_ptr[i],
|
||||
static_cast<uint8_t>(grey8_image_frame.PixelData()[i]) / 255.0);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorConverterCpuTest, ConvertMatrixToTensorOnCpu) {
|
||||
auto test_matrix = CreateTestMatrix(/*num_rows=*/3, /*num_columns=*/4);
|
||||
|
||||
MP_ASSERT_OK_AND_ASSIGN(
|
||||
Tensor output, ConvertMatrixToTensorOnCpu(test_matrix,
|
||||
/*row_major_matrix=*/false));
|
||||
|
||||
const auto cpu_read_view = output.GetCpuReadView();
|
||||
const float* tensor_ptr = cpu_read_view.buffer<float>();
|
||||
for (int i = 0; i < test_matrix.size(); ++i) {
|
||||
EXPECT_FLOAT_EQ(tensor_ptr[i], test_matrix.data()[i]);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
} // namespace mediapipe
|
|
@ -879,6 +879,13 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
|
|||
if (max_results_ > 0 && output_detections->size() == max_results_) {
|
||||
break;
|
||||
}
|
||||
if (options_.has_min_score_thresh() &&
|
||||
detection_scores[i] < options_.min_score_thresh()) {
|
||||
continue;
|
||||
}
|
||||
if (!IsClassIndexAllowed(detection_classes[i])) {
|
||||
continue;
|
||||
}
|
||||
const int box_offset = i * num_coords_;
|
||||
Detection detection = ConvertToDetection(
|
||||
/*box_ymin=*/detection_boxes[box_offset + box_indices_[0]],
|
||||
|
@ -888,11 +895,6 @@ absl::Status TensorsToDetectionsCalculator::ConvertToDetections(
|
|||
absl::MakeConstSpan(detection_scores + i, classes_per_detection_),
|
||||
absl::MakeConstSpan(detection_classes + i, classes_per_detection_),
|
||||
options_.flip_vertically());
|
||||
// if all the scores and classes are filtered out, we skip the empty
|
||||
// detection.
|
||||
if (detection.score().empty()) {
|
||||
continue;
|
||||
}
|
||||
const auto& bbox = detection.location_data().relative_bounding_box();
|
||||
if (bbox.width() < 0 || bbox.height() < 0 || std::isnan(bbox.width()) ||
|
||||
std::isnan(bbox.height())) {
|
||||
|
@ -930,10 +932,6 @@ Detection TensorsToDetectionsCalculator::ConvertToDetection(
|
|||
if (!IsClassIndexAllowed(class_ids[i])) {
|
||||
continue;
|
||||
}
|
||||
if (options_.has_min_score_thresh() &&
|
||||
scores[i] < options_.min_score_thresh()) {
|
||||
continue;
|
||||
}
|
||||
detection.add_score(scores[i]);
|
||||
detection.add_label_id(class_ids[i]);
|
||||
}
|
||||
|
|
|
@ -124,7 +124,7 @@ absl::Status TensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
|
|||
kFlipVertically(cc).IsConnected())) {
|
||||
RET_CHECK(options_.has_input_image_height() &&
|
||||
options_.has_input_image_width())
|
||||
<< "Must provide input width/height for using flipping when outputting "
|
||||
<< "Must provide input width/height for using flipping when outputing "
|
||||
"landmarks in absolute coordinates.";
|
||||
}
|
||||
return absl::OkStatus();
|
||||
|
|
|
@ -12,35 +12,32 @@
|
|||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
#include "absl/strings/str_format.h"
|
||||
#include "absl/types/span.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
#include "mediapipe/framework/port/statusor.h"
|
||||
#include "mediapipe/gpu/gpu_origin.pb.h"
|
||||
#include "mediapipe/util/resource_util.h"
|
||||
#include "tensorflow/lite/interpreter.h"
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#include "mediapipe/gpu/gl_calculator_helper.h"
|
||||
#include "mediapipe/gpu/gl_simple_shaders.h"
|
||||
#include "mediapipe/gpu/gpu_buffer_format.h"
|
||||
#include "mediapipe/gpu/gpu_buffer.h"
|
||||
#include "mediapipe/gpu/shader_util.h"
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
|
||||
#include "mediapipe/framework/formats/image_opencv.h"
|
||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
|
@ -65,9 +62,37 @@ namespace {
|
|||
constexpr int kWorkgroupSize = 8; // Block size for GPU shader.
|
||||
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
|
||||
|
||||
// Commonly used to compute the number of blocks to launch in a kernel.
|
||||
int NumGroups(const int size, const int group_size) { // NOLINT
|
||||
return (size + group_size - 1) / group_size;
|
||||
}
|
||||
|
||||
bool CanUseGpu() {
|
||||
#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
|
||||
// TODO: Configure GPU usage policy in individual calculators.
|
||||
constexpr bool kAllowGpuProcessing = true;
|
||||
return kAllowGpuProcessing;
|
||||
#else
|
||||
return false;
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
|
||||
}
|
||||
|
||||
constexpr char kTensorsTag[] = "TENSORS";
|
||||
constexpr char kOutputSizeTag[] = "OUTPUT_SIZE";
|
||||
constexpr char kMaskTag[] = "MASK";
|
||||
|
||||
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
|
||||
const std::vector<int>& dims) {
|
||||
if (dims.size() == 3) {
|
||||
return std::make_tuple(dims[0], dims[1], dims[2]);
|
||||
} else if (dims.size() == 4) {
|
||||
// BHWC format check B == 1
|
||||
RET_CHECK_EQ(1, dims[0]) << "Expected batch to be 1 for BHWC heatmap";
|
||||
return std::make_tuple(dims[1], dims[2], dims[3]);
|
||||
} else {
|
||||
RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
namespace mediapipe {
|
||||
|
@ -131,28 +156,19 @@ class TensorsToSegmentationCalculator : public CalculatorBase {
|
|||
private:
|
||||
absl::Status LoadOptions(CalculatorContext* cc);
|
||||
absl::Status InitGpu(CalculatorContext* cc);
|
||||
absl::Status ProcessGpu(CalculatorContext* cc,
|
||||
const std::vector<Tensor>& input_tensors,
|
||||
std::tuple<int, int, int> hwc, int output_width,
|
||||
int output_height);
|
||||
absl::Status ProcessGpu(CalculatorContext* cc);
|
||||
absl::Status ProcessCpu(CalculatorContext* cc);
|
||||
void GlRender();
|
||||
|
||||
bool DoesGpuTextureStartAtBottom() {
|
||||
return options_.gpu_origin() != mediapipe::GpuOrigin_Mode_TOP_LEFT;
|
||||
}
|
||||
absl::Status InitConverterIfNecessary() {
|
||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||
if (!cpu_converter_) {
|
||||
MP_ASSIGN_OR_RETURN(cpu_converter_, CreateOpenCvConverter(options_));
|
||||
}
|
||||
#else
|
||||
RET_CHECK_FAIL() << "OpenCV processing disabled.";
|
||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
mediapipe::TensorsToSegmentationCalculatorOptions options_;
|
||||
std::unique_ptr<TensorsToSegmentationConverter> cpu_converter_;
|
||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||
template <class T>
|
||||
absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat);
|
||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||
::mediapipe::TensorsToSegmentationCalculatorOptions options_;
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
mediapipe::GlCalculatorHelper gpu_helper_;
|
||||
|
@ -245,7 +261,7 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
|
|||
MP_ASSIGN_OR_RETURN(auto hwc,
|
||||
GetHwcFromDims(input_tensors[0].shape().dims));
|
||||
int tensor_channels = std::get<2>(hwc);
|
||||
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
|
||||
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
|
||||
switch (options_.activation()) {
|
||||
case Options::NONE:
|
||||
RET_CHECK_EQ(tensor_channels, 1);
|
||||
|
@ -259,17 +275,6 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
|
|||
}
|
||||
}
|
||||
|
||||
// Get dimensions.
|
||||
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
|
||||
auto [tensor_height, tensor_width, tensor_channels] = hwc;
|
||||
int output_width = tensor_width, output_height = tensor_height;
|
||||
if (cc->Inputs().HasTag(kOutputSizeTag)) {
|
||||
const auto& size =
|
||||
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
|
||||
output_width = size.first;
|
||||
output_height = size.second;
|
||||
}
|
||||
|
||||
if (use_gpu) {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
if (!gpu_initialized_) {
|
||||
|
@ -281,25 +286,16 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) {
|
|||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
MP_RETURN_IF_ERROR(
|
||||
gpu_helper_.RunInGlContext([this, cc, &input_tensors, output_width,
|
||||
output_height, hwc]() -> absl::Status {
|
||||
MP_RETURN_IF_ERROR(
|
||||
ProcessGpu(cc, input_tensors, hwc, output_width, output_height));
|
||||
return absl::OkStatus();
|
||||
}));
|
||||
MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status {
|
||||
MP_RETURN_IF_ERROR(ProcessGpu(cc));
|
||||
return absl::OkStatus();
|
||||
}));
|
||||
#else
|
||||
RET_CHECK_FAIL() << "GPU processing disabled.";
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU
|
||||
} else {
|
||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||
// Lazily initialize converter.
|
||||
MP_RETURN_IF_ERROR(InitConverterIfNecessary());
|
||||
MP_ASSIGN_OR_RETURN(
|
||||
std::unique_ptr<Image> output_mask,
|
||||
cpu_converter_->Convert(input_tensors, output_width, output_height));
|
||||
cc->Outputs().Tag(kMaskTag).Add(output_mask.release(),
|
||||
cc->InputTimestamp());
|
||||
MP_RETURN_IF_ERROR(ProcessCpu(cc));
|
||||
#else
|
||||
RET_CHECK_FAIL() << "OpenCV processing disabled.";
|
||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||
|
@ -333,15 +329,132 @@ absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) {
|
|||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::Status TensorsToSegmentationCalculator::ProcessCpu(
|
||||
CalculatorContext* cc) {
|
||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||
// Get input streams, and dimensions.
|
||||
const auto& input_tensors =
|
||||
cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
|
||||
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
|
||||
auto [tensor_height, tensor_width, tensor_channels] = hwc;
|
||||
int output_width = tensor_width, output_height = tensor_height;
|
||||
if (cc->Inputs().HasTag(kOutputSizeTag)) {
|
||||
const auto& size =
|
||||
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
|
||||
output_width = size.first;
|
||||
output_height = size.second;
|
||||
}
|
||||
|
||||
// Create initial working mask.
|
||||
cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
|
||||
|
||||
// Wrap input tensor.
|
||||
auto raw_input_tensor = &input_tensors[0];
|
||||
auto raw_input_view = raw_input_tensor->GetCpuReadView();
|
||||
const float* raw_input_data = raw_input_view.buffer<float>();
|
||||
cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
|
||||
CV_MAKETYPE(CV_32F, tensor_channels),
|
||||
const_cast<float*>(raw_input_data));
|
||||
|
||||
// Process mask tensor and apply activation function.
|
||||
if (tensor_channels == 2) {
|
||||
MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
|
||||
} else if (tensor_channels == 1) {
|
||||
RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
|
||||
options_.activation()); // Requires 2 channels.
|
||||
if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
|
||||
options_.activation()) // Pass-through optimization.
|
||||
tensor_mat.copyTo(small_mask_mat);
|
||||
else
|
||||
MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
|
||||
} else {
|
||||
RET_CHECK_FAIL() << "Unsupported number of tensor channels "
|
||||
<< tensor_channels;
|
||||
}
|
||||
|
||||
// Send out image as CPU packet.
|
||||
std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
|
||||
ImageFormat::VEC32F1, output_width, output_height);
|
||||
std::unique_ptr<Image> output_mask = absl::make_unique<Image>(mask_frame);
|
||||
auto output_mat = formats::MatView(output_mask.get());
|
||||
// Upsample small mask into output.
|
||||
cv::resize(small_mask_mat, *output_mat,
|
||||
cv::Size(output_width, output_height));
|
||||
cc->Outputs().Tag(kMaskTag).Add(output_mask.release(), cc->InputTimestamp());
|
||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_OPENCV
|
||||
template <class T>
|
||||
absl::Status TensorsToSegmentationCalculator::ApplyActivation(
|
||||
cv::Mat& tensor_mat, cv::Mat* small_mask_mat) {
|
||||
// Configure activation function.
|
||||
const int output_layer_index = options_.output_layer_index();
|
||||
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
|
||||
const auto activation_fn = [&](const cv::Vec2f& mask_value) {
|
||||
float new_mask_value = 0;
|
||||
// TODO consider moving switch out of the loop,
|
||||
// and also avoid float/Vec2f casting.
|
||||
switch (options_.activation()) {
|
||||
case Options::NONE: {
|
||||
new_mask_value = mask_value[0];
|
||||
break;
|
||||
}
|
||||
case Options::SIGMOID: {
|
||||
const float pixel0 = mask_value[0];
|
||||
new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
|
||||
break;
|
||||
}
|
||||
case Options::SOFTMAX: {
|
||||
const float pixel0 = mask_value[0];
|
||||
const float pixel1 = mask_value[1];
|
||||
const float max_pixel = std::max(pixel0, pixel1);
|
||||
const float min_pixel = std::min(pixel0, pixel1);
|
||||
const float softmax_denom =
|
||||
/*exp(max_pixel - max_pixel)=*/1.0f +
|
||||
std::exp(min_pixel - max_pixel);
|
||||
new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
|
||||
softmax_denom;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new_mask_value;
|
||||
};
|
||||
|
||||
// Process mask tensor.
|
||||
for (int i = 0; i < tensor_mat.rows; ++i) {
|
||||
for (int j = 0; j < tensor_mat.cols; ++j) {
|
||||
const T& input_pix = tensor_mat.at<T>(i, j);
|
||||
const float mask_value = activation_fn(input_pix);
|
||||
small_mask_mat->at<float>(i, j) = mask_value;
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
#endif // !MEDIAPIPE_DISABLE_OPENCV
|
||||
|
||||
// Steps:
|
||||
// 1. receive tensor
|
||||
// 2. process segmentation tensor into small mask
|
||||
// 3. upsample small mask into output mask to be same size as input image
|
||||
absl::Status TensorsToSegmentationCalculator::ProcessGpu(
|
||||
CalculatorContext* cc, const std::vector<Tensor>& input_tensors,
|
||||
std::tuple<int, int, int> hwc, int output_width, int output_height) {
|
||||
CalculatorContext* cc) {
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
// Get input streams, and dimensions.
|
||||
const auto& input_tensors =
|
||||
cc->Inputs().Tag(kTensorsTag).Get<std::vector<Tensor>>();
|
||||
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
|
||||
auto [tensor_height, tensor_width, tensor_channels] = hwc;
|
||||
int output_width = tensor_width, output_height = tensor_height;
|
||||
if (cc->Inputs().HasTag(kOutputSizeTag)) {
|
||||
const auto& size =
|
||||
cc->Inputs().Tag(kOutputSizeTag).Get<std::pair<int, int>>();
|
||||
output_width = size.first;
|
||||
output_height = size.second;
|
||||
}
|
||||
|
||||
// Create initial working mask texture.
|
||||
#if !(MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31)
|
||||
|
@ -519,7 +632,7 @@ void TensorsToSegmentationCalculator::GlRender() {
|
|||
absl::Status TensorsToSegmentationCalculator::LoadOptions(
|
||||
CalculatorContext* cc) {
|
||||
// Get calculator options specified in the graph.
|
||||
options_ = cc->Options<mediapipe::TensorsToSegmentationCalculatorOptions>();
|
||||
options_ = cc->Options<::mediapipe::TensorsToSegmentationCalculatorOptions>();
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
@ -713,7 +826,7 @@ void main() {
|
|||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
|
||||
// Shader defines.
|
||||
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
|
||||
typedef mediapipe::TensorsToSegmentationCalculatorOptions Options;
|
||||
const std::string output_layer_index =
|
||||
"\n#define OUTPUT_LAYER_INDEX int(" +
|
||||
std::to_string(options_.output_layer_index()) + ")";
|
||||
|
|
|
@ -17,8 +17,10 @@
|
|||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/log/absl_log.h"
|
||||
#include "absl/log/log.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
|
@ -28,6 +30,7 @@
|
|||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/packet.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
#include "mediapipe/framework/timestamp.h"
|
||||
|
||||
|
@ -37,17 +40,58 @@ namespace {
|
|||
using ::testing::SizeIs;
|
||||
using ::testing::TestWithParam;
|
||||
using Options = mediapipe::TensorsToSegmentationCalculatorOptions;
|
||||
namespace test_utils = ::mediapipe::tensors_to_segmentation_utils;
|
||||
|
||||
using TensorsToSegmentationCalculatorTest =
|
||||
TestWithParam<test_utils::FormattingTestCase>;
|
||||
std::string ActivationTypeToString(Options::Activation activation) {
|
||||
switch (activation) {
|
||||
case Options::NONE:
|
||||
return "NONE";
|
||||
case Options::SIGMOID:
|
||||
return "SIGMOID";
|
||||
case Options::SOFTMAX:
|
||||
return "SOFTMAX";
|
||||
default:
|
||||
ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
|
||||
return "UNKNOWN";
|
||||
}
|
||||
}
|
||||
|
||||
struct FormattingTestCase {
|
||||
std::string test_name;
|
||||
std::vector<float> inputs;
|
||||
std::vector<float> expected_outputs;
|
||||
Options::Activation activation;
|
||||
int rows = 1;
|
||||
int cols = 1;
|
||||
int rows_new = 1;
|
||||
int cols_new = 1;
|
||||
int channels = 1;
|
||||
double max_abs_diff = 1e-7;
|
||||
};
|
||||
|
||||
using TensorsToSegmentationCalculatorTest = TestWithParam<FormattingTestCase>;
|
||||
|
||||
TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
|
||||
const auto& [test_name, inputs, expected_outputs, activation, rows, cols,
|
||||
rows_new, cols_new, channels, max_abs_diff] = GetParam();
|
||||
|
||||
auto graph_config =
|
||||
test_utils::CreateGraphConfigForTest(/*test_gpu=*/false, activation);
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(absl::Substitute(
|
||||
R"pb(
|
||||
input_stream: "tensors"
|
||||
input_stream: "size"
|
||||
node {
|
||||
calculator: "TensorsToSegmentationCalculator"
|
||||
input_stream: "TENSORS:tensors"
|
||||
input_stream: "OUTPUT_SIZE:size"
|
||||
output_stream: "MASK:image_as_mask"
|
||||
options: {
|
||||
[mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
|
||||
activation: $0
|
||||
}
|
||||
}
|
||||
}
|
||||
)pb",
|
||||
ActivationTypeToString(activation)));
|
||||
|
||||
std::vector<Packet> output_packets;
|
||||
tool::AddVectorSink("image_as_mask", &graph_config, &output_packets);
|
||||
|
@ -107,7 +151,7 @@ TEST_P(TensorsToSegmentationCalculatorTest, ParameterizedTests) {
|
|||
|
||||
INSTANTIATE_TEST_SUITE_P(
|
||||
TensorsToSegmentationCalculatorTests, TensorsToSegmentationCalculatorTest,
|
||||
testing::ValuesIn<test_utils::FormattingTestCase>({
|
||||
testing::ValuesIn<FormattingTestCase>({
|
||||
{.test_name = "NoActivationAndNoOutputResize",
|
||||
.inputs = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0,
|
||||
12.0, 13.0, 14.0, 15.0, 16.0},
|
||||
|
|
|
@ -1,111 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/log/absl_log.h"
|
||||
#include "absl/strings/substitute.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator.pb.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace tensors_to_segmentation_utils {
|
||||
|
||||
std::string ActivationTypeToString(
|
||||
const TensorsToSegmentationCalculatorOptions::Activation& activation) {
|
||||
switch (activation) {
|
||||
case TensorsToSegmentationCalculatorOptions::NONE:
|
||||
return "NONE";
|
||||
case TensorsToSegmentationCalculatorOptions::SIGMOID:
|
||||
return "SIGMOID";
|
||||
case TensorsToSegmentationCalculatorOptions::SOFTMAX:
|
||||
return "SOFTMAX";
|
||||
}
|
||||
ABSL_LOG(FATAL) << "Unknown activation type: " << activation;
|
||||
return "UNKNOWN";
|
||||
}
|
||||
|
||||
std::vector<unsigned char> ArrayFloatToUnsignedChar(
|
||||
const std::vector<float>& array) {
|
||||
std::vector<unsigned char> result;
|
||||
result.reserve(array.size());
|
||||
for (int i = 0; i < array.size(); ++i) {
|
||||
result.push_back(static_cast<unsigned char>(array[i]));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values) {
|
||||
std::vector<float> result;
|
||||
result.reserve(values.size() * 4);
|
||||
for (const float& value : values) {
|
||||
result.push_back(value);
|
||||
result.push_back(0);
|
||||
result.push_back(0);
|
||||
result.push_back(value);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// For GPU tests, the input tensor needs to be moved to GPU, using
|
||||
// TensorViewRequestor. After calculation, the output needs to be moved back
|
||||
// to CPU, using ToImageCalculator. The output is an ImageFrame.
|
||||
mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
|
||||
bool test_gpu,
|
||||
const TensorsToSegmentationCalculatorOptions::Activation& activation) {
|
||||
std::string pre_process = R"pb(
|
||||
node {
|
||||
calculator: "mediapipe.aimatter.TensorViewRequestor"
|
||||
input_stream: "TENSORS:tensors"
|
||||
output_stream: "TENSORS:tensors_gpu"
|
||||
options {
|
||||
[mediapipe.aimatter.TensorViewRequestorOptions.ext] { gpu {} }
|
||||
}
|
||||
}
|
||||
)pb";
|
||||
std::string post_process = R"pb(
|
||||
node {
|
||||
calculator: "FromImageCalculator"
|
||||
input_stream: "IMAGE:image_as_mask_gpu"
|
||||
output_stream: "IMAGE_CPU:image_as_mask"
|
||||
}
|
||||
)pb";
|
||||
return mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
|
||||
absl::Substitute(
|
||||
R"pb(
|
||||
input_stream: "tensors"
|
||||
input_stream: "size" $0
|
||||
node {
|
||||
calculator: "TensorsToSegmentationCalculator"
|
||||
input_stream: "TENSORS:tensors$1"
|
||||
input_stream: "OUTPUT_SIZE:size"
|
||||
output_stream: "MASK:image_as_mask$2"
|
||||
options: {
|
||||
[mediapipe.TensorsToSegmentationCalculatorOptions.ext] {
|
||||
activation: $3
|
||||
gpu_origin: TOP_LEFT
|
||||
}
|
||||
}
|
||||
} $4
|
||||
)pb",
|
||||
test_gpu ? pre_process : "", test_gpu ? "_gpu" : "",
|
||||
test_gpu ? "_gpu" : "", ActivationTypeToString(activation),
|
||||
test_gpu ? post_process : ""));
|
||||
}
|
||||
} // namespace tensors_to_segmentation_utils
|
||||
} // namespace mediapipe
|
|
@ -1,57 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace tensors_to_segmentation_utils {
|
||||
std::string ActivationTypeToString(
|
||||
const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
|
||||
activation);
|
||||
|
||||
std::vector<unsigned char> ArrayFloatToUnsignedChar(
|
||||
const std::vector<float>& array);
|
||||
|
||||
std::vector<float> MakeRedAlphaMatrix(const std::vector<float>& values);
|
||||
|
||||
mediapipe::CalculatorGraphConfig CreateGraphConfigForTest(
|
||||
bool test_gpu,
|
||||
const mediapipe::TensorsToSegmentationCalculatorOptions::Activation&
|
||||
activation);
|
||||
|
||||
struct FormattingTestCase {
|
||||
std::string test_name;
|
||||
std::vector<float> inputs;
|
||||
std::vector<float> expected_outputs;
|
||||
mediapipe::TensorsToSegmentationCalculatorOptions::Activation activation;
|
||||
int rows = 1;
|
||||
int cols = 1;
|
||||
int rows_new = 1;
|
||||
int cols_new = 1;
|
||||
int channels = 1;
|
||||
double max_abs_diff = 1e-7;
|
||||
};
|
||||
} // namespace tensors_to_segmentation_utils
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CALCULATOR_TEST_UTILS_H_
|
|
@ -1,50 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator_test_utils.h"
|
||||
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
|
||||
namespace mediapipe::tensors_to_segmentation_utils {
|
||||
namespace {
|
||||
|
||||
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
|
||||
|
||||
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
|
||||
ActivationTypeToStringWorksCorrectly) {
|
||||
EXPECT_EQ(ActivationTypeToString(Options::NONE), "NONE");
|
||||
EXPECT_EQ(ActivationTypeToString(Options::SIGMOID), "SIGMOID");
|
||||
EXPECT_EQ(ActivationTypeToString(Options::SOFTMAX), "SOFTMAX");
|
||||
}
|
||||
|
||||
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
|
||||
ArrayFloatToUnsignedCharWorksCorrectly) {
|
||||
std::vector<float> input = {1.0, 2.0, 3.0};
|
||||
std::vector<unsigned char> expected = {1, 2, 3};
|
||||
EXPECT_EQ(ArrayFloatToUnsignedChar(input), expected);
|
||||
}
|
||||
|
||||
TEST(TensorsToSegmentationCalculatorTestUtilsTest,
|
||||
MakeRedAlphaMatrixWorksCorrectly) {
|
||||
std::vector<float> input = {1.0, 2.0, 3.0};
|
||||
std::vector<float> expected = {1.0, 0.0, 0.0, 1.0, 2.0, 0.0,
|
||||
0.0, 2.0, 3.0, 0.0, 0.0, 3.0};
|
||||
EXPECT_EQ(MakeRedAlphaMatrix(input), expected);
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe::tensors_to_segmentation_utils
|
|
@ -1,43 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
class TensorsToSegmentationConverter {
|
||||
public:
|
||||
virtual ~TensorsToSegmentationConverter() = default;
|
||||
|
||||
// Converts tensors to image mask.
|
||||
// Returns a unique pointer containing the converted image.
|
||||
// @input_tensors contains the tensors needed to be processed.
|
||||
// @output_width/height describes output dimensions to reshape the output mask
|
||||
// into.
|
||||
virtual absl::StatusOr<std::unique_ptr<Image>> Convert(
|
||||
const std::vector<Tensor>& input_tensors, int output_width,
|
||||
int output_height) = 0;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_H_
|
|
@ -1,157 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter_opencv.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
|
||||
#include "mediapipe/framework/formats/image.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
#include "mediapipe/framework/formats/image_opencv.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/opencv_core_inc.h"
|
||||
#include "mediapipe/framework/port/opencv_imgproc_inc.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
class OpenCvProcessor : public TensorsToSegmentationConverter {
|
||||
public:
|
||||
absl::Status Init(const TensorsToSegmentationCalculatorOptions& options) {
|
||||
options_ = options;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::StatusOr<std::unique_ptr<Image>> Convert(
|
||||
const std::vector<Tensor>& input_tensors, int output_width,
|
||||
int output_height) override;
|
||||
|
||||
private:
|
||||
template <class T>
|
||||
absl::Status ApplyActivation(cv::Mat& tensor_mat, cv::Mat* small_mask_mat);
|
||||
|
||||
TensorsToSegmentationCalculatorOptions options_;
|
||||
};
|
||||
|
||||
absl::StatusOr<std::unique_ptr<Image>> OpenCvProcessor::Convert(
|
||||
const std::vector<Tensor>& input_tensors, int output_width,
|
||||
int output_height) {
|
||||
MP_ASSIGN_OR_RETURN(auto hwc, GetHwcFromDims(input_tensors[0].shape().dims));
|
||||
auto [tensor_height, tensor_width, tensor_channels] = hwc;
|
||||
// Create initial working mask.
|
||||
cv::Mat small_mask_mat(cv::Size(tensor_width, tensor_height), CV_32FC1);
|
||||
|
||||
// Wrap input tensor.
|
||||
auto raw_input_tensor = &input_tensors[0];
|
||||
auto raw_input_view = raw_input_tensor->GetCpuReadView();
|
||||
const float* raw_input_data = raw_input_view.buffer<float>();
|
||||
cv::Mat tensor_mat(cv::Size(tensor_width, tensor_height),
|
||||
CV_MAKETYPE(CV_32F, tensor_channels),
|
||||
const_cast<float*>(raw_input_data));
|
||||
|
||||
// Process mask tensor and apply activation function.
|
||||
if (tensor_channels == 2) {
|
||||
MP_RETURN_IF_ERROR(ApplyActivation<cv::Vec2f>(tensor_mat, &small_mask_mat));
|
||||
} else if (tensor_channels == 1) {
|
||||
RET_CHECK(mediapipe::TensorsToSegmentationCalculatorOptions::SOFTMAX !=
|
||||
options_.activation()); // Requires 2 channels.
|
||||
if (mediapipe::TensorsToSegmentationCalculatorOptions::NONE ==
|
||||
options_.activation()) // Pass-through optimization.
|
||||
tensor_mat.copyTo(small_mask_mat);
|
||||
else
|
||||
MP_RETURN_IF_ERROR(ApplyActivation<float>(tensor_mat, &small_mask_mat));
|
||||
} else {
|
||||
RET_CHECK_FAIL() << "Unsupported number of tensor channels "
|
||||
<< tensor_channels;
|
||||
}
|
||||
|
||||
// Send out image as CPU packet.
|
||||
std::shared_ptr<ImageFrame> mask_frame = std::make_shared<ImageFrame>(
|
||||
ImageFormat::VEC32F1, output_width, output_height);
|
||||
auto output_mask = std::make_unique<Image>(mask_frame);
|
||||
auto output_mat = formats::MatView(output_mask.get());
|
||||
// Upsample small mask into output.
|
||||
cv::resize(small_mask_mat, *output_mat,
|
||||
cv::Size(output_width, output_height));
|
||||
return output_mask;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
absl::Status OpenCvProcessor::ApplyActivation(cv::Mat& tensor_mat,
|
||||
cv::Mat* small_mask_mat) {
|
||||
// Configure activation function.
|
||||
const int output_layer_index = options_.output_layer_index();
|
||||
using Options = ::mediapipe::TensorsToSegmentationCalculatorOptions;
|
||||
const auto activation_fn = [&](const cv::Vec2f& mask_value) {
|
||||
float new_mask_value = 0;
|
||||
// TODO consider moving switch out of the loop,
|
||||
// and also avoid float/Vec2f casting.
|
||||
switch (options_.activation()) {
|
||||
case Options::NONE: {
|
||||
new_mask_value = mask_value[0];
|
||||
break;
|
||||
}
|
||||
case Options::SIGMOID: {
|
||||
const float pixel0 = mask_value[0];
|
||||
new_mask_value = 1.0 / (std::exp(-pixel0) + 1.0);
|
||||
break;
|
||||
}
|
||||
case Options::SOFTMAX: {
|
||||
const float pixel0 = mask_value[0];
|
||||
const float pixel1 = mask_value[1];
|
||||
const float max_pixel = std::max(pixel0, pixel1);
|
||||
const float min_pixel = std::min(pixel0, pixel1);
|
||||
const float softmax_denom =
|
||||
/*exp(max_pixel - max_pixel)=*/1.0f +
|
||||
std::exp(min_pixel - max_pixel);
|
||||
new_mask_value = std::exp(mask_value[output_layer_index] - max_pixel) /
|
||||
softmax_denom;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return new_mask_value;
|
||||
};
|
||||
|
||||
// Process mask tensor.
|
||||
for (int i = 0; i < tensor_mat.rows; ++i) {
|
||||
for (int j = 0; j < tensor_mat.cols; ++j) {
|
||||
const T& input_pix = tensor_mat.at<T>(i, j);
|
||||
const float mask_value = activation_fn(input_pix);
|
||||
small_mask_mat->at<float>(i, j) = mask_value;
|
||||
}
|
||||
}
|
||||
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
|
||||
CreateOpenCvConverter(const TensorsToSegmentationCalculatorOptions& options) {
|
||||
auto converter = std::make_unique<OpenCvProcessor>();
|
||||
MP_RETURN_IF_ERROR(converter->Init(options));
|
||||
return converter;
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
|
@ -1,31 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_converter.h"
|
||||
|
||||
namespace mediapipe {
|
||||
// Creates OpenCV tensors-to-segmentation converter.
|
||||
absl::StatusOr<std::unique_ptr<TensorsToSegmentationConverter>>
|
||||
CreateOpenCvConverter(
|
||||
const mediapipe::TensorsToSegmentationCalculatorOptions& options);
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_CONVERTER_OPENCV_H_
|
|
@ -1,52 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
|
||||
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/port.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
int NumGroups(int size, int group_size) {
|
||||
return (size + group_size - 1) / group_size;
|
||||
}
|
||||
|
||||
bool CanUseGpu() {
|
||||
#if !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
|
||||
// TODO: Configure GPU usage policy in individual calculators.
|
||||
constexpr bool kAllowGpuProcessing = true;
|
||||
return kAllowGpuProcessing;
|
||||
#else
|
||||
return false;
|
||||
#endif // !MEDIAPIPE_DISABLE_GPU || MEDIAPIPE_METAL_ENABLED
|
||||
}
|
||||
|
||||
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
|
||||
const std::vector<int>& dims) {
|
||||
if (dims.size() == 3) {
|
||||
return std::make_tuple(dims[0], dims[1], dims[2]);
|
||||
} else if (dims.size() == 4) {
|
||||
// BHWC format check B == 1
|
||||
RET_CHECK_EQ(dims[0], 1) << "Expected batch to be 1 for BHWC heatmap";
|
||||
return std::make_tuple(dims[1], dims[2], dims[3]);
|
||||
} else {
|
||||
RET_CHECK(false) << "Invalid shape for segmentation tensor " << dims.size();
|
||||
}
|
||||
}
|
||||
} // namespace mediapipe
|
|
@ -1,34 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
|
||||
#define MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
|
||||
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Commonly used to compute the number of blocks to launch in a kernel.
|
||||
int NumGroups(const int size, const int group_size); // NOLINT
|
||||
|
||||
bool CanUseGpu();
|
||||
|
||||
absl::StatusOr<std::tuple<int, int, int>> GetHwcFromDims(
|
||||
const std::vector<int>& dims);
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_CALCULATORS_TENSOR_TENSORS_TO_SEGMENTATION_UTILS_H_
|
|
@ -1,63 +0,0 @@
|
|||
// Copyright 2023 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "mediapipe/calculators/tensor/tensors_to_segmentation_utils.h"
|
||||
|
||||
#include <tuple>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/status_matchers.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
using ::testing::HasSubstr;
|
||||
|
||||
TEST(TensorsToSegmentationUtilsTest, NumGroupsWorksProperly) {
|
||||
EXPECT_EQ(NumGroups(13, 4), 4);
|
||||
EXPECT_EQ(NumGroups(4, 13), 1);
|
||||
}
|
||||
|
||||
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsWorksProperly) {
|
||||
std::vector<int> dims_3 = {2, 3, 4};
|
||||
absl::StatusOr<std::tuple<int, int, int>> result_1 = GetHwcFromDims(dims_3);
|
||||
MP_ASSERT_OK(result_1);
|
||||
EXPECT_EQ(result_1.value(), (std::make_tuple(2, 3, 4)));
|
||||
std::vector<int> dims_4 = {1, 3, 4, 5};
|
||||
absl::StatusOr<std::tuple<int, int, int>> result_2 = GetHwcFromDims(dims_4);
|
||||
MP_ASSERT_OK(result_2);
|
||||
EXPECT_EQ(result_2.value(), (std::make_tuple(3, 4, 5)));
|
||||
}
|
||||
|
||||
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsBatchCheckFail) {
|
||||
std::vector<int> dims_4 = {2, 3, 4, 5};
|
||||
absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_4);
|
||||
EXPECT_FALSE(result.ok());
|
||||
EXPECT_THAT(result.status().message(),
|
||||
HasSubstr("Expected batch to be 1 for BHWC heatmap"));
|
||||
}
|
||||
|
||||
TEST(TensorsToSegmentationUtilsTest, GetHwcFromDimsInvalidShape) {
|
||||
std::vector<int> dims_5 = {1, 2, 3, 4, 5};
|
||||
absl::StatusOr<std::tuple<int, int, int>> result = GetHwcFromDims(dims_5);
|
||||
EXPECT_FALSE(result.ok());
|
||||
EXPECT_THAT(result.status().message(),
|
||||
HasSubstr("Invalid shape for segmentation tensor"));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
|
@ -79,7 +79,7 @@ namespace mpms = mediapipe::mediasequence;
|
|||
// and label and label_id are optional but at least one of them should be set.
|
||||
// "IMAGE_${NAME}", "BBOX_${NAME}", and "KEYPOINTS_${NAME}" will also store
|
||||
// prefixed versions of each stream, which allows for multiple image streams to
|
||||
// be included. However, the default names are supported by more tools.
|
||||
// be included. However, the default names are suppored by more tools.
|
||||
//
|
||||
// Example config:
|
||||
// node {
|
||||
|
|
|
@ -67,8 +67,8 @@ absl::Status FillTimeSeriesHeaderIfValid(const Packet& header_packet,
|
|||
// -- 1-D or 2-D Tensor
|
||||
// Output:
|
||||
// -- Matrix with the same values as the Tensor
|
||||
// If input tensor is 1 dimensional, the output Matrix is of (1xn) shape.
|
||||
// If input tensor is 2 dimensional (batched), the output Matrix is (mxn) shape.
|
||||
// If input tensor is 1 dimensional, the ouput Matrix is of (1xn) shape.
|
||||
// If input tensor is 2 dimensional (batched), the ouput Matrix is (mxn) shape.
|
||||
//
|
||||
// Example Config
|
||||
// node: {
|
||||
|
|
|
@ -15,9 +15,9 @@
|
|||
// Calculator converts from one-dimensional Tensor of DT_FLOAT to vector<float>
|
||||
// OR from (batched) two-dimensional Tensor of DT_FLOAT to vector<vector<float>.
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
|
||||
#include "absl/base/integral_types.h"
|
||||
#include "mediapipe/calculators/tensorflow/tensor_to_vector_int_calculator_options.pb.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
|
|
|
@ -111,8 +111,8 @@ class InferenceState {
|
|||
// input_side_packet.
|
||||
//
|
||||
// The input and output streams are TensorFlow tensors labeled by tags. The tags
|
||||
// for the streams are matched to feeds and fetches in a TensorFlow session
|
||||
// using a named_signature.generic_signature in the ModelManifest. The
|
||||
// for the streams are matched to feeds and fetchs in a TensorFlow session using
|
||||
// a named_signature.generic_signature in the ModelManifest. The
|
||||
// generic_signature is used as key-value pairs between the MediaPipe tag and
|
||||
// the TensorFlow tensor. The signature_name in the options proto determines
|
||||
// which named_signature is used. The keys in the generic_signature must be
|
||||
|
@ -128,7 +128,7 @@ class InferenceState {
|
|||
// addition. Once batch_size inputs have been provided, the batch will be run
|
||||
// and the output tensors sent out on the output streams with timestamps
|
||||
// corresponding to the input stream packets. Setting the batch_size to 1
|
||||
// completely disables batching, but is independent of add_batch_dim_to_tensors.
|
||||
// completely disables batching, but is indepdent of add_batch_dim_to_tensors.
|
||||
//
|
||||
// The TensorFlowInferenceCalculator also support feeding states recurrently for
|
||||
// RNNs and LSTMs. Simply set the recurrent_tag_pair options to define the
|
||||
|
|
|
@ -42,7 +42,7 @@ message TensorFlowInferenceCalculatorOptions {
|
|||
// If the 0th dimension is the batch dimension, then the tensors are
|
||||
// concatenated on that dimension. If the 0th is a data dimension, then a 0th
|
||||
// dimension is added before concatenating. If added, the extra dimension is
|
||||
// removed before outputting the tensor. Examples of each case: If you want
|
||||
// removed before outputing the tensor. Examples of each case: If you want
|
||||
// to batch spectra of audio over time for an LSTM, a time-frequency
|
||||
// representation has a 0th dimension as the batch dimension. If you want to
|
||||
// batch frames of video that are [width, height, channels], the batch
|
||||
|
|
|
@ -1,2 +1,2 @@
|
|||
The model files add.bin, add_quantized.bin
|
||||
(and corresponding metadata json files) come from tensorflow/lite/testdata/
|
||||
(and corresponding metatada json files) come from tensorflow/lite/testdata/
|
||||
|
|
|
@ -95,7 +95,7 @@ struct GPUData {
|
|||
// into a TfLiteTensor (float 32) or a GpuBuffer to a tflite::gpu::GlBuffer
|
||||
// or MTLBuffer.
|
||||
//
|
||||
// This calculator is designed to be used with the TfLiteInferenceCalculator,
|
||||
// This calculator is designed to be used with the TfLiteInferenceCalcualtor,
|
||||
// as a pre-processing step for calculator inputs.
|
||||
//
|
||||
// IMAGE and IMAGE_GPU inputs are normalized to [-1,1] (default) or [0,1],
|
||||
|
|
|
@ -31,7 +31,7 @@ message TfLiteConverterCalculatorOptions {
|
|||
// Custom settings to override the internal scaling factors `div` and `sub`.
|
||||
// Both values must be set to non-negative values. Will only take effect on
|
||||
// CPU AND when |use_custom_normalization| is set to true. When these custom
|
||||
// values take effect, the |zero_center| setting above will be overridden, and
|
||||
// values take effect, the |zero_center| setting above will be overriden, and
|
||||
// the normalized_value will be calculated as:
|
||||
// normalized_value = input / custom_div - custom_sub.
|
||||
optional bool use_custom_normalization = 6 [default = false];
|
||||
|
|
|
@ -25,7 +25,7 @@ message TfLiteTensorsToClassificationCalculatorOptions {
|
|||
optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463;
|
||||
}
|
||||
|
||||
// Score threshold for preserving the class.
|
||||
// Score threshold for perserving the class.
|
||||
optional float min_score_threshold = 1;
|
||||
// Number of highest scoring labels to output. If top_k is not positive then
|
||||
// all labels are used.
|
||||
|
|
|
@ -116,7 +116,7 @@ void ConvertAnchorsToRawValues(const std::vector<Anchor>& anchors,
|
|||
// tensors can have 2 or 3 tensors. First tensor is the predicted
|
||||
// raw boxes/keypoints. The size of the values must be (num_boxes
|
||||
// * num_predicted_values). Second tensor is the score tensor. The
|
||||
// size of the values must be (num_boxes * num_classes). It's
|
||||
// size of the valuse must be (num_boxes * num_classes). It's
|
||||
// optional to pass in a third tensor for anchors (e.g. for SSD
|
||||
// models) depend on the outputs of the detection model. The size
|
||||
// of anchor tensor must be (num_boxes * 4).
|
||||
|
|
|
@ -69,6 +69,6 @@ message TfLiteTensorsToDetectionsCalculatorOptions {
|
|||
// representation has a bottom-left origin (e.g., in OpenGL).
|
||||
optional bool flip_vertically = 18 [default = false];
|
||||
|
||||
// Score threshold for preserving decoded detections.
|
||||
// Score threshold for perserving decoded detections.
|
||||
optional float min_score_thresh = 19;
|
||||
}
|
||||
|
|
|
@ -158,7 +158,7 @@ absl::Status TfLiteTensorsToLandmarksCalculator::Open(CalculatorContext* cc) {
|
|||
RET_CHECK(options_.has_input_image_height() &&
|
||||
options_.has_input_image_width())
|
||||
<< "Must provide input width/height for using flip_vertically option "
|
||||
"when outputting landmarks in absolute coordinates.";
|
||||
"when outputing landmarks in absolute coordinates.";
|
||||
}
|
||||
|
||||
flip_horizontally_ =
|
||||
|
|
Binary file not shown.
|
@ -1,7 +1,6 @@
|
|||
distributionBase=GRADLE_USER_HOME
|
||||
distributionPath=wrapper/dists
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
|
||||
distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip
|
||||
networkTimeout=10000
|
||||
validateDistributionUrl=true
|
||||
zipStoreBase=GRADLE_USER_HOME
|
||||
zipStorePath=wrapper/dists
|
||||
|
|
29
mediapipe/examples/android/solutions/gradlew
vendored
29
mediapipe/examples/android/solutions/gradlew
vendored
|
@ -83,8 +83,10 @@ done
|
|||
# This is normally unused
|
||||
# shellcheck disable=SC2034
|
||||
APP_BASE_NAME=${0##*/}
|
||||
# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
|
||||
APP_HOME=$( cd "${APP_HOME:-./}" > /dev/null && pwd -P ) || exit
|
||||
APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
||||
MAX_FD=maximum
|
||||
|
@ -131,13 +133,10 @@ location of your Java installation."
|
|||
fi
|
||||
else
|
||||
JAVACMD=java
|
||||
if ! command -v java >/dev/null 2>&1
|
||||
then
|
||||
die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
||||
|
||||
Please set the JAVA_HOME variable in your environment to match the
|
||||
location of your Java installation."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Increase the maximum file descriptors if we can.
|
||||
|
@ -145,7 +144,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
|||
case $MAX_FD in #(
|
||||
max*)
|
||||
# In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
# shellcheck disable=SC3045
|
||||
MAX_FD=$( ulimit -H -n ) ||
|
||||
warn "Could not query maximum file descriptor limit"
|
||||
esac
|
||||
|
@ -153,7 +152,7 @@ if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
|
|||
'' | soft) :;; #(
|
||||
*)
|
||||
# In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
|
||||
# shellcheck disable=SC2039,SC3045
|
||||
# shellcheck disable=SC3045
|
||||
ulimit -n "$MAX_FD" ||
|
||||
warn "Could not set maximum file descriptor limit to $MAX_FD"
|
||||
esac
|
||||
|
@ -198,15 +197,11 @@ if "$cygwin" || "$msys" ; then
|
|||
done
|
||||
fi
|
||||
|
||||
|
||||
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
||||
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
|
||||
|
||||
# Collect all arguments for the java command:
|
||||
# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
|
||||
# and any embedded shellness will be escaped.
|
||||
# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
|
||||
# treated as '${Hostname}' itself on the command line.
|
||||
# Collect all arguments for the java command;
|
||||
# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
|
||||
# shell script including quotes and variable substitutions, so put them in
|
||||
# double quotes to make sure that they get re-expanded; and
|
||||
# * put everything else in single quotes, so that it's not re-expanded.
|
||||
|
||||
set -- \
|
||||
"-Dorg.gradle.appname=$APP_BASE_NAME" \
|
||||
|
|
|
@ -43,9 +43,9 @@ class KinematicPathSolver {
|
|||
initialized_(false),
|
||||
pixels_per_degree_(pixels_per_degree) {}
|
||||
// Add an observation (detection) at a position and time.
|
||||
absl::Status AddObservation(int position, const uint64_t time_us);
|
||||
absl::Status AddObservation(int position, const uint64 time_us);
|
||||
// Get the predicted position at a time.
|
||||
absl::Status UpdatePrediction(const int64_t time_us);
|
||||
absl::Status UpdatePrediction(const int64 time_us);
|
||||
// Get the state at a time, as an int.
|
||||
absl::Status GetState(int* position);
|
||||
// Get the state at a time, as a float.
|
||||
|
@ -63,7 +63,7 @@ class KinematicPathSolver {
|
|||
bool IsMotionTooSmall(double delta_degs);
|
||||
// Check if a position measurement will cause the camera to be in motion
|
||||
// without updating the internal state.
|
||||
absl::Status PredictMotionState(int position, const uint64_t time_us,
|
||||
absl::Status PredictMotionState(int position, const uint64 time_us,
|
||||
bool* state);
|
||||
// Clear any history buffer of positions that are used when
|
||||
// filtering_time_window_us is set to a non-zero value.
|
||||
|
@ -85,9 +85,9 @@ class KinematicPathSolver {
|
|||
double current_position_px_;
|
||||
double prior_position_px_;
|
||||
double current_velocity_deg_per_s_;
|
||||
uint64_t current_time_ = 0;
|
||||
uint64 current_time_ = 0;
|
||||
// History of observations (second) and their time (first).
|
||||
std::deque<std::pair<uint64_t, int>> raw_positions_at_time_;
|
||||
std::deque<std::pair<uint64, int>> raw_positions_at_time_;
|
||||
// Current target position.
|
||||
double target_position_px_;
|
||||
// Defines if the camera is moving to a target (true) or reached a target
|
||||
|
|
|
@ -67,7 +67,7 @@ class SceneCameraMotionAnalyzer {
|
|||
const KeyFrameCropOptions& key_frame_crop_options,
|
||||
const std::vector<KeyFrameCropResult>& key_frame_crop_results,
|
||||
const int scene_frame_width, const int scene_frame_height,
|
||||
const std::vector<int64_t>& scene_frame_timestamps,
|
||||
const std::vector<int64>& scene_frame_timestamps,
|
||||
const bool has_solid_color_background,
|
||||
SceneKeyFrameCropSummary* scene_summary,
|
||||
std::vector<FocusPointFrame>* focus_point_frames,
|
||||
|
@ -78,7 +78,7 @@ class SceneCameraMotionAnalyzer {
|
|||
// crop window in SceneKeyFrameCropSummary in the case of steady motion.
|
||||
absl::Status DecideCameraMotionType(
|
||||
const KeyFrameCropOptions& key_frame_crop_options,
|
||||
const double scene_span_sec, const int64_t end_time_us,
|
||||
const double scene_span_sec, const int64 end_time_us,
|
||||
SceneKeyFrameCropSummary* scene_summary,
|
||||
SceneCameraMotion* scene_camera_motion) const;
|
||||
|
||||
|
@ -87,7 +87,7 @@ class SceneCameraMotionAnalyzer {
|
|||
absl::Status PopulateFocusPointFrames(
|
||||
const SceneKeyFrameCropSummary& scene_summary,
|
||||
const SceneCameraMotion& scene_camera_motion,
|
||||
const std::vector<int64_t>& scene_frame_timestamps,
|
||||
const std::vector<int64>& scene_frame_timestamps,
|
||||
std::vector<FocusPointFrame>* focus_point_frames) const;
|
||||
|
||||
private:
|
||||
|
@ -118,7 +118,7 @@ class SceneCameraMotionAnalyzer {
|
|||
absl::Status PopulateFocusPointFramesForTracking(
|
||||
const SceneKeyFrameCropSummary& scene_summary,
|
||||
const FocusPointFrameType focus_point_frame_type,
|
||||
const std::vector<int64_t>& scene_frame_timestamps,
|
||||
const std::vector<int64>& scene_frame_timestamps,
|
||||
std::vector<FocusPointFrame>* focus_point_frames) const;
|
||||
|
||||
// Decide to use steady motion.
|
||||
|
@ -142,7 +142,7 @@ class SceneCameraMotionAnalyzer {
|
|||
|
||||
// Last position
|
||||
SceneCameraMotion last_scene_with_salient_region_;
|
||||
int64_t time_since_last_salient_region_us_;
|
||||
int64 time_since_last_salient_region_us_;
|
||||
|
||||
// Scene has solid color background.
|
||||
bool has_solid_color_background_;
|
||||
|
|
|
@ -62,7 +62,7 @@ class SceneCropper {
|
|||
// TODO: split this function into two separate functions.
|
||||
absl::Status CropFrames(
|
||||
const SceneKeyFrameCropSummary& scene_summary,
|
||||
const std::vector<int64_t>& scene_timestamps,
|
||||
const std::vector<int64>& scene_timestamps,
|
||||
const std::vector<bool>& is_key_frames,
|
||||
const std::vector<cv::Mat>& scene_frames_or_empty,
|
||||
const std::vector<FocusPointFrame>& focus_point_frames,
|
||||
|
@ -73,7 +73,7 @@ class SceneCropper {
|
|||
|
||||
absl::Status ProcessKinematicPathSolver(
|
||||
const SceneKeyFrameCropSummary& scene_summary,
|
||||
const std::vector<int64_t>& scene_timestamps,
|
||||
const std::vector<int64>& scene_timestamps,
|
||||
const std::vector<bool>& is_key_frames,
|
||||
const std::vector<FocusPointFrame>& focus_point_frames,
|
||||
const bool continue_last_scene, std::vector<cv::Mat>* all_xforms);
|
||||
|
|
|
@ -29,7 +29,7 @@ namespace autoflip {
|
|||
// Packs detected features and timestamp (ms) into a KeyFrameInfo object. Scales
|
||||
// features back to the original frame size if features have been detected on a
|
||||
// different frame size.
|
||||
absl::Status PackKeyFrameInfo(const int64_t frame_timestamp_ms,
|
||||
absl::Status PackKeyFrameInfo(const int64 frame_timestamp_ms,
|
||||
const DetectionSet& detections,
|
||||
const int original_frame_width,
|
||||
const int original_frame_height,
|
||||
|
@ -71,7 +71,7 @@ absl::Status ComputeSceneStaticBordersSize(
|
|||
// interpolation functions in Lab space using input timestamps.
|
||||
absl::Status FindSolidBackgroundColor(
|
||||
const std::vector<StaticFeatures>& static_features,
|
||||
const std::vector<int64_t>& static_features_timestamps,
|
||||
const std::vector<int64>& static_features_timestamps,
|
||||
const double min_fraction_solid_background_color,
|
||||
bool* has_solid_background,
|
||||
PiecewiseLinearFunction* background_color_l_function,
|
||||
|
|
|
@ -11,6 +11,7 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
|
||||
|
@ -367,7 +368,6 @@ cc_library(
|
|||
"@com_google_absl//absl/container:flat_hash_set",
|
||||
"@com_google_absl//absl/log:absl_check",
|
||||
"@com_google_absl//absl/log:absl_log",
|
||||
"@com_google_absl//absl/log:check",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
|
|
|
@ -81,13 +81,15 @@ cc_library(
|
|||
srcs = ["node.cc"],
|
||||
hdrs = ["node.h"],
|
||||
deps = [
|
||||
":const_str",
|
||||
":contract",
|
||||
":packet",
|
||||
":port",
|
||||
"//mediapipe/framework:calculator_base",
|
||||
"//mediapipe/framework:calculator_context",
|
||||
"//mediapipe/framework:calculator_contract",
|
||||
"//mediapipe/framework:subgraph",
|
||||
"//mediapipe/framework/deps:registration",
|
||||
"@com_google_absl//absl/status",
|
||||
"//mediapipe/framework/deps:no_destructor",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
@ -330,14 +330,6 @@ using MultiSideDestination = MultiPort<SideDestination<T>>;
|
|||
|
||||
class NodeBase {
|
||||
public:
|
||||
NodeBase() = default;
|
||||
~NodeBase() = default;
|
||||
NodeBase(NodeBase&&) = default;
|
||||
NodeBase& operator=(NodeBase&&) = default;
|
||||
// Explicitly delete copies to improve error messages.
|
||||
NodeBase(const NodeBase&) = delete;
|
||||
NodeBase& operator=(const NodeBase&) = delete;
|
||||
|
||||
// TODO: right now access to an indexed port is made directly by
|
||||
// specifying both a tag and an index. It would be better to represent this
|
||||
// as a two-step lookup, first getting a multi-port, and then accessing one
|
||||
|
@ -593,14 +585,6 @@ class PacketGenerator {
|
|||
|
||||
class Graph {
|
||||
public:
|
||||
Graph() = default;
|
||||
~Graph() = default;
|
||||
Graph(Graph&&) = default;
|
||||
Graph& operator=(Graph&&) = default;
|
||||
// Explicitly delete copies to improve error messages.
|
||||
Graph(const Graph&) = delete;
|
||||
Graph& operator=(const Graph&) = delete;
|
||||
|
||||
void SetType(std::string type) { type_ = std::move(type); }
|
||||
|
||||
// Creates a node of a specific type. Should be used for calculators whose
|
||||
|
|
|
@ -1,15 +1,17 @@
|
|||
#ifndef MEDIAPIPE_FRAMEWORK_API2_NODE_H_
|
||||
#define MEDIAPIPE_FRAMEWORK_API2_NODE_H_
|
||||
|
||||
#include <memory>
|
||||
#include <type_traits>
|
||||
#include <functional>
|
||||
#include <string>
|
||||
|
||||
#include "absl/status/status.h"
|
||||
#include "mediapipe/framework/api2/const_str.h"
|
||||
#include "mediapipe/framework/api2/contract.h"
|
||||
#include "mediapipe/framework/api2/packet.h"
|
||||
#include "mediapipe/framework/api2/port.h"
|
||||
#include "mediapipe/framework/calculator_base.h"
|
||||
#include "mediapipe/framework/calculator_context.h"
|
||||
#include "mediapipe/framework/calculator_contract.h"
|
||||
#include "mediapipe/framework/deps/registration.h"
|
||||
#include "mediapipe/framework/deps/no_destructor.h"
|
||||
#include "mediapipe/framework/subgraph.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
@ -176,10 +178,7 @@ class SubgraphImpl : public Subgraph,
|
|||
absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>)
|
||||
|
||||
// This macro is used to register a non-split-contract calculator. Deprecated.
|
||||
#define MEDIAPIPE_REGISTER_NODE(name) \
|
||||
MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED( \
|
||||
mediapipe::CalculatorBaseRegistry, calculator_registration, #name, \
|
||||
absl::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<name>>)
|
||||
#define MEDIAPIPE_REGISTER_NODE(name) REGISTER_CALCULATOR(name)
|
||||
|
||||
// This macro is used to define a subgraph that does not use automatic
|
||||
// registration. Deprecated.
|
||||
|
|
|
@ -28,7 +28,6 @@
|
|||
#include "absl/container/flat_hash_set.h"
|
||||
#include "absl/log/absl_check.h"
|
||||
#include "absl/log/absl_log.h"
|
||||
#include "absl/log/check.h"
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/strings/str_cat.h"
|
||||
|
@ -891,12 +890,12 @@ absl::Status CalculatorGraph::WaitForObservedOutput() {
|
|||
}
|
||||
|
||||
absl::Status CalculatorGraph::AddPacketToInputStream(
|
||||
absl::string_view stream_name, const Packet& packet) {
|
||||
const std::string& stream_name, const Packet& packet) {
|
||||
return AddPacketToInputStreamInternal(stream_name, packet);
|
||||
}
|
||||
|
||||
absl::Status CalculatorGraph::AddPacketToInputStream(
|
||||
absl::string_view stream_name, Packet&& packet) {
|
||||
const std::string& stream_name, Packet&& packet) {
|
||||
return AddPacketToInputStreamInternal(stream_name, std::move(packet));
|
||||
}
|
||||
|
||||
|
@ -919,18 +918,14 @@ absl::Status CalculatorGraph::SetInputStreamTimestampBound(
|
|||
// std::forward will deduce the correct type as we pass along packet.
|
||||
template <typename T>
|
||||
absl::Status CalculatorGraph::AddPacketToInputStreamInternal(
|
||||
absl::string_view stream_name, T&& packet) {
|
||||
auto stream_it = graph_input_streams_.find(stream_name);
|
||||
const std::string& stream_name, T&& packet) {
|
||||
std::unique_ptr<GraphInputStream>* stream =
|
||||
stream_it == graph_input_streams_.end() ? nullptr : &stream_it->second;
|
||||
mediapipe::FindOrNull(graph_input_streams_, stream_name);
|
||||
RET_CHECK(stream).SetNoLogging() << absl::Substitute(
|
||||
"AddPacketToInputStream called on input stream \"$0\" which is not a "
|
||||
"graph input stream.",
|
||||
stream_name);
|
||||
auto node_id_it = graph_input_stream_node_ids_.find(stream_name);
|
||||
ABSL_CHECK(node_id_it != graph_input_stream_node_ids_.end())
|
||||
<< "Map key not found: " << stream_name;
|
||||
int node_id = node_id_it->second;
|
||||
int node_id = mediapipe::FindOrDie(graph_input_stream_node_ids_, stream_name);
|
||||
ABSL_CHECK_GE(node_id, validated_graph_->CalculatorInfos().size());
|
||||
{
|
||||
absl::MutexLock lock(&full_input_streams_mutex_);
|
||||
|
|
|
@ -32,7 +32,6 @@
|
|||
#include "absl/container/flat_hash_set.h"
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "absl/synchronization/mutex.h"
|
||||
#include "mediapipe/framework/calculator.pb.h"
|
||||
#include "mediapipe/framework/calculator_base.h"
|
||||
|
@ -256,7 +255,7 @@ class CalculatorGraph {
|
|||
// sizes of the queues in the graph. The input stream must have been specified
|
||||
// in the configuration as a graph level input_stream. On error, nothing is
|
||||
// added.
|
||||
absl::Status AddPacketToInputStream(absl::string_view stream_name,
|
||||
absl::Status AddPacketToInputStream(const std::string& stream_name,
|
||||
const Packet& packet);
|
||||
|
||||
// Same as the l-value version of this function by the same name, but moves
|
||||
|
@ -266,7 +265,7 @@ class CalculatorGraph {
|
|||
// packet may remain valid. In particular, when using the ADD_IF_NOT_FULL
|
||||
// mode with a full queue, this will return StatusUnavailable and the caller
|
||||
// may try adding the packet again later.
|
||||
absl::Status AddPacketToInputStream(absl::string_view stream_name,
|
||||
absl::Status AddPacketToInputStream(const std::string& stream_name,
|
||||
Packet&& packet);
|
||||
|
||||
// Indicates that input will arrive no earlier than a certain timestamp.
|
||||
|
@ -510,7 +509,7 @@ class CalculatorGraph {
|
|||
// AddPacketToInputStream(Packet&& packet) or
|
||||
// AddPacketToInputStream(const Packet& packet).
|
||||
template <typename T>
|
||||
absl::Status AddPacketToInputStreamInternal(absl::string_view stream_name,
|
||||
absl::Status AddPacketToInputStreamInternal(const std::string& stream_name,
|
||||
T&& packet);
|
||||
|
||||
// Sets the executor that will run the nodes assigned to the executor
|
||||
|
|
|
@ -31,14 +31,14 @@
|
|||
// A StrongInt<T> with a NullStrongIntValidator should compile away to a raw T
|
||||
// in optimized mode. What this means is that the generated assembly for:
|
||||
//
|
||||
// int64_t foo = 123;
|
||||
// int64_t bar = 456;
|
||||
// int64_t baz = foo + bar;
|
||||
// constexpr int64_t fubar = 789;
|
||||
// int64 foo = 123;
|
||||
// int64 bar = 456;
|
||||
// int64 baz = foo + bar;
|
||||
// constexpr int64 fubar = 789;
|
||||
//
|
||||
// ...should be identical to the generated assembly for:
|
||||
//
|
||||
// DEFINE_STRONG_INT_TYPE(MyStrongInt, int64_t);
|
||||
// DEFINE_STRONG_INT_TYPE(MyStrongInt, int64);
|
||||
// MyStrongInt foo(123);
|
||||
// MyStrongInt bar(456);
|
||||
// MyStrongInt baz = foo + bar;
|
||||
|
@ -97,7 +97,6 @@
|
|||
#ifndef MEDIAPIPE_DEPS_STRONG_INT_H_
|
||||
#define MEDIAPIPE_DEPS_STRONG_INT_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <iosfwd>
|
||||
#include <limits>
|
||||
#include <ostream>
|
||||
|
@ -180,11 +179,11 @@ struct NullStrongIntValidator {
|
|||
}
|
||||
// Verify lhs << rhs.
|
||||
template <typename T>
|
||||
static void ValidateLeftShift(T lhs, int64_t rhs) { /* do nothing */
|
||||
static void ValidateLeftShift(T lhs, int64 rhs) { /* do nothing */
|
||||
}
|
||||
// Verify lhs >> rhs.
|
||||
template <typename T>
|
||||
static void ValidateRightShift(T lhs, int64_t rhs) { /* do nothing */
|
||||
static void ValidateRightShift(T lhs, int64 rhs) { /* do nothing */
|
||||
}
|
||||
// Verify lhs & rhs.
|
||||
template <typename T>
|
||||
|
@ -225,8 +224,8 @@ class StrongInt {
|
|||
//
|
||||
// Example: Assume you have two StrongInt types.
|
||||
//
|
||||
// DEFINE_STRONG_INT_TYPE(Bytes, int64_t);
|
||||
// DEFINE_STRONG_INT_TYPE(Megabytes, int64_t);
|
||||
// DEFINE_STRONG_INT_TYPE(Bytes, int64);
|
||||
// DEFINE_STRONG_INT_TYPE(Megabytes, int64);
|
||||
//
|
||||
// If you want to be able to (explicitly) construct an instance of Bytes from
|
||||
// an instance of Megabytes, simply define a converter function in the same
|
||||
|
@ -338,12 +337,12 @@ class StrongInt {
|
|||
value_ %= arg;
|
||||
return *this;
|
||||
}
|
||||
StrongInt &operator<<=(int64_t arg) { // NOLINT(whitespace/operators)
|
||||
StrongInt &operator<<=(int64 arg) { // NOLINT(whitespace/operators)
|
||||
ValidatorType::template ValidateLeftShift<ValueType>(value_, arg);
|
||||
value_ <<= arg;
|
||||
return *this;
|
||||
}
|
||||
StrongInt &operator>>=(int64_t arg) { // NOLINT(whitespace/operators)
|
||||
StrongInt &operator>>=(int64 arg) { // NOLINT(whitespace/operators)
|
||||
ValidatorType::template ValidateRightShift<ValueType>(value_, arg);
|
||||
value_ >>= arg;
|
||||
return *this;
|
||||
|
@ -379,19 +378,19 @@ std::ostream &operator<<(std::ostream &os,
|
|||
return os << arg.value();
|
||||
}
|
||||
|
||||
// Provide the << operator, primarily for logging purposes. Specialized for
|
||||
// int8_t so that an integer and not a character is printed.
|
||||
// Provide the << operator, primarily for logging purposes. Specialized for int8
|
||||
// so that an integer and not a character is printed.
|
||||
template <typename TagType, typename ValidatorType>
|
||||
std::ostream &operator<<(std::ostream &os,
|
||||
StrongInt<TagType, int8_t, ValidatorType> arg) {
|
||||
StrongInt<TagType, int8, ValidatorType> arg) {
|
||||
return os << static_cast<int>(arg.value());
|
||||
}
|
||||
|
||||
// Provide the << operator, primarily for logging purposes. Specialized for
|
||||
// uint8_t so that an integer and not a character is printed.
|
||||
// uint8 so that an integer and not a character is printed.
|
||||
template <typename TagType, typename ValidatorType>
|
||||
std::ostream &operator<<(std::ostream &os,
|
||||
StrongInt<TagType, uint8_t, ValidatorType> arg) {
|
||||
StrongInt<TagType, uint8, ValidatorType> arg) {
|
||||
return os << static_cast<unsigned int>(arg.value());
|
||||
}
|
||||
|
||||
|
|
|
@ -76,12 +76,10 @@ def _get_proto_provider(dep):
|
|||
"""
|
||||
if ProtoInfo in dep:
|
||||
return dep[ProtoInfo]
|
||||
|
||||
elif hasattr(dep, "proto"):
|
||||
return dep.proto
|
||||
else:
|
||||
fail("cannot happen, rule definition requires .proto" +
|
||||
" or ProtoInfo")
|
||||
fail("cannot happen, rule definition requires .proto or ProtoInfo")
|
||||
|
||||
def _encode_binary_proto_impl(ctx):
|
||||
"""Implementation of the encode_binary_proto rule."""
|
||||
|
@ -144,10 +142,7 @@ _encode_binary_proto = rule(
|
|||
cfg = "exec",
|
||||
),
|
||||
"deps": attr.label_list(
|
||||
providers = [
|
||||
[ProtoInfo],
|
||||
["proto"],
|
||||
],
|
||||
providers = [[ProtoInfo], ["proto"]],
|
||||
),
|
||||
"input": attr.label(
|
||||
mandatory = True,
|
||||
|
@ -187,10 +182,7 @@ def _generate_proto_descriptor_set_impl(ctx):
|
|||
all_protos = depset(transitive = [
|
||||
_get_proto_provider(dep).transitive_sources
|
||||
for dep in ctx.attr.deps
|
||||
if (
|
||||
ProtoInfo in dep or
|
||||
hasattr(dep, "proto")
|
||||
)
|
||||
if ProtoInfo in dep or hasattr(dep, "proto")
|
||||
])
|
||||
descriptor = ctx.outputs.output
|
||||
|
||||
|
@ -221,10 +213,7 @@ generate_proto_descriptor_set = rule(
|
|||
cfg = "exec",
|
||||
),
|
||||
"deps": attr.label_list(
|
||||
providers = [
|
||||
[ProtoInfo],
|
||||
["proto"],
|
||||
],
|
||||
providers = [[ProtoInfo], ["proto"]],
|
||||
),
|
||||
},
|
||||
outputs = {"output": "%{name}.proto.bin"},
|
||||
|
|
|
@ -124,15 +124,6 @@ cc_library(
|
|||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "ahwb_view",
|
||||
hdrs = ["ahwb_view.h"],
|
||||
deps = [
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/gpu:gpu_buffer_storage",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "affine_transform",
|
||||
srcs = ["affine_transform.cc"],
|
||||
|
@ -155,27 +146,6 @@ cc_library(
|
|||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "hardware_buffer",
|
||||
srcs = ["hardware_buffer_android.cc"],
|
||||
hdrs = ["hardware_buffer.h"],
|
||||
linkopts = select({
|
||||
"//conditions:default": [],
|
||||
# Option for vendor binaries to avoid linking libandroid.so.
|
||||
"//mediapipe/framework:android_no_jni": [],
|
||||
"//mediapipe:android": ["-landroid"],
|
||||
":android_link_native_window": [
|
||||
"-lnativewindow", # Provides <android/hardware_buffer.h> to vendor binaries on Android API >= 26.
|
||||
],
|
||||
}),
|
||||
visibility = ["//visibility:private"],
|
||||
deps = [
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//mediapipe/framework/port:statusor",
|
||||
"@com_google_absl//absl/log:absl_check",
|
||||
],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "image_frame",
|
||||
srcs = ["image_frame.cc"],
|
||||
|
@ -514,31 +484,28 @@ cc_library(
|
|||
"//conditions:default": [],
|
||||
# Option for vendor binaries to avoid linking libandroid.so.
|
||||
"//mediapipe/framework:android_no_jni": [],
|
||||
"//mediapipe:android": ["-landroid"],
|
||||
":android_link_native_window": [
|
||||
"-lnativewindow", # Provides <android/hardware_buffer.h> to vendor binaries on Android API >= 26.
|
||||
],
|
||||
}),
|
||||
deps = [
|
||||
"//mediapipe/framework:port",
|
||||
"@com_google_absl//absl/container:flat_hash_map",
|
||||
"@com_google_absl//absl/log:absl_check",
|
||||
"@com_google_absl//absl/log:absl_log",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/synchronization",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/gpu:gl_base",
|
||||
"//mediapipe/gpu:gl_context",
|
||||
],
|
||||
"//mediapipe:android": [
|
||||
":hardware_buffer",
|
||||
"//mediapipe/gpu:gl_base",
|
||||
"//mediapipe/gpu:gl_context",
|
||||
],
|
||||
":android_link_native_window": [
|
||||
":hardware_buffer",
|
||||
"//mediapipe/gpu:gl_base",
|
||||
"//mediapipe/gpu:gl_context",
|
||||
],
|
||||
}),
|
||||
"//mediapipe/framework:port",
|
||||
"@com_google_absl//absl/container:flat_hash_map",
|
||||
"@com_google_absl//absl/log:absl_check",
|
||||
"@com_google_absl//absl/log:absl_log",
|
||||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/synchronization",
|
||||
] + select({
|
||||
"//mediapipe/gpu:disable_gpu": [],
|
||||
"//conditions:default": [
|
||||
"//mediapipe/gpu:gl_base",
|
||||
"//mediapipe/gpu:gl_context",
|
||||
],
|
||||
}) +
|
||||
select({
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
)
|
||||
|
||||
cc_test(
|
||||
|
|
|
@ -1,54 +0,0 @@
|
|||
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
|
||||
#define MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
|
||||
|
||||
#include "mediapipe/framework/port.h"
|
||||
#ifdef MEDIAPIPE_GPU_BUFFER_USE_AHWB
|
||||
#include <android/hardware_buffer.h>
|
||||
|
||||
#include "mediapipe/gpu/gpu_buffer_storage.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
// Wrapper to facilitate short lived access to Android Hardware Buffer objects.
|
||||
// Intended use cases:
|
||||
// - Extracting an AHWB for processing in another library after it's produced by
|
||||
// MediaPipe.
|
||||
// - Sending AHWBs to compute devices that are able to map the memory for their
|
||||
// own usage.
|
||||
// The AHWB abstractions in GpuBuffer and Tensor are likely more suitable for
|
||||
// other CPU/GPU uses of AHWBs.
|
||||
class AhwbView {
|
||||
public:
|
||||
explicit AhwbView(AHardwareBuffer* handle) : handle_(handle) {}
|
||||
// Non-copyable
|
||||
AhwbView(const AhwbView&) = delete;
|
||||
AhwbView& operator=(const AhwbView&) = delete;
|
||||
// Non-movable
|
||||
AhwbView(AhwbView&&) = delete;
|
||||
|
||||
// Only supports synchronous usage. All users of GetHandle must finish
|
||||
// accessing the buffer before this view object is destroyed to avoid race
|
||||
// conditions.
|
||||
// TODO: Support asynchronous usage.
|
||||
const AHardwareBuffer* GetHandle() const { return handle_; }
|
||||
|
||||
private:
|
||||
const AHardwareBuffer* handle_;
|
||||
};
|
||||
|
||||
namespace internal {
|
||||
// Makes this class available as a GpuBuffer view.
|
||||
template <>
|
||||
class ViewProvider<AhwbView> {
|
||||
public:
|
||||
virtual ~ViewProvider() = default;
|
||||
virtual const AhwbView GetReadView(types<AhwbView>) const = 0;
|
||||
virtual AhwbView GetWriteView(types<AhwbView>) = 0;
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_GPU_BUFFER_USE_AHWB
|
||||
#endif // MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_VIEW_H_
|
|
@ -17,25 +17,10 @@
|
|||
|
||||
#include <stdio.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
#include "absl/log/absl_log.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
DeletingFile::DeletingFile(DeletingFile&& other)
|
||||
: path_(std::move(other.path_)),
|
||||
delete_on_destruction_(other.delete_on_destruction_) {
|
||||
other.delete_on_destruction_ = false;
|
||||
}
|
||||
|
||||
DeletingFile& DeletingFile::operator=(DeletingFile&& other) {
|
||||
path_ = std::move(other.path_);
|
||||
delete_on_destruction_ = other.delete_on_destruction_;
|
||||
other.delete_on_destruction_ = false;
|
||||
return *this;
|
||||
}
|
||||
|
||||
DeletingFile::DeletingFile(const std::string& path, bool delete_on_destruction)
|
||||
: path_(path), delete_on_destruction_(delete_on_destruction) {}
|
||||
|
||||
|
|
|
@ -28,11 +28,6 @@ class DeletingFile {
|
|||
DeletingFile(const DeletingFile&) = delete;
|
||||
DeletingFile& operator=(const DeletingFile&) = delete;
|
||||
|
||||
// DeletingFile is movable. The moved-from object remains in valid but
|
||||
// unspecified state and will not perform any operations on destruction.
|
||||
DeletingFile(DeletingFile&& other);
|
||||
DeletingFile& operator=(DeletingFile&& other);
|
||||
|
||||
// Provide the path to the file and whether the file should be deleted
|
||||
// when this object is destroyed.
|
||||
DeletingFile(const std::string& path, bool delete_on_destruction);
|
||||
|
|
|
@ -16,11 +16,11 @@ limitations under the License.
|
|||
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_FRAME_BUFFER_H_
|
||||
#define MEDIAPIPE_FRAMEWORK_FORMATS_FRAME_BUFFER_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/log/absl_check.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
|
@ -76,13 +76,13 @@ class FrameBuffer {
|
|||
|
||||
// Plane encapsulates buffer and stride information.
|
||||
struct Plane {
|
||||
Plane(uint8_t* buffer, Stride stride) : buffer_(buffer), stride_(stride) {}
|
||||
const uint8_t* buffer() const { return buffer_; }
|
||||
uint8_t* mutable_buffer() { return buffer_; }
|
||||
Plane(uint8* buffer, Stride stride) : buffer_(buffer), stride_(stride) {}
|
||||
const uint8* buffer() const { return buffer_; }
|
||||
uint8* mutable_buffer() { return buffer_; }
|
||||
Stride stride() const { return stride_; }
|
||||
|
||||
private:
|
||||
uint8_t* buffer_;
|
||||
uint8* buffer_;
|
||||
Stride stride_;
|
||||
};
|
||||
|
||||
|
@ -121,9 +121,9 @@ class FrameBuffer {
|
|||
|
||||
// YUV data structure.
|
||||
struct YuvData {
|
||||
const uint8_t* y_buffer;
|
||||
const uint8_t* u_buffer;
|
||||
const uint8_t* v_buffer;
|
||||
const uint8* y_buffer;
|
||||
const uint8* u_buffer;
|
||||
const uint8* v_buffer;
|
||||
// Y buffer row stride in bytes.
|
||||
int y_row_stride;
|
||||
// U/V buffer row stride in bytes.
|
||||
|
|
|
@ -1,167 +0,0 @@
|
|||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_HARDWARE_BUFFER_H_
|
||||
#define MEDIAPIPE_FRAMEWORK_FORMATS_HARDWARE_BUFFER_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
|
||||
#include "absl/base/attributes.h"
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
|
||||
typedef struct AHardwareBuffer AHardwareBuffer;
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
struct HardwareBufferSpec {
|
||||
// Buffer pixel formats. See NDK's hardware_buffer.h for descriptions.
|
||||
enum {
|
||||
// This must be kept in sync with NDK's hardware_buffer.h
|
||||
AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM = 0x01,
|
||||
AHARDWAREBUFFER_FORMAT_R8G8B8_UNORM = 0x03,
|
||||
AHARDWAREBUFFER_FORMAT_R16G16B16A16_FLOAT = 0x16,
|
||||
AHARDWAREBUFFER_FORMAT_BLOB = 0x21,
|
||||
AHARDWAREBUFFER_FORMAT_R8_UNORM = 0x38,
|
||||
};
|
||||
|
||||
// Buffer usage descriptions. See NDK's hardware_buffer.h for descriptions.
|
||||
enum {
|
||||
// This must be kept in sync with NDK's hardware_buffer.h
|
||||
AHARDWAREBUFFER_USAGE_CPU_READ_NEVER = 0x0UL,
|
||||
AHARDWAREBUFFER_USAGE_CPU_READ_RARELY = 0x2UL,
|
||||
AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN = 0x3UL,
|
||||
AHARDWAREBUFFER_USAGE_CPU_WRITE_NEVER = UINT64_C(0) << 4,
|
||||
AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY = UINT64_C(2) << 4,
|
||||
AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN = UINT64_C(3) << 4,
|
||||
AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE = UINT64_C(1) << 8,
|
||||
AHARDWAREBUFFER_USAGE_GPU_FRAMEBUFFER = UINT64_C(1) << 9,
|
||||
AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER = UINT64_C(1) << 24,
|
||||
};
|
||||
|
||||
// Hashing required to use HardwareBufferSpec as key in buffer pools. See
|
||||
// absl::Hash for details.
|
||||
template <typename H>
|
||||
friend H AbslHashValue(H h, const HardwareBufferSpec& spec) {
|
||||
return H::combine(std::move(h), spec.width, spec.height, spec.layers,
|
||||
spec.format, spec.usage);
|
||||
}
|
||||
|
||||
uint32_t width = 0;
|
||||
uint32_t height = 0;
|
||||
uint32_t layers = 0;
|
||||
uint32_t format = 0;
|
||||
uint64_t usage = 0;
|
||||
};
|
||||
|
||||
// Equality operators
|
||||
inline bool operator==(const HardwareBufferSpec& lhs,
|
||||
const HardwareBufferSpec& rhs) {
|
||||
return lhs.width == rhs.width && lhs.height == rhs.height &&
|
||||
lhs.layers == rhs.layers && lhs.format == rhs.format &&
|
||||
lhs.usage == rhs.usage;
|
||||
}
|
||||
inline bool operator!=(const HardwareBufferSpec& lhs,
|
||||
const HardwareBufferSpec& rhs) {
|
||||
return !operator==(lhs, rhs);
|
||||
}
|
||||
|
||||
// For internal use only. Thinly wraps the Android NDK AHardwareBuffer.
|
||||
class HardwareBuffer {
|
||||
public:
|
||||
// Constructs a HardwareBuffer instance from a newly allocated Android NDK
|
||||
// AHardwareBuffer.
|
||||
static absl::StatusOr<HardwareBuffer> Create(const HardwareBufferSpec& spec);
|
||||
|
||||
// Destructs the HardwareBuffer, releasing the AHardwareBuffer.
|
||||
~HardwareBuffer();
|
||||
|
||||
// Support HardwareBuffer moves.
|
||||
HardwareBuffer(HardwareBuffer&& other);
|
||||
|
||||
// Delete assignment and copy constructors.
|
||||
HardwareBuffer(HardwareBuffer& other) = delete;
|
||||
HardwareBuffer(const HardwareBuffer& other) = delete;
|
||||
HardwareBuffer& operator=(const HardwareBuffer&) = delete;
|
||||
|
||||
// Returns true if AHWB is supported.
|
||||
static bool IsSupported();
|
||||
|
||||
// Lock the hardware buffer for the given usage flags. fence_file_descriptor
|
||||
// specifies a fence file descriptor on which to wait before locking the
|
||||
// buffer. Returns raw memory address if lock is successful, nullptr
|
||||
// otherwise.
|
||||
ABSL_MUST_USE_RESULT absl::StatusOr<void*> Lock(
|
||||
uint64_t usage, std::optional<int> fence_file_descriptor = std::nullopt);
|
||||
|
||||
// Unlocks the hardware buffer synchronously. This method blocks until
|
||||
// unlocking is complete.
|
||||
absl::Status Unlock();
|
||||
|
||||
// Unlocks the hardware buffer asynchronously. It returns a file_descriptor
|
||||
// which can be used as a fence that is signaled once unlocking is complete.
|
||||
absl::StatusOr<int> UnlockAsync();
|
||||
|
||||
// Returns the underlying raw AHardwareBuffer pointer to be used directly with
|
||||
// AHardwareBuffer APIs.
|
||||
AHardwareBuffer* GetAHardwareBuffer() const { return ahw_buffer_; }
|
||||
|
||||
// Returns whether this HardwareBuffer contains a valid AHardwareBuffer.
|
||||
bool IsValid() const { return ahw_buffer_ != nullptr; }
|
||||
|
||||
// Returns whether this HardwareBuffer is locked.
|
||||
bool IsLocked() const { return is_locked_; }
|
||||
|
||||
// Releases the AHardwareBuffer.
|
||||
void Reset();
|
||||
|
||||
// Ahwb's are aligned to an implementation specific cacheline size.
|
||||
uint32_t GetAlignedWidth() const;
|
||||
|
||||
// Returns buffer spec.
|
||||
const HardwareBufferSpec& spec() const { return spec_; }
|
||||
|
||||
private:
|
||||
// Allocates an AHardwareBuffer instance;
|
||||
static absl::StatusOr<AHardwareBuffer*> AllocateAHardwareBuffer(
|
||||
const HardwareBufferSpec& spec);
|
||||
|
||||
// Constructs a HardwareBuffer instance from an already aquired
|
||||
// AHardwareBuffer instance and its spec.
|
||||
HardwareBuffer(const HardwareBufferSpec& spec, AHardwareBuffer* ahwb);
|
||||
|
||||
// Unlocks the hardware buffer. If fence_file_descriptor_ptr is not nullptr,
|
||||
// the function won't block and instead fence_file_descriptor_ptr will be set
|
||||
// to a file descriptor to become signaled once unlocking is complete.
|
||||
absl::Status UnlockInternal(int* fence_file_descriptor_ptr);
|
||||
|
||||
// Releases ahw_buffer_ AHardwareBuffer instance;
|
||||
absl::Status ReleaseAHardwareBuffer();
|
||||
|
||||
// Buffer spec.
|
||||
HardwareBufferSpec spec_ = {};
|
||||
|
||||
// Android NDK AHardwareBuffer.
|
||||
AHardwareBuffer* ahw_buffer_ = nullptr;
|
||||
|
||||
// Indicates if AHardwareBuffer is locked for reading or writing.
|
||||
bool is_locked_ = false;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_FRAMEWORK_FORMATS_AHWB_BUFFER_H_
|
|
@ -1,152 +0,0 @@
|
|||
// Copyright 2019 The MediaPipe Authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#if !defined(MEDIAPIPE_NO_JNI) && \
|
||||
(__ANDROID_API__ >= 26 || \
|
||||
defined(__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__))
|
||||
|
||||
#include <android/hardware_buffer.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "absl/log/absl_check.h"
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/status/statusor.h"
|
||||
#include "mediapipe/framework/formats/hardware_buffer.h"
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
HardwareBuffer::HardwareBuffer(HardwareBuffer &&other) {
|
||||
spec_ = std::exchange(other.spec_, {});
|
||||
ahw_buffer_ = std::exchange(other.ahw_buffer_, nullptr);
|
||||
is_locked_ = std::exchange(other.is_locked_, false);
|
||||
}
|
||||
|
||||
HardwareBuffer::HardwareBuffer(const HardwareBufferSpec &spec,
|
||||
AHardwareBuffer *ahwb)
|
||||
: spec_(spec), ahw_buffer_(ahwb), is_locked_(false) {}
|
||||
|
||||
HardwareBuffer::~HardwareBuffer() { Reset(); }
|
||||
|
||||
absl::StatusOr<HardwareBuffer> HardwareBuffer::Create(
|
||||
const HardwareBufferSpec &spec) {
|
||||
MP_ASSIGN_OR_RETURN(AHardwareBuffer * ahwb, AllocateAHardwareBuffer(spec));
|
||||
return HardwareBuffer(spec, ahwb);
|
||||
}
|
||||
|
||||
bool HardwareBuffer::IsSupported() {
|
||||
if (__builtin_available(android 26, *)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
absl::StatusOr<AHardwareBuffer *> HardwareBuffer::AllocateAHardwareBuffer(
|
||||
const HardwareBufferSpec &spec) {
|
||||
RET_CHECK(IsSupported()) << "AndroidHWBuffers not supported";
|
||||
|
||||
AHardwareBuffer *output = nullptr;
|
||||
int error = 0;
|
||||
if (__builtin_available(android 26, *)) {
|
||||
AHardwareBuffer_Desc desc = {
|
||||
.width = spec.width,
|
||||
.height = spec.height,
|
||||
.layers = spec.layers,
|
||||
.format = spec.format,
|
||||
.usage = spec.usage,
|
||||
};
|
||||
error = AHardwareBuffer_allocate(&desc, &output);
|
||||
}
|
||||
RET_CHECK(!error && output != nullptr) << "AHardwareBuffer_allocate failed";
|
||||
return output;
|
||||
}
|
||||
|
||||
absl::Status HardwareBuffer::ReleaseAHardwareBuffer() {
|
||||
if (ahw_buffer_ == nullptr) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
if (is_locked_) {
|
||||
MP_RETURN_IF_ERROR(Unlock());
|
||||
}
|
||||
if (__builtin_available(android 26, *)) {
|
||||
AHardwareBuffer_release(ahw_buffer_);
|
||||
}
|
||||
spec_ = {};
|
||||
ahw_buffer_ = nullptr;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
absl::StatusOr<void *> HardwareBuffer::Lock(
|
||||
uint64_t usage, std::optional<int> fence_file_descriptor) {
|
||||
RET_CHECK(ahw_buffer_ != nullptr) << "Hardware Buffer not allocated";
|
||||
RET_CHECK(!is_locked_) << "Hardware Buffer already locked";
|
||||
void *mem = nullptr;
|
||||
if (__builtin_available(android 26, *)) {
|
||||
const int error = AHardwareBuffer_lock(
|
||||
ahw_buffer_, usage,
|
||||
fence_file_descriptor.has_value() ? *fence_file_descriptor : -1,
|
||||
nullptr, &mem);
|
||||
RET_CHECK(error == 0) << "Hardware Buffer lock failed. Error: " << error;
|
||||
}
|
||||
is_locked_ = true;
|
||||
return mem;
|
||||
}
|
||||
|
||||
absl::Status HardwareBuffer::Unlock() {
|
||||
return UnlockInternal(/*fence_file_descriptor=*/nullptr);
|
||||
}
|
||||
|
||||
absl::StatusOr<int> HardwareBuffer::UnlockAsync() {
|
||||
int fence_file_descriptor = -1;
|
||||
MP_RETURN_IF_ERROR(UnlockInternal(&fence_file_descriptor));
|
||||
return fence_file_descriptor;
|
||||
}
|
||||
|
||||
absl::Status HardwareBuffer::UnlockInternal(int *fence_file_descriptor) {
|
||||
RET_CHECK(ahw_buffer_ != nullptr) << "Hardware Buffer not allocated";
|
||||
if (!is_locked_) {
|
||||
return absl::OkStatus();
|
||||
}
|
||||
if (__builtin_available(android 26, *)) {
|
||||
const int error =
|
||||
AHardwareBuffer_unlock(ahw_buffer_, fence_file_descriptor);
|
||||
RET_CHECK(error == 0) << "Hardware Buffer unlock failed. error: " << error;
|
||||
}
|
||||
is_locked_ = false;
|
||||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
uint32_t HardwareBuffer::GetAlignedWidth() const {
|
||||
if (__builtin_available(android 26, *)) {
|
||||
ABSL_CHECK(ahw_buffer_ != nullptr) << "Hardware Buffer not allocated";
|
||||
AHardwareBuffer_Desc desc = {};
|
||||
AHardwareBuffer_describe(ahw_buffer_, &desc);
|
||||
ABSL_CHECK_GT(desc.stride, 0);
|
||||
return desc.stride;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void HardwareBuffer::Reset() {
|
||||
const auto success = ReleaseAHardwareBuffer();
|
||||
if (!success.ok()) {
|
||||
ABSL_LOG(DFATAL) << "Failed to release AHardwareBuffer: " << success;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // !defined(MEDIAPIPE_NO_JNI) && (__ANDROID_API__>= 26 ||
|
||||
// defined(__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__))
|
|
@ -1,131 +0,0 @@
|
|||
#include "mediapipe/framework/formats/hardware_buffer.h"
|
||||
|
||||
#include <android/hardware_buffer.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "base/logging.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
#include "testing/base/public/gmock.h"
|
||||
#include "testing/base/public/gunit.h"
|
||||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace {
|
||||
|
||||
HardwareBufferSpec GetTestHardwareBufferSpec(uint32_t size_bytes) {
|
||||
return {.width = size_bytes,
|
||||
.height = 1,
|
||||
.layers = 1,
|
||||
.format = HardwareBufferSpec::AHARDWAREBUFFER_FORMAT_BLOB,
|
||||
.usage = HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY |
|
||||
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN |
|
||||
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN |
|
||||
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER};
|
||||
}
|
||||
|
||||
TEST(HardwareBufferTest, ShouldConstructValidAHardwareBuffer) {
|
||||
MP_ASSERT_OK_AND_ASSIGN(
|
||||
HardwareBuffer hardware_buffer,
|
||||
HardwareBuffer::Create(GetTestHardwareBufferSpec(/*size_bytes=*/123)));
|
||||
EXPECT_NE(hardware_buffer.GetAHardwareBuffer(), nullptr);
|
||||
EXPECT_TRUE(hardware_buffer.IsValid());
|
||||
}
|
||||
|
||||
TEST(HardwareBufferTest, ShouldResetValidAHardwareBuffer) {
|
||||
MP_ASSERT_OK_AND_ASSIGN(
|
||||
HardwareBuffer hardware_buffer,
|
||||
HardwareBuffer::Create(GetTestHardwareBufferSpec(/*size_bytes=*/123)));
|
||||
EXPECT_TRUE(hardware_buffer.IsValid());
|
||||
EXPECT_NE(*hardware_buffer.Lock(
|
||||
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY),
|
||||
nullptr);
|
||||
EXPECT_TRUE(hardware_buffer.IsLocked());
|
||||
|
||||
hardware_buffer.Reset();
|
||||
|
||||
EXPECT_FALSE(hardware_buffer.IsValid());
|
||||
EXPECT_FALSE(hardware_buffer.IsLocked());
|
||||
}
|
||||
|
||||
TEST(HardwareBufferTest, ShouldAllocateRequestedBufferSize) {
|
||||
constexpr int kBufferSize = 123;
|
||||
const HardwareBufferSpec spec = GetTestHardwareBufferSpec(kBufferSize);
|
||||
MP_ASSERT_OK_AND_ASSIGN(HardwareBuffer hardware_buffer,
|
||||
HardwareBuffer::Create(spec));
|
||||
|
||||
EXPECT_TRUE(hardware_buffer.IsValid());
|
||||
if (__builtin_available(android 26, *)) {
|
||||
AHardwareBuffer_Desc desc;
|
||||
AHardwareBuffer_describe(hardware_buffer.GetAHardwareBuffer(), &desc);
|
||||
EXPECT_EQ(desc.width, spec.width);
|
||||
EXPECT_EQ(desc.height, spec.height);
|
||||
EXPECT_EQ(desc.layers, spec.layers);
|
||||
EXPECT_EQ(desc.format, spec.format);
|
||||
EXPECT_EQ(desc.usage, spec.usage);
|
||||
}
|
||||
EXPECT_EQ(hardware_buffer.spec().width, spec.width);
|
||||
EXPECT_EQ(hardware_buffer.spec().height, spec.height);
|
||||
EXPECT_EQ(hardware_buffer.spec().layers, spec.layers);
|
||||
EXPECT_EQ(hardware_buffer.spec().format, spec.format);
|
||||
EXPECT_EQ(hardware_buffer.spec().usage, spec.usage);
|
||||
}
|
||||
|
||||
TEST(HardwareBufferTest, ShouldSupportMoveConstructor) {
|
||||
constexpr int kBufferSize = 123;
|
||||
const auto spec = GetTestHardwareBufferSpec(kBufferSize);
|
||||
MP_ASSERT_OK_AND_ASSIGN(HardwareBuffer hardware_buffer_a,
|
||||
HardwareBuffer::Create(spec));
|
||||
EXPECT_TRUE(hardware_buffer_a.IsValid());
|
||||
void* const ahardware_buffer_ptr_a = hardware_buffer_a.GetAHardwareBuffer();
|
||||
EXPECT_NE(ahardware_buffer_ptr_a, nullptr);
|
||||
EXPECT_FALSE(hardware_buffer_a.IsLocked());
|
||||
MP_ASSERT_OK_AND_ASSIGN(
|
||||
void* const hardware_buffer_a_locked_ptr,
|
||||
hardware_buffer_a.Lock(
|
||||
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY));
|
||||
EXPECT_NE(hardware_buffer_a_locked_ptr, nullptr);
|
||||
EXPECT_TRUE(hardware_buffer_a.IsLocked());
|
||||
|
||||
HardwareBuffer hardware_buffer_b(std::move(hardware_buffer_a));
|
||||
|
||||
EXPECT_FALSE(hardware_buffer_a.IsValid());
|
||||
EXPECT_FALSE(hardware_buffer_a.IsLocked());
|
||||
void* const ahardware_buffer_ptr_b = hardware_buffer_b.GetAHardwareBuffer();
|
||||
EXPECT_EQ(ahardware_buffer_ptr_a, ahardware_buffer_ptr_b);
|
||||
EXPECT_TRUE(hardware_buffer_b.IsValid());
|
||||
EXPECT_TRUE(hardware_buffer_b.IsLocked());
|
||||
|
||||
EXPECT_EQ(hardware_buffer_a.spec(), HardwareBufferSpec());
|
||||
EXPECT_EQ(hardware_buffer_b.spec(), spec);
|
||||
|
||||
MP_ASSERT_OK(hardware_buffer_b.Unlock());
|
||||
}
|
||||
|
||||
TEST(HardwareBufferTest, ShouldSupportReadWrite) {
|
||||
constexpr std::string_view kTestString = "TestString";
|
||||
constexpr int kBufferSize = kTestString.size();
|
||||
MP_ASSERT_OK_AND_ASSIGN(
|
||||
HardwareBuffer hardware_buffer,
|
||||
HardwareBuffer::Create(GetTestHardwareBufferSpec(kBufferSize)));
|
||||
|
||||
// Write test string.
|
||||
MP_ASSERT_OK_AND_ASSIGN(
|
||||
void* const write_ptr,
|
||||
hardware_buffer.Lock(
|
||||
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY));
|
||||
memcpy(write_ptr, kTestString.data(), kBufferSize);
|
||||
MP_ASSERT_OK(hardware_buffer.Unlock());
|
||||
|
||||
// Read test string.
|
||||
MP_ASSERT_OK_AND_ASSIGN(
|
||||
void* const read_ptr,
|
||||
hardware_buffer.Lock(
|
||||
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_READ_RARELY));
|
||||
EXPECT_EQ(memcmp(read_ptr, kTestString.data(), kBufferSize), 0);
|
||||
MP_ASSERT_OK(hardware_buffer.Unlock());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
} // namespace mediapipe
|
|
@ -15,7 +15,6 @@
|
|||
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_H_
|
||||
#define MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
|
||||
#include "absl/synchronization/mutex.h"
|
||||
|
@ -208,7 +207,7 @@ inline void Image::UnlockPixels() const {}
|
|||
// Image buf = ...
|
||||
// {
|
||||
// PixelLock lock(&buf);
|
||||
// uint8_t* buf_ptr = lock.Pixels();
|
||||
// uint8* buf_ptr = lock.Pixels();
|
||||
// ... use buf_ptr to access pixel data ...
|
||||
// ... lock released automatically at end of scope ...
|
||||
// }
|
||||
|
@ -229,7 +228,7 @@ class PixelReadLock {
|
|||
}
|
||||
PixelReadLock(const PixelReadLock&) = delete;
|
||||
|
||||
const uint8_t* Pixels() const {
|
||||
const uint8* Pixels() const {
|
||||
if (frame_) return frame_->PixelData();
|
||||
return nullptr;
|
||||
}
|
||||
|
@ -255,7 +254,7 @@ class PixelWriteLock {
|
|||
}
|
||||
PixelWriteLock(const PixelWriteLock&) = delete;
|
||||
|
||||
uint8_t* Pixels() {
|
||||
uint8* Pixels() {
|
||||
if (frame_) return frame_->MutablePixelData();
|
||||
return nullptr;
|
||||
}
|
||||
|
|
|
@ -35,13 +35,13 @@
|
|||
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_H_
|
||||
#define MEDIAPIPE_FRAMEWORK_FORMATS_IMAGE_FRAME_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "mediapipe/framework/formats/image_format.pb.h"
|
||||
#include "mediapipe/framework/port.h"
|
||||
#include "mediapipe/framework/port/integral_types.h"
|
||||
#include "mediapipe/framework/tool/type_util.h"
|
||||
|
||||
#define IMAGE_FRAME_RAW_IMAGE MEDIAPIPE_HAS_RTTI
|
||||
|
@ -63,7 +63,7 @@ namespace mediapipe {
|
|||
// stored with row padding for alignment purposes.
|
||||
class ImageFrame {
|
||||
public:
|
||||
typedef std::function<void(uint8_t*)> Deleter;
|
||||
typedef std::function<void(uint8*)> Deleter;
|
||||
|
||||
// This class offers a few standard delete functions and retains
|
||||
// compatibility with the previous API.
|
||||
|
@ -78,13 +78,13 @@ class ImageFrame {
|
|||
// Use a default alignment boundary of 16 because Intel SSE2 instructions may
|
||||
// incur performance penalty when accessing data not aligned on a 16-byte
|
||||
// boundary. FFmpeg requires at least this level of alignment.
|
||||
static const uint32_t kDefaultAlignmentBoundary = 16;
|
||||
static const uint32 kDefaultAlignmentBoundary = 16;
|
||||
|
||||
// If the pixel data of an ImageFrame will be passed to an OpenGL function
|
||||
// such as glTexImage2D() or glReadPixels(), use a four-byte alignment
|
||||
// boundary because that is the initial value of the OpenGL GL_PACK_ALIGNMENT
|
||||
// and GL_UNPACK_ALIGNMENT parameters.
|
||||
static const uint32_t kGlDefaultAlignmentBoundary = 4;
|
||||
static const uint32 kGlDefaultAlignmentBoundary = 4;
|
||||
|
||||
// Returns number of channels for an ImageFormat.
|
||||
static int NumberOfChannelsForFormat(ImageFormat::Format format);
|
||||
|
@ -104,7 +104,7 @@ class ImageFrame {
|
|||
// must be a power of 2 (the number 1 is valid, and means the data will
|
||||
// be stored contiguously).
|
||||
ImageFrame(ImageFormat::Format format, int width, int height,
|
||||
uint32_t alignment_boundary);
|
||||
uint32 alignment_boundary);
|
||||
// Same as above, but use kDefaultAlignmentBoundary for alignment_boundary.
|
||||
ImageFrame(ImageFormat::Format format, int width, int height);
|
||||
|
||||
|
@ -115,8 +115,8 @@ class ImageFrame {
|
|||
// width*num_channels*depth. Both width_step and depth are in units
|
||||
// of bytes.
|
||||
ImageFrame(ImageFormat::Format format, int width, int height, int width_step,
|
||||
uint8_t* pixel_data,
|
||||
Deleter deleter = std::default_delete<uint8_t[]>());
|
||||
uint8* pixel_data,
|
||||
Deleter deleter = std::default_delete<uint8[]>());
|
||||
|
||||
ImageFrame(ImageFrame&& move_from);
|
||||
ImageFrame& operator=(ImageFrame&& move_from);
|
||||
|
@ -142,7 +142,7 @@ class ImageFrame {
|
|||
// alignment_boundary. If IsAligned(16) is true then so are
|
||||
// IsAligned(8), IsAligned(4), IsAligned(2), and IsAligned(1).
|
||||
// alignment_boundary must be 1 or a power of 2.
|
||||
bool IsAligned(uint32_t alignment_boundary) const;
|
||||
bool IsAligned(uint32 alignment_boundary) const;
|
||||
|
||||
// Returns the image / video format.
|
||||
ImageFormat::Format Format() const { return format_; }
|
||||
|
@ -167,13 +167,13 @@ class ImageFrame {
|
|||
// Reset the current image frame and copy the data from image_frame into
|
||||
// this image frame. The alignment_boundary must be given (and won't
|
||||
// necessarily match the alignment_boundary of the input image_frame).
|
||||
void CopyFrom(const ImageFrame& image_frame, uint32_t alignment_boundary);
|
||||
void CopyFrom(const ImageFrame& image_frame, uint32 alignment_boundary);
|
||||
|
||||
// Get a mutable pointer to the underlying image data. The ImageFrame
|
||||
// retains ownership.
|
||||
uint8_t* MutablePixelData() { return pixel_data_.get(); }
|
||||
uint8* MutablePixelData() { return pixel_data_.get(); }
|
||||
// Get a const pointer to the underlying image data.
|
||||
const uint8_t* PixelData() const { return pixel_data_.get(); }
|
||||
const uint8* PixelData() const { return pixel_data_.get(); }
|
||||
|
||||
// Returns the total size of the pixel data.
|
||||
int PixelDataSize() const { return Height() * WidthStep(); }
|
||||
|
@ -187,41 +187,41 @@ class ImageFrame {
|
|||
// ImageFrame takes ownership of pixel_data. See the Constructor
|
||||
// with the same arguments for details.
|
||||
void AdoptPixelData(ImageFormat::Format format, int width, int height,
|
||||
int width_step, uint8_t* pixel_data,
|
||||
Deleter deleter = std::default_delete<uint8_t[]>());
|
||||
int width_step, uint8* pixel_data,
|
||||
Deleter deleter = std::default_delete<uint8[]>());
|
||||
|
||||
// Resets the ImageFrame and makes it a copy of the provided pixel
|
||||
// data, which is assumed to be stored contiguously. The ImageFrame
|
||||
// will use the given alignment_boundary.
|
||||
void CopyPixelData(ImageFormat::Format format, int width, int height,
|
||||
const uint8_t* pixel_data, uint32_t alignment_boundary);
|
||||
const uint8* pixel_data, uint32 alignment_boundary);
|
||||
|
||||
// Resets the ImageFrame and makes it a copy of the provided pixel
|
||||
// data, with given width_step. The ImageFrame
|
||||
// will use the given alignment_boundary.
|
||||
void CopyPixelData(ImageFormat::Format format, int width, int height,
|
||||
int width_step, const uint8_t* pixel_data,
|
||||
uint32_t alignment_boundary);
|
||||
int width_step, const uint8* pixel_data,
|
||||
uint32 alignment_boundary);
|
||||
|
||||
// Allocates a frame of the specified format, width, height, and alignment,
|
||||
// without clearing any current pixel data. See the constructor with the same
|
||||
// argument list.
|
||||
void Reset(ImageFormat::Format format, int width, int height,
|
||||
uint32_t alignment_boundary);
|
||||
uint32 alignment_boundary);
|
||||
|
||||
// Relinquishes ownership of the pixel data. Notice that the unique_ptr
|
||||
// uses a non-standard deleter.
|
||||
std::unique_ptr<uint8_t[], Deleter> Release();
|
||||
std::unique_ptr<uint8[], Deleter> Release();
|
||||
|
||||
// Copy the 8-bit ImageFrame into a contiguous, pre-allocated buffer. Note
|
||||
// that ImageFrame does not necessarily store its data contiguously (i.e. do
|
||||
// not use copy_n to move image data).
|
||||
void CopyToBuffer(uint8_t* buffer, int buffer_size) const;
|
||||
void CopyToBuffer(uint8* buffer, int buffer_size) const;
|
||||
|
||||
// A version of CopyToBuffer for 16-bit pixel data. Note that buffer_size
|
||||
// stores the number of 16-bit elements in the buffer, not the number of
|
||||
// bytes.
|
||||
void CopyToBuffer(uint16_t* buffer, int buffer_size) const;
|
||||
void CopyToBuffer(uint16* buffer, int buffer_size) const;
|
||||
|
||||
// A version of CopyToBuffer for float pixel data. Note that buffer_size
|
||||
// stores the number of float elements in the buffer, not the number of
|
||||
|
@ -233,12 +233,12 @@ class ImageFrame {
|
|||
|
||||
private:
|
||||
// Returns true if alignment_number is 1 or a power of 2.
|
||||
static bool IsValidAlignmentNumber(uint32_t alignment_boundary);
|
||||
static bool IsValidAlignmentNumber(uint32 alignment_boundary);
|
||||
|
||||
// The internal implementation of copying data from the provided pixel data.
|
||||
// If width_step is 0, then calculates width_step assuming no padding.
|
||||
void InternalCopyFrom(int width, int height, int width_step, int channel_size,
|
||||
const uint8_t* pixel_data);
|
||||
const uint8* pixel_data);
|
||||
|
||||
// The internal implementation of copying data to the provided buffer.
|
||||
// If width_step is 0, then calculates width_step assuming no padding.
|
||||
|
@ -249,7 +249,7 @@ class ImageFrame {
|
|||
int height_;
|
||||
int width_step_;
|
||||
|
||||
std::unique_ptr<uint8_t[], Deleter> pixel_data_;
|
||||
std::unique_ptr<uint8[], Deleter> pixel_data_;
|
||||
};
|
||||
|
||||
} // namespace mediapipe
|
||||
|
|
|
@ -16,7 +16,6 @@
|
|||
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
|
||||
#include "absl/log/absl_check.h"
|
||||
|
|
|
@ -215,7 +215,7 @@ Location CreateCvMaskLocation(const cv::Mat_<T>& mask) {
|
|||
return Location(location_data);
|
||||
}
|
||||
|
||||
template Location CreateCvMaskLocation(const cv::Mat_<uint8_t>& mask);
|
||||
template Location CreateCvMaskLocation(const cv::Mat_<uint8>& mask);
|
||||
template Location CreateCvMaskLocation(const cv::Mat_<float>& mask);
|
||||
|
||||
} // namespace mediapipe
|
||||
|
|
|
@ -24,9 +24,6 @@
|
|||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
|
||||
#include "mediapipe/gpu/gl_base.h"
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
|
||||
#ifdef MEDIAPIPE_TENSOR_USE_AHWB
|
||||
#include "mediapipe/framework/formats/hardware_buffer.h"
|
||||
#endif // MEDIAPIPE_TENSOR_USE_AHWB
|
||||
|
||||
#if MEDIAPIPE_METAL_ENABLED
|
||||
#import <Metal/Metal.h>
|
||||
|
@ -342,14 +339,6 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferReadView() const {
|
|||
<< "Tensor conversion between different GPU backing formats is not "
|
||||
"supported yet.";
|
||||
auto lock(absl::make_unique<absl::MutexLock>(&view_mutex_));
|
||||
if ((valid_ & kValidOpenGlBuffer) && gl_context_ != nullptr &&
|
||||
!gl_context_->IsCurrent() && GlContext::IsAnyContextCurrent()) {
|
||||
ABSL_LOG_FIRST_N(WARNING, 1)
|
||||
<< "Tensor::GetOpenGlBufferReadView is not executed on the same GL "
|
||||
"context where GL buffer was created. Note that Tensor has "
|
||||
"limited synchronization support when sharing OpenGl objects "
|
||||
"between multiple OpenGL contexts.";
|
||||
}
|
||||
AllocateOpenGlBuffer();
|
||||
if (!(valid_ & kValidOpenGlBuffer)) {
|
||||
// If the call succeeds then AHWB -> SSBO are synchronized so any usage of
|
||||
|
@ -367,13 +356,7 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferReadView() const {
|
|||
}
|
||||
return {opengl_buffer_, std::move(lock),
|
||||
#ifdef MEDIAPIPE_TENSOR_USE_AHWB
|
||||
// ssbo_read_ is passed to be populated on OpenGlBufferView
|
||||
// destruction in order to perform delayed resources releasing (see
|
||||
// tensor_ahwb.cc/DelayedReleaser) only when AHWB is in use.
|
||||
//
|
||||
// Not passing for the case when AHWB is not in use to avoid creation
|
||||
// of unnecessary sync object and memory leak.
|
||||
use_ahwb_ ? &ssbo_read_ : nullptr
|
||||
&ssbo_read_
|
||||
#else
|
||||
nullptr
|
||||
#endif // MEDIAPIPE_TENSOR_USE_AHWB
|
||||
|
@ -384,14 +367,6 @@ Tensor::OpenGlBufferView Tensor::GetOpenGlBufferWriteView(
|
|||
uint64_t source_location_hash) const {
|
||||
auto lock(absl::make_unique<absl::MutexLock>(&view_mutex_));
|
||||
TrackAhwbUsage(source_location_hash);
|
||||
if ((valid_ & kValidOpenGlBuffer) && gl_context_ != nullptr &&
|
||||
!gl_context_->IsCurrent() && GlContext::IsAnyContextCurrent()) {
|
||||
ABSL_LOG_FIRST_N(WARNING, 1)
|
||||
<< "Tensor::GetOpenGlBufferWriteView is not executed on the same GL "
|
||||
"context where GL buffer was created. Note that Tensor has "
|
||||
"limited synchronization support when sharing OpenGl objects "
|
||||
"between multiple OpenGL contexts.";
|
||||
}
|
||||
AllocateOpenGlBuffer();
|
||||
valid_ = kValidOpenGlBuffer;
|
||||
return {opengl_buffer_, std::move(lock), nullptr};
|
||||
|
@ -561,8 +536,9 @@ Tensor::CpuReadView Tensor::GetCpuReadView() const {
|
|||
void* ptr = MapAhwbToCpuRead();
|
||||
if (ptr) {
|
||||
valid_ |= kValidCpu;
|
||||
return {ptr, std::move(lock), [ahwb = ahwb_.get()] {
|
||||
ABSL_CHECK_OK(ahwb->Unlock()) << "Unlock failed.";
|
||||
return {ptr, std::move(lock), [ahwb = ahwb_] {
|
||||
auto error = AHardwareBuffer_unlock(ahwb, nullptr);
|
||||
ABSL_CHECK(error == 0) << "AHardwareBuffer_unlock " << error;
|
||||
}};
|
||||
}
|
||||
}
|
||||
|
@ -644,11 +620,9 @@ Tensor::CpuWriteView Tensor::GetCpuWriteView(
|
|||
if (__builtin_available(android 26, *)) {
|
||||
void* ptr = MapAhwbToCpuWrite();
|
||||
if (ptr) {
|
||||
return {ptr, std::move(lock),
|
||||
[ahwb = ahwb_.get(), fence_fd = &fence_fd_] {
|
||||
auto fence_fd_status = ahwb->UnlockAsync();
|
||||
ABSL_CHECK_OK(fence_fd_status) << "Unlock failed.";
|
||||
*fence_fd = fence_fd_status.value();
|
||||
return {ptr, std::move(lock), [ahwb = ahwb_, fence_fd = &fence_fd_] {
|
||||
auto error = AHardwareBuffer_unlock(ahwb, fence_fd);
|
||||
ABSL_CHECK(error == 0) << "AHardwareBuffer_unlock " << error;
|
||||
}};
|
||||
}
|
||||
}
|
||||
|
|
|
@ -44,8 +44,7 @@
|
|||
#ifdef MEDIAPIPE_TENSOR_USE_AHWB
|
||||
#include <EGL/egl.h>
|
||||
#include <EGL/eglext.h>
|
||||
|
||||
#include "mediapipe/framework/formats/hardware_buffer.h"
|
||||
#include <android/hardware_buffer.h>
|
||||
#endif // MEDIAPIPE_TENSOR_USE_AHWB
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_30
|
||||
#include "mediapipe/gpu/gl_base.h"
|
||||
|
@ -196,11 +195,9 @@ class Tensor {
|
|||
using FinishingFunc = std::function<bool(bool)>;
|
||||
class AHardwareBufferView : public View {
|
||||
public:
|
||||
AHardwareBuffer* handle() const {
|
||||
return hardware_buffer_->GetAHardwareBuffer();
|
||||
}
|
||||
AHardwareBuffer* handle() const { return handle_; }
|
||||
AHardwareBufferView(AHardwareBufferView&& src) : View(std::move(src)) {
|
||||
hardware_buffer_ = std::move(src.hardware_buffer_);
|
||||
handle_ = std::exchange(src.handle_, nullptr);
|
||||
file_descriptor_ = src.file_descriptor_;
|
||||
fence_fd_ = std::exchange(src.fence_fd_, nullptr);
|
||||
ahwb_written_ = std::exchange(src.ahwb_written_, nullptr);
|
||||
|
@ -225,17 +222,17 @@ class Tensor {
|
|||
|
||||
protected:
|
||||
friend class Tensor;
|
||||
AHardwareBufferView(HardwareBuffer* hardware_buffer, int file_descriptor,
|
||||
AHardwareBufferView(AHardwareBuffer* handle, int file_descriptor,
|
||||
int* fence_fd, FinishingFunc* ahwb_written,
|
||||
std::function<void()>* release_callback,
|
||||
std::unique_ptr<absl::MutexLock>&& lock)
|
||||
: View(std::move(lock)),
|
||||
hardware_buffer_(hardware_buffer),
|
||||
handle_(handle),
|
||||
file_descriptor_(file_descriptor),
|
||||
fence_fd_(fence_fd),
|
||||
ahwb_written_(ahwb_written),
|
||||
release_callback_(release_callback) {}
|
||||
HardwareBuffer* hardware_buffer_;
|
||||
AHardwareBuffer* handle_;
|
||||
int file_descriptor_;
|
||||
// The view sets some Tensor's fields. The view is released prior to tensor.
|
||||
int* fence_fd_;
|
||||
|
@ -288,22 +285,18 @@ class Tensor {
|
|||
class OpenGlBufferView : public View {
|
||||
public:
|
||||
GLuint name() const { return name_; }
|
||||
|
||||
OpenGlBufferView(OpenGlBufferView&& src) : View(std::move(src)) {
|
||||
name_ = std::exchange(src.name_, GL_INVALID_INDEX);
|
||||
ssbo_read_ = std::exchange(src.ssbo_read_, nullptr);
|
||||
}
|
||||
~OpenGlBufferView() {
|
||||
if (ssbo_read_) {
|
||||
// TODO: update tensor to properly handle cases when
|
||||
// multiple views were requested multiple sync fence may be needed.
|
||||
*ssbo_read_ = glFenceSync(GL_SYNC_GPU_COMMANDS_COMPLETE, 0);
|
||||
}
|
||||
}
|
||||
|
||||
protected:
|
||||
friend class Tensor;
|
||||
|
||||
OpenGlBufferView(GLuint name, std::unique_ptr<absl::MutexLock>&& lock,
|
||||
GLsync* ssbo_read)
|
||||
: View(std::move(lock)), name_(name), ssbo_read_(ssbo_read) {}
|
||||
|
@ -391,7 +384,7 @@ class Tensor {
|
|||
mutable std::unique_ptr<MtlResources> mtl_resources_;
|
||||
|
||||
#ifdef MEDIAPIPE_TENSOR_USE_AHWB
|
||||
mutable std::unique_ptr<HardwareBuffer> ahwb_;
|
||||
mutable AHardwareBuffer* ahwb_ = nullptr;
|
||||
// Signals when GPU finished writing into SSBO so AHWB can be used then. Or
|
||||
// signals when writing into AHWB has been finished so GPU can read from SSBO.
|
||||
// Sync and FD are bound together.
|
||||
|
|
|
@ -10,7 +10,7 @@
|
|||
#include "absl/log/absl_check.h"
|
||||
#include "absl/log/absl_log.h"
|
||||
#include "absl/synchronization/mutex.h"
|
||||
#include "mediapipe/framework/formats/hardware_buffer.h"
|
||||
#include "mediapipe/framework/port.h"
|
||||
#include "mediapipe/gpu/gl_base.h"
|
||||
#endif // MEDIAPIPE_TENSOR_USE_AHWB
|
||||
|
||||
|
@ -97,7 +97,7 @@ class DelayedReleaser {
|
|||
DelayedReleaser(DelayedReleaser&&) = delete;
|
||||
DelayedReleaser& operator=(DelayedReleaser&&) = delete;
|
||||
|
||||
static void Add(std::unique_ptr<HardwareBuffer> ahwb, GLuint opengl_buffer,
|
||||
static void Add(AHardwareBuffer* ahwb, GLuint opengl_buffer,
|
||||
EGLSyncKHR ssbo_sync, GLsync ssbo_read,
|
||||
Tensor::FinishingFunc&& ahwb_written,
|
||||
std::shared_ptr<mediapipe::GlContext> gl_context,
|
||||
|
@ -115,8 +115,8 @@ class DelayedReleaser {
|
|||
|
||||
// Using `new` to access a non-public constructor.
|
||||
to_release_local.emplace_back(absl::WrapUnique(new DelayedReleaser(
|
||||
std::move(ahwb), opengl_buffer, ssbo_sync, ssbo_read,
|
||||
std::move(ahwb_written), gl_context, std::move(callback))));
|
||||
ahwb, opengl_buffer, ssbo_sync, ssbo_read, std::move(ahwb_written),
|
||||
gl_context, std::move(callback))));
|
||||
for (auto it = to_release_local.begin(); it != to_release_local.end();) {
|
||||
if ((*it)->IsSignaled()) {
|
||||
it = to_release_local.erase(it);
|
||||
|
@ -136,6 +136,9 @@ class DelayedReleaser {
|
|||
|
||||
~DelayedReleaser() {
|
||||
if (release_callback_) release_callback_();
|
||||
if (__builtin_available(android 26, *)) {
|
||||
AHardwareBuffer_release(ahwb_);
|
||||
}
|
||||
}
|
||||
|
||||
bool IsSignaled() {
|
||||
|
@ -178,7 +181,7 @@ class DelayedReleaser {
|
|||
}
|
||||
|
||||
protected:
|
||||
std::unique_ptr<HardwareBuffer> ahwb_;
|
||||
AHardwareBuffer* ahwb_;
|
||||
GLuint opengl_buffer_;
|
||||
// TODO: use wrapper instead.
|
||||
EGLSyncKHR fence_sync_;
|
||||
|
@ -189,12 +192,12 @@ class DelayedReleaser {
|
|||
std::function<void()> release_callback_;
|
||||
static inline std::deque<std::unique_ptr<DelayedReleaser>> to_release_;
|
||||
|
||||
DelayedReleaser(std::unique_ptr<HardwareBuffer> ahwb, GLuint opengl_buffer,
|
||||
DelayedReleaser(AHardwareBuffer* ahwb, GLuint opengl_buffer,
|
||||
EGLSyncKHR fence_sync, GLsync ssbo_read,
|
||||
Tensor::FinishingFunc&& ahwb_written,
|
||||
std::shared_ptr<mediapipe::GlContext> gl_context,
|
||||
std::function<void()>&& callback)
|
||||
: ahwb_(std::move(ahwb)),
|
||||
: ahwb_(ahwb),
|
||||
opengl_buffer_(opengl_buffer),
|
||||
fence_sync_(fence_sync),
|
||||
ssbo_read_(ssbo_read),
|
||||
|
@ -211,7 +214,7 @@ Tensor::AHardwareBufferView Tensor::GetAHardwareBufferReadView() const {
|
|||
ABSL_CHECK(!(valid_ & kValidOpenGlTexture2d))
|
||||
<< "Tensor conversion between OpenGL texture and AHardwareBuffer is not "
|
||||
"supported.";
|
||||
bool transfer = ahwb_ == nullptr;
|
||||
bool transfer = !ahwb_;
|
||||
ABSL_CHECK(AllocateAHardwareBuffer())
|
||||
<< "AHardwareBuffer is not supported on the target system.";
|
||||
valid_ |= kValidAHardwareBuffer;
|
||||
|
@ -220,10 +223,12 @@ Tensor::AHardwareBufferView Tensor::GetAHardwareBufferReadView() const {
|
|||
} else {
|
||||
if (valid_ & kValidOpenGlBuffer) CreateEglSyncAndFd();
|
||||
}
|
||||
return {ahwb_.get(), ssbo_written_,
|
||||
return {ahwb_,
|
||||
ssbo_written_,
|
||||
&fence_fd_, // The FD is created for SSBO -> AHWB synchronization.
|
||||
&ahwb_written_, // Filled by SetReadingFinishedFunc.
|
||||
&release_callback_, std::move(lock)};
|
||||
&release_callback_,
|
||||
std::move(lock)};
|
||||
}
|
||||
|
||||
void Tensor::CreateEglSyncAndFd() const {
|
||||
|
@ -253,11 +258,12 @@ Tensor::AHardwareBufferView Tensor::GetAHardwareBufferWriteView(
|
|||
ABSL_CHECK(AllocateAHardwareBuffer(size_alignment))
|
||||
<< "AHardwareBuffer is not supported on the target system.";
|
||||
valid_ = kValidAHardwareBuffer;
|
||||
return {ahwb_.get(),
|
||||
return {ahwb_,
|
||||
/*ssbo_written=*/-1,
|
||||
&fence_fd_, // For SetWritingFinishedFD.
|
||||
&ahwb_written_, // Filled by SetReadingFinishedFunc.
|
||||
&release_callback_, std::move(lock)};
|
||||
&fence_fd_, // For SetWritingFinishedFD.
|
||||
&ahwb_written_,
|
||||
&release_callback_,
|
||||
std::move(lock)};
|
||||
}
|
||||
|
||||
bool Tensor::AllocateAHardwareBuffer(int size_alignment) const {
|
||||
|
@ -270,43 +276,40 @@ bool Tensor::AllocateAHardwareBuffer(int size_alignment) const {
|
|||
}
|
||||
use_ahwb_ = true;
|
||||
|
||||
if (ahwb_ == nullptr) {
|
||||
HardwareBufferSpec spec = {};
|
||||
if (size_alignment == 0) {
|
||||
spec.width = bytes();
|
||||
} else {
|
||||
// We expect allocations to be page-aligned, implicitly satisfying any
|
||||
// requirements from Edge TPU. No need to add a check for this,
|
||||
// since Edge TPU will check for us.
|
||||
spec.width = AlignedToPowerOf2(bytes(), size_alignment);
|
||||
if (__builtin_available(android 26, *)) {
|
||||
if (ahwb_ == nullptr) {
|
||||
AHardwareBuffer_Desc desc = {};
|
||||
if (size_alignment == 0) {
|
||||
desc.width = bytes();
|
||||
} else {
|
||||
// We expect allocations to be page-aligned, implicitly satisfying any
|
||||
// requirements from Edge TPU. No need to add a check for this,
|
||||
// since Edge TPU will check for us.
|
||||
desc.width = AlignedToPowerOf2(bytes(), size_alignment);
|
||||
}
|
||||
desc.height = 1;
|
||||
desc.layers = 1;
|
||||
desc.format = AHARDWAREBUFFER_FORMAT_BLOB;
|
||||
desc.usage = AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN |
|
||||
AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN |
|
||||
AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER;
|
||||
return AHardwareBuffer_allocate(&desc, &ahwb_) == 0;
|
||||
}
|
||||
spec.height = 1;
|
||||
spec.layers = 1;
|
||||
spec.format = HardwareBufferSpec::AHARDWAREBUFFER_FORMAT_BLOB;
|
||||
spec.usage = HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN |
|
||||
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN |
|
||||
HardwareBufferSpec::AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER;
|
||||
auto new_ahwb = HardwareBuffer::Create(spec);
|
||||
if (!new_ahwb.ok()) {
|
||||
ABSL_LOG(ERROR) << "Allocation of NDK Hardware Buffer failed: "
|
||||
<< new_ahwb.status();
|
||||
return false;
|
||||
}
|
||||
ahwb_ = std::make_unique<HardwareBuffer>(std::move(*new_ahwb));
|
||||
return true;
|
||||
}
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Tensor::AllocateAhwbMapToSsbo() const {
|
||||
if (__builtin_available(android 26, *)) {
|
||||
if (AllocateAHardwareBuffer()) {
|
||||
if (MapAHardwareBufferToGlBuffer(ahwb_->GetAHardwareBuffer(), bytes())
|
||||
.ok()) {
|
||||
if (MapAHardwareBufferToGlBuffer(ahwb_, bytes()).ok()) {
|
||||
glBindBuffer(GL_SHADER_STORAGE_BUFFER, 0);
|
||||
return true;
|
||||
}
|
||||
// Unable to make OpenGL <-> AHWB binding. Use regular SSBO instead.
|
||||
ahwb_.reset();
|
||||
AHardwareBuffer_release(ahwb_);
|
||||
ahwb_ = nullptr;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
|
@ -314,11 +317,14 @@ bool Tensor::AllocateAhwbMapToSsbo() const {
|
|||
|
||||
// Moves Cpu/Ssbo resource under the Ahwb backed memory.
|
||||
void Tensor::MoveCpuOrSsboToAhwb() const {
|
||||
auto dest =
|
||||
ahwb_->Lock(HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY);
|
||||
ABSL_CHECK_OK(dest) << "Lock of AHWB failed";
|
||||
void* dest = nullptr;
|
||||
if (__builtin_available(android 26, *)) {
|
||||
auto error = AHardwareBuffer_lock(
|
||||
ahwb_, AHARDWAREBUFFER_USAGE_CPU_WRITE_RARELY, -1, nullptr, &dest);
|
||||
ABSL_CHECK(error == 0) << "AHardwareBuffer_lock " << error;
|
||||
}
|
||||
if (valid_ & kValidCpu) {
|
||||
std::memcpy(*dest, cpu_buffer_, bytes());
|
||||
std::memcpy(dest, cpu_buffer_, bytes());
|
||||
// Free CPU memory because next time AHWB is mapped instead.
|
||||
free(cpu_buffer_);
|
||||
cpu_buffer_ = nullptr;
|
||||
|
@ -328,7 +334,7 @@ void Tensor::MoveCpuOrSsboToAhwb() const {
|
|||
glBindBuffer(GL_SHADER_STORAGE_BUFFER, opengl_buffer_);
|
||||
const void* src = glMapBufferRange(GL_SHADER_STORAGE_BUFFER, 0, bytes(),
|
||||
GL_MAP_READ_BIT);
|
||||
std::memcpy(*dest, src, bytes());
|
||||
std::memcpy(dest, src, bytes());
|
||||
glUnmapBuffer(GL_SHADER_STORAGE_BUFFER);
|
||||
glDeleteBuffers(1, &opengl_buffer_);
|
||||
});
|
||||
|
@ -341,7 +347,10 @@ void Tensor::MoveCpuOrSsboToAhwb() const {
|
|||
ABSL_LOG(FATAL) << "Can't convert tensor with mask " << valid_
|
||||
<< " into AHWB.";
|
||||
}
|
||||
ABSL_CHECK_OK(ahwb_->Unlock()) << "Unlock of AHWB failed";
|
||||
if (__builtin_available(android 26, *)) {
|
||||
auto error = AHardwareBuffer_unlock(ahwb_, nullptr);
|
||||
ABSL_CHECK(error == 0) << "AHardwareBuffer_unlock " << error;
|
||||
}
|
||||
}
|
||||
|
||||
// SSBO is created on top of AHWB. A fence is inserted into the GPU queue before
|
||||
|
@ -394,52 +403,59 @@ void Tensor::ReleaseAhwbStuff() {
|
|||
if (ahwb_) {
|
||||
if (ssbo_read_ != 0 || fence_sync_ != EGL_NO_SYNC_KHR || ahwb_written_) {
|
||||
if (ssbo_written_ != -1) close(ssbo_written_);
|
||||
DelayedReleaser::Add(std::move(ahwb_), opengl_buffer_, fence_sync_,
|
||||
ssbo_read_, std::move(ahwb_written_), gl_context_,
|
||||
DelayedReleaser::Add(ahwb_, opengl_buffer_, fence_sync_, ssbo_read_,
|
||||
std::move(ahwb_written_), gl_context_,
|
||||
std::move(release_callback_));
|
||||
opengl_buffer_ = GL_INVALID_INDEX;
|
||||
} else {
|
||||
if (release_callback_) release_callback_();
|
||||
ahwb_.reset();
|
||||
AHardwareBuffer_release(ahwb_);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void* Tensor::MapAhwbToCpuRead() const {
|
||||
if (ahwb_ != nullptr) {
|
||||
if (!(valid_ & kValidCpu)) {
|
||||
if ((valid_ & kValidOpenGlBuffer) && ssbo_written_ == -1) {
|
||||
// EGLSync is failed. Use another synchronization method.
|
||||
// TODO: Use tflite::gpu::GlBufferSync and GlActiveSync.
|
||||
gl_context_->Run([]() { glFinish(); });
|
||||
} else if (valid_ & kValidAHardwareBuffer) {
|
||||
ABSL_CHECK(ahwb_written_) << "Ahwb-to-Cpu synchronization requires the "
|
||||
"completion function to be set";
|
||||
ABSL_CHECK(ahwb_written_(true))
|
||||
<< "An error oqcured while waiting for the buffer to be written";
|
||||
if (__builtin_available(android 26, *)) {
|
||||
if (ahwb_) {
|
||||
if (!(valid_ & kValidCpu)) {
|
||||
if ((valid_ & kValidOpenGlBuffer) && ssbo_written_ == -1) {
|
||||
// EGLSync is failed. Use another synchronization method.
|
||||
// TODO: Use tflite::gpu::GlBufferSync and GlActiveSync.
|
||||
gl_context_->Run([]() { glFinish(); });
|
||||
} else if (valid_ & kValidAHardwareBuffer) {
|
||||
ABSL_CHECK(ahwb_written_)
|
||||
<< "Ahwb-to-Cpu synchronization requires the "
|
||||
"completion function to be set";
|
||||
ABSL_CHECK(ahwb_written_(true))
|
||||
<< "An error oqcured while waiting for the buffer to be written";
|
||||
}
|
||||
}
|
||||
void* ptr;
|
||||
auto error =
|
||||
AHardwareBuffer_lock(ahwb_, AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN,
|
||||
ssbo_written_, nullptr, &ptr);
|
||||
ABSL_CHECK(error == 0) << "AHardwareBuffer_lock " << error;
|
||||
close(ssbo_written_);
|
||||
ssbo_written_ = -1;
|
||||
return ptr;
|
||||
}
|
||||
auto ptr =
|
||||
ahwb_->Lock(HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN,
|
||||
ssbo_written_);
|
||||
ABSL_CHECK_OK(ptr) << "Lock of AHWB failed";
|
||||
close(ssbo_written_);
|
||||
ssbo_written_ = -1;
|
||||
return *ptr;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void* Tensor::MapAhwbToCpuWrite() const {
|
||||
if (ahwb_ != nullptr) {
|
||||
// TODO: If previously acquired view is GPU write view then need
|
||||
// to be sure that writing is finished. That's a warning: two consequent
|
||||
// write views should be interleaved with read view.
|
||||
auto locked_ptr =
|
||||
ahwb_->Lock(HardwareBufferSpec::AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN);
|
||||
ABSL_CHECK_OK(locked_ptr) << "Lock of AHWB failed";
|
||||
return *locked_ptr;
|
||||
if (__builtin_available(android 26, *)) {
|
||||
if (ahwb_) {
|
||||
// TODO: If previously acquired view is GPU write view then need
|
||||
// to be sure that writing is finished. That's a warning: two consequent
|
||||
// write views should be interleaved with read view.
|
||||
void* ptr;
|
||||
auto error = AHardwareBuffer_lock(
|
||||
ahwb_, AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN, -1, nullptr, &ptr);
|
||||
ABSL_CHECK(error == 0) << "AHardwareBuffer_lock " << error;
|
||||
return ptr;
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
|
|
@ -6,7 +6,6 @@
|
|||
|
||||
#include <cstdint>
|
||||
|
||||
#include "absl/algorithm/container.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/formats/tensor/views/data_types.h"
|
||||
#include "mediapipe/gpu/gpu_test_base.h"
|
||||
|
@ -19,7 +18,7 @@
|
|||
// Then the test requests the CPU view and compares the values.
|
||||
// Float32 and Float16 tests are there.
|
||||
|
||||
namespace mediapipe {
|
||||
namespace {
|
||||
|
||||
using mediapipe::Float16;
|
||||
using mediapipe::Tensor;
|
||||
|
@ -28,16 +27,6 @@ MATCHER_P(NearWithPrecision, precision, "") {
|
|||
return std::abs(std::get<0>(arg) - std::get<1>(arg)) < precision;
|
||||
}
|
||||
|
||||
template <typename F = float>
|
||||
std::vector<F> CreateReferenceData(int num_elements) {
|
||||
std::vector<F> reference;
|
||||
reference.resize(num_elements);
|
||||
for (int i = 0; i < num_elements; i++) {
|
||||
reference[i] = static_cast<float>(i) / 10.0f;
|
||||
}
|
||||
return reference;
|
||||
}
|
||||
|
||||
#if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
|
||||
// Utility function to fill the GPU buffer.
|
||||
|
@ -121,7 +110,11 @@ TEST_F(TensorAhwbGpuTest, TestGpuToCpuFloat32) {
|
|||
});
|
||||
auto ptr = tensor.GetCpuReadView().buffer<float>();
|
||||
ASSERT_NE(ptr, nullptr);
|
||||
std::vector<float> reference = CreateReferenceData(num_elements);
|
||||
std::vector<float> reference;
|
||||
reference.resize(num_elements);
|
||||
for (int i = 0; i < num_elements; i++) {
|
||||
reference[i] = static_cast<float>(i) / 10.0f;
|
||||
}
|
||||
EXPECT_THAT(absl::Span<const float>(ptr, num_elements),
|
||||
testing::Pointwise(testing::FloatEq(), reference));
|
||||
}
|
||||
|
@ -144,7 +137,11 @@ TEST_F(TensorAhwbGpuTest, TestGpuToCpuFloat16) {
|
|||
});
|
||||
auto ptr = tensor.GetCpuReadView().buffer<Float16>();
|
||||
ASSERT_NE(ptr, nullptr);
|
||||
std::vector<Float16> reference = CreateReferenceData<Float16>(num_elements);
|
||||
std::vector<Float16> reference;
|
||||
reference.resize(num_elements);
|
||||
for (int i = 0; i < num_elements; i++) {
|
||||
reference[i] = static_cast<float>(i) / 10.0f;
|
||||
}
|
||||
// Precision is set to a reasonable value for Float16.
|
||||
EXPECT_THAT(absl::Span<const Float16>(ptr, num_elements),
|
||||
testing::Pointwise(NearWithPrecision(0.001), reference));
|
||||
|
@ -169,7 +166,11 @@ TEST_F(TensorAhwbGpuTest, TestReplacingCpuByAhwb) {
|
|||
}
|
||||
auto ptr = tensor.GetCpuReadView().buffer<float>();
|
||||
ASSERT_NE(ptr, nullptr);
|
||||
std::vector<float> reference = CreateReferenceData(num_elements);
|
||||
std::vector<float> reference;
|
||||
reference.resize(num_elements);
|
||||
for (int i = 0; i < num_elements; i++) {
|
||||
reference[i] = static_cast<float>(i) / 10.0f;
|
||||
}
|
||||
EXPECT_THAT(absl::Span<const float>(ptr, num_elements),
|
||||
testing::Pointwise(testing::FloatEq(), reference));
|
||||
}
|
||||
|
@ -193,107 +194,17 @@ TEST_F(TensorAhwbGpuTest, TestReplacingGpuByAhwb) {
|
|||
}
|
||||
auto ptr = tensor.GetCpuReadView().buffer<float>();
|
||||
ASSERT_NE(ptr, nullptr);
|
||||
std::vector<float> reference = CreateReferenceData(num_elements);
|
||||
std::vector<float> reference;
|
||||
reference.resize(num_elements);
|
||||
for (int i = 0; i < num_elements; i++) {
|
||||
reference[i] = static_cast<float>(i) / 10.0f;
|
||||
}
|
||||
EXPECT_THAT(absl::Span<const float>(ptr, num_elements),
|
||||
testing::Pointwise(testing::FloatEq(), reference));
|
||||
}
|
||||
|
||||
std::vector<float> ReadGlBufferView(const Tensor::OpenGlBufferView& view,
|
||||
int num_elements) {
|
||||
glBindBuffer(GL_SHADER_STORAGE_BUFFER, view.name());
|
||||
int bytes = num_elements * sizeof(float);
|
||||
void* ptr =
|
||||
glMapBufferRange(GL_SHADER_STORAGE_BUFFER, 0, bytes, GL_MAP_READ_BIT);
|
||||
ABSL_CHECK(ptr) << "glMapBufferRange failed: " << glGetError();
|
||||
|
||||
std::vector<float> data;
|
||||
data.resize(num_elements);
|
||||
std::memcpy(data.data(), ptr, bytes);
|
||||
glUnmapBuffer(GL_SHADER_STORAGE_BUFFER);
|
||||
return data;
|
||||
}
|
||||
|
||||
TEST_F(TensorAhwbGpuTest, TestGetOpenGlBufferReadViewNoAhwb) {
|
||||
constexpr size_t kNumElements = 20;
|
||||
std::vector<float> reference = CreateReferenceData(kNumElements);
|
||||
|
||||
Tensor tensor(Tensor::ElementType::kFloat32, Tensor::Shape({kNumElements}));
|
||||
{
|
||||
// Populate tensor on CPU and make sure view is destroyed
|
||||
absl::c_copy(reference, tensor.GetCpuWriteView().buffer<float>());
|
||||
}
|
||||
|
||||
RunInGlContext([&] {
|
||||
// Triggers conversion to GL buffer.
|
||||
auto ssbo_view = tensor.GetOpenGlBufferReadView();
|
||||
ASSERT_NE(ssbo_view.name(), 0);
|
||||
// ssbo_read_ must NOT be populated, as there's no AHWB associated with
|
||||
// GL buffer
|
||||
ASSERT_EQ(ssbo_view.ssbo_read_, nullptr);
|
||||
|
||||
std::vector<float> output = ReadGlBufferView(ssbo_view, kNumElements);
|
||||
EXPECT_THAT(output, testing::Pointwise(testing::FloatEq(), reference));
|
||||
});
|
||||
}
|
||||
|
||||
TEST_F(TensorAhwbGpuTest, TestGetOpenGlBufferReadViewAhwbFromCpu) {
|
||||
constexpr size_t kNumElements = 20;
|
||||
std::vector<float> reference = CreateReferenceData(kNumElements);
|
||||
|
||||
Tensor tensor(Tensor::ElementType::kFloat32, Tensor::Shape({kNumElements}));
|
||||
{
|
||||
// Populate tensor on CPU and make sure view is destroyed
|
||||
absl::c_copy(reference, tensor.GetCpuWriteView().buffer<float>());
|
||||
}
|
||||
|
||||
{
|
||||
// Make tensor to allocate ahwb and make sure view is destroyed.
|
||||
ASSERT_NE(tensor.GetAHardwareBufferReadView().handle(), nullptr);
|
||||
}
|
||||
|
||||
RunInGlContext([&] {
|
||||
// Triggers conversion to GL buffer.
|
||||
auto ssbo_view = tensor.GetOpenGlBufferReadView();
|
||||
ASSERT_NE(ssbo_view.name(), 0);
|
||||
// ssbo_read_ must be populated, so during view destruction it's set
|
||||
// properly for further AHWB destruction
|
||||
ASSERT_NE(ssbo_view.ssbo_read_, nullptr);
|
||||
|
||||
std::vector<float> output = ReadGlBufferView(ssbo_view, kNumElements);
|
||||
EXPECT_THAT(output, testing::Pointwise(testing::FloatEq(), reference));
|
||||
});
|
||||
}
|
||||
|
||||
TEST_F(TensorAhwbGpuTest, TestGetOpenGlBufferReadViewAhwbFromGpu) {
|
||||
constexpr size_t kNumElements = 20;
|
||||
std::vector<float> reference = CreateReferenceData(kNumElements);
|
||||
|
||||
Tensor tensor(Tensor::ElementType::kFloat32, Tensor::Shape({kNumElements}));
|
||||
{
|
||||
// Make tensor to allocate ahwb and make sure view is destroyed.
|
||||
ASSERT_NE(tensor.GetAHardwareBufferWriteView().handle(), nullptr);
|
||||
}
|
||||
|
||||
RunInGlContext([&] {
|
||||
FillGpuBuffer(tensor.GetOpenGlBufferWriteView().name(),
|
||||
tensor.shape().num_elements(), tensor.element_type());
|
||||
});
|
||||
|
||||
RunInGlContext([&] {
|
||||
// Triggers conversion to GL buffer.
|
||||
auto ssbo_view = tensor.GetOpenGlBufferReadView();
|
||||
ASSERT_NE(ssbo_view.name(), 0);
|
||||
// ssbo_read_ must be populated, so during view destruction it's set
|
||||
// properly for further AHWB destruction
|
||||
ASSERT_NE(ssbo_view.ssbo_read_, nullptr);
|
||||
|
||||
std::vector<float> output = ReadGlBufferView(ssbo_view, kNumElements);
|
||||
EXPECT_THAT(output, testing::Pointwise(testing::FloatEq(), reference));
|
||||
});
|
||||
}
|
||||
|
||||
#endif // MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31
|
||||
} // namespace mediapipe
|
||||
} // namespace
|
||||
|
||||
#endif // !defined(MEDIAPIPE_NO_JNI) && (__ANDROID_API__ >= 26 ||
|
||||
// defined(__ANDROID_UNAVAILABLE_SYMBOLS_ARE_WEAK__))
|
||||
|
|
|
@ -1,5 +1,3 @@
|
|||
#include <android/hardware_buffer.h>
|
||||
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "testing/base/public/gmock.h"
|
||||
#include "testing/base/public/gunit.h"
|
||||
|
|
|
@ -15,7 +15,6 @@
|
|||
#ifndef MEDIAPIPE_FRAMEWORK_FORMATS_YUV_IMAGE_H_
|
||||
#define MEDIAPIPE_FRAMEWORK_FORMATS_YUV_IMAGE_H_
|
||||
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
|
||||
|
@ -120,13 +119,13 @@ class YUVImage {
|
|||
~YUVImage() { Clear(); }
|
||||
|
||||
// Convenience constructor
|
||||
YUVImage(libyuv::FourCC fourcc, //
|
||||
std::unique_ptr<uint8_t[]> data_location, //
|
||||
uint8_t* data0, int stride0, //
|
||||
uint8_t* data1, int stride1, //
|
||||
uint8_t* data2, int stride2, //
|
||||
YUVImage(libyuv::FourCC fourcc, //
|
||||
std::unique_ptr<uint8[]> data_location, //
|
||||
uint8* data0, int stride0, //
|
||||
uint8* data1, int stride1, //
|
||||
uint8* data2, int stride2, //
|
||||
int width, int height, int bit_depth = 8) {
|
||||
uint8_t* tmp = data_location.release();
|
||||
uint8* tmp = data_location.release();
|
||||
std::function<void()> deallocate = [tmp]() { delete[] tmp; };
|
||||
Initialize(fourcc, //
|
||||
deallocate, //
|
||||
|
@ -138,14 +137,14 @@ class YUVImage {
|
|||
|
||||
// Convenience constructor to construct the YUVImage with data stored
|
||||
// in three unique_ptrs.
|
||||
YUVImage(libyuv::FourCC fourcc, //
|
||||
std::unique_ptr<uint8_t[]> data0, int stride0, //
|
||||
std::unique_ptr<uint8_t[]> data1, int stride1, //
|
||||
std::unique_ptr<uint8_t[]> data2, int stride2, //
|
||||
YUVImage(libyuv::FourCC fourcc, //
|
||||
std::unique_ptr<uint8[]> data0, int stride0, //
|
||||
std::unique_ptr<uint8[]> data1, int stride1, //
|
||||
std::unique_ptr<uint8[]> data2, int stride2, //
|
||||
int width, int height, int bit_depth = 8) {
|
||||
uint8_t* tmp0 = data0.release();
|
||||
uint8_t* tmp1 = data1.release();
|
||||
uint8_t* tmp2 = data2.release();
|
||||
uint8* tmp0 = data0.release();
|
||||
uint8* tmp1 = data1.release();
|
||||
uint8* tmp2 = data2.release();
|
||||
std::function<void()> deallocate = [tmp0, tmp1, tmp2]() {
|
||||
delete[] tmp0;
|
||||
delete[] tmp1;
|
||||
|
@ -178,9 +177,9 @@ class YUVImage {
|
|||
// pixel format it holds.
|
||||
void Initialize(libyuv::FourCC fourcc, //
|
||||
std::function<void()> deallocation_function, //
|
||||
uint8_t* data0, int stride0, //
|
||||
uint8_t* data1, int stride1, //
|
||||
uint8_t* data2, int stride2, //
|
||||
uint8* data0, int stride0, //
|
||||
uint8* data1, int stride1, //
|
||||
uint8* data2, int stride2, //
|
||||
int width, int height, int bit_depth = 8) {
|
||||
Clear();
|
||||
deallocation_function_ = deallocation_function;
|
||||
|
@ -215,7 +214,7 @@ class YUVImage {
|
|||
|
||||
// Getters.
|
||||
libyuv::FourCC fourcc() const { return fourcc_; }
|
||||
const uint8_t* data(int index) const { return data_[index]; }
|
||||
const uint8* data(int index) const { return data_[index]; }
|
||||
int stride(int index) const { return stride_[index]; }
|
||||
int width() const { return width_; }
|
||||
int height() const { return height_; }
|
||||
|
@ -227,7 +226,7 @@ class YUVImage {
|
|||
|
||||
// Setters.
|
||||
void set_fourcc(libyuv::FourCC fourcc) { fourcc_ = fourcc; }
|
||||
uint8_t* mutable_data(int index) { return data_[index]; }
|
||||
uint8* mutable_data(int index) { return data_[index]; }
|
||||
void set_stride(int index, int stride) { stride_[index] = stride; }
|
||||
void set_width(int width) { width_ = width; }
|
||||
void set_height(int height) { height_ = height; }
|
||||
|
@ -242,7 +241,7 @@ class YUVImage {
|
|||
std::function<void()> deallocation_function_;
|
||||
|
||||
libyuv::FourCC fourcc_ = libyuv::FOURCC_ANY;
|
||||
uint8_t* data_[kMaxNumPlanes];
|
||||
uint8* data_[kMaxNumPlanes];
|
||||
int stride_[kMaxNumPlanes];
|
||||
int width_ = 0;
|
||||
int height_ = 0;
|
||||
|
|
|
@ -50,7 +50,7 @@
|
|||
// but may or may not still be able to run other OpenGL code.
|
||||
#if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) && \
|
||||
(defined(__APPLE__) || defined(__EMSCRIPTEN__) || MEDIAPIPE_DISABLE_GPU || \
|
||||
MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
|
||||
MEDIAPIPE_USING_SWIFTSHADER)
|
||||
#define MEDIAPIPE_DISABLE_GL_COMPUTE
|
||||
#endif
|
||||
|
||||
|
@ -104,9 +104,4 @@
|
|||
#endif
|
||||
#endif // MEDIAPIPE_HAS_RTTI
|
||||
|
||||
// AHardware buffers are only available since Android API 26.
|
||||
#if (__ANDROID_API__ >= 26)
|
||||
#define MEDIAPIPE_GPU_BUFFER_USE_AHWB 1
|
||||
#endif
|
||||
|
||||
#endif // MEDIAPIPE_FRAMEWORK_PORT_H_
|
||||
|
|
|
@ -26,7 +26,7 @@ def replace_suffix(string, old, new):
|
|||
|
||||
def mediapipe_ts_library(
|
||||
name,
|
||||
srcs = [],
|
||||
srcs,
|
||||
visibility = None,
|
||||
deps = [],
|
||||
testonly = 0,
|
||||
|
|
|
@ -13,13 +13,13 @@
|
|||
# limitations under the License.
|
||||
#
|
||||
|
||||
load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test")
|
||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
|
||||
load(
|
||||
"//mediapipe/framework/tool:mediapipe_graph.bzl",
|
||||
"data_as_c_string",
|
||||
"mediapipe_binary_graph",
|
||||
)
|
||||
load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test")
|
||||
load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
|
||||
|
||||
licenses(["notice"])
|
||||
|
@ -616,7 +616,6 @@ cc_test(
|
|||
"//mediapipe/framework:calculator_runner",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/port:parse_text_proto",
|
||||
"@com_google_absl//absl/functional:bind_front",
|
||||
"@com_google_absl//absl/strings",
|
||||
],
|
||||
)
|
||||
|
@ -857,7 +856,6 @@ cc_library(
|
|||
mediapipe_cc_test(
|
||||
name = "switch_demux_calculator_test",
|
||||
srcs = ["switch_demux_calculator_test.cc"],
|
||||
requires_full_emulation = False,
|
||||
deps = [
|
||||
":container_util",
|
||||
":switch_demux_calculator",
|
||||
|
@ -893,7 +891,6 @@ cc_library(
|
|||
mediapipe_cc_test(
|
||||
name = "switch_mux_calculator_test",
|
||||
srcs = ["switch_mux_calculator_test.cc"],
|
||||
requires_full_emulation = False,
|
||||
deps = [
|
||||
":container_util",
|
||||
":switch_mux_calculator",
|
||||
|
|
|
@ -17,7 +17,6 @@
|
|||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/functional/bind_front.h"
|
||||
#include "absl/strings/string_view.h"
|
||||
#include "mediapipe/framework/calculator_framework.h"
|
||||
#include "mediapipe/framework/calculator_runner.h"
|
||||
|
|
|
@ -196,7 +196,6 @@ cc_library(
|
|||
":gpu_buffer_format",
|
||||
"//mediapipe/framework:executor",
|
||||
"//mediapipe/framework:mediapipe_profiling",
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/framework:timestamp",
|
||||
"//mediapipe/framework/port:logging",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
|
@ -210,7 +209,6 @@ cc_library(
|
|||
"@com_google_absl//absl/memory",
|
||||
"@com_google_absl//absl/status",
|
||||
"@com_google_absl//absl/status:statusor",
|
||||
"@com_google_absl//absl/strings:str_format",
|
||||
"@com_google_absl//absl/synchronization",
|
||||
] + select({
|
||||
"//conditions:default": [],
|
||||
|
@ -513,19 +511,11 @@ cc_library(
|
|||
],
|
||||
}),
|
||||
deps = [
|
||||
":gl_base_hdr",
|
||||
":gl_context",
|
||||
":gl_texture_buffer",
|
||||
":gl_texture_view",
|
||||
":gpu_buffer_format",
|
||||
":gpu_buffer_storage",
|
||||
":image_frame_view",
|
||||
"//mediapipe/framework:port",
|
||||
"//mediapipe/framework/formats:ahwb_view",
|
||||
"//mediapipe/framework/formats:image_frame",
|
||||
"//mediapipe/framework/port:ret_check",
|
||||
"//third_party/GL:EGL_headers",
|
||||
"@com_google_absl//absl/log:absl_check",
|
||||
"@com_google_absl//absl/strings:str_format",
|
||||
],
|
||||
)
|
||||
|
@ -671,8 +661,6 @@ cc_library(
|
|||
"//mediapipe/framework/port:ret_check",
|
||||
"@com_google_absl//absl/base:core_headers",
|
||||
"@com_google_absl//absl/log:absl_check",
|
||||
"@com_google_absl//absl/log:absl_log",
|
||||
"@com_google_absl//absl/status",
|
||||
] + select({
|
||||
"//conditions:default": [],
|
||||
"//mediapipe:apple": [
|
||||
|
@ -1235,13 +1223,9 @@ mediapipe_cc_test(
|
|||
],
|
||||
requires_full_emulation = True,
|
||||
deps = [
|
||||
":gl_texture_buffer",
|
||||
":gl_texture_util",
|
||||
":gpu_buffer_format",
|
||||
":gpu_buffer_storage_ahwb",
|
||||
":gpu_test_base",
|
||||
"//mediapipe/framework/port:gtest_main",
|
||||
"//mediapipe/framework/tool:test_util",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
@ -57,8 +57,8 @@ void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) {
|
|||
|
||||
// static
|
||||
absl::Status GlCalculatorHelper::UpdateContract(CalculatorContract* cc,
|
||||
bool request_gpu_as_optional) {
|
||||
if (request_gpu_as_optional) {
|
||||
bool requesst_gpu_as_optional) {
|
||||
if (requesst_gpu_as_optional) {
|
||||
cc->UseService(kGpuService).Optional();
|
||||
} else {
|
||||
cc->UseService(kGpuService);
|
||||
|
|
|
@ -68,7 +68,7 @@ class GlCalculatorHelper {
|
|||
// This method can be called from GetContract to set up the needed GPU
|
||||
// resources.
|
||||
static absl::Status UpdateContract(CalculatorContract* cc,
|
||||
bool request_gpu_as_optional = false);
|
||||
bool requesst_gpu_as_optional = false);
|
||||
|
||||
// This method can be called from FillExpectations to set the correct types
|
||||
// for the shared GL input side packet(s).
|
||||
|
|
|
@ -26,9 +26,7 @@
|
|||
#include "absl/log/absl_log.h"
|
||||
#include "absl/memory/memory.h"
|
||||
#include "absl/status/status.h"
|
||||
#include "absl/strings/str_format.h"
|
||||
#include "absl/synchronization/mutex.h"
|
||||
#include "mediapipe/framework/port.h" // IWYU pragma: keep
|
||||
#include "mediapipe/framework/port/ret_check.h"
|
||||
#include "mediapipe/framework/port/status.h"
|
||||
#include "mediapipe/framework/port/status_builder.h"
|
||||
|
@ -50,17 +48,6 @@
|
|||
|
||||
namespace mediapipe {
|
||||
|
||||
namespace internal_gl_context {
|
||||
|
||||
bool IsOpenGlVersionSameOrAbove(const OpenGlVersion& version,
|
||||
const OpenGlVersion& expected_version) {
|
||||
return (version.major == expected_version.major &&
|
||||
version.minor >= expected_version.minor) ||
|
||||
version.major > expected_version.major;
|
||||
}
|
||||
|
||||
} // namespace internal_gl_context
|
||||
|
||||
static void SetThreadName(const char* name) {
|
||||
#if defined(__GLIBC_PREREQ)
|
||||
#define LINUX_STYLE_SETNAME_NP __GLIBC_PREREQ(2, 12)
|
||||
|
@ -651,11 +638,6 @@ class GlSyncWrapper {
|
|||
// TODO: do something if the wait fails?
|
||||
}
|
||||
|
||||
// This method exists only for investigation purposes to distinguish stack
|
||||
// traces: external vs. internal context.
|
||||
// TODO: remove after glWaitSync crashes are resolved.
|
||||
void WaitOnGpuExternalContext() { glWaitSync(sync_, 0, GL_TIMEOUT_IGNORED); }
|
||||
|
||||
void WaitOnGpu() {
|
||||
if (!sync_) return;
|
||||
// WebGL2 specifies a waitSync call, but since cross-context
|
||||
|
@ -663,33 +645,6 @@ class GlSyncWrapper {
|
|||
// a warning when it's called, so let's just skip the call. See
|
||||
// b/184637485 for details.
|
||||
#ifndef __EMSCRIPTEN__
|
||||
|
||||
if (!GlContext::IsAnyContextCurrent()) {
|
||||
// glWaitSync must be called on with some context current. Doing the
|
||||
// opposite doesn't necessarily result in a crash or GL error. Hence,
|
||||
// just logging an error and skipping the call.
|
||||
ABSL_LOG_FIRST_N(ERROR, 1)
|
||||
<< "An attempt to wait for a sync without any context current.";
|
||||
return;
|
||||
}
|
||||
|
||||
auto context = GlContext::GetCurrent();
|
||||
if (context == nullptr) {
|
||||
// This can happen when WaitOnGpu is invoked on an external context,
|
||||
// created by other than GlContext::Create means.
|
||||
WaitOnGpuExternalContext();
|
||||
return;
|
||||
}
|
||||
|
||||
// GlContext::ShouldUseFenceSync guards creation of sync objects, so this
|
||||
// CHECK should never fail if clients use MediaPipe APIs in an intended way.
|
||||
// TODO: remove after glWaitSync crashes are resolved.
|
||||
ABSL_CHECK(context->ShouldUseFenceSync()) << absl::StrFormat(
|
||||
"An attempt to wait for a sync when it should not be used. (OpenGL "
|
||||
"Version "
|
||||
"%d.%d)",
|
||||
context->gl_major_version(), context->gl_minor_version());
|
||||
|
||||
glWaitSync(sync_, 0, GL_TIMEOUT_IGNORED);
|
||||
#endif
|
||||
}
|
||||
|
@ -742,13 +697,10 @@ class GlFenceSyncPoint : public GlSyncPoint {
|
|||
|
||||
void Wait() override {
|
||||
if (!sync_) return;
|
||||
if (GlContext::IsAnyContextCurrent()) {
|
||||
gl_context_->Run([this] {
|
||||
// TODO: must this run on the original context??
|
||||
sync_.Wait();
|
||||
return;
|
||||
}
|
||||
// In case a current GL context is not available, we fall back using the
|
||||
// captured gl_context_.
|
||||
gl_context_->Run([this] { sync_.Wait(); });
|
||||
});
|
||||
}
|
||||
|
||||
void WaitOnGpu() override {
|
||||
|
@ -860,25 +812,15 @@ class GlNopSyncPoint : public GlSyncPoint {
|
|||
#endif
|
||||
|
||||
bool GlContext::ShouldUseFenceSync() const {
|
||||
using internal_gl_context::OpenGlVersion;
|
||||
#if defined(__EMSCRIPTEN__)
|
||||
#ifdef __EMSCRIPTEN__
|
||||
// In Emscripten the glWaitSync function is non-null depending on linkopts,
|
||||
// but only works in a WebGL2 context.
|
||||
constexpr OpenGlVersion kMinVersionSyncAvaiable = {.major = 3, .minor = 0};
|
||||
#elif defined(MEDIAPIPE_MOBILE)
|
||||
// OpenGL ES, glWaitSync is available since 3.0
|
||||
constexpr OpenGlVersion kMinVersionSyncAvaiable = {.major = 3, .minor = 0};
|
||||
// but only works in a WebGL2 context, so fall back to use Finish if it is a
|
||||
// WebGL1/ES2 context.
|
||||
// TODO: apply this more generally once b/152794517 is fixed.
|
||||
return gl_major_version() > 2;
|
||||
#else
|
||||
// TODO: specify major/minor version per remaining platforms.
|
||||
// By default, ignoring major/minor version requirement for backward
|
||||
// compatibility.
|
||||
constexpr OpenGlVersion kMinVersionSyncAvaiable = {.major = 0, .minor = 0};
|
||||
#endif
|
||||
|
||||
return SymbolAvailable(&glWaitSync) &&
|
||||
internal_gl_context::IsOpenGlVersionSameOrAbove(
|
||||
{.major = gl_major_version(), .minor = gl_minor_version()},
|
||||
kMinVersionSyncAvaiable);
|
||||
return SymbolAvailable(&glWaitSync);
|
||||
#endif // __EMSCRIPTEN__
|
||||
}
|
||||
|
||||
std::shared_ptr<GlSyncPoint> GlContext::CreateSyncToken() {
|
||||
|
|
|
@ -71,8 +71,6 @@ typedef std::function<void()> GlVoidFunction;
|
|||
typedef std::function<absl::Status()> GlStatusFunction;
|
||||
|
||||
class GlContext;
|
||||
// TODO: remove after glWaitSync crashes are resolved.
|
||||
class GlSyncWrapper;
|
||||
|
||||
// Generic interface for synchronizing access to a shared resource from a
|
||||
// different context. This is an abstract class to keep users from
|
||||
|
@ -192,7 +190,8 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
|
|||
// Like Run, but does not wait.
|
||||
void RunWithoutWaiting(GlVoidFunction gl_func);
|
||||
|
||||
// Returns a synchronization token for this GlContext.
|
||||
// Returns a synchronization token.
|
||||
// This should not be called outside of the GlContext thread.
|
||||
std::shared_ptr<GlSyncPoint> CreateSyncToken();
|
||||
|
||||
// If another part of the framework calls glFinish, it should call this
|
||||
|
@ -331,9 +330,6 @@ class GlContext : public std::enable_shared_from_this<GlContext> {
|
|||
SyncTokenTypeForTest type);
|
||||
|
||||
private:
|
||||
// TODO: remove after glWaitSync crashes are resolved.
|
||||
friend GlSyncWrapper;
|
||||
|
||||
GlContext();
|
||||
|
||||
bool ShouldUseFenceSync() const;
|
||||
|
@ -492,18 +488,6 @@ ABSL_DEPRECATED(
|
|||
const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
|
||||
int plane);
|
||||
|
||||
namespace internal_gl_context {
|
||||
|
||||
struct OpenGlVersion {
|
||||
int major;
|
||||
int minor;
|
||||
};
|
||||
|
||||
bool IsOpenGlVersionSameOrAbove(const OpenGlVersion& version,
|
||||
const OpenGlVersion& expected_version);
|
||||
|
||||
} // namespace internal_gl_context
|
||||
|
||||
} // namespace mediapipe
|
||||
|
||||
#endif // MEDIAPIPE_GPU_GL_CONTEXT_H_
|
||||
|
|
|
@ -32,7 +32,7 @@ namespace mediapipe {
|
|||
// TODO: Handle webGL "context lost" and "context restored" events.
|
||||
GlContext::StatusOrGlContext GlContext::Create(std::nullptr_t nullp,
|
||||
bool create_thread) {
|
||||
return Create(static_cast<EMSCRIPTEN_WEBGL_CONTEXT_HANDLE>(0), create_thread);
|
||||
return Create(0, create_thread);
|
||||
}
|
||||
|
||||
GlContext::StatusOrGlContext GlContext::Create(const GlContext& share_context,
|
||||
|
|
|
@ -14,8 +14,6 @@
|
|||
|
||||
#include "mediapipe/gpu/gl_texture_buffer.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include "absl/log/absl_check.h"
|
||||
#include "absl/log/absl_log.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
|
@ -133,13 +131,6 @@ bool GlTextureBuffer::CreateInternal(const void* data, int alignment) {
|
|||
SymbolAvailable(&glTexStorage2D)) {
|
||||
ABSL_CHECK(data == nullptr) << "unimplemented";
|
||||
glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
|
||||
} else if (info.immutable) {
|
||||
ABSL_CHECK(SymbolAvailable(&glTexStorage2D) &&
|
||||
context->GetGlVersion() != GlVersion::kGLES2)
|
||||
<< "Immutable GpuBuffer format requested is not supported in this "
|
||||
<< "GlContext. Format was " << static_cast<uint32_t>(format_);
|
||||
ABSL_CHECK(data == nullptr) << "unimplemented";
|
||||
glTexStorage2D(target_, 1, info.gl_internal_format, width_, height_);
|
||||
} else {
|
||||
glTexImage2D(target_, 0 /* level */, info.gl_internal_format, width_,
|
||||
height_, 0 /* border */, info.gl_format, info.gl_type, data);
|
||||
|
|
|
@ -19,7 +19,6 @@
|
|||
#define MEDIAPIPE_GPU_GL_TEXTURE_BUFFER_H_
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
|
||||
#include "absl/memory/memory.h"
|
||||
#include "mediapipe/framework/formats/image_frame.h"
|
||||
|
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
#include <cstdlib>
|
||||
|
||||
#if defined(MEDIAPIPE_USING_LEGACY_SWIFTSHADER)
|
||||
#if defined(MEDIAPIPE_USING_SWIFTSHADER)
|
||||
#define MEDIAPIPE_NEEDS_GL_THREAD_COLLECTOR 1
|
||||
#endif
|
||||
|
||||
|
|
|
@ -35,10 +35,6 @@ namespace mediapipe {
|
|||
#endif // GL_HALF_FLOAT_OES
|
||||
#endif // __EMSCRIPTEN__
|
||||
|
||||
#ifndef GL_RGBA8
|
||||
#define GL_RGBA8 0x8058
|
||||
#endif // GL_RGBA8
|
||||
|
||||
#if !MEDIAPIPE_DISABLE_GPU
|
||||
#ifdef GL_ES_VERSION_2_0
|
||||
static void AdaptGlTextureInfoForGLES2(GlTextureInfo* info) {
|
||||
|
@ -167,14 +163,6 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
|
|||
{
|
||||
{GL_RGBA32F, GL_RGBA, GL_FLOAT, 1},
|
||||
}},
|
||||
{GpuBufferFormat::kImmutableRGBAFloat128,
|
||||
{
|
||||
{GL_RGBA32F, GL_RGBA, GL_FLOAT, 1, true /* immutable */},
|
||||
}},
|
||||
{GpuBufferFormat::kImmutableRGBA32,
|
||||
{
|
||||
{GL_RGBA8, GL_RGBA, GL_UNSIGNED_BYTE, 1, true /* immutable */},
|
||||
}},
|
||||
}};
|
||||
|
||||
static const auto* gles2_format_info = ([] {
|
||||
|
@ -218,7 +206,6 @@ const GlTextureInfo& GlTextureInfoForGpuBufferFormat(GpuBufferFormat format,
|
|||
|
||||
ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
|
||||
switch (format) {
|
||||
case GpuBufferFormat::kImmutableRGBA32:
|
||||
case GpuBufferFormat::kBGRA32:
|
||||
// TODO: verify we are handling order of channels correctly.
|
||||
return ImageFormat::SRGBA;
|
||||
|
@ -234,11 +221,10 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
|
|||
return ImageFormat::SRGB;
|
||||
case GpuBufferFormat::kTwoComponentFloat32:
|
||||
return ImageFormat::VEC32F2;
|
||||
case GpuBufferFormat::kImmutableRGBAFloat128:
|
||||
case GpuBufferFormat::kRGBAFloat128:
|
||||
return ImageFormat::VEC32F4;
|
||||
case GpuBufferFormat::kRGBA32:
|
||||
return ImageFormat::SRGBA;
|
||||
// TODO: this likely maps to ImageFormat::SRGBA
|
||||
case GpuBufferFormat::kGrayHalf16:
|
||||
case GpuBufferFormat::kOneComponent8Alpha:
|
||||
case GpuBufferFormat::kOneComponent8Red:
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user