Merge Master
This commit is contained in:
commit
a223b2f6cc
|
@ -37,8 +37,10 @@ class TensorToVectorIntCalculator : public CalculatorBase {
|
||||||
|
|
||||||
private:
|
private:
|
||||||
void TokenizeVector(std::vector<int64>* vector) const;
|
void TokenizeVector(std::vector<int64>* vector) const;
|
||||||
|
void RemoveOverlapVector(std::vector<int64>* vector);
|
||||||
|
|
||||||
TensorToVectorIntCalculatorOptions options_;
|
TensorToVectorIntCalculatorOptions options_;
|
||||||
|
int32_t overlapping_values_;
|
||||||
};
|
};
|
||||||
REGISTER_CALCULATOR(TensorToVectorIntCalculator);
|
REGISTER_CALCULATOR(TensorToVectorIntCalculator);
|
||||||
|
|
||||||
|
@ -66,6 +68,7 @@ absl::Status TensorToVectorIntCalculator::GetContract(CalculatorContract* cc) {
|
||||||
|
|
||||||
absl::Status TensorToVectorIntCalculator::Open(CalculatorContext* cc) {
|
absl::Status TensorToVectorIntCalculator::Open(CalculatorContext* cc) {
|
||||||
options_ = cc->Options<TensorToVectorIntCalculatorOptions>();
|
options_ = cc->Options<TensorToVectorIntCalculatorOptions>();
|
||||||
|
overlapping_values_ = 0;
|
||||||
|
|
||||||
// Inform mediapipe that this calculator produces an output at time t for
|
// Inform mediapipe that this calculator produces an output at time t for
|
||||||
// each input received at time t (i.e. this calculator does not buffer
|
// each input received at time t (i.e. this calculator does not buffer
|
||||||
|
@ -106,6 +109,7 @@ absl::Status TensorToVectorIntCalculator::Process(CalculatorContext* cc) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
TokenizeVector(&instance_output);
|
TokenizeVector(&instance_output);
|
||||||
|
RemoveOverlapVector(&instance_output);
|
||||||
}
|
}
|
||||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||||
} else {
|
} else {
|
||||||
|
@ -128,12 +132,28 @@ absl::Status TensorToVectorIntCalculator::Process(CalculatorContext* cc) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
TokenizeVector(output.get());
|
TokenizeVector(output.get());
|
||||||
|
RemoveOverlapVector(output.get());
|
||||||
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
|
||||||
}
|
}
|
||||||
|
|
||||||
return absl::OkStatus();
|
return absl::OkStatus();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void TensorToVectorIntCalculator::RemoveOverlapVector(
|
||||||
|
std::vector<int64>* vector) {
|
||||||
|
if (options_.overlap() <= 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (overlapping_values_ > 0) {
|
||||||
|
if (vector->size() < overlapping_values_) {
|
||||||
|
vector->clear();
|
||||||
|
} else {
|
||||||
|
vector->erase(vector->begin(), vector->begin() + overlapping_values_);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
overlapping_values_ = options_.overlap();
|
||||||
|
}
|
||||||
|
|
||||||
void TensorToVectorIntCalculator::TokenizeVector(
|
void TensorToVectorIntCalculator::TokenizeVector(
|
||||||
std::vector<int64>* vector) const {
|
std::vector<int64>* vector) const {
|
||||||
if (!options_.tensor_is_token()) {
|
if (!options_.tensor_is_token()) {
|
||||||
|
|
|
@ -36,4 +36,8 @@ message TensorToVectorIntCalculatorOptions {
|
||||||
optional bool tensor_is_token = 3 [default = false];
|
optional bool tensor_is_token = 3 [default = false];
|
||||||
// Threshold for the token generation.
|
// Threshold for the token generation.
|
||||||
optional float token_threshold = 4 [default = 0.5];
|
optional float token_threshold = 4 [default = 0.5];
|
||||||
|
|
||||||
|
// Values which overlap between timely following vectors. They are removed
|
||||||
|
// from the output to reduce redundancy.
|
||||||
|
optional int32 overlap = 5 [default = 0];
|
||||||
}
|
}
|
||||||
|
|
|
@ -28,7 +28,8 @@ namespace tf = ::tensorflow;
|
||||||
class TensorToVectorIntCalculatorTest : public ::testing::Test {
|
class TensorToVectorIntCalculatorTest : public ::testing::Test {
|
||||||
protected:
|
protected:
|
||||||
void SetUpRunner(const bool tensor_is_2d, const bool flatten_nd,
|
void SetUpRunner(const bool tensor_is_2d, const bool flatten_nd,
|
||||||
const bool tensor_is_token = false) {
|
const bool tensor_is_token = false,
|
||||||
|
const int32_t overlap = 0) {
|
||||||
CalculatorGraphConfig::Node config;
|
CalculatorGraphConfig::Node config;
|
||||||
config.set_calculator("TensorToVectorIntCalculator");
|
config.set_calculator("TensorToVectorIntCalculator");
|
||||||
config.add_input_stream("input_tensor");
|
config.add_input_stream("input_tensor");
|
||||||
|
@ -38,6 +39,7 @@ class TensorToVectorIntCalculatorTest : public ::testing::Test {
|
||||||
options->set_tensor_is_2d(tensor_is_2d);
|
options->set_tensor_is_2d(tensor_is_2d);
|
||||||
options->set_flatten_nd(flatten_nd);
|
options->set_flatten_nd(flatten_nd);
|
||||||
options->set_tensor_is_token(tensor_is_token);
|
options->set_tensor_is_token(tensor_is_token);
|
||||||
|
options->set_overlap(overlap);
|
||||||
runner_ = absl::make_unique<CalculatorRunner>(config);
|
runner_ = absl::make_unique<CalculatorRunner>(config);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -188,5 +190,54 @@ TEST_F(TensorToVectorIntCalculatorTest, FlattenShouldTakeAllDimensions) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TEST_F(TensorToVectorIntCalculatorTest, Overlap) {
|
||||||
|
SetUpRunner(false, false, false, 2);
|
||||||
|
for (int time = 0; time < 3; ++time) {
|
||||||
|
const tf::TensorShape tensor_shape(std::vector<tf::int64>{5});
|
||||||
|
auto tensor = absl::make_unique<tf::Tensor>(tf::DT_INT64, tensor_shape);
|
||||||
|
auto tensor_vec = tensor->vec<int64>();
|
||||||
|
for (int i = 0; i < 5; ++i) {
|
||||||
|
// 2^i can be represented exactly in floating point numbers if 'i' is
|
||||||
|
// small.
|
||||||
|
tensor_vec(i) = static_cast<int64>(time + (1 << i));
|
||||||
|
}
|
||||||
|
|
||||||
|
runner_->MutableInputs()->Index(0).packets.push_back(
|
||||||
|
Adopt(tensor.release()).At(Timestamp(time)));
|
||||||
|
}
|
||||||
|
|
||||||
|
ASSERT_TRUE(runner_->Run().ok());
|
||||||
|
const std::vector<Packet>& output_packets =
|
||||||
|
runner_->Outputs().Index(0).packets;
|
||||||
|
EXPECT_EQ(3, output_packets.size());
|
||||||
|
|
||||||
|
{
|
||||||
|
// First vector in full.
|
||||||
|
int time = 0;
|
||||||
|
EXPECT_EQ(time, output_packets[time].Timestamp().Value());
|
||||||
|
const std::vector<int64>& output_vector =
|
||||||
|
output_packets[time].Get<std::vector<int64>>();
|
||||||
|
|
||||||
|
EXPECT_EQ(5, output_vector.size());
|
||||||
|
for (int i = 0; i < 5; ++i) {
|
||||||
|
const int64 expected = static_cast<int64>(time + (1 << i));
|
||||||
|
EXPECT_EQ(expected, output_vector[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// All following vectors the overlap removed
|
||||||
|
for (int time = 1; time < 3; ++time) {
|
||||||
|
EXPECT_EQ(time, output_packets[time].Timestamp().Value());
|
||||||
|
const std::vector<int64>& output_vector =
|
||||||
|
output_packets[time].Get<std::vector<int64>>();
|
||||||
|
|
||||||
|
EXPECT_EQ(3, output_vector.size());
|
||||||
|
for (int i = 0; i < 3; ++i) {
|
||||||
|
const int64 expected = static_cast<int64>(time + (1 << (i + 2)));
|
||||||
|
EXPECT_EQ(expected, output_vector[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
} // namespace
|
} // namespace
|
||||||
} // namespace mediapipe
|
} // namespace mediapipe
|
||||||
|
|
|
@ -18,6 +18,8 @@ licenses(["notice"])
|
||||||
|
|
||||||
package(default_visibility = [
|
package(default_visibility = [
|
||||||
"//mediapipe/examples:__subpackages__",
|
"//mediapipe/examples:__subpackages__",
|
||||||
|
"//photos/editing/mobile/mediapipe/calculators:__subpackages__",
|
||||||
|
"//photos/editing/mobile/mediapipe/proto:__subpackages__",
|
||||||
])
|
])
|
||||||
|
|
||||||
proto_library(
|
proto_library(
|
||||||
|
@ -45,6 +47,8 @@ mediapipe_cc_proto_library(
|
||||||
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
|
||||||
visibility = [
|
visibility = [
|
||||||
"//mediapipe/examples:__subpackages__",
|
"//mediapipe/examples:__subpackages__",
|
||||||
|
"//photos/editing/mobile/mediapipe/calculators:__pkg__",
|
||||||
|
"//photos/editing/mobile/mediapipe/calculators:__subpackages__",
|
||||||
],
|
],
|
||||||
deps = [":autoflip_messages_proto"],
|
deps = [":autoflip_messages_proto"],
|
||||||
)
|
)
|
||||||
|
|
|
@ -68,8 +68,9 @@ void FillGpuBuffer(GLuint name, std::size_t size,
|
||||||
MP_ASSERT_OK(TFLITE_GPU_CALL_GL(glGetShaderiv, shader, GL_INFO_LOG_LENGTH,
|
MP_ASSERT_OK(TFLITE_GPU_CALL_GL(glGetShaderiv, shader, GL_INFO_LOG_LENGTH,
|
||||||
&max_length));
|
&max_length));
|
||||||
std::vector<GLchar> error_log(max_length);
|
std::vector<GLchar> error_log(max_length);
|
||||||
glGetShaderInfoLog(shader, max_length, &max_length, error_log.data());
|
MP_ASSERT_OK(TFLITE_GPU_CALL_GL(glGetShaderInfoLog, shader, max_length,
|
||||||
glDeleteShader(shader);
|
&max_length, error_log.data()));
|
||||||
|
MP_EXPECT_OK(TFLITE_GPU_CALL_GL(glDeleteShader, shader));
|
||||||
FAIL() << error_log.data();
|
FAIL() << error_log.data();
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
@ -83,8 +84,8 @@ void FillGpuBuffer(GLuint name, std::size_t size,
|
||||||
TFLITE_GPU_CALL_GL(glBindBufferBase, GL_SHADER_STORAGE_BUFFER, 0, name));
|
TFLITE_GPU_CALL_GL(glBindBufferBase, GL_SHADER_STORAGE_BUFFER, 0, name));
|
||||||
MP_ASSERT_OK(TFLITE_GPU_CALL_GL(glUseProgram, to_buffer_program));
|
MP_ASSERT_OK(TFLITE_GPU_CALL_GL(glUseProgram, to_buffer_program));
|
||||||
MP_ASSERT_OK(TFLITE_GPU_CALL_GL(glDispatchCompute, size / 2, 1, 1));
|
MP_ASSERT_OK(TFLITE_GPU_CALL_GL(glDispatchCompute, size / 2, 1, 1));
|
||||||
MP_ASSERT_OK(TFLITE_GPU_CALL_GL(glBindBuffer, GL_SHADER_STORAGE_BUFFER, 0));
|
MP_EXPECT_OK(TFLITE_GPU_CALL_GL(glBindBuffer, GL_SHADER_STORAGE_BUFFER, 0));
|
||||||
MP_ASSERT_OK(TFLITE_GPU_CALL_GL(glDeleteProgram, to_buffer_program));
|
MP_EXPECT_OK(TFLITE_GPU_CALL_GL(glDeleteProgram, to_buffer_program));
|
||||||
}
|
}
|
||||||
|
|
||||||
class TensorAhwbGpuTest : public mediapipe::GpuTestBase {
|
class TensorAhwbGpuTest : public mediapipe::GpuTestBase {
|
||||||
|
@ -97,18 +98,18 @@ TEST_F(TensorAhwbGpuTest, TestGpuToCpuFloat32) {
|
||||||
{
|
{
|
||||||
// Request Ahwb first to get Ahwb storage allocated internally.
|
// Request Ahwb first to get Ahwb storage allocated internally.
|
||||||
auto view = tensor.GetAHardwareBufferWriteView();
|
auto view = tensor.GetAHardwareBufferWriteView();
|
||||||
EXPECT_NE(view.handle(), nullptr);
|
ASSERT_NE(view.handle(), nullptr);
|
||||||
view.SetWritingFinishedFD(-1, [](bool) { return true; });
|
view.SetWritingFinishedFD(-1, [](bool) { return true; });
|
||||||
}
|
}
|
||||||
RunInGlContext([&tensor] {
|
RunInGlContext([&tensor] {
|
||||||
auto ssbo_view = tensor.GetOpenGlBufferWriteView();
|
auto ssbo_view = tensor.GetOpenGlBufferWriteView();
|
||||||
auto ssbo_name = ssbo_view.name();
|
auto ssbo_name = ssbo_view.name();
|
||||||
EXPECT_GT(ssbo_name, 0);
|
ASSERT_GT(ssbo_name, 0);
|
||||||
FillGpuBuffer(ssbo_name, tensor.shape().num_elements(),
|
FillGpuBuffer(ssbo_name, tensor.shape().num_elements(),
|
||||||
tensor.element_type());
|
tensor.element_type());
|
||||||
});
|
});
|
||||||
auto ptr = tensor.GetCpuReadView().buffer<float>();
|
auto ptr = tensor.GetCpuReadView().buffer<float>();
|
||||||
EXPECT_NE(ptr, nullptr);
|
ASSERT_NE(ptr, nullptr);
|
||||||
std::vector<float> reference;
|
std::vector<float> reference;
|
||||||
reference.resize(num_elements);
|
reference.resize(num_elements);
|
||||||
for (int i = 0; i < num_elements; i++) {
|
for (int i = 0; i < num_elements; i++) {
|
||||||
|
@ -124,18 +125,18 @@ TEST_F(TensorAhwbGpuTest, TestGpuToCpuFloat16) {
|
||||||
{
|
{
|
||||||
// Request Ahwb first to get Ahwb storage allocated internally.
|
// Request Ahwb first to get Ahwb storage allocated internally.
|
||||||
auto view = tensor.GetAHardwareBufferWriteView();
|
auto view = tensor.GetAHardwareBufferWriteView();
|
||||||
EXPECT_NE(view.handle(), nullptr);
|
ASSERT_NE(view.handle(), nullptr);
|
||||||
view.SetReadingFinishedFunc([](bool) { return true; });
|
view.SetReadingFinishedFunc([](bool) { return true; });
|
||||||
}
|
}
|
||||||
RunInGlContext([&tensor] {
|
RunInGlContext([&tensor] {
|
||||||
auto ssbo_view = tensor.GetOpenGlBufferWriteView();
|
auto ssbo_view = tensor.GetOpenGlBufferWriteView();
|
||||||
auto ssbo_name = ssbo_view.name();
|
auto ssbo_name = ssbo_view.name();
|
||||||
EXPECT_GT(ssbo_name, 0);
|
ASSERT_GT(ssbo_name, 0);
|
||||||
FillGpuBuffer(ssbo_name, tensor.shape().num_elements(),
|
FillGpuBuffer(ssbo_name, tensor.shape().num_elements(),
|
||||||
tensor.element_type());
|
tensor.element_type());
|
||||||
});
|
});
|
||||||
auto ptr = tensor.GetCpuReadView().buffer<Float16>();
|
auto ptr = tensor.GetCpuReadView().buffer<Float16>();
|
||||||
EXPECT_NE(ptr, nullptr);
|
ASSERT_NE(ptr, nullptr);
|
||||||
std::vector<Float16> reference;
|
std::vector<Float16> reference;
|
||||||
reference.resize(num_elements);
|
reference.resize(num_elements);
|
||||||
for (int i = 0; i < num_elements; i++) {
|
for (int i = 0; i < num_elements; i++) {
|
||||||
|
@ -153,18 +154,18 @@ TEST_F(TensorAhwbGpuTest, TestReplacingCpuByAhwb) {
|
||||||
Tensor tensor{Tensor::ElementType::kFloat32, Tensor::Shape({num_elements})};
|
Tensor tensor{Tensor::ElementType::kFloat32, Tensor::Shape({num_elements})};
|
||||||
{
|
{
|
||||||
auto ptr = tensor.GetCpuWriteView().buffer<float>();
|
auto ptr = tensor.GetCpuWriteView().buffer<float>();
|
||||||
EXPECT_NE(ptr, nullptr);
|
ASSERT_NE(ptr, nullptr);
|
||||||
for (int i = 0; i < num_elements; i++) {
|
for (int i = 0; i < num_elements; i++) {
|
||||||
ptr[i] = static_cast<float>(i) / 10.0f;
|
ptr[i] = static_cast<float>(i) / 10.0f;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
{
|
{
|
||||||
auto view = tensor.GetAHardwareBufferReadView();
|
auto view = tensor.GetAHardwareBufferReadView();
|
||||||
EXPECT_NE(view.handle(), nullptr);
|
ASSERT_NE(view.handle(), nullptr);
|
||||||
view.SetReadingFinishedFunc([](bool) { return true; });
|
view.SetReadingFinishedFunc([](bool) { return true; });
|
||||||
}
|
}
|
||||||
auto ptr = tensor.GetCpuReadView().buffer<float>();
|
auto ptr = tensor.GetCpuReadView().buffer<float>();
|
||||||
EXPECT_NE(ptr, nullptr);
|
ASSERT_NE(ptr, nullptr);
|
||||||
std::vector<float> reference;
|
std::vector<float> reference;
|
||||||
reference.resize(num_elements);
|
reference.resize(num_elements);
|
||||||
for (int i = 0; i < num_elements; i++) {
|
for (int i = 0; i < num_elements; i++) {
|
||||||
|
@ -182,17 +183,17 @@ TEST_F(TensorAhwbGpuTest, TestReplacingGpuByAhwb) {
|
||||||
RunInGlContext([&tensor] {
|
RunInGlContext([&tensor] {
|
||||||
auto ssbo_view = tensor.GetOpenGlBufferWriteView();
|
auto ssbo_view = tensor.GetOpenGlBufferWriteView();
|
||||||
auto ssbo_name = ssbo_view.name();
|
auto ssbo_name = ssbo_view.name();
|
||||||
EXPECT_GT(ssbo_name, 0);
|
ASSERT_GT(ssbo_name, 0);
|
||||||
FillGpuBuffer(ssbo_name, tensor.shape().num_elements(),
|
FillGpuBuffer(ssbo_name, tensor.shape().num_elements(),
|
||||||
tensor.element_type());
|
tensor.element_type());
|
||||||
});
|
});
|
||||||
{
|
{
|
||||||
auto view = tensor.GetAHardwareBufferReadView();
|
auto view = tensor.GetAHardwareBufferReadView();
|
||||||
EXPECT_NE(view.handle(), nullptr);
|
ASSERT_NE(view.handle(), nullptr);
|
||||||
view.SetReadingFinishedFunc([](bool) { return true; });
|
view.SetReadingFinishedFunc([](bool) { return true; });
|
||||||
}
|
}
|
||||||
auto ptr = tensor.GetCpuReadView().buffer<float>();
|
auto ptr = tensor.GetCpuReadView().buffer<float>();
|
||||||
EXPECT_NE(ptr, nullptr);
|
ASSERT_NE(ptr, nullptr);
|
||||||
std::vector<float> reference;
|
std::vector<float> reference;
|
||||||
reference.resize(num_elements);
|
reference.resize(num_elements);
|
||||||
for (int i = 0; i < num_elements; i++) {
|
for (int i = 0; i < num_elements; i++) {
|
||||||
|
|
|
@ -34,7 +34,7 @@ TEST(TensorAhwbTest, TestAhwbAlignment) {
|
||||||
Tensor tensor(Tensor::ElementType::kFloat32, Tensor::Shape{5});
|
Tensor tensor(Tensor::ElementType::kFloat32, Tensor::Shape{5});
|
||||||
{
|
{
|
||||||
auto view = tensor.GetAHardwareBufferWriteView(16);
|
auto view = tensor.GetAHardwareBufferWriteView(16);
|
||||||
EXPECT_NE(view.handle(), nullptr);
|
ASSERT_NE(view.handle(), nullptr);
|
||||||
if (__builtin_available(android 26, *)) {
|
if (__builtin_available(android 26, *)) {
|
||||||
AHardwareBuffer_Desc desc;
|
AHardwareBuffer_Desc desc;
|
||||||
AHardwareBuffer_describe(view.handle(), &desc);
|
AHardwareBuffer_describe(view.handle(), &desc);
|
||||||
|
|
|
@ -132,9 +132,9 @@ setuptools.setup(
|
||||||
'Operating System :: MacOS :: MacOS X',
|
'Operating System :: MacOS :: MacOS X',
|
||||||
'Operating System :: Microsoft :: Windows',
|
'Operating System :: Microsoft :: Windows',
|
||||||
'Operating System :: POSIX :: Linux',
|
'Operating System :: POSIX :: Linux',
|
||||||
'Programming Language :: Python :: 3.7',
|
|
||||||
'Programming Language :: Python :: 3.8',
|
'Programming Language :: Python :: 3.8',
|
||||||
'Programming Language :: Python :: 3.9',
|
'Programming Language :: Python :: 3.9',
|
||||||
|
'Programming Language :: Python :: 3.10',
|
||||||
'Programming Language :: Python :: 3 :: Only',
|
'Programming Language :: Python :: 3 :: Only',
|
||||||
'Topic :: Scientific/Engineering',
|
'Topic :: Scientific/Engineering',
|
||||||
'Topic :: Scientific/Engineering :: Artificial Intelligence',
|
'Topic :: Scientific/Engineering :: Artificial Intelligence',
|
||||||
|
|
|
@ -91,8 +91,8 @@ objc_library(
|
||||||
"//mediapipe/framework:calculator_cc_proto",
|
"//mediapipe/framework:calculator_cc_proto",
|
||||||
"//mediapipe/tasks/cc/core:task_runner",
|
"//mediapipe/tasks/cc/core:task_runner",
|
||||||
"//mediapipe/tasks/cc/core:mediapipe_builtin_op_resolver",
|
"//mediapipe/tasks/cc/core:mediapipe_builtin_op_resolver",
|
||||||
"@org_tensorflow//tensorflow/lite/core/api:op_resolver",
|
|
||||||
"//mediapipe/tasks/ios/common/utils:MPPCommonUtils",
|
"//mediapipe/tasks/ios/common/utils:MPPCommonUtils",
|
||||||
|
"@org_tensorflow//tensorflow/lite/core/api:op_resolver",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user