Project import generated by Copybara.
GitOrigin-RevId: 947096e4fc99d6b974e9f50d360d7c0a75072c5d
This commit is contained in:
parent
a908d668c7
commit
8f69af91fe
|
@ -50,6 +50,10 @@ Object Detection
|
|||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
|
||||
## MediaPipe on the Web
|
||||
|
||||
MediaPipe on the Web is an effort to run the same ML solutions built for mobile
|
||||
|
@ -89,7 +93,8 @@ run code search using
|
|||
|
||||
## Publications
|
||||
|
||||
* [Face AR with MediaPipe Face Mesh](https://mediapipe.page.link/face-geometry-blog) in Google Developers Blog
|
||||
* [MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
|
||||
in Google Developers Blog
|
||||
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
|
||||
in Google Developers Blog
|
||||
* [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
|
|
|
@ -50,6 +50,10 @@ Object Detection
|
|||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
|
||||
## MediaPipe on the Web
|
||||
|
||||
MediaPipe on the Web is an effort to run the same ML solutions built for mobile
|
||||
|
@ -89,7 +93,8 @@ run code search using
|
|||
|
||||
## Publications
|
||||
|
||||
* [Face AR with MediaPipe Face Mesh](https://mediapipe.page.link/face-geometry-blog) in Google Developers Blog
|
||||
* [MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
|
||||
in Google Developers Blog
|
||||
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
|
||||
in Google Developers Blog
|
||||
* [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)
|
||||
|
|
|
@ -277,7 +277,7 @@ only works for a single face. For visual reference, please refer to *Fig. 4*.
|
|||
* TensorFlow Blog:
|
||||
[Face and hand tracking in the browser with MediaPipe and TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
|
||||
* Google Developers Blog:
|
||||
[Face AR with MediaPipe Face Mesh](https://mediapipe.page.link/face-geometry-blog)
|
||||
[MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
|
||||
* Paper:
|
||||
[Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs](https://arxiv.org/abs/1907.06724)
|
||||
([poster](https://docs.google.com/presentation/d/1-LWwOMO9TzEVdrZ1CS1ndJzciRHfYDJfbSxH_ke_JRg/present?slide=id.g5986dd4b4c_4_212))
|
||||
|
|
|
@ -5,7 +5,7 @@ parent: Solutions
|
|||
nav_order: 30
|
||||
---
|
||||
|
||||
# Models and Model Cards
|
||||
# MediaPipe Models and Model Cards
|
||||
{: .no_toc }
|
||||
|
||||
1. TOC
|
||||
|
|
|
@ -32,3 +32,7 @@ has_toc: false
|
|||
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
|
||||
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
|
||||
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
|
||||
|
||||
See also
|
||||
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
|
||||
for ML models released in MediaPipe.
|
||||
|
|
|
@ -42,16 +42,16 @@ namespace tf = tensorflow;
|
|||
// a flag controls whether a new first dimension is inserted before
|
||||
// concatenation.
|
||||
//
|
||||
// Currently, the number of tensors output will be buffer_size less than the
|
||||
// number of input tensors because no padding is implemented and only full
|
||||
// buffers are output.
|
||||
// The number of tensors output will be buffer_size less than the
|
||||
// number of input tensors unless padding is set to a non-zero value in the
|
||||
// options proto.
|
||||
//
|
||||
// The timestamp of the output batch will match the timestamp of the first
|
||||
// tensor in that batch by default. (e.g. when buffer_size frames are added, the
|
||||
// output tensor will have the timestamp of the first input.). This behavior can
|
||||
// be adjusted by the timestamp_offset option.
|
||||
//
|
||||
// Example config:
|
||||
// Example config without padding:
|
||||
// node {
|
||||
// calculator: "LappedTensorBufferCalculator"
|
||||
// input_stream: "input_tensor"
|
||||
|
@ -64,26 +64,50 @@ namespace tf = tensorflow;
|
|||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// Example config with padding and timestamp output:
|
||||
// node {
|
||||
// calculator: "LappedTensorBufferCalculator"
|
||||
// input_stream: "input_tensor"
|
||||
// output_stream: "output_tensor"
|
||||
// output_stream: "output_timestamp"
|
||||
// options {
|
||||
// [mediapipe.LappedTensorBufferCalculatorOptions.ext] {
|
||||
// buffer_size: 100
|
||||
// overlap: 50
|
||||
// add_batch_dim_to_tensors: true
|
||||
// timestamp_offset: 25
|
||||
// padding: 25
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
class LappedTensorBufferCalculator : public CalculatorBase {
|
||||
public:
|
||||
static ::mediapipe::Status GetContract(CalculatorContract* cc);
|
||||
|
||||
::mediapipe::Status Open(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Process(CalculatorContext* cc) override;
|
||||
::mediapipe::Status Close(CalculatorContext* cc) override;
|
||||
|
||||
private:
|
||||
// Adds a batch dimension to the input tensor if specified in the calculator
|
||||
// options.
|
||||
// Adds a batch dimension to the input tensor if specified in the
|
||||
// calculator options.
|
||||
::mediapipe::Status AddBatchDimension(tf::Tensor* input_tensor);
|
||||
// Sends the current buffer downstream.
|
||||
::mediapipe::Status ProcessBuffer(CalculatorContext* cc);
|
||||
|
||||
int steps_until_output_;
|
||||
int buffer_size_;
|
||||
int overlap_;
|
||||
int timestamp_offset_;
|
||||
int initialized_;
|
||||
|
||||
std::unique_ptr<CircularBuffer<Timestamp>> timestamp_buffer_;
|
||||
std::unique_ptr<CircularBuffer<tf::Tensor>> buffer_;
|
||||
LappedTensorBufferCalculatorOptions options_;
|
||||
};
|
||||
|
||||
REGISTER_CALCULATOR(LappedTensorBufferCalculator);
|
||||
|
||||
::mediapipe::Status LappedTensorBufferCalculator::GetContract(
|
||||
|
@ -93,8 +117,8 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
|
|||
cc->Inputs().Index(0).Set<tf::Tensor>(
|
||||
// tensorflow::Tensor stream.
|
||||
);
|
||||
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
|
||||
<< "Only one output stream is supported.";
|
||||
RET_CHECK_LE(cc->Outputs().NumEntries(), 2)
|
||||
<< "Only one or two output stream(s) is/are supported.";
|
||||
|
||||
if (cc->InputSidePackets().HasTag(kBufferSize)) {
|
||||
cc->InputSidePackets().Tag(kBufferSize).Set<int>();
|
||||
|
@ -108,11 +132,15 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
|
|||
if (cc->InputSidePackets().HasTag(kCalculatorOptions)) {
|
||||
cc->InputSidePackets()
|
||||
.Tag(kCalculatorOptions)
|
||||
.Set<LappedTensorBufferCalculatorOptions>();
|
||||
.Set<LappedTensorBufferCalculator>();
|
||||
}
|
||||
cc->Outputs().Index(0).Set<tf::Tensor>(
|
||||
// Output tensorflow::Tensor stream with possibly overlapping steps.
|
||||
);
|
||||
// Output timestamp stream with possibly overlapping steps.
|
||||
if (cc->Outputs().NumEntries() > 1) {
|
||||
cc->Outputs().Index(1).Set<std::vector<Timestamp>>();
|
||||
}
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
|
@ -141,10 +169,13 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
|
|||
<< "Negative timestamp_offset is not allowed.";
|
||||
RET_CHECK_LT(timestamp_offset_, buffer_size_)
|
||||
<< "output_frame_num_offset has to be less than buffer_size.";
|
||||
RET_CHECK_LT(options_.padding(), buffer_size_)
|
||||
<< "padding option must be smaller than buffer size.";
|
||||
timestamp_buffer_ =
|
||||
absl::make_unique<CircularBuffer<Timestamp>>(buffer_size_);
|
||||
buffer_ = absl::make_unique<CircularBuffer<tf::Tensor>>(buffer_size_);
|
||||
steps_until_output_ = buffer_size_;
|
||||
steps_until_output_ = buffer_size_ - options_.padding();
|
||||
initialized_ = false;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
|
@ -156,23 +187,36 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
|
|||
if (options_.add_batch_dim_to_tensors()) {
|
||||
RET_CHECK_OK(AddBatchDimension(&input_tensor));
|
||||
}
|
||||
// Pad frames at the beginning with the first frame.
|
||||
if (!initialized_) {
|
||||
for (int i = 0; i < options_.padding(); ++i) {
|
||||
buffer_->push_back(input_tensor);
|
||||
timestamp_buffer_->push_back(cc->InputTimestamp());
|
||||
}
|
||||
initialized_ = true;
|
||||
}
|
||||
buffer_->push_back(input_tensor);
|
||||
timestamp_buffer_->push_back(cc->InputTimestamp());
|
||||
--steps_until_output_;
|
||||
|
||||
if (steps_until_output_ <= 0) {
|
||||
auto concatenated = ::absl::make_unique<tf::Tensor>();
|
||||
|
||||
const tf::Status concat_status = tf::tensor::Concat(
|
||||
std::vector<tf::Tensor>(buffer_->begin(), buffer_->end()),
|
||||
concatenated.get());
|
||||
RET_CHECK(concat_status.ok()) << concat_status.ToString();
|
||||
|
||||
cc->Outputs().Index(0).Add(concatenated.release(),
|
||||
timestamp_buffer_->Get(timestamp_offset_));
|
||||
|
||||
steps_until_output_ = buffer_size_ - overlap_;
|
||||
MP_RETURN_IF_ERROR(ProcessBuffer(cc));
|
||||
}
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
::mediapipe::Status LappedTensorBufferCalculator::Close(CalculatorContext* cc) {
|
||||
if (!initialized_ || options_.padding() == 0) {
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
int last_frame = buffer_size_ - steps_until_output_ - 1;
|
||||
const auto& pad_frame = buffer_->Get(last_frame);
|
||||
for (int i = 0; i < steps_until_output_ + options_.padding(); ++i) {
|
||||
buffer_->push_back(pad_frame);
|
||||
timestamp_buffer_->push_back(cc->InputTimestamp());
|
||||
}
|
||||
MP_RETURN_IF_ERROR(ProcessBuffer(cc));
|
||||
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
|
@ -190,4 +234,29 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
|
|||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
// Process buffer
|
||||
::mediapipe::Status LappedTensorBufferCalculator::ProcessBuffer(
|
||||
CalculatorContext* cc) {
|
||||
auto concatenated = ::absl::make_unique<tf::Tensor>();
|
||||
const tf::Status concat_status = tf::tensor::Concat(
|
||||
std::vector<tf::Tensor>(buffer_->begin(), buffer_->end()),
|
||||
concatenated.get());
|
||||
RET_CHECK(concat_status.ok()) << concat_status.ToString();
|
||||
// Output cancatenated tensor.
|
||||
cc->Outputs().Index(0).Add(concatenated.release(),
|
||||
timestamp_buffer_->Get(timestamp_offset_));
|
||||
if (cc->Outputs().NumEntries() > 1) {
|
||||
auto output_timestamp = ::absl::make_unique<std::vector<Timestamp>>();
|
||||
// Output timestamp vector.
|
||||
*output_timestamp = std::vector<Timestamp>(timestamp_buffer_->begin(),
|
||||
timestamp_buffer_->end());
|
||||
RET_CHECK_EQ(output_timestamp->size(), buffer_size_)
|
||||
<< "Output timestamp size is not correct.";
|
||||
cc->Outputs().Index(1).Add(output_timestamp.release(),
|
||||
timestamp_buffer_->Get(timestamp_offset_));
|
||||
}
|
||||
steps_until_output_ = buffer_size_ - overlap_;
|
||||
return ::mediapipe::OkStatus();
|
||||
}
|
||||
|
||||
} // namespace mediapipe
|
||||
|
|
|
@ -45,4 +45,8 @@ message LappedTensorBufferCalculatorOptions {
|
|||
// This is useful for aligning the timestamp to be centered on the input
|
||||
// range.
|
||||
optional int32 timestamp_offset = 4 [default = 0];
|
||||
|
||||
// Amount of padding (repeating of first/last value) to add to the beginning
|
||||
// and end of the input stream.
|
||||
optional int32 padding = 5;
|
||||
}
|
||||
|
|
|
@ -31,11 +31,15 @@ namespace tf = ::tensorflow;
|
|||
class LappedTensorBufferCalculatorTest : public ::testing::Test {
|
||||
protected:
|
||||
void SetUpCalculator(int buffer_size, int overlap, bool add_dim,
|
||||
int timestamp_offset) {
|
||||
int timestamp_offset, int padding,
|
||||
bool timestamp_output) {
|
||||
CalculatorGraphConfig::Node config;
|
||||
config.set_calculator("LappedTensorBufferCalculator");
|
||||
config.add_input_stream("input_tensor");
|
||||
config.add_output_stream("output_tensor");
|
||||
if (timestamp_output) {
|
||||
config.add_output_stream("output_timestamp");
|
||||
}
|
||||
auto options = config.mutable_options()->MutableExtension(
|
||||
LappedTensorBufferCalculatorOptions::ext);
|
||||
options->set_buffer_size(buffer_size);
|
||||
|
@ -44,13 +48,14 @@ class LappedTensorBufferCalculatorTest : public ::testing::Test {
|
|||
options->set_add_batch_dim_to_tensors(true);
|
||||
}
|
||||
options->set_timestamp_offset(timestamp_offset);
|
||||
options->set_padding(padding);
|
||||
runner_ = ::absl::make_unique<CalculatorRunner>(config);
|
||||
}
|
||||
std::unique_ptr<CalculatorRunner> runner_;
|
||||
};
|
||||
|
||||
TEST_F(LappedTensorBufferCalculatorTest, OneToOne) {
|
||||
SetUpCalculator(1, 0, false, 0);
|
||||
SetUpCalculator(1, 0, false, 0, 0, false);
|
||||
int num_timesteps = 3;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto input = ::absl::make_unique<tensorflow::Tensor>(
|
||||
|
@ -74,7 +79,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToTwo) {
|
|||
int buffer_size = 2;
|
||||
int overlap = 1;
|
||||
bool add_dim = false;
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, 0);
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
|
||||
int num_timesteps = 3;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto input = ::absl::make_unique<tensorflow::Tensor>(
|
||||
|
@ -100,7 +105,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThree) {
|
|||
int buffer_size = 3;
|
||||
int overlap = 2;
|
||||
bool add_dim = false;
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, 0);
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
|
||||
int num_timesteps = 3;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto input = ::absl::make_unique<tensorflow::Tensor>(
|
||||
|
@ -126,7 +131,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeSkip) {
|
|||
int buffer_size = 3;
|
||||
int overlap = 1;
|
||||
bool add_dim = false;
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, 0);
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
|
||||
int num_timesteps = 3;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto input = ::absl::make_unique<tensorflow::Tensor>(
|
||||
|
@ -152,7 +157,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeBatch) {
|
|||
int buffer_size = 3;
|
||||
int overlap = 2;
|
||||
bool add_dim = true;
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, 0);
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
|
||||
int num_timesteps = 3;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto input = ::absl::make_unique<tensorflow::Tensor>(
|
||||
|
@ -180,7 +185,7 @@ TEST_F(LappedTensorBufferCalculatorTest, NegativeTimestampOffsetFails) {
|
|||
int overlap = 15;
|
||||
bool add_dim = true;
|
||||
int timestamp_offset = -7;
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset);
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false);
|
||||
int num_timesteps = 20;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto input = ::absl::make_unique<tensorflow::Tensor>(
|
||||
|
@ -197,7 +202,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OutOfRangeTimestampOffsetFails) {
|
|||
int overlap = 15;
|
||||
bool add_dim = true;
|
||||
int timestamp_offset = buffer_size;
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset);
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false);
|
||||
int num_timesteps = 20;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto input = ::absl::make_unique<tensorflow::Tensor>(
|
||||
|
@ -214,7 +219,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeBatchTimestampOffset) {
|
|||
int overlap = 15;
|
||||
bool add_dim = true;
|
||||
int timestamp_offset = 7;
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset);
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false);
|
||||
int num_timesteps = 20;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto input = ::absl::make_unique<tensorflow::Tensor>(
|
||||
|
@ -236,5 +241,37 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeBatchTimestampOffset) {
|
|||
}
|
||||
}
|
||||
|
||||
TEST_F(LappedTensorBufferCalculatorTest,
|
||||
OneToThreeBatchTimestampOffsetPadding) {
|
||||
int buffer_size = 12;
|
||||
int overlap = 6;
|
||||
bool add_dim = true;
|
||||
int timestamp_offset = 3;
|
||||
int padding = 0;
|
||||
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, padding,
|
||||
true);
|
||||
int num_timesteps = 20;
|
||||
for (int i = 0; i < num_timesteps; ++i) {
|
||||
auto input = ::absl::make_unique<tensorflow::Tensor>(
|
||||
tensorflow::DT_FLOAT, tensorflow::TensorShape({1}));
|
||||
input->tensor<float, 1>()(0) = i;
|
||||
runner_->MutableInputs()->Index(0).packets.push_back(
|
||||
Adopt(input.release()).At(Timestamp(i)));
|
||||
}
|
||||
ASSERT_TRUE(runner_->Run().ok());
|
||||
|
||||
const int output_size = num_timesteps / buffer_size + 1;
|
||||
const std::vector<Packet>& output_packets =
|
||||
runner_->Outputs().Index(0).packets;
|
||||
ASSERT_EQ(output_size, output_packets.size());
|
||||
for (int i = 0; i < output_size; ++i) {
|
||||
int64 value = output_packets[i].Timestamp().Value();
|
||||
ASSERT_EQ(i * overlap + timestamp_offset, value);
|
||||
}
|
||||
const std::vector<Packet>& output_timestamps =
|
||||
runner_->Outputs().Index(1).packets;
|
||||
ASSERT_EQ(output_size, output_timestamps.size());
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace mediapipe
|
||||
|
|
Loading…
Reference in New Issue
Block a user