cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: 947096e4fc99d6b974e9f50d360d7c0a75072c5d
This commit is contained in:
MediaPipe Team 2020-09-17 11:43:21 -04:00 committed by chuoling
parent a908d668c7
commit 8f69af91fe
8 changed files with 159 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@ -50,6 +50,10 @@ Object Detection
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
See also
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
for ML models released in MediaPipe.
## MediaPipe on the Web
MediaPipe on the Web is an effort to run the same ML solutions built for mobile
@ -89,7 +93,8 @@ run code search using
## Publications
* [Face AR with MediaPipe Face Mesh](https://mediapipe.page.link/face-geometry-blog) in Google Developers Blog
* [MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
in Google Developers Blog
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
in Google Developers Blog
* [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)

7
docs/index.md
View File

@ -50,6 +50,10 @@ Object Detection
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
See also
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
for ML models released in MediaPipe.
## MediaPipe on the Web
MediaPipe on the Web is an effort to run the same ML solutions built for mobile
@ -89,7 +93,8 @@ run code search using
## Publications
* [Face AR with MediaPipe Face Mesh](https://mediapipe.page.link/face-geometry-blog) in Google Developers Blog
* [MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
in Google Developers Blog
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
in Google Developers Blog
* [BlazePose - On-device Real-time Body Pose Tracking](https://ai.googleblog.com/2020/08/on-device-real-time-body-pose-tracking.html)

2
docs/solutions/face_mesh.md
View File

@ -277,7 +277,7 @@ only works for a single face. For visual reference, please refer to *Fig. 4*.
* TensorFlow Blog:
[Face and hand tracking in the browser with MediaPipe and TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
* Google Developers Blog:
[Face AR with MediaPipe Face Mesh](https://mediapipe.page.link/face-geometry-blog)
[MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
* Paper:
[Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs](https://arxiv.org/abs/1907.06724)
([poster](https://docs.google.com/presentation/d/1-LWwOMO9TzEVdrZ1CS1ndJzciRHfYDJfbSxH_ke_JRg/present?slide=id.g5986dd4b4c_4_212))

2
docs/solutions/models.md
View File

@ -5,7 +5,7 @@ parent: Solutions
nav_order: 30
---
# Models and Model Cards
# MediaPipe Models and Model Cards
{: .no_toc }
1. TOC

4
docs/solutions/solutions.md
View File

@ -32,3 +32,7 @@ has_toc: false
[AutoFlip](https://google.github.io/mediapipe/solutions/autoflip) | | | ✅ | | |
[MediaSequence](https://google.github.io/mediapipe/solutions/media_sequence) | | | ✅ | | |
[YouTube 8M](https://google.github.io/mediapipe/solutions/youtube_8m) | | | ✅ | | |
See also
[MediaPipe Models and Model Cards](https://google.github.io/mediapipe/solutions/models)
for ML models released in MediaPipe.

113
mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.cc
View File

@ -42,16 +42,16 @@ namespace tf = tensorflow;
// a flag controls whether a new first dimension is inserted before
// concatenation.
//
// Currently, the number of tensors output will be buffer_size less than the
// number of input tensors because no padding is implemented and only full
// buffers are output.
// The number of tensors output will be buffer_size less than the
// number of input tensors unless padding is set to a non-zero value in the
// options proto.
//
// The timestamp of the output batch will match the timestamp of the first
// tensor in that batch by default. (e.g. when buffer_size frames are added, the
// output tensor will have the timestamp of the first input.). This behavior can
// be adjusted by the timestamp_offset option.
//
// Example config:
// Example config without padding:
// node {
// calculator: "LappedTensorBufferCalculator"
// input_stream: "input_tensor"
@ -64,26 +64,50 @@ namespace tf = tensorflow;
// }
// }
// }
//
// Example config with padding and timestamp output:
// node {
// calculator: "LappedTensorBufferCalculator"
// input_stream: "input_tensor"
// output_stream: "output_tensor"
// output_stream: "output_timestamp"
// options {
// [mediapipe.LappedTensorBufferCalculatorOptions.ext] {
// buffer_size: 100
// overlap: 50
// add_batch_dim_to_tensors: true
// timestamp_offset: 25
// padding: 25
// }
// }
// }
class LappedTensorBufferCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
private:
// Adds a batch dimension to the input tensor if specified in the calculator
// options.
// Adds a batch dimension to the input tensor if specified in the
// calculator options.
::mediapipe::Status AddBatchDimension(tf::Tensor* input_tensor);
// Sends the current buffer downstream.
::mediapipe::Status ProcessBuffer(CalculatorContext* cc);
int steps_until_output_;
int buffer_size_;
int overlap_;
int timestamp_offset_;
int initialized_;
std::unique_ptr<CircularBuffer<Timestamp>> timestamp_buffer_;
std::unique_ptr<CircularBuffer<tf::Tensor>> buffer_;
LappedTensorBufferCalculatorOptions options_;
};
REGISTER_CALCULATOR(LappedTensorBufferCalculator);
::mediapipe::Status LappedTensorBufferCalculator::GetContract(
@ -93,8 +117,8 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
cc->Inputs().Index(0).Set<tf::Tensor>(
// tensorflow::Tensor stream.
);
RET_CHECK_EQ(cc->Outputs().NumEntries(), 1)
<< "Only one output stream is supported.";
RET_CHECK_LE(cc->Outputs().NumEntries(), 2)
<< "Only one or two output stream(s) is/are supported.";
if (cc->InputSidePackets().HasTag(kBufferSize)) {
cc->InputSidePackets().Tag(kBufferSize).Set<int>();
@ -108,11 +132,15 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
if (cc->InputSidePackets().HasTag(kCalculatorOptions)) {
cc->InputSidePackets()
.Tag(kCalculatorOptions)
.Set<LappedTensorBufferCalculatorOptions>();
.Set<LappedTensorBufferCalculator>();
}
cc->Outputs().Index(0).Set<tf::Tensor>(
// Output tensorflow::Tensor stream with possibly overlapping steps.
);
// Output timestamp stream with possibly overlapping steps.
if (cc->Outputs().NumEntries() > 1) {
cc->Outputs().Index(1).Set<std::vector<Timestamp>>();
}
return ::mediapipe::OkStatus();
}
@ -141,10 +169,13 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
<< "Negative timestamp_offset is not allowed.";
RET_CHECK_LT(timestamp_offset_, buffer_size_)
<< "output_frame_num_offset has to be less than buffer_size.";
RET_CHECK_LT(options_.padding(), buffer_size_)
<< "padding option must be smaller than buffer size.";
timestamp_buffer_ =
absl::make_unique<CircularBuffer<Timestamp>>(buffer_size_);
buffer_ = absl::make_unique<CircularBuffer<tf::Tensor>>(buffer_size_);
steps_until_output_ = buffer_size_;
steps_until_output_ = buffer_size_ - options_.padding();
initialized_ = false;
return ::mediapipe::OkStatus();
}
@ -156,23 +187,36 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
if (options_.add_batch_dim_to_tensors()) {
RET_CHECK_OK(AddBatchDimension(&input_tensor));
}
// Pad frames at the beginning with the first frame.
if (!initialized_) {
for (int i = 0; i < options_.padding(); ++i) {
buffer_->push_back(input_tensor);
timestamp_buffer_->push_back(cc->InputTimestamp());
}
initialized_ = true;
}
buffer_->push_back(input_tensor);
timestamp_buffer_->push_back(cc->InputTimestamp());
--steps_until_output_;
if (steps_until_output_ <= 0) {
auto concatenated = ::absl::make_unique<tf::Tensor>();
const tf::Status concat_status = tf::tensor::Concat(
std::vector<tf::Tensor>(buffer_->begin(), buffer_->end()),
concatenated.get());
RET_CHECK(concat_status.ok()) << concat_status.ToString();
cc->Outputs().Index(0).Add(concatenated.release(),
timestamp_buffer_->Get(timestamp_offset_));
steps_until_output_ = buffer_size_ - overlap_;
MP_RETURN_IF_ERROR(ProcessBuffer(cc));
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status LappedTensorBufferCalculator::Close(CalculatorContext* cc) {
if (!initialized_ || options_.padding() == 0) {
return ::mediapipe::OkStatus();
}
int last_frame = buffer_size_ - steps_until_output_ - 1;
const auto& pad_frame = buffer_->Get(last_frame);
for (int i = 0; i < steps_until_output_ + options_.padding(); ++i) {
buffer_->push_back(pad_frame);
timestamp_buffer_->push_back(cc->InputTimestamp());
}
MP_RETURN_IF_ERROR(ProcessBuffer(cc));
return ::mediapipe::OkStatus();
}
@ -190,4 +234,29 @@ REGISTER_CALCULATOR(LappedTensorBufferCalculator);
return ::mediapipe::OkStatus();
}
// Process buffer
::mediapipe::Status LappedTensorBufferCalculator::ProcessBuffer(
CalculatorContext* cc) {
auto concatenated = ::absl::make_unique<tf::Tensor>();
const tf::Status concat_status = tf::tensor::Concat(
std::vector<tf::Tensor>(buffer_->begin(), buffer_->end()),
concatenated.get());
RET_CHECK(concat_status.ok()) << concat_status.ToString();
// Output cancatenated tensor.
cc->Outputs().Index(0).Add(concatenated.release(),
timestamp_buffer_->Get(timestamp_offset_));
if (cc->Outputs().NumEntries() > 1) {
auto output_timestamp = ::absl::make_unique<std::vector<Timestamp>>();
// Output timestamp vector.
*output_timestamp = std::vector<Timestamp>(timestamp_buffer_->begin(),
timestamp_buffer_->end());
RET_CHECK_EQ(output_timestamp->size(), buffer_size_)
<< "Output timestamp size is not correct.";
cc->Outputs().Index(1).Add(output_timestamp.release(),
timestamp_buffer_->Get(timestamp_offset_));
}
steps_until_output_ = buffer_size_ - overlap_;
return ::mediapipe::OkStatus();
}
} // namespace mediapipe

4
mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator.proto
View File

@ -45,4 +45,8 @@ message LappedTensorBufferCalculatorOptions {
// This is useful for aligning the timestamp to be centered on the input
// range.
optional int32 timestamp_offset = 4 [default = 0];
// Amount of padding (repeating of first/last value) to add to the beginning
// and end of the input stream.
optional int32 padding = 5;
}

55
mediapipe/calculators/tensorflow/lapped_tensor_buffer_calculator_test.cc
View File

@ -31,11 +31,15 @@ namespace tf = ::tensorflow;
class LappedTensorBufferCalculatorTest : public ::testing::Test {
protected:
void SetUpCalculator(int buffer_size, int overlap, bool add_dim,
int timestamp_offset) {
int timestamp_offset, int padding,
bool timestamp_output) {
CalculatorGraphConfig::Node config;
config.set_calculator("LappedTensorBufferCalculator");
config.add_input_stream("input_tensor");
config.add_output_stream("output_tensor");
if (timestamp_output) {
config.add_output_stream("output_timestamp");
}
auto options = config.mutable_options()->MutableExtension(
LappedTensorBufferCalculatorOptions::ext);
options->set_buffer_size(buffer_size);
@ -44,13 +48,14 @@ class LappedTensorBufferCalculatorTest : public ::testing::Test {
options->set_add_batch_dim_to_tensors(true);
}
options->set_timestamp_offset(timestamp_offset);
options->set_padding(padding);
runner_ = ::absl::make_unique<CalculatorRunner>(config);
}
std::unique_ptr<CalculatorRunner> runner_;
};
TEST_F(LappedTensorBufferCalculatorTest, OneToOne) {
SetUpCalculator(1, 0, false, 0);
SetUpCalculator(1, 0, false, 0, 0, false);
int num_timesteps = 3;
for (int i = 0; i < num_timesteps; ++i) {
auto input = ::absl::make_unique<tensorflow::Tensor>(
@ -74,7 +79,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToTwo) {
int buffer_size = 2;
int overlap = 1;
bool add_dim = false;
SetUpCalculator(buffer_size, overlap, add_dim, 0);
SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
int num_timesteps = 3;
for (int i = 0; i < num_timesteps; ++i) {
auto input = ::absl::make_unique<tensorflow::Tensor>(
@ -100,7 +105,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThree) {
int buffer_size = 3;
int overlap = 2;
bool add_dim = false;
SetUpCalculator(buffer_size, overlap, add_dim, 0);
SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
int num_timesteps = 3;
for (int i = 0; i < num_timesteps; ++i) {
auto input = ::absl::make_unique<tensorflow::Tensor>(
@ -126,7 +131,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeSkip) {
int buffer_size = 3;
int overlap = 1;
bool add_dim = false;
SetUpCalculator(buffer_size, overlap, add_dim, 0);
SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
int num_timesteps = 3;
for (int i = 0; i < num_timesteps; ++i) {
auto input = ::absl::make_unique<tensorflow::Tensor>(
@ -152,7 +157,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeBatch) {
int buffer_size = 3;
int overlap = 2;
bool add_dim = true;
SetUpCalculator(buffer_size, overlap, add_dim, 0);
SetUpCalculator(buffer_size, overlap, add_dim, 0, 0, false);
int num_timesteps = 3;
for (int i = 0; i < num_timesteps; ++i) {
auto input = ::absl::make_unique<tensorflow::Tensor>(
@ -180,7 +185,7 @@ TEST_F(LappedTensorBufferCalculatorTest, NegativeTimestampOffsetFails) {
int overlap = 15;
bool add_dim = true;
int timestamp_offset = -7;
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset);
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false);
int num_timesteps = 20;
for (int i = 0; i < num_timesteps; ++i) {
auto input = ::absl::make_unique<tensorflow::Tensor>(
@ -197,7 +202,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OutOfRangeTimestampOffsetFails) {
int overlap = 15;
bool add_dim = true;
int timestamp_offset = buffer_size;
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset);
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false);
int num_timesteps = 20;
for (int i = 0; i < num_timesteps; ++i) {
auto input = ::absl::make_unique<tensorflow::Tensor>(
@ -214,7 +219,7 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeBatchTimestampOffset) {
int overlap = 15;
bool add_dim = true;
int timestamp_offset = 7;
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset);
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, 0, false);
int num_timesteps = 20;
for (int i = 0; i < num_timesteps; ++i) {
auto input = ::absl::make_unique<tensorflow::Tensor>(
@ -236,5 +241,37 @@ TEST_F(LappedTensorBufferCalculatorTest, OneToThreeBatchTimestampOffset) {
}
}
TEST_F(LappedTensorBufferCalculatorTest,
OneToThreeBatchTimestampOffsetPadding) {
int buffer_size = 12;
int overlap = 6;
bool add_dim = true;
int timestamp_offset = 3;
int padding = 0;
SetUpCalculator(buffer_size, overlap, add_dim, timestamp_offset, padding,
true);
int num_timesteps = 20;
for (int i = 0; i < num_timesteps; ++i) {
auto input = ::absl::make_unique<tensorflow::Tensor>(
tensorflow::DT_FLOAT, tensorflow::TensorShape({1}));
input->tensor<float, 1>()(0) = i;
runner_->MutableInputs()->Index(0).packets.push_back(
Adopt(input.release()).At(Timestamp(i)));
}
ASSERT_TRUE(runner_->Run().ok());
const int output_size = num_timesteps / buffer_size + 1;
const std::vector<Packet>& output_packets =
runner_->Outputs().Index(0).packets;
ASSERT_EQ(output_size, output_packets.size());
for (int i = 0; i < output_size; ++i) {
int64 value = output_packets[i].Timestamp().Value();
ASSERT_EQ(i * overlap + timestamp_offset, value);
}
const std::vector<Packet>& output_timestamps =
runner_->Outputs().Index(1).packets;
ASSERT_EQ(output_size, output_timestamps.size());
}
} // namespace
} // namespace mediapipe