cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
PiperOrigin-RevId: 267274408
This commit is contained in:
MediaPipe Team 2019-09-04 18:19:36 -07:00 committed by jqtang
parent 731d2b9536
commit af67642055
80 changed files with 3181 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Dockerfile
View File

@ -24,6 +24,10 @@ ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
ca-certificates \
<<<<<<< HEAD
=======
curl \
>>>>>>> Project import generated by Copybara.
git \
wget \
unzip \
@ -35,7 +39,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libopencv-video-dev \
software-properties-common && \
add-apt-repository -y ppa:openjdk-r/ppa && \
<<<<<<< HEAD
apt-get update && apt-get install -y openjdk-11-jdk && \
=======
apt-get update && apt-get install -y openjdk-8-jdk && \
>>>>>>> Project import generated by Copybara.
apt-get clean && \
rm -rf /var/lib/apt/lists/*

60
mediapipe/calculators/audio/BUILD
View File

@ -68,6 +68,26 @@ mediapipe_cc_proto_library(
)
proto_library(
<<<<<<< HEAD
=======
name = "stabilized_log_calculator_proto",
srcs = ["stabilized_log_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "stabilized_log_calculator_cc_proto",
srcs = ["stabilized_log_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":stabilized_log_calculator_proto"],
)
proto_library(
>>>>>>> Project import generated by Copybara.
name = "time_series_framer_calculator_proto",
srcs = ["time_series_framer_calculator.proto"],
visibility = ["//visibility:public"],
@ -157,6 +177,25 @@ cc_library(
)
cc_library(
<<<<<<< HEAD
=======
name = "stabilized_log_calculator",
srcs = ["stabilized_log_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":stabilized_log_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:core_proto",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_util",
],
alwayslink = 1,
)
cc_library(
>>>>>>> Project import generated by Copybara.
name = "spectrogram_calculator",
srcs = ["spectrogram_calculator.cc"],
visibility = ["//visibility:public"],
@ -267,6 +306,27 @@ cc_test(
)
cc_test(
<<<<<<< HEAD
=======
name = "stabilized_log_calculator_test",
srcs = ["stabilized_log_calculator_test.cc"],
deps = [
":stabilized_log_calculator",
":stabilized_log_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:time_series_header_cc_proto",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:status",
"//mediapipe/util:time_series_test_util",
"@eigen_archive//:eigen",
],
)
cc_test(
>>>>>>> Project import generated by Copybara.
name = "time_series_framer_calculator_test",
srcs = ["time_series_framer_calculator_test.cc"],
deps = [

4
mediapipe/calculators/audio/audio_decoder_calculator.cc
View File

@ -64,7 +64,11 @@ class AudioDecoderCalculator : public CalculatorBase {
cc->Outputs().Tag("AUDIO").Set<Matrix>();
if (cc->Outputs().HasTag("AUDIO_HEADER")) {
<<<<<<< HEAD
cc->Outputs().Tag("AUDIO_HEADER").Set<mediapipe::TimeSeriesHeader>();
=======
cc->Outputs().Tag("AUDIO_HEADER").SetNone();
>>>>>>> Project import generated by Copybara.
}
return ::mediapipe::OkStatus();
}

22
mediapipe/calculators/audio/mfcc_mel_calculators.cc
View File

@ -90,8 +90,13 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
private:
// Takes header and options, and sets up state including calling
// set_num_output_channels() on the base object.
<<<<<<< HEAD
virtual ::mediapipe::Status ConfigureTransform(
const TimeSeriesHeader& header, const CalculatorOptions& options) = 0;
=======
virtual ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
CalculatorContext* cc) = 0;
>>>>>>> Project import generated by Copybara.
// Takes a vector<double> corresponding to an input frame, and
// perform the specific transformation to produce an output frame.
@ -108,7 +113,11 @@ class FramewiseTransformCalculatorBase : public CalculatorBase {
RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
cc->Inputs().Index(0).Header(), &input_header));
<<<<<<< HEAD
::mediapipe::Status status = ConfigureTransform(input_header, cc->Options());
=======
::mediapipe::Status status = ConfigureTransform(input_header, cc);
>>>>>>> Project import generated by Copybara.
auto output_header = new TimeSeriesHeader(input_header);
output_header->set_num_channels(num_output_channels_);
@ -175,11 +184,17 @@ class MfccCalculator : public FramewiseTransformCalculatorBase {
}
private:
<<<<<<< HEAD
::mediapipe::Status ConfigureTransform(
const TimeSeriesHeader& header,
const CalculatorOptions& options) override {
MfccCalculatorOptions mfcc_options;
time_series_util::FillOptionsExtensionOrDie(options, &mfcc_options);
=======
::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
CalculatorContext* cc) override {
MfccCalculatorOptions mfcc_options = cc->Options<MfccCalculatorOptions>();
>>>>>>> Project import generated by Copybara.
mfcc_.reset(new audio_dsp::Mfcc());
int input_length = header.num_channels();
// Set up the parameters to the Mfcc object.
@ -235,11 +250,18 @@ class MelSpectrumCalculator : public FramewiseTransformCalculatorBase {
}
private:
<<<<<<< HEAD
::mediapipe::Status ConfigureTransform(
const TimeSeriesHeader& header,
const CalculatorOptions& options) override {
MelSpectrumCalculatorOptions mel_spectrum_options;
time_series_util::FillOptionsExtensionOrDie(options, &mel_spectrum_options);
=======
::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header,
CalculatorContext* cc) override {
MelSpectrumCalculatorOptions mel_spectrum_options =
cc->Options<MelSpectrumCalculatorOptions>();
>>>>>>> Project import generated by Copybara.
mel_filterbank_.reset(new audio_dsp::MelFilterbank());
int input_length = header.num_channels();
set_num_output_channels(mel_spectrum_options.channel_count());

5
mediapipe/calculators/audio/rational_factor_resample_calculator.cc
View File

@ -64,8 +64,13 @@ void CopyVectorToChannel(const std::vector<float>& vec, Matrix* matrix,
::mediapipe::Status RationalFactorResampleCalculator::Open(
CalculatorContext* cc) {
<<<<<<< HEAD
RationalFactorResampleCalculatorOptions resample_options;
time_series_util::FillOptionsExtensionOrDie(cc->Options(), &resample_options);
=======
RationalFactorResampleCalculatorOptions resample_options =
cc->Options<RationalFactorResampleCalculatorOptions>();
>>>>>>> Project import generated by Copybara.
if (!resample_options.has_target_sample_rate()) {
return tool::StatusInvalid(

17
mediapipe/calculators/audio/spectrogram_calculator.cc
View File

@ -71,10 +71,15 @@ class SpectrogramCalculator : public CalculatorBase {
// Input stream with TimeSeriesHeader.
);
<<<<<<< HEAD
SpectrogramCalculatorOptions spectrogram_options;
time_series_util::FillOptionsExtensionOrDie(cc->Options(),
&spectrogram_options);
=======
SpectrogramCalculatorOptions spectrogram_options =
cc->Options<SpectrogramCalculatorOptions>();
>>>>>>> Project import generated by Copybara.
if (!spectrogram_options.allow_multichannel_input()) {
if (spectrogram_options.output_type() ==
SpectrogramCalculatorOptions::COMPLEX) {
@ -172,9 +177,14 @@ REGISTER_CALCULATOR(SpectrogramCalculator);
const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
::mediapipe::Status SpectrogramCalculator::Open(CalculatorContext* cc) {
<<<<<<< HEAD
SpectrogramCalculatorOptions spectrogram_options;
time_series_util::FillOptionsExtensionOrDie(cc->Options(),
&spectrogram_options);
=======
SpectrogramCalculatorOptions spectrogram_options =
cc->Options<SpectrogramCalculatorOptions>();
>>>>>>> Project import generated by Copybara.
if (spectrogram_options.frame_duration_seconds() <= 0.0) {
::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC)
@ -223,6 +233,13 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518;
std::vector<double> window;
switch (spectrogram_options.window_type()) {
<<<<<<< HEAD
=======
case SpectrogramCalculatorOptions::COSINE:
audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_,
&window);
break;
>>>>>>> Project import generated by Copybara.
case SpectrogramCalculatorOptions::HANN:
audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_,
&window);

4
mediapipe/calculators/audio/spectrogram_calculator.proto
View File

@ -58,6 +58,10 @@ message SpectrogramCalculatorOptions {
enum WindowType {
HANN = 0;
HAMMING = 1;
<<<<<<< HEAD
=======
COSINE = 2;
>>>>>>> Project import generated by Copybara.
}
optional WindowType window_type = 6 [default = HANN];

94
mediapipe/calculators/audio/stabilized_log_calculator.cc Normal file
View File

@ -0,0 +1,94 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Defines StabilizedLogCalculator.
#include <cmath>
#include <memory>
#include <string>
#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/time_series_header.pb.h"
#include "mediapipe/framework/port/proto_ns.h"
#include "mediapipe/util/time_series_util.h"
namespace mediapipe {
// Example config:
// node {
// calculator: "StabilizedLogCalculator"
// input_stream: "input_time_series"
// output_stream: "stabilized_log_time_series"
// options {
// [mediapipe.StabilizedLogCalculatorOptions.ext] {
// stabilizer: .00001
// check_nonnegativity: true
// }
// }
// }
class StabilizedLogCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<Matrix>(
// Input stream with TimeSeriesHeader.
);
cc->Outputs().Index(0).Set<Matrix>(
// Output stabilized log stream with TimeSeriesHeader.
);
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
StabilizedLogCalculatorOptions stabilized_log_calculator_options =
cc->Options<StabilizedLogCalculatorOptions>();
stabilizer_ = stabilized_log_calculator_options.stabilizer();
output_scale_ = stabilized_log_calculator_options.output_scale();
check_nonnegativity_ =
stabilized_log_calculator_options.check_nonnegativity();
CHECK_GE(stabilizer_, 0.0)
<< "stabilizer must be >= 0.0, received a value of " << stabilizer_;
// If the input packets have a header, propagate the header to the output.
if (!cc->Inputs().Index(0).Header().IsEmpty()) {
TimeSeriesHeader input_header;
RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid(
cc->Inputs().Index(0).Header(), &input_header));
cc->Outputs().Index(0).SetHeader(
Adopt(new TimeSeriesHeader(input_header)));
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
auto input_matrix = cc->Inputs().Index(0).Get<Matrix>();
if (check_nonnegativity_) {
CHECK_GE(input_matrix.minCoeff(), 0);
}
std::unique_ptr<Matrix> output_frame(new Matrix(
output_scale_ * (input_matrix.array() + stabilizer_).log().matrix()));
cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
private:
float stabilizer_;
bool check_nonnegativity_;
double output_scale_;
};
REGISTER_CALCULATOR(StabilizedLogCalculator);
} // namespace mediapipe

37
mediapipe/calculators/audio/stabilized_log_calculator.proto Normal file
View File

@ -0,0 +1,37 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message StabilizedLogCalculatorOptions {
extend CalculatorOptions {
optional StabilizedLogCalculatorOptions ext = 101978339;
}
// The calculator computes log(x + stabilizer). stabilizer must be >=
// 0, with 0 indicating a lack of stabilization.
optional float stabilizer = 1 [default = .00001];
// If true, CHECK that all input values in are >= 0. If false, the
// code will take the log of the potentially negative input values
// plus the stabilizer.
optional bool check_nonnegativity = 2 [default = true];
// Support a fixed multiplicative scaling of the output.
optional double output_scale = 3 [default = 1.0];
}

131
mediapipe/calculators/audio/stabilized_log_calculator_test.cc Normal file
View File

@ -0,0 +1,131 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "Eigen/Core"
#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/time_series_header.pb.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/util/time_series_test_util.h"
namespace mediapipe {
const float kStabilizer = 0.1;
const int kNumChannels = 3;
const int kNumSamples = 10;
class StabilizedLogCalculatorTest
: public TimeSeriesCalculatorTest<StabilizedLogCalculatorOptions> {
protected:
void SetUp() override {
calculator_name_ = "StabilizedLogCalculator";
options_.set_stabilizer(kStabilizer);
input_sample_rate_ = 8000.0;
num_input_channels_ = kNumChannels;
num_input_samples_ = kNumSamples;
}
void RunGraphNoReturn() { MEDIAPIPE_ASSERT_OK(RunGraph()); }
};
TEST_F(StabilizedLogCalculatorTest, BasicOperation) {
const int kNumPackets = 5;
InitializeGraph();
FillInputHeader();
std::vector<Matrix> input_data_matrices;
for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
Matrix input_data_matrix =
Matrix::Random(kNumChannels, kNumSamples).array().abs();
input_data_matrices.push_back(input_data_matrix);
AppendInputPacket(new Matrix(input_data_matrix), timestamp);
}
MEDIAPIPE_ASSERT_OK(RunGraph());
ExpectOutputHeaderEqualsInputHeader();
for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) {
ExpectApproximatelyEqual(
(input_data_matrices[output_packet].array() + kStabilizer).log(),
runner_->Outputs().Index(0).packets[output_packet].Get<Matrix>());
}
}
TEST_F(StabilizedLogCalculatorTest, OutputScaleWorks) {
const int kNumPackets = 5;
double output_scale = 2.5;
options_.set_output_scale(output_scale);
InitializeGraph();
FillInputHeader();
std::vector<Matrix> input_data_matrices;
for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) {
const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond;
Matrix input_data_matrix =
Matrix::Random(kNumChannels, kNumSamples).array().abs();
input_data_matrices.push_back(input_data_matrix);
AppendInputPacket(new Matrix(input_data_matrix), timestamp);
}
MEDIAPIPE_ASSERT_OK(RunGraph());
ExpectOutputHeaderEqualsInputHeader();
for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) {
ExpectApproximatelyEqual(
output_scale *
((input_data_matrices[output_packet].array() + kStabilizer).log()),
runner_->Outputs().Index(0).packets[output_packet].Get<Matrix>());
}
}
TEST_F(StabilizedLogCalculatorTest, ZerosAreStabilized) {
InitializeGraph();
FillInputHeader();
AppendInputPacket(new Matrix(Matrix::Zero(kNumChannels, kNumSamples)),
0 /* timestamp */);
MEDIAPIPE_ASSERT_OK(RunGraph());
ExpectOutputHeaderEqualsInputHeader();
ExpectApproximatelyEqual(
Matrix::Constant(kNumChannels, kNumSamples, kStabilizer).array().log(),
runner_->Outputs().Index(0).packets[0].Get<Matrix>());
}
TEST_F(StabilizedLogCalculatorTest, NegativeValuesCheckFail) {
InitializeGraph();
FillInputHeader();
AppendInputPacket(
new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)),
0 /* timestamp */);
ASSERT_DEATH(RunGraphNoReturn(), "");
}
TEST_F(StabilizedLogCalculatorTest, NegativeValuesDoNotCheckFailIfCheckIsOff) {
options_.set_check_nonnegativity(false);
InitializeGraph();
FillInputHeader();
AppendInputPacket(
new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)),
0 /* timestamp */);
MEDIAPIPE_ASSERT_OK(RunGraph());
// Results are undefined.
}
} // namespace mediapipe

5
mediapipe/calculators/audio/time_series_framer_calculator.cc
View File

@ -206,8 +206,13 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) {
}
::mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) {
<<<<<<< HEAD
TimeSeriesFramerCalculatorOptions framer_options;
time_series_util::FillOptionsExtensionOrDie(cc->Options(), &framer_options);
=======
TimeSeriesFramerCalculatorOptions framer_options =
cc->Options<TimeSeriesFramerCalculatorOptions>();
>>>>>>> Project import generated by Copybara.
RET_CHECK_GT(framer_options.frame_duration_seconds(), 0.0)
<< "Invalid or missing frame_duration_seconds. "

46
mediapipe/calculators/core/BUILD
View File

@ -162,6 +162,10 @@ cc_library(
deps = [
":concatenate_vector_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
<<<<<<< HEAD
=======
"//mediapipe/framework/formats:landmark_cc_proto",
>>>>>>> Project import generated by Copybara.
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"@org_tensorflow//tensorflow/lite:framework",
@ -523,6 +527,10 @@ cc_library(
deps = [
":split_vector_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
<<<<<<< HEAD
=======
"//mediapipe/framework/formats:landmark_cc_proto",
>>>>>>> Project import generated by Copybara.
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/util:resource_util",
@ -628,6 +636,44 @@ cc_test(
)
cc_library(
<<<<<<< HEAD
=======
name = "matrix_to_vector_calculator",
srcs = ["matrix_to_vector_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/tool:status_util",
"//mediapipe/util:time_series_util",
"@com_google_absl//absl/memory",
"@eigen_archive//:eigen",
],
alwayslink = 1,
)
cc_test(
name = "matrix_to_vector_calculator_test",
srcs = ["matrix_to_vector_calculator_test.cc"],
deps = [
":matrix_to_vector_calculator",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:integral_types",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:validate_type",
"//mediapipe/util:time_series_test_util",
"//mediapipe/util:time_series_util",
],
)
cc_library(
>>>>>>> Project import generated by Copybara.
name = "merge_calculator",
srcs = ["merge_calculator.cc"],
visibility = ["//visibility:public"],

10
mediapipe/calculators/core/concatenate_vector_calculator.cc
View File

@ -16,6 +16,10 @@
#include <vector>
<<<<<<< HEAD
=======
#include "mediapipe/framework/formats/landmark.pb.h"
>>>>>>> Project import generated by Copybara.
#include "tensorflow/lite/interpreter.h"
namespace mediapipe {
@ -41,4 +45,10 @@ typedef ConcatenateVectorCalculator<TfLiteTensor>
ConcatenateTfLiteTensorVectorCalculator;
REGISTER_CALCULATOR(ConcatenateTfLiteTensorVectorCalculator);
<<<<<<< HEAD
=======
typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark>
ConcatenateLandmarkVectorCalculator;
REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator);
>>>>>>> Project import generated by Copybara.
} // namespace mediapipe

83
mediapipe/calculators/core/matrix_to_vector_calculator.cc Normal file
View File

@ -0,0 +1,83 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Defines MatrixToVectorCalculator.
#include <math.h>
#include <deque>
#include <memory>
#include <string>
#include "Eigen/Core"
#include "absl/memory/memory.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/tool/status_util.h"
#include "mediapipe/util/time_series_util.h"
namespace mediapipe {
// A calculator that converts a Matrix M to a vector containing all the
// entries of M in column-major order.
//
// Example config:
// node {
// calculator: "MatrixToVectorCalculator"
// input_stream: "input_matrix"
// output_stream: "column_major_vector"
// }
class MatrixToVectorCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Index(0).Set<Matrix>(
// Input Packet containing a Matrix.
);
cc->Outputs().Index(0).Set<std::vector<float>>(
// Output Packet containing a vector, one for each input Packet.
);
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override;
// Outputs a packet containing a vector for each input packet.
::mediapipe::Status Process(CalculatorContext* cc) override;
};
REGISTER_CALCULATOR(MatrixToVectorCalculator);
::mediapipe::Status MatrixToVectorCalculator::Open(CalculatorContext* cc) {
// Inform the framework that we don't alter timestamps.
cc->SetOffset(mediapipe::TimestampDiff(0));
return ::mediapipe::OkStatus();
}
::mediapipe::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) {
const Matrix& input = cc->Inputs().Index(0).Get<Matrix>();
auto output = absl::make_unique<std::vector<float>>();
// The following lines work to convert the Matrix to a vector because Matrix
// is an Eigen::MatrixXf and Eigen uses column-major layout by default.
output->resize(input.rows() * input.cols());
auto output_as_matrix =
Eigen::Map<Matrix>(output->data(), input.rows(), input.cols());
output_as_matrix = input;
cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
} // namespace mediapipe

88
mediapipe/calculators/core/matrix_to_vector_calculator_test.cc Normal file
View File

@ -0,0 +1,88 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include <vector>
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/integral_types.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/tool/validate_type.h"
#include "mediapipe/util/time_series_test_util.h"
#include "mediapipe/util/time_series_util.h"
namespace mediapipe {
namespace {
class MatrixToVectorCalculatorTest
: public mediapipe::TimeSeriesCalculatorTest<mediapipe::NoOptions> {
protected:
void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; }
void AppendInput(const std::vector<float>& column_major_data,
int64 timestamp) {
ASSERT_EQ(num_input_samples_ * num_input_channels_,
column_major_data.size());
Eigen::Map<const Matrix> data_map(&column_major_data[0],
num_input_channels_, num_input_samples_);
AppendInputPacket(new Matrix(data_map), timestamp);
}
void SetInputStreamParameters(int num_channels, int num_samples) {
num_input_channels_ = num_channels;
num_input_samples_ = num_samples;
input_sample_rate_ = 100;
input_packet_rate_ = 20.0;
}
void SetInputHeader(int num_channels, int num_samples) {
SetInputStreamParameters(num_channels, num_samples);
FillInputHeader();
}
void CheckOutputPacket(int packet, std::vector<float> expected_vector) {
const auto& actual_vector =
runner_->Outputs().Index(0).packets[packet].Get<std::vector<float>>();
EXPECT_THAT(actual_vector, testing::ContainerEq(expected_vector));
}
};
TEST_F(MatrixToVectorCalculatorTest, SingleRow) {
InitializeGraph();
SetInputHeader(1, 4); // 1 channel x 4 samples
const std::vector<float>& data_vector = {1.0, 2.0, 3.0, 4.0};
AppendInput(data_vector, 0);
MEDIAPIPE_ASSERT_OK(RunGraph());
CheckOutputPacket(0, data_vector);
}
TEST_F(MatrixToVectorCalculatorTest, RegularMatrix) {
InitializeGraph();
SetInputHeader(4, 2); // 4 channels x 2 samples
// Actual data matrix is the transpose of the appearance below.
const std::vector<float>& data_vector = {1.0, 2.0, 3.0, 4.0,
5.0, 6.0, 7.0, 8.0};
AppendInput(data_vector, 0);
MEDIAPIPE_ASSERT_OK(RunGraph());
CheckOutputPacket(0, data_vector);
}
} // namespace
} // namespace mediapipe

10
mediapipe/calculators/core/split_vector_calculator.cc
View File

@ -16,6 +16,10 @@
#include <vector>
<<<<<<< HEAD
=======
#include "mediapipe/framework/formats/landmark.pb.h"
>>>>>>> Project import generated by Copybara.
#include "tensorflow/lite/interpreter.h"
namespace mediapipe {
@ -37,4 +41,10 @@ namespace mediapipe {
typedef SplitVectorCalculator<TfLiteTensor> SplitTfLiteTensorVectorCalculator;
REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator);
<<<<<<< HEAD
=======
typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark>
SplitLandmarkVectorCalculator;
REGISTER_CALCULATOR(SplitLandmarkVectorCalculator);
>>>>>>> Project import generated by Copybara.
} // namespace mediapipe

4
mediapipe/calculators/image/image_transformation_calculator.cc
View File

@ -244,7 +244,11 @@ REGISTER_CALCULATOR(ImageTransformationCalculator);
rotation_ = DegreesToRotationMode(
cc->InputSidePackets().Tag("ROTATION_DEGREES").Get<int>());
} else {
<<<<<<< HEAD
rotation_ = DegreesToRotationMode(options_.rotation_mode());
=======
rotation_ = options_.rotation_mode();
>>>>>>> Project import generated by Copybara.
}
scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE);

79
mediapipe/calculators/tensorflow/BUILD
View File

@ -188,6 +188,20 @@ mediapipe_cc_proto_library(
)
mediapipe_cc_proto_library(
<<<<<<< HEAD
=======
name = "tensorflow_session_from_frozen_graph_calculator_cc_proto",
srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"@org_tensorflow//tensorflow/core:protos_all_cc",
],
visibility = ["//mediapipe:__subpackages__"],
deps = [":tensorflow_session_from_frozen_graph_calculator_proto"],
)
mediapipe_cc_proto_library(
>>>>>>> Project import generated by Copybara.
name = "tensorflow_session_from_saved_model_generator_cc_proto",
srcs = ["tensorflow_session_from_saved_model_generator.proto"],
cc_deps = ["//mediapipe/framework:packet_generator_cc_proto"],
@ -445,6 +459,38 @@ cc_library(
)
cc_library(
<<<<<<< HEAD
=======
name = "tensorflow_session_from_frozen_graph_calculator",
srcs = ["tensorflow_session_from_frozen_graph_calculator.cc"],
features = ["no_layering_check"],
visibility = ["//visibility:public"],
deps = [
":tensorflow_session",
"//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/tool:status_util",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:ret_check",
] + select({
"//conditions:default": [
"//mediapipe/framework/port:file_helpers",
"@org_tensorflow//tensorflow/core:core",
],
"//mediapipe:android": [
"@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos",
"//mediapipe/android/file/base",
],
"//mediapipe:ios": [
"@org_tensorflow//tensorflow/core:ios_tensorflow_lib",
"//mediapipe/android/file/base",
],
}),
alwayslink = 1,
)
cc_library(
>>>>>>> Project import generated by Copybara.
name = "tensorflow_session_from_frozen_graph_generator",
srcs = ["tensorflow_session_from_frozen_graph_generator.cc"],
features = ["no_layering_check"],
@ -738,6 +784,39 @@ cc_test(
)
cc_test(
<<<<<<< HEAD
=======
name = "tensorflow_session_from_frozen_graph_calculator_test",
srcs = ["tensorflow_session_from_frozen_graph_calculator_test.cc"],
data = [":test_frozen_graph"],
linkstatic = 1,
deps = [
":tensorflow_inference_calculator",
":tensorflow_session",
":tensorflow_session_from_frozen_graph_calculator",
"//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework:packet",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/framework/tool:tag_map_helper",
"//mediapipe/framework/tool:validate_type",
"@com_google_absl//absl/strings",
"@org_tensorflow//tensorflow/core:direct_session",
"@org_tensorflow//tensorflow/core:framework",
"@org_tensorflow//tensorflow/core:protos_all_cc",
"@org_tensorflow//tensorflow/core:testlib",
"@org_tensorflow//tensorflow/core/kernels:conv_ops",
"@org_tensorflow//tensorflow/core/kernels:math",
],
)
cc_test(
>>>>>>> Project import generated by Copybara.
name = "tensorflow_session_from_frozen_graph_generator_test",
srcs = ["tensorflow_session_from_frozen_graph_generator_test.cc"],
data = [":test_frozen_graph"],

19
mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc
View File

@ -34,6 +34,13 @@
#include "tensorflow/core/framework/tensor_shape.h"
#include "tensorflow/core/framework/tensor_util.h"
<<<<<<< HEAD
=======
#if !defined(__ANDROID__) && !defined(__APPLE__)
#include "tensorflow/core/profiler/lib/traceme.h"
#endif
>>>>>>> Project import generated by Copybara.
namespace tf = ::tensorflow;
namespace mediapipe {
@ -435,9 +442,21 @@ class TensorFlowInferenceCalculator : public CalculatorBase {
session_run_throttle->Acquire(1);
}
const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow());
<<<<<<< HEAD
const tf::Status tf_status =
session_->Run(input_tensors, output_tensor_names,
{} /* target_node_names */, &outputs);
=======
tf::Status tf_status;
{
#if !defined(__ANDROID__) && !defined(__APPLE__)
tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName()));
#endif
tf_status = session_->Run(input_tensors, output_tensor_names,
{} /* target_node_names */, &outputs);
}
>>>>>>> Project import generated by Copybara.
if (session_run_throttle != nullptr) {
session_run_throttle->Release(1);
}

136
mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc Normal file
View File

@ -0,0 +1,136 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Reads serialized GraphDef proto. There are three ways to load a model:
// 1. Specify the path to a graph.pb in the calculator options.
// 2. Specify the path to the graph.pb through the
// input_side_packet:STRING_MODEL_FILE_PATH
// 3. Provide a serialized GraphDef through input_side_packet:STRING_MODEL,
// typically provided by EmbeddingFilePacketFactory.
//
// Produces a SessionBundle that TensorFlowInferenceCalculator can use.
#include <string>
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/tool/status_util.h"
#include "tensorflow/core/public/session_options.h"
#if defined(MEDIAPIPE_LITE) || defined(__ANDROID__) || \
defined(__APPLE__) && !TARGET_OS_OSX
#include "mediapipe/util/android/file/base/helpers.h"
#else
#include "mediapipe/framework/port/file_helpers.h"
#endif
namespace mediapipe {
namespace tf = ::tensorflow;
class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc) {
const auto& options =
cc->Options<TensorFlowSessionFromFrozenGraphCalculatorOptions>();
bool has_exactly_one_model =
!options.graph_proto_path().empty()
? !(cc->InputSidePackets().HasTag("STRING_MODEL") |
cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH"))
: (cc->InputSidePackets().HasTag("STRING_MODEL") ^
cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH"));
RET_CHECK(has_exactly_one_model)
<< "Must have exactly one of graph_proto_path in options or "
"input_side_packets STRING_MODEL or STRING_MODEL_FILE_PATH";
if (cc->InputSidePackets().HasTag("STRING_MODEL")) {
cc->InputSidePackets()
.Tag("STRING_MODEL")
.Set<std::string>(
// String model from embedded path
);
} else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) {
cc->InputSidePackets()
.Tag("STRING_MODEL_FILE_PATH")
.Set<std::string>(
// Filename of std::string model.
);
}
cc->OutputSidePackets().Tag("SESSION").Set<TensorFlowSession>(
// A TensorFlow model loaded and ready for use along with
// a map from tags to tensor names.
);
RET_CHECK_GT(options.tag_to_tensor_names().size(), 0);
return ::mediapipe::OkStatus();
}
::mediapipe::Status Open(CalculatorContext* cc) override {
const auto& options =
cc->Options<TensorFlowSessionFromFrozenGraphCalculatorOptions>();
// Output bundle packet.
auto session = ::absl::make_unique<TensorFlowSession>();
tf::SessionOptions session_options;
session_options.config.CopyFrom(options.config());
std::vector<mediapipe::ProtoString> initialization_op_names;
initialization_op_names.reserve(options.initialization_op_names_size());
for (int i = 0; i < options.initialization_op_names_size(); ++i) {
initialization_op_names.emplace_back(options.initialization_op_names(i));
}
session->session.reset(tf::NewSession(session_options));
std::string graph_def_serialized;
if (cc->InputSidePackets().HasTag("STRING_MODEL")) {
graph_def_serialized =
cc->InputSidePackets().Tag("STRING_MODEL").Get<std::string>();
} else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) {
const std::string& frozen_graph = cc->InputSidePackets()
.Tag("STRING_MODEL_FILE_PATH")
.Get<std::string>();
RET_CHECK_OK(
mediapipe::file::GetContents(frozen_graph, &graph_def_serialized));
} else {
RET_CHECK_OK(mediapipe::file::GetContents(options.graph_proto_path(),
&graph_def_serialized));
}
tensorflow::GraphDef graph_def;
RET_CHECK(graph_def.ParseFromString(graph_def_serialized));
const tf::Status tf_status = session->session->Create(graph_def);
RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.error_message();
for (const auto& key_value : options.tag_to_tensor_names()) {
session->tag_to_tensor_map[key_value.first] = key_value.second;
}
if (!initialization_op_names.empty()) {
const tf::Status tf_status =
session->session->Run({}, {}, initialization_op_names, {});
// RET_CHECK on the tf::Status object itself in order to print an
// informative error message.
RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.error_message();
}
cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release()));
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) override {
return ::mediapipe::OkStatus();
}
};
REGISTER_CALCULATOR(TensorFlowSessionFromFrozenGraphCalculator);
} // namespace mediapipe

72
mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.proto Normal file
View File

@ -0,0 +1,72 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
import "tensorflow/core/protobuf/config.proto";
message TensorFlowSessionFromFrozenGraphCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional TensorFlowSessionFromFrozenGraphCalculatorOptions ext = 266997877;
}
// Path to file containing serialized proto of type tensorflow::GraphDef.
optional string graph_proto_path = 1;
// To run inference with MediaPipe inputs MediaPipe streams need to be mapped
// to TensorFlow tensors. This map defines the which streams are fed into
// which tensors in the model. The MediaPipe tag of the stream is the map key.
// Tags must be capitalized, matching regex [A-Z0-9_]+. Examples: "JPG_STRING"
// and "SOFTMAX". Then, those tags can be used as the MediaPipe tags of
// input_stream or output_stream of the TensorflowInferenceCalculator
// consuming the packet produced by this calculator. The tensor names must
// match the tensor names in the graph that you want to feed or fetch into or
// out of. Examples: "DecodeJpeg/contents:0" or "softmax:0". For example, a
// mediapipe graph can include the nodes:
//
// node {
// calculator: "TensorFlowSessionFromFrozenGraphCalculator"
// output_side_packet: "SESSION:session"
// options {
// [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
// graph_proto_path: "[PATH]"
// tag_to_tensor_names {
// key: "JPG_STRING"
// value: "input:0"
// }
// tag_to_tensor_names {
// key: "SOFTMAX"
// value: "softmax:0"
// }
// }
// }
// }
// node {
// calculator: "TensorflowInferenceCalculator"
// input_side_packet: "SESSION:graph_with_bindings"
// input_stream: "JPG_STRING:jpg_string_tensor"
// output_stream: "SOFTMAX:softmax_tensor"
// }
map<string, string> tag_to_tensor_names = 2;
// Tensorflow session config options.
optional tensorflow.ConfigProto config = 3;
// Graph nodes to run to initialize the model. Any output of these ops is
// ignored.
repeated string initialization_op_names = 4;
}

316
mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc Normal file
View File

@ -0,0 +1,316 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "absl/strings/substitute.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session.h"
#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "mediapipe/framework/tool/tag_map_helper.h"
#include "mediapipe/framework/tool/validate_type.h"
#include "tensorflow/core/framework/tensor.h"
#include "tensorflow/core/protobuf/config.pb.h"
namespace mediapipe {
namespace {
namespace tf = ::tensorflow;
std::string GetGraphDefPath() {
return mediapipe::file::JoinPath("./",
"mediapipe/calculators/tensorflow/"
"testdata/frozen_graph_def.pb");
}
// Helper function that creates Tensor INT32 matrix with size 1x3.
tf::Tensor TensorMatrix1x3(const int v1, const int v2, const int v3) {
tf::Tensor tensor(tf::DT_INT32,
tf::TensorShape(std::vector<tf::int64>({1, 3})));
auto matrix = tensor.matrix<int32>();
matrix(0, 0) = v1;
matrix(0, 1) = v2;
matrix(0, 2) = v3;
return tensor;
}
class TensorFlowSessionFromFrozenGraphCalculatorTest : public ::testing::Test {
protected:
void SetUp() override {
extendable_options_.Clear();
calculator_options_ = extendable_options_.MutableExtension(
TensorFlowSessionFromFrozenGraphCalculatorOptions::ext);
calculator_options_->set_graph_proto_path(GetGraphDefPath());
(*calculator_options_->mutable_tag_to_tensor_names())["MULTIPLIED"] =
"multiplied:0";
(*calculator_options_->mutable_tag_to_tensor_names())["A"] = "a:0";
(*calculator_options_->mutable_tag_to_tensor_names())["B"] = "b:0";
calculator_options_->mutable_config()->set_intra_op_parallelism_threads(1);
calculator_options_->mutable_config()->set_inter_op_parallelism_threads(2);
}
void VerifySignatureMap(const TensorFlowSession& session) {
// Session must be set.
ASSERT_NE(session.session, nullptr);
// Bindings are inserted.
EXPECT_EQ(session.tag_to_tensor_map.size(), 3);
// For some reason, EXPECT_EQ and EXPECT_NE are not working with iterators.
EXPECT_FALSE(session.tag_to_tensor_map.find("A") ==
session.tag_to_tensor_map.end());
EXPECT_FALSE(session.tag_to_tensor_map.find("B") ==
session.tag_to_tensor_map.end());
EXPECT_FALSE(session.tag_to_tensor_map.find("MULTIPLIED") ==
session.tag_to_tensor_map.end());
// Sanity: find() actually returns a reference to end() if element not
// found.
EXPECT_TRUE(session.tag_to_tensor_map.find("Z") ==
session.tag_to_tensor_map.end());
EXPECT_EQ(session.tag_to_tensor_map.at("A"), "a:0");
EXPECT_EQ(session.tag_to_tensor_map.at("B"), "b:0");
EXPECT_EQ(session.tag_to_tensor_map.at("MULTIPLIED"), "multiplied:0");
}
CalculatorOptions extendable_options_;
TensorFlowSessionFromFrozenGraphCalculatorOptions* calculator_options_;
};
TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
CreatesPacketWithGraphAndBindings) {
CalculatorRunner runner(absl::Substitute(R"(
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
output_side_packet: "SESSION:tf_model"
options {
[mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
$0
}
})",
calculator_options_->DebugString()));
MEDIAPIPE_ASSERT_OK(runner.Run());
const TensorFlowSession& session =
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
VerifySignatureMap(session);
}
// Integration test. Verifies that TensorFlowInferenceCalculator correctly
// consumes the Packet emitted by this calculator.
TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
ProducesPacketUsableByTensorFlowInferenceCalculator) {
CalculatorGraphConfig config =
::mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
absl::Substitute(R"(
node {
calculator: "TensorFlowInferenceCalculator"
input_side_packet: "SESSION:session"
input_stream: "A:a_tensor"
output_stream: "MULTIPLIED:multiplied_tensor"
options {
[mediapipe.TensorFlowInferenceCalculatorOptions.ext] {
batch_size: 5
add_batch_dim_to_tensors: false
}
}
}
node {
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
output_side_packet: "SESSION:session"
options {
[mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
$0
}
}
}
input_stream: "a_tensor"
)",
calculator_options_->DebugString()));
CalculatorGraph graph;
MEDIAPIPE_ASSERT_OK(graph.Initialize(config));
StatusOrPoller status_or_poller =
graph.AddOutputStreamPoller("multiplied_tensor");
ASSERT_TRUE(status_or_poller.ok());
OutputStreamPoller poller = std::move(status_or_poller.ValueOrDie());
MEDIAPIPE_ASSERT_OK(graph.StartRun({}));
MEDIAPIPE_ASSERT_OK(graph.AddPacketToInputStream(
"a_tensor",
Adopt(new auto(TensorMatrix1x3(1, -1, 10))).At(Timestamp(0))));
MEDIAPIPE_ASSERT_OK(graph.CloseInputStream("a_tensor"));
Packet packet;
ASSERT_TRUE(poller.Next(&packet));
// input tensor gets multiplied by [[3, 2, 1]]. Expected output:
tf::Tensor expected_multiplication = TensorMatrix1x3(3, -2, 10);
EXPECT_EQ(expected_multiplication.DebugString(),
packet.Get<tf::Tensor>().DebugString());
ASSERT_FALSE(poller.Next(&packet));
MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone());
}
TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
CreatesPacketWithGraphAndBindingsFromInputSidePacket) {
calculator_options_->clear_graph_proto_path();
CalculatorRunner runner(absl::Substitute(R"(
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
input_side_packet: "STRING_MODEL:model"
output_side_packet: "SESSION:session"
options {
[mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
$0
}
})",
calculator_options_->DebugString()));
std::string serialized_graph_contents;
MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
&serialized_graph_contents));
runner.MutableSidePackets()->Tag("STRING_MODEL") =
Adopt(new std::string(serialized_graph_contents));
MEDIAPIPE_ASSERT_OK(runner.Run());
const TensorFlowSession& session =
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
VerifySignatureMap(session);
}
TEST_F(
TensorFlowSessionFromFrozenGraphCalculatorTest,
CreatesPacketWithGraphAndBindingsFromInputSidePacketStringModelFilePath) {
calculator_options_->clear_graph_proto_path();
CalculatorRunner runner(absl::Substitute(R"(
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
output_side_packet: "SESSION:session"
options {
[mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
$0
}
})",
calculator_options_->DebugString()));
runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
Adopt(new std::string(GetGraphDefPath()));
MEDIAPIPE_ASSERT_OK(runner.Run());
const TensorFlowSession& session =
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
VerifySignatureMap(session);
}
TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
CheckFailureForOptionsAndInputsProvideGraphDefProto) {
CalculatorRunner runner(absl::Substitute(R"(
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
output_side_packet: "SESSION:session"
options {
[mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
$0
}
})",
calculator_options_->DebugString()));
runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
Adopt(new std::string(GetGraphDefPath()));
auto run_status = runner.Run();
EXPECT_THAT(
run_status.message(),
::testing::HasSubstr("Must have exactly one of graph_proto_path"));
}
TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
CheckFailureForAllInputsProvideGraphDefProto) {
CalculatorRunner runner(absl::Substitute(R"(
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
input_side_packet: "STRING_MODEL:model"
output_side_packet: "SESSION:session"
options {
[mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
$0
}
})",
calculator_options_->DebugString()));
runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
Adopt(new std::string(GetGraphDefPath()));
std::string serialized_graph_contents;
MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
&serialized_graph_contents));
runner.MutableSidePackets()->Tag("STRING_MODEL") =
Adopt(new std::string(serialized_graph_contents));
auto run_status = runner.Run();
EXPECT_THAT(
run_status.message(),
::testing::HasSubstr("Must have exactly one of graph_proto_path"));
}
TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
CheckFailureForOnlyBothInputSidePacketsProvideGraphDefProto) {
calculator_options_->clear_graph_proto_path();
CalculatorRunner runner(absl::Substitute(R"(
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
input_side_packet: "STRING_MODEL_FILE_PATH:file_path"
input_side_packet: "STRING_MODEL:model"
output_side_packet: "SESSION:session"
options {
[mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
$0
}
})",
calculator_options_->DebugString()));
runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") =
Adopt(new std::string(GetGraphDefPath()));
std::string serialized_graph_contents;
MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(),
&serialized_graph_contents));
runner.MutableSidePackets()->Tag("STRING_MODEL") =
Adopt(new std::string(serialized_graph_contents));
auto run_status = runner.Run();
EXPECT_THAT(
run_status.message(),
::testing::HasSubstr("Must have exactly one of graph_proto_path"));
}
TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest,
CheckInitializationOpName) {
calculator_options_->add_initialization_op_names("multiplied:0");
CalculatorRunner runner(absl::Substitute(R"(
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
output_side_packet: "SESSION:session"
options {
[mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: {
$0
}
})",
calculator_options_->DebugString()));
MEDIAPIPE_ASSERT_OK(runner.Run());
const TensorFlowSession& session =
runner.OutputSidePackets().Tag("SESSION").Get<TensorFlowSession>();
VerifySignatureMap(session);
}
} // namespace
} // namespace mediapipe

77
mediapipe/calculators/tflite/BUILD
View File

@ -62,6 +62,16 @@ proto_library(
)
proto_library(
<<<<<<< HEAD
=======
name = "tflite_tensors_to_classification_calculator_proto",
srcs = ["tflite_tensors_to_classification_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
)
proto_library(
>>>>>>> Project import generated by Copybara.
name = "tflite_tensors_to_landmarks_calculator_proto",
srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
visibility = ["//visibility:public"],
@ -117,6 +127,17 @@ mediapipe_cc_proto_library(
)
mediapipe_cc_proto_library(
<<<<<<< HEAD
=======
name = "tflite_tensors_to_classification_calculator_cc_proto",
srcs = ["tflite_tensors_to_classification_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//mediapipe:__subpackages__"],
deps = [":tflite_tensors_to_classification_calculator_proto"],
)
mediapipe_cc_proto_library(
>>>>>>> Project import generated by Copybara.
name = "tflite_tensors_to_landmarks_calculator_cc_proto",
srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
@ -311,6 +332,28 @@ cc_library(
alwayslink = 1,
)
<<<<<<< HEAD
=======
cc_test(
name = "tflite_tensors_to_classification_calculator_test",
srcs = ["tflite_tensors_to_classification_calculator_test.cc"],
data = ["testdata/labelmap.txt"],
deps = [
":tflite_tensors_to_classification_calculator",
":tflite_tensors_to_classification_calculator_cc_proto",
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_runner",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/port:parse_text_proto",
"@com_google_absl//absl/memory",
"@com_google_googletest//:gtest_main",
"@org_tensorflow//tensorflow/lite:framework",
],
)
>>>>>>> Project import generated by Copybara.
cc_library(
name = "tflite_tensors_to_detections_calculator",
srcs = ["tflite_tensors_to_detections_calculator.cc"],
@ -340,6 +383,40 @@ cc_library(
)
cc_library(
<<<<<<< HEAD
=======
name = "tflite_tensors_to_classification_calculator",
srcs = ["tflite_tensors_to_classification_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":tflite_tensors_to_classification_calculator_cc_proto",
"@com_google_absl//absl/strings:str_format",
"@com_google_absl//absl/types:span",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:location",
"//mediapipe/framework/port:ret_check",
"//mediapipe/util:resource_util",
"@org_tensorflow//tensorflow/lite:framework",
] + select({
"//mediapipe:android": [
"//mediapipe/util/android/file/base",
],
"//mediapipe:apple": [
"//mediapipe/util/android/file/base",
],
"//mediapipe:macos": [
"//mediapipe/framework/port:file_helpers",
],
"//conditions:default": [
"//mediapipe/framework/port:file_helpers",
],
}),
alwayslink = 1,
)
cc_library(
>>>>>>> Project import generated by Copybara.
name = "tflite_tensors_to_landmarks_calculator",
srcs = ["tflite_tensors_to_landmarks_calculator.cc"],
visibility = ["//visibility:public"],

3
mediapipe/calculators/tflite/testdata/labelmap.txt vendored Normal file
View File

@ -0,0 +1,3 @@
classA
classB
classC

6
mediapipe/calculators/tflite/tflite_inference_calculator.cc
View File

@ -434,8 +434,14 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator);
use_quantized_tensors_ = false;
} else {
RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk);
<<<<<<< HEAD
use_quantized_tensors_ = (interpreter_->tensor(0)->quantization.type ==
kTfLiteAffineQuantization);
=======
use_quantized_tensors_ =
(interpreter_->tensor(interpreter_->inputs()[0])->quantization.type ==
kTfLiteAffineQuantization);
>>>>>>> Project import generated by Copybara.
if (use_quantized_tensors_) gpu_inference_ = false;
}

176
mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc Normal file
View File

@ -0,0 +1,176 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <algorithm>
#include <unordered_map>
#include <vector>
#include "absl/strings/str_format.h"
#include "absl/types/span.h"
#include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/util/resource_util.h"
#include "tensorflow/lite/interpreter.h"
#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX)
#include "mediapipe/util/android/file/base/file.h"
#include "mediapipe/util/android/file/base/helpers.h"
#else
#include "mediapipe/framework/port/file_helpers.h"
#endif
namespace mediapipe {
// Convert result TFLite tensors from classification models into MediaPipe
// classifications.
//
// Input:
// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32 containing one
// tensor, the size of which must be (1, * num_classes).
// Output:
// CLASSIFICATIONS - Result MediaPipe ClassificationList. The score and index
// fields of each classification are set, while the label
// field is only set if label_map_path is provided.
//
// Usage example:
// node {
// calculator: "TfLiteTensorsToClassificationCalculator"
// input_stream: "TENSORS:tensors"
// output_stream: "CLASSIFICATIONS:classifications"
// options: {
// [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {
// num_classes: 1024
// min_score_threshold: 0.1
// label_map_path: "labelmap.txt"
// }
// }
// }
class TfLiteTensorsToClassificationCalculator : public CalculatorBase {
public:
static ::mediapipe::Status GetContract(CalculatorContract* cc);
::mediapipe::Status Open(CalculatorContext* cc) override;
::mediapipe::Status Process(CalculatorContext* cc) override;
::mediapipe::Status Close(CalculatorContext* cc) override;
private:
int top_k_ = 0;
double min_score_threshold_ = 0;
std::unordered_map<int, std::string> label_map_;
bool label_map_loaded_ = false;
};
REGISTER_CALCULATOR(TfLiteTensorsToClassificationCalculator);
::mediapipe::Status TfLiteTensorsToClassificationCalculator::GetContract(
CalculatorContract* cc) {
RET_CHECK(!cc->Inputs().GetTags().empty());
RET_CHECK(!cc->Outputs().GetTags().empty());
if (cc->Inputs().HasTag("TENSORS")) {
cc->Inputs().Tag("TENSORS").Set<std::vector<TfLiteTensor>>();
}
if (cc->Outputs().HasTag("CLASSIFICATIONS")) {
cc->Outputs().Tag("CLASSIFICATIONS").Set<ClassificationList>();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status TfLiteTensorsToClassificationCalculator::Open(
CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
auto options = cc->Options<
::mediapipe::TfLiteTensorsToClassificationCalculatorOptions>();
top_k_ = options.top_k();
min_score_threshold_ = options.min_score_threshold();
if (options.has_label_map_path()) {
std::string string_path;
ASSIGN_OR_RETURN(string_path,
PathToResourceAsFile(options.label_map_path()));
std::string label_map_string;
RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string));
std::istringstream stream(label_map_string);
std::string line;
int i = 0;
while (std::getline(stream, line)) {
label_map_[i++] = line;
}
label_map_loaded_ = true;
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status TfLiteTensorsToClassificationCalculator::Process(
CalculatorContext* cc) {
const auto& input_tensors =
cc->Inputs().Tag("TENSORS").Get<std::vector<TfLiteTensor>>();
RET_CHECK_EQ(input_tensors.size(), 1);
const TfLiteTensor* raw_score_tensor = &input_tensors[0];
RET_CHECK_EQ(raw_score_tensor->dims->size, 2);
RET_CHECK_EQ(raw_score_tensor->dims->data[0], 1);
int num_classes = raw_score_tensor->dims->data[1];
if (label_map_loaded_) {
RET_CHECK_EQ(num_classes, label_map_.size());
}
const float* raw_scores = raw_score_tensor->data.f;
auto classification_list = absl::make_unique<ClassificationList>();
for (int i = 0; i < num_classes; ++i) {
if (raw_scores[i] < min_score_threshold_) {
continue;
}
Classification* classification = classification_list->add_classification();
classification->set_index(i);
classification->set_score(raw_scores[i]);
if (label_map_loaded_) {
classification->set_label(label_map_[i]);
}
}
// Note that partial_sort will raise error when top_k_ >
// classification_list->classification_size().
auto raw_classification_list = classification_list->mutable_classification();
if (top_k_ > 0 && classification_list->classification_size() >= top_k_) {
std::partial_sort(raw_classification_list->begin(),
raw_classification_list->begin() + top_k_,
raw_classification_list->end(),
[](const Classification a, const Classification b) {
return a.score() > b.score();
});
// Resizes the underlying list to have only top_k_ classifications.
raw_classification_list->DeleteSubrange(
top_k_, raw_classification_list->size() - top_k_);
}
cc->Outputs()
.Tag("CLASSIFICATIONS")
.Add(classification_list.release(), cc->InputTimestamp());
return ::mediapipe::OkStatus();
}
::mediapipe::Status TfLiteTensorsToClassificationCalculator::Close(
CalculatorContext* cc) {
return ::mediapipe::OkStatus();
}
} // namespace mediapipe

35
mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto Normal file
View File

@ -0,0 +1,35 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// The option proto for the TfLiteTensorsToClassificationCalculator.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message TfLiteTensorsToClassificationCalculatorOptions {
extend .mediapipe.CalculatorOptions {
optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463;
}
// Score threshold for perserving the class.
optional float min_score_threshold = 1;
// Number of highest scoring labels to output. If top_k is not positive then
// all labels are used.
optional int32 top_k = 2;
// Path to a label map file for getting the actual name of class ids.
optional string label_map_path = 3;
}

194
mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator_test.cc Normal file
View File

@ -0,0 +1,194 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <vector>
#include "absl/memory/memory.h"
#include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status_matchers.h"
#include "tensorflow/lite/interpreter.h"
namespace mediapipe {
using ::mediapipe::ParseTextProtoOrDie;
using ::tflite::Interpreter;
using Node = ::mediapipe::CalculatorGraphConfig::Node;
class TfLiteTensorsToClassificationCalculatorTest : public ::testing::Test {
protected:
void BuildGraph(mediapipe::CalculatorRunner* runner,
const std::vector<float>& scores) {
interpreter_ = absl::make_unique<Interpreter>();
std::vector<int> dims(2);
dims[0] = 1;
dims[1] = scores.size();
interpreter_->AddTensors(1);
interpreter_->SetInputs({0});
interpreter_->SetTensorParametersReadWrite(0, kTfLiteFloat32, "", dims,
TfLiteQuantization());
int t = interpreter_->inputs()[0];
TfLiteTensor* tensor = interpreter_->tensor(t);
interpreter_->ResizeInputTensor(t, dims);
interpreter_->AllocateTensors();
float* tensor_buffer = tensor->data.f;
ASSERT_NE(tensor_buffer, nullptr);
for (int i = 0; i < scores.size(); ++i) {
tensor_buffer[i] = scores[i];
}
auto tensors = absl::make_unique<std::vector<TfLiteTensor>>();
tensors->emplace_back(*tensor);
int64 stream_timestamp = 0;
auto& input_stream_packets =
runner->MutableInputs()->Tag("TENSORS").packets;
input_stream_packets.push_back(
mediapipe::Adopt(tensors.release())
.At(mediapipe::Timestamp(stream_timestamp++)));
}
std::unique_ptr<Interpreter> interpreter_;
};
TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutput) {
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
calculator: "TfLiteTensorsToClassificationCalculator"
input_stream: "TENSORS:tensors"
output_stream: "CLASSIFICATIONS:classifications"
options {
[mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {}
}
)"));
BuildGraph(&runner, {0, 0.5, 1});
MEDIAPIPE_ASSERT_OK(runner.Run());
const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
EXPECT_EQ(1, output_packets_.size());
const auto& classification_list =
output_packets_[0].Get<ClassificationList>();
EXPECT_EQ(3, classification_list.classification_size());
// Verify that the label_id and score fields are set correctly.
for (int i = 0; i < classification_list.classification_size(); ++i) {
EXPECT_EQ(i, classification_list.classification(i).index());
EXPECT_EQ(i * 0.5, classification_list.classification(i).score());
ASSERT_FALSE(classification_list.classification(i).has_label());
}
}
TEST_F(TfLiteTensorsToClassificationCalculatorTest,
CorrectOutputWithLabelMapPath) {
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
calculator: "TfLiteTensorsToClassificationCalculator"
input_stream: "TENSORS:tensors"
output_stream: "CLASSIFICATIONS:classifications"
options {
[mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {
label_map_path: "mediapipe/calculators/tflite/testdata/labelmap.txt"
}
}
)"));
BuildGraph(&runner, {0, 0.5, 1});
MEDIAPIPE_ASSERT_OK(runner.Run());
const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
EXPECT_EQ(1, output_packets_.size());
const auto& classification_list =
output_packets_[0].Get<ClassificationList>();
EXPECT_EQ(3, classification_list.classification_size());
// Verify that the label field is set.
for (int i = 0; i < classification_list.classification_size(); ++i) {
EXPECT_EQ(i, classification_list.classification(i).index());
EXPECT_EQ(i * 0.5, classification_list.classification(i).score());
ASSERT_TRUE(classification_list.classification(i).has_label());
}
}
TEST_F(TfLiteTensorsToClassificationCalculatorTest,
CorrectOutputWithLabelMinScoreThreshold) {
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
calculator: "TfLiteTensorsToClassificationCalculator"
input_stream: "TENSORS:tensors"
output_stream: "CLASSIFICATIONS:classifications"
options {
[mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {
min_score_threshold: 0.6
}
}
)"));
BuildGraph(&runner, {0, 0.5, 1});
MEDIAPIPE_ASSERT_OK(runner.Run());
const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
EXPECT_EQ(1, output_packets_.size());
const auto& classification_list =
output_packets_[0].Get<ClassificationList>();
// Verify that the low score labels are filtered out.
EXPECT_EQ(1, classification_list.classification_size());
EXPECT_EQ(1, classification_list.classification(0).score());
}
TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutputWithTopK) {
mediapipe::CalculatorRunner runner(ParseTextProtoOrDie<Node>(R"(
calculator: "TfLiteTensorsToClassificationCalculator"
input_stream: "TENSORS:tensors"
output_stream: "CLASSIFICATIONS:classifications"
options {
[mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {
top_k: 2
}
}
)"));
BuildGraph(&runner, {0, 0.5, 1});
MEDIAPIPE_ASSERT_OK(runner.Run());
const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets;
EXPECT_EQ(1, output_packets_.size());
const auto& classification_list =
output_packets_[0].Get<ClassificationList>();
// Verify that the only top2 labels are left.
EXPECT_EQ(2, classification_list.classification_size());
for (int i = 0; i < classification_list.classification_size(); ++i) {
EXPECT_EQ((classification_list.classification_size() - i) * 0.5,
classification_list.classification(i).score());
}
}
} // namespace mediapipe

15
mediapipe/calculators/video/opencv_video_decoder_calculator.cc
View File

@ -154,8 +154,19 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
cv::COLOR_BGRA2RGBA);
}
}
<<<<<<< HEAD
cc->Outputs().Tag("VIDEO").Add(image_frame.release(), timestamp);
decoded_frames_++;
=======
// If the timestamp of the current frame is not greater than the one of the
// previous frame, the new frame will be discarded.
if (prev_timestamp_ < timestamp) {
cc->Outputs().Tag("VIDEO").Add(image_frame.release(), timestamp);
prev_timestamp_ = timestamp;
decoded_frames_++;
}
>>>>>>> Project import generated by Copybara.
return ::mediapipe::OkStatus();
}
@ -178,6 +189,10 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase {
int frame_count_;
int decoded_frames_ = 0;
ImageFormat::Format format_;
<<<<<<< HEAD
=======
Timestamp prev_timestamp_ = Timestamp::Unset();
>>>>>>> Project import generated by Copybara.
};
REGISTER_CALCULATOR(OpenCvVideoDecoderCalculator);

4
mediapipe/docs/cycles.md
View File

@ -61,7 +61,11 @@ ordering and ignores packet timestamps, which will eliminate this inconvenience.
By default, MediaPipe calls the `Close()` method of a non-source calculator when
all of its input streams are done. In the example graph, we want to stop the
adder node as soon as the integer source is done. This is accomplished by
<<<<<<< HEAD
configuring the adder node with an alternative input stream hander,
=======
configuring the adder node with an alternative input stream handler,
>>>>>>> Project import generated by Copybara.
`EarlyCloseInputStreamHandler`.
## Relevant Source Code

8
mediapipe/docs/examples.md
View File

@ -85,6 +85,14 @@ and model details are described in the
[Hello World for C++](./hello_world_desktop.md) shows how to run a simple graph
using the MediaPipe C++ APIs.
<<<<<<< HEAD
=======
### Feature Extration for YouTube-8M Challenge
[Feature Extration for YouTube-8M Challenge](./youtube_8m.md) shows how to use
MediaPipe to prepare training data for the YouTube-8M Challenge.
>>>>>>> Project import generated by Copybara.
### Preparing Data Sets with MediaSequence
[Preparing Data Sets with MediaSequence](./media_sequence.md) shows how to use

8
mediapipe/docs/framework_concepts.md
View File

@ -17,7 +17,11 @@ packets and produces zero or more output streams and/or side packets.
### CalculatorBase
A calculator is created by defining a new sub-class of the
<<<<<<< HEAD
[`CalculatorBase`](http://github.com/google/mediapipe/mediapipe/framework/calculator_base.cc)
=======
[`CalculatorBase`](https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator_base.cc)
>>>>>>> Project import generated by Copybara.
class, implementing a number of methods, and registering the new sub-class with
Mediapipe. At a minimum, a new calculator must implement the below four methods
@ -31,7 +35,11 @@ Mediapipe. At a minimum, a new calculator must implement the below four methods
* After all calls to `Process()` finish or when all input streams close, the framework calls `Close()`. This function is always called if `Open()` was called and succeeded and even if the graph run terminated because of an error. No inputs are available via any input streams during `Close()`, but it still has access to input side packets and therefore may write outputs. After `Close()` returns, the calculator should be considered a dead node. The calculator object is destroyed as soon as the graph finishes running.
The following are code snippets from
<<<<<<< HEAD
[CalculatorBase.h](http://github.com/google/mediapipe/mediapipe/framework/calculator_base.h).
=======
[CalculatorBase.h](https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator_base.h).
>>>>>>> Project import generated by Copybara.
```c++
class CalculatorBase {

4
mediapipe/docs/hello_world_android.md
View File

@ -643,7 +643,11 @@ Initialize the asset manager in `onCreate(Bundle)` before initializing
`eglManager`:
```
<<<<<<< HEAD
// Initilize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
=======
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
>>>>>>> Project import generated by Copybara.
// binary graphs.
AndroidAssetUtil.initializeNativeAssetManager(this);
```

4
mediapipe/docs/hello_world_ios.md
View File

@ -6,7 +6,11 @@ This codelab uses MediaPipe on an iOS device.
### What you will learn
<<<<<<< HEAD
How to develop an Android application that uses MediaPipe and run a MediaPipe
=======
How to develop an iOS application that uses MediaPipe and run a MediaPipe
>>>>>>> Project import generated by Copybara.
graph on iOS.
### What you will build

40
mediapipe/docs/help.md
View File

@ -1,9 +1,18 @@
<<<<<<< HEAD
## Getting help
- [Technical questions](#technical-questions)
- [Bugs and Feature requests](#bugs-and-feature-requests)
Below are the various ways to get help
=======
## Getting Help
- [Technical questions](#technical-questions)
- [Bugs and feature requests](#bugs-and-feature-requests)
Below are the various ways to get help:
>>>>>>> Project import generated by Copybara.
### Technical questions
@ -11,9 +20,40 @@ For help with technical or algorithmic questions, visit
[Stack Overflow](https://stackoverflow.com/questions/tagged/mediapipe) to find
answers and support from the MediaPipe community.
<<<<<<< HEAD
### Bugs and Feature requests
To report bugs or make feature requests,
[file an issue on GitHub](https://github.com/google/mediapipe/mediapipe/issues).
Please choose the appropriate repository for the project from the
[MediaPipe repo](https://github.com/google/mediapipe/mediapipe)
=======
### Bugs and feature requests
To report bugs or make feature requests,
[file an issue on GitHub](https://github.com/google/mediapipe/issues).
If you open a GitHub issue, here is our policy:
1. It must be a bug, a feature request, or a significant problem with documentation (for small doc fixes please send a PR instead).
2. The form below must be filled out.
**Here's why we have that policy**: MediaPipe developers respond to issues. We want to focus on work that benefits the whole community, e.g., fixing bugs and adding features. Support only helps individuals. GitHub also notifies thousands of people when issues are filed. We want them to see you communicating an interesting problem, rather than being redirected to Stack Overflow.
------------------------
### System information
- **Have I written custom code**:
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**:
- **Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device**:
- **Bazel version**:
- **Android Studio, NDK, SDK versions (if issue is related to building in mobile dev enviroment)**:
- **Xcode & Tulsi version (if issue is related to building in mobile dev enviroment)**:
- **Exact steps to reproduce**:
### Describe the problem
Describe the problem clearly here. Be sure to convey here why it's a bug in MediaPipe or a feature request.
### Source code / logs
Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached instead of being pasted into the issue as text.
>>>>>>> Project import generated by Copybara.

170
mediapipe/docs/install.md
View File

@ -49,10 +49,17 @@ To build and run iOS apps:
[documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
to install any version of Bazel manually.
<<<<<<< HEAD
3. Install OpenCV.
Option 1. Use package manager tool to install the pre-compiled OpenCV
libraries.
=======
3. Install OpenCV and FFmpeg.
Option 1. Use package manager tool to install the pre-compiled OpenCV
libraries. FFmpeg will be installed via libopencv-video-dev.
>>>>>>> Project import generated by Copybara.
Note: Debian 9 and Ubuntu 16.04 provide OpenCV 2.4.9. You may want to take
option 2 or 3 to install OpenCV 3 or above.
@ -83,6 +90,7 @@ To build and run iOS apps:
)
cc_library(
<<<<<<< HEAD
name = "opencv",
srcs = glob(
[
@ -101,6 +109,24 @@ To build and run iOS apps:
visibility = ["//visibility:public"],
)
=======
name = "opencv",
srcs = glob(
[
"lib/libopencv_core.so",
"lib/libopencv_highgui.so",
"lib/libopencv_imgcodecs.so",
"lib/libopencv_imgproc.so",
"lib/libopencv_video.so",
"lib/libopencv_videoio.so",
],
),
hdrs = glob(["include/opencv4/**/*.h*"]),
includes = ["include/opencv4/"],
linkstatic = 1,
visibility = ["//visibility:public"],
)
>>>>>>> Project import generated by Copybara.
```
4. Run the [Hello World desktop example](./hello_world_desktop.md).
@ -168,6 +194,7 @@ To build and run iOS apps:
)
cc_library(
<<<<<<< HEAD
name = "opencv",
srcs = glob(
[
@ -186,6 +213,24 @@ To build and run iOS apps:
visibility = ["//visibility:public"],
)
=======
name = "opencv",
srcs = glob(
[
"lib/libopencv_core.so",
"lib/libopencv_highgui.so",
"lib/libopencv_imgcodecs.so",
"lib/libopencv_imgproc.so",
"lib/libopencv_video.so",
"lib/libopencv_videoio.so",
],
),
hdrs = glob(["include/opencv4/**/*.h*"]),
includes = ["include/opencv4/"],
linkstatic = 1,
visibility = ["//visibility:public"],
)
>>>>>>> Project import generated by Copybara.
```
4. Run the [Hello World desktop example](./hello_world_desktop.md).
@ -239,10 +284,17 @@ To build and run iOS apps:
[documentation](https://docs.bazel.build/versions/master/install-ubuntu.html)
to install any version of Bazel manually.
<<<<<<< HEAD
4. Install OpenCV.
Option 1. Use HomeBrew package manager tool to install the pre-compiled
OpenCV libraries.
=======
4. Install OpenCV and FFmpeg.
Option 1. Use HomeBrew package manager tool to install the pre-compiled
OpenCV libraries. FFmpeg will be installed via OpenCV.
>>>>>>> Project import generated by Copybara.
```bash
$ brew install opencv
@ -254,6 +306,7 @@ To build and run iOS apps:
$ port install opencv
```
<<<<<<< HEAD
Note: when using MacPorts, please edit the [`WORKSAPCE`] and
[`opencv_linux.BUILD`] files like the following:
@ -281,6 +334,60 @@ To build and run iOS apps:
linkstatic = 1,
visibility = ["//visibility:public"],
)
=======
Note: when using MacPorts, please edit the [`WORKSAPCE`],
[`opencv_macos.BUILD`], and [`ffmpeg_macos.BUILD`] files like the following:
```bash
new_local_repository(
name = "macos_opencv",
build_file = "@//third_party:opencv_macos.BUILD",
path = "/opt",
)
new_local_repository(
name = "macos_ffmpeg",
build_file = "@//third_party:ffmpeg_macos.BUILD",
path = "/opt",
)
cc_library(
name = "opencv",
srcs = glob(
[
"local/lib/libopencv_core.dylib",
"local/lib/libopencv_highgui.dylib",
"local/lib/libopencv_imgcodecs.dylib",
"local/lib/libopencv_imgproc.dylib",
"local/lib/libopencv_video.dylib",
"local/lib/libopencv_videoio.dylib",
],
),
hdrs = glob(["local/include/opencv2/**/*.h*"]),
includes = ["local/include/"],
linkstatic = 1,
visibility = ["//visibility:public"],
)
cc_library(
name = "libffmpeg",
srcs = glob(
[
"local/lib/libav*.dylib",
],
),
hdrs = glob(["local/include/libav*/*.h"]),
includes = ["local/include/"],
linkopts = [
"-lavcodec",
"-lavformat",
"-lavutil",
],
linkstatic = 1,
visibility = ["//visibility:public"],
)
>>>>>>> Project import generated by Copybara.
```
5. Run the [Hello World desktop example](./hello_world_desktop.md).
@ -350,10 +457,17 @@ To build and run iOS apps:
username@DESKTOP-TMVLBJ1:~$ cd mediapipe
```
<<<<<<< HEAD
7. Install OpenCV.
Option 1. Use package manager tool to install the pre-compiled OpenCV
libraries.
=======
7. Install OpenCV and FFmpeg.
Option 1. Use package manager tool to install the pre-compiled OpenCV
libraries. FFmpeg will be installed via libopencv-video-dev.
>>>>>>> Project import generated by Copybara.
```bash
username@DESKTOP-TMVLBJ1:~/mediapipe$ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \
@ -381,6 +495,7 @@ To build and run iOS apps:
)
cc_library(
<<<<<<< HEAD
name = "opencv",
srcs = glob(
[
@ -399,6 +514,24 @@ To build and run iOS apps:
visibility = ["//visibility:public"],
)
=======
name = "opencv",
srcs = glob(
[
"lib/libopencv_core.so",
"lib/libopencv_highgui.so",
"lib/libopencv_imgcodecs.so",
"lib/libopencv_imgproc.so",
"lib/libopencv_video.so",
"lib/libopencv_videoio.so",
],
),
hdrs = glob(["include/opencv4/**/*.h*"]),
includes = ["include/opencv4/"],
linkstatic = 1,
visibility = ["//visibility:public"],
)
>>>>>>> Project import generated by Copybara.
```
8. Run the [Hello World desktop example](./hello_world_desktop.md).
@ -428,7 +561,11 @@ To build and run iOS apps:
This will use a Docker image that will isolate mediapipe's installation from the rest of the system.
1. [Install Docker](https://docs.docker.com/install/#supported-platforms) on
<<<<<<< HEAD
your host sytem.
=======
your host system.
>>>>>>> Project import generated by Copybara.
2. Build a docker image with tag "mediapipe".
@ -473,7 +610,35 @@ This will use a Docker image that will isolate mediapipe's installation from the
# Hello World!
```
<<<<<<< HEAD
<!-- 4. Uncomment the last line of the Dockerfile
=======
4. Build Mediapipe [Android demos](./examples.md).
```bash
$ docker run -it --name mediapipe mediapipe:latest
root@bca08b91ff63:/mediapipe# bash ./setup_android_sdk_and_ndk
# Should print:
# Android NDK is now installed. Consider setting $ANDROID_NDK_HOME environment variable to be /root/Android/Sdk/ndk-bundle/android-ndk-r18b
# Set android_ndk_repository and android_sdk_repository in WORKSPACE
# Done
root@bca08b91ff63:/mediapipe# bazel build -c opt --config=android_arm64 mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu:objectdetectiongpu
# Should print:
# Target //mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu:objectdetectiongpu up-to-date:
# bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/objectdetectiongpu_deploy.jar
# bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/objectdetectiongpu_unsigned.apk
# bazel-bin/mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/objectdetectiongpu.apk
# INFO: Elapsed time: 144.462s, Critical Path: 79.47s
# INFO: 1958 processes: 1 local, 1863 processwrapper-sandbox, 94 worker.
# INFO: Build completed successfully, 2028 total actions
```
<!-- 5. Uncomment the last line of the Dockerfile
>>>>>>> Project import generated by Copybara.
```bash
RUN bazel build -c opt --define MEDIAPIPE_DISABLE_GPU=1 mediapipe/examples/desktop/demo:object_detection_tensorflow_demo
@ -563,5 +728,10 @@ The steps below use Android Studio to build and install a MediaPipe example app.
[`WORKSAPCE`]: https://github.com/google/mediapipe/tree/master/WORKSPACE
[`opencv_linux.BUILD`]: https://github.com/google/mediapipe/tree/master/third_party/opencv_linux.BUILD
<<<<<<< HEAD
=======
[`opencv_macos.BUILD`]: https://github.com/google/mediapipe/tree/master/third_party/opencv_macos.BUILD
[`ffmpeg_macos.BUILD`]:https://github.com/google/mediapipe/tree/master/third_party/ffmpeg_macos.BUILD
>>>>>>> Project import generated by Copybara.
[`setup_opencv.sh`]: https://github.com/google/mediapipe/tree/master/setup_opencv.sh
[`setup_android_sdk_and_ndk.sh`]: https://github.com/google/mediapipe/tree/master/setup_android_sdk_and_ndk.sh

8
mediapipe/docs/media_sequence.md
View File

@ -1,6 +1,10 @@
## Preparing Data Sets with MediaSequence
<<<<<<< HEAD
MediaPipe is useful and general framework for media processing that can
=======
MediaPipe is a useful and general framework for media processing that can
>>>>>>> Project import generated by Copybara.
assist with research, development, and deployment of ML models. This example
focuses on development by demonstrating how to prepare video data for training
a TensorFlow model.
@ -18,7 +22,11 @@ process new data sets, in the documentation of
### Preparing an example data set
<<<<<<< HEAD
1. Checkout mediapipe repository
=======
1. Checkout the mediapipe repository
>>>>>>> Project import generated by Copybara.
```bash
git clone https://github.com/google/mediapipe.git

16
mediapipe/docs/mediapipe_ios_setup.md
View File

@ -78,7 +78,13 @@ Debug to Release. Note that this is set independently for each target.
## Building an iOS app from the command line
<<<<<<< HEAD
1. Build one of the example apps for iOS. We will be using the
=======
1. Modify the `bundle_id` field of the app's ios_application rule to use your own identifier, e.g. for [Face Detection GPU App example](./face_detection_mobile_gpu.md), you need to modify the line 26 of the [BUILD file](https://github.com/google/mediapipe/blob/master/mediapipe/examples/ios/facedetectiongpu/BUILD).
2. Build one of the example apps for iOS. We will be using the
>>>>>>> Project import generated by Copybara.
[Face Detection GPU App example](./face_detection_mobile_gpu.md)
```bash
@ -88,6 +94,7 @@ Debug to Release. Note that this is set independently for each target.
You may see a permission request from `codesign` in order to sign the app.
<<<<<<< HEAD
2. In Xcode, open the `Devices and Simulators` window (command-shift-2).
3. Make sure your device is connected. You will see a list of installed apps.
@ -95,3 +102,12 @@ Debug to Release. Note that this is set independently for each target.
Bazel.
4. You can now run the app on your device.
=======
3. In Xcode, open the `Devices and Simulators` window (command-shift-2).
4. Make sure your device is connected. You will see a list of installed apps.
Press the "+" button under the list, and select the `.ipa` file built by
Bazel.
5. You can now run the app on your device.
>>>>>>> Project import generated by Copybara.

4
mediapipe/docs/troubleshooting.md
View File

@ -41,7 +41,11 @@ When the library for a calculator is linked into an application binary, the
calculator is automatically registered by name through the
[`REGISTER_CALCULATOR`] macro using the [`registration.h`] library. Note that
[`REGISTER_CALCULATOR`] can register a calculator with a namespace prefix,
<<<<<<< HEAD
identical to its C++ namespace. In this case, the calcultor graph must also use
=======
identical to its C++ namespace. In this case, the calculator graph must also use
>>>>>>> Project import generated by Copybara.
the same namespace prefix.
## Out Of Memory error

83
mediapipe/docs/youtube_8m.md Normal file
View File

@ -0,0 +1,83 @@
## Extracting Video Features for YouTube-8M Challenge
MediaPipe is a useful and general framework for media processing that can assist
with research, development, and deployment of ML models. This example focuses on
model development by demonstrating how to prepare training data for the
YouTube-8M Challenge.
[Youtube-8M Challenge](https://www.kaggle.com/c/youtube8m-2019) is an annual
video classification challenge hosted by Google. Over the last two years, the
first two challenges have collectively drawn 1000+ teams from 60+ countries to
further advance large-scale video understanding research. In addition to the
feature extraction Python code released in the
[google/youtube-8m](https://github.com/google/youtube-8m/tree/master/feature_extractor)
repo, we release a MediaPipe based feature extraction pipeline that can extract
both video and audio features from a local video. The MediaPipe based pipeline
utilizes two machine learning models,
[Inception v3](https://github.com/tensorflow/models/tree/master/research/inception)
and
[VGGish](https://github.com/tensorflow/models/tree/master/research/audioset/vggish),
to extract features from video and audio respectively.
To visualize the
[graph](https://github.com/google/mediapipe/tree/master/mediapipe/graphs/youtube8m/feature_extraction.pbtxt),
copy the text specification of the graph and paste it into
[MediaPipe Visualizer](https://viz.mediapipe.dev/). The feature extraction
pipeline is highly customizable. You are welcome to add new calculators or use
your own machine learning models to extract more advanced features from the
videos.
### Steps to run the YouTube-8M feature extraction graph
1. Checkout the mediapipe repository
```bash
git clone https://github.com/google/mediapipe.git
cd mediapipe
```
2. Download the PCA and model data
```bash
mkdir /tmp/mediapipe
cd /tmp/mediapipe
curl -O http://data.yt8m.org/pca_matrix_data/inception3_mean_matrix_data.pb
curl -O http://data.yt8m.org/pca_matrix_data/inception3_projection_matrix_data.pb
curl -O http://data.yt8m.org/pca_matrix_data/vggish_mean_matrix_data.pb
curl -O http://data.yt8m.org/pca_matrix_data/vggish_projection_matrix_data.pb
curl -O http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz
tar -xvf /tmp/mediapipe/inception-2015-12-05.tgz
```
3. Get the VGGish frozen graph
Note: To run step 3 and step 4, you must have Python 2.7 or 3.5+ installed
with the TensorFlow 1.14+ package installed.
```bash
# cd to the root directory of the MediaPipe repo
cd -
python -m mediapipe.examples.desktop.youtube8m.generate_vggish_frozen_graph
```
4. Generate a MediaSequence metadata from the input video
Note: the output file is /tmp/mediapipe/metadata.tfrecord
```bash
python -m mediapipe.examples.desktop.youtube8m.generate_input_sequence_example \
--path_to_input_video=/absolute/path/to/the/local/video/file
```
5. Run the MediaPipe binary to extract the features
```bash
bazel build -c opt \
--define MEDIAPIPE_DISABLE_GPU=1 --define no_aws_support=true \
mediapipe/examples/desktop/youtube8m:extract_yt8m_features
./bazel-bin/mediapipe/examples/desktop/youtube8m/extract_yt8m_features
--calculator_graph_config_file=mediapipe/graphs/youtube8m/feature_extraction.pbtxt \
--input_side_packets=input_sequence_example=/tmp/mediapipe/metadata.tfrecord \
--output_side_packets=output_sequence_example=/tmp/mediapipe/output.tfrecord
```

4
mediapipe/examples/android/src/java/com/google/mediapipe/apps/edgedetectiongpu/MainActivity.java
View File

@ -75,7 +75,11 @@ public class MainActivity extends AppCompatActivity {
previewDisplayView = new SurfaceView(this);
setupPreviewDisplayView();
<<<<<<< HEAD
// Initilize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
=======
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
>>>>>>> Project import generated by Copybara.
// binary graphs.
AndroidAssetUtil.initializeNativeAssetManager(this);

4
mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectioncpu/MainActivity.java
View File

@ -76,7 +76,11 @@ public class MainActivity extends AppCompatActivity {
previewDisplayView = new SurfaceView(this);
setupPreviewDisplayView();
<<<<<<< HEAD
// Initilize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
=======
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
>>>>>>> Project import generated by Copybara.
// binary graphs.
AndroidAssetUtil.initializeNativeAssetManager(this);

4
mediapipe/examples/android/src/java/com/google/mediapipe/apps/facedetectiongpu/MainActivity.java
View File

@ -76,7 +76,11 @@ public class MainActivity extends AppCompatActivity {
previewDisplayView = new SurfaceView(this);
setupPreviewDisplayView();
<<<<<<< HEAD
// Initilize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
=======
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
>>>>>>> Project import generated by Copybara.
// binary graphs.
AndroidAssetUtil.initializeNativeAssetManager(this);

4
mediapipe/examples/android/src/java/com/google/mediapipe/apps/hairsegmentationgpu/MainActivity.java
View File

@ -76,7 +76,11 @@ public class MainActivity extends AppCompatActivity {
previewDisplayView = new SurfaceView(this);
setupPreviewDisplayView();
<<<<<<< HEAD
// Initilize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
=======
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
>>>>>>> Project import generated by Copybara.
// binary graphs.
AndroidAssetUtil.initializeNativeAssetManager(this);

4
mediapipe/examples/android/src/java/com/google/mediapipe/apps/handdetectiongpu/MainActivity.java
View File

@ -76,7 +76,11 @@ public class MainActivity extends AppCompatActivity {
previewDisplayView = new SurfaceView(this);
setupPreviewDisplayView();
<<<<<<< HEAD
// Initilize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
=======
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
>>>>>>> Project import generated by Copybara.
// binary graphs.
AndroidAssetUtil.initializeNativeAssetManager(this);

4
mediapipe/examples/android/src/java/com/google/mediapipe/apps/handtrackinggpu/MainActivity.java
View File

@ -76,7 +76,11 @@ public class MainActivity extends AppCompatActivity {
previewDisplayView = new SurfaceView(this);
setupPreviewDisplayView();
<<<<<<< HEAD
// Initilize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
=======
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
>>>>>>> Project import generated by Copybara.
// binary graphs.
AndroidAssetUtil.initializeNativeAssetManager(this);

4
mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectioncpu/MainActivity.java
View File

@ -76,7 +76,11 @@ public class MainActivity extends AppCompatActivity {
previewDisplayView = new SurfaceView(this);
setupPreviewDisplayView();
<<<<<<< HEAD
// Initilize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
=======
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
>>>>>>> Project import generated by Copybara.
// binary graphs.
AndroidAssetUtil.initializeNativeAssetManager(this);

4
mediapipe/examples/android/src/java/com/google/mediapipe/apps/objectdetectiongpu/MainActivity.java
View File

@ -76,7 +76,11 @@ public class MainActivity extends AppCompatActivity {
previewDisplayView = new SurfaceView(this);
setupPreviewDisplayView();
<<<<<<< HEAD
// Initilize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
=======
// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g.,
>>>>>>> Project import generated by Copybara.
// binary graphs.
AndroidAssetUtil.initializeNativeAssetManager(this);

3
mediapipe/examples/desktop/README.md
View File

@ -46,6 +46,7 @@ bazel-bin/mediapipe/examples/desktop/object_detection/object_detection_tensorflo
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file
--alsologtostderr
```
<<<<<<< HEAD
**TFlite Face Detection**
@ -97,3 +98,5 @@ bazel-bin/mediapipe/examples/desktop/hand_tracking/hand_tracking_tflite \
--input_side_packets=input_video_path=/path/to/input/file,output_video_path=/path/to/output/file \
--alsologtostderr
```
=======
>>>>>>> Project import generated by Copybara.

35
mediapipe/examples/desktop/youtube8m/BUILD Normal file
View File

@ -0,0 +1,35 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
cc_binary(
name = "extract_yt8m_features",
srcs = ["extract_yt8m_features.cc"],
deps = [
"@com_google_absl//absl/strings",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/framework/port:commandlineflags",
"//mediapipe/framework/port:file_helpers",
"//mediapipe/framework/port:map_util",
"//mediapipe/framework/port:parse_text_proto",
"//mediapipe/framework/port:status",
"//mediapipe/graphs/youtube8m:yt8m_calculators_deps",
# TODO: Figure out the minimum set of the kernels needed by this example.
"@org_tensorflow//tensorflow/core:all_kernels",
"@org_tensorflow//tensorflow/core:direct_session",
],
)

54
mediapipe/examples/desktop/youtube8m/README.md Normal file
View File

@ -0,0 +1,54 @@
### Steps to run the YouTube-8M feature extraction graph
1. Checkout the mediapipe repository
```bash
git clone https://github.com/google/mediapipe.git
cd mediapipe
```
2. Download the PCA and model data
```bash
mkdir /tmp/mediapipe
cd /tmp/mediapipe
curl -O http://data.yt8m.org/pca_matrix_data/inception3_mean_matrix_data.pb
curl -O http://data.yt8m.org/pca_matrix_data/inception3_projection_matrix_data.pb
curl -O http://data.yt8m.org/pca_matrix_data/vggish_mean_matrix_data.pb
curl -O http://data.yt8m.org/pca_matrix_data/vggish_projection_matrix_data.pb
curl -O http://download.tensorflow.org/models/image/imagenet/inception-2015-12-05.tgz
tar -xvf /tmp/mediapipe/inception-2015-12-05.tgz
```
3. Get the VGGish frozen graph
Note: To run step 3 and step 4, you must have Python 2.7 or 3.5+ installed
with the TensorFlow 1.14+ package installed.
```bash
# cd to the root directory of the MediaPipe repo
cd -
python -m mediapipe.examples.desktop.youtube8m.generate_vggish_frozen_graph
```
4. Generate a MediaSequence metadata from the input video
Note: the output file is /tmp/mediapipe/metadata.tfrecord
```bash
python -m mediapipe.examples.desktop.youtube8m.generate_input_sequence_example \
--path_to_input_video=/absolute/path/to/the/local/video/file
```
5. Run the MediaPipe binary to extract the features
```bash
bazel build -c opt \
--define MEDIAPIPE_DISABLE_GPU=1 --define no_aws_support=true \
mediapipe/examples/desktop/youtube8m:extract_yt8m_features
./bazel-bin/mediapipe/examples/desktop/youtube8m/extract_yt8m_features
--calculator_graph_config_file=mediapipe/graphs/youtube8m/feature_extraction.pbtxt \
--input_side_packets=input_sequence_example=/tmp/mediapipe/metadata.tfrecord \
--output_side_packets=output_sequence_example=/tmp/mediapipe/output.tfrecord
```

14
mediapipe/examples/desktop/youtube8m/__init__.py Normal file
View File

@ -0,0 +1,14 @@
"""Copyright 2019 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

135
mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc Normal file
View File

@ -0,0 +1,135 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// A simple main function to run a MediaPipe graph. Input side packets are read
// from files provided via the command line and output side packets are written
// to disk.
#include "absl/strings/str_split.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/port/commandlineflags.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/map_util.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/port/status.h"
DEFINE_string(
calculator_graph_config_file, "",
"Name of file containing text format CalculatorGraphConfig proto.");
DEFINE_string(input_side_packets, "",
"Comma-separated list of key=value pairs specifying side packets "
"and corresponding file paths for the CalculatorGraph. The side "
"packets are read from the files and fed to the graph as strings "
"even if they represent doubles, floats, etc.");
DEFINE_string(output_side_packets, "",
"Comma-separated list of key=value pairs specifying the output "
"side packets and paths to write to disk for the "
"CalculatorGraph.");
::mediapipe::Status RunMPPGraph() {
std::string calculator_graph_config_contents;
RETURN_IF_ERROR(mediapipe::file::GetContents(
FLAGS_calculator_graph_config_file, &calculator_graph_config_contents));
LOG(INFO) << "Get calculator graph config contents: "
<< calculator_graph_config_contents;
mediapipe::CalculatorGraphConfig config =
mediapipe::ParseTextProtoOrDie<mediapipe::CalculatorGraphConfig>(
calculator_graph_config_contents);
std::map<std::string, ::mediapipe::Packet> input_side_packets;
std::vector<std::string> kv_pairs =
absl::StrSplit(FLAGS_input_side_packets, ',');
for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2);
RET_CHECK(!::mediapipe::ContainsKey(input_side_packets, name_and_value[0]));
std::string input_side_packet_contents;
RETURN_IF_ERROR(mediapipe::file::GetContents(name_and_value[1],
&input_side_packet_contents));
input_side_packets[name_and_value[0]] =
::mediapipe::MakePacket<std::string>(input_side_packet_contents);
}
mediapipe::MatrixData inc3_pca_mean_matrix_data,
inc3_pca_projection_matrix_data, vggish_pca_mean_matrix_data,
vggish_pca_projection_matrix_data;
mediapipe::Matrix inc3_pca_mean_matrix, inc3_pca_projection_matrix,
vggish_pca_mean_matrix, vggish_pca_projection_matrix;
std::string content;
RETURN_IF_ERROR(mediapipe::file::GetContents(
"/tmp/mediapipe/inception3_mean_matrix_data.pb", &content));
inc3_pca_mean_matrix_data.ParseFromString(content);
mediapipe::MatrixFromMatrixDataProto(inc3_pca_mean_matrix_data,
&inc3_pca_mean_matrix);
input_side_packets["inception3_pca_mean_matrix"] =
::mediapipe::MakePacket<mediapipe::Matrix>(inc3_pca_mean_matrix);
RETURN_IF_ERROR(mediapipe::file::GetContents(
"/tmp/mediapipe/inception3_projection_matrix_data.pb", &content));
inc3_pca_projection_matrix_data.ParseFromString(content);
mediapipe::MatrixFromMatrixDataProto(inc3_pca_projection_matrix_data,
&inc3_pca_projection_matrix);
input_side_packets["inception3_pca_projection_matrix"] =
::mediapipe::MakePacket<mediapipe::Matrix>(inc3_pca_projection_matrix);
RETURN_IF_ERROR(mediapipe::file::GetContents(
"/tmp/mediapipe/vggish_mean_matrix_data.pb", &content));
vggish_pca_mean_matrix_data.ParseFromString(content);
mediapipe::MatrixFromMatrixDataProto(vggish_pca_mean_matrix_data,
&vggish_pca_mean_matrix);
input_side_packets["vggish_pca_mean_matrix"] =
::mediapipe::MakePacket<mediapipe::Matrix>(vggish_pca_mean_matrix);
RETURN_IF_ERROR(mediapipe::file::GetContents(
"/tmp/mediapipe/vggish_projection_matrix_data.pb", &content));
vggish_pca_projection_matrix_data.ParseFromString(content);
mediapipe::MatrixFromMatrixDataProto(vggish_pca_projection_matrix_data,
&vggish_pca_projection_matrix);
input_side_packets["vggish_pca_projection_matrix"] =
::mediapipe::MakePacket<mediapipe::Matrix>(vggish_pca_projection_matrix);
LOG(INFO) << "Initialize the calculator graph.";
mediapipe::CalculatorGraph graph;
RETURN_IF_ERROR(graph.Initialize(config, input_side_packets));
LOG(INFO) << "Start running the calculator graph.";
RETURN_IF_ERROR(graph.Run());
LOG(INFO) << "Gathering output side packets.";
kv_pairs = absl::StrSplit(FLAGS_output_side_packets, ',');
for (const std::string& kv_pair : kv_pairs) {
std::vector<std::string> name_and_value = absl::StrSplit(kv_pair, '=');
RET_CHECK(name_and_value.size() == 2);
::mediapipe::StatusOr<::mediapipe::Packet> output_packet =
graph.GetOutputSidePacket(name_and_value[0]);
RET_CHECK(output_packet.ok())
<< "Packet " << name_and_value[0] << " was not available.";
const std::string& serialized_string =
output_packet.ValueOrDie().Get<std::string>();
RETURN_IF_ERROR(
mediapipe::file::SetContents(name_and_value[1], serialized_string));
}
return ::mediapipe::OkStatus();
}
int main(int argc, char** argv) {
google::InitGoogleLogging(argv[0]);
gflags::ParseCommandLineFlags(&argc, &argv, true);
::mediapipe::Status run_status = RunMPPGraph();
if (!run_status.ok()) {
LOG(ERROR) << "Failed to run the graph: " << run_status.message();
} else {
LOG(INFO) << "Success!";
}
return 0;
}

56
mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py Normal file
View File

@ -0,0 +1,56 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Generate a MediaSequence metadata for MediaPipe input."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
from absl import app
from absl import flags
import tensorflow as tf
from mediapipe.util.sequence import media_sequence as ms
FLAGS = flags.FLAGS
SECONDS_TO_MICROSECONDS = 1000000
def bytes23(string):
"""Creates a bytes string in either Python 2 or 3."""
if sys.version_info >= (3, 0):
return bytes(string, 'utf8')
else:
return bytes(string)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
if not flags.FLAGS.path_to_input_video:
raise ValueError('You must specify the path to the input video.')
metadata = tf.train.SequenceExample()
ms.set_clip_data_path(bytes23(flags.FLAGS.path_to_input_video), metadata)
ms.set_clip_start_timestamp(0, metadata)
ms.set_clip_end_timestamp(
int(float(300 * SECONDS_TO_MICROSECONDS)), metadata)
with open('/tmp/mediapipe/metadata.tfrecord', 'wb') as writer:
writer.write(metadata.SerializeToString())
if __name__ == '__main__':
flags.DEFINE_string('path_to_input_video', '', 'Path to the input video.')
app.run(main)

70
mediapipe/examples/desktop/youtube8m/generate_vggish_frozen_graph.py Normal file
View File

@ -0,0 +1,70 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
r"""Code to clone the github repository, download the checkpoint and generate the frozen graph.
The frozen VGGish checkpoint will be saved to `/tmp/mediapipe/vggish_new.pb`.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import sys
from absl import app
import tensorflow as tf
from tensorflow.python.tools import freeze_graph
BASE_DIR = '/tmp/mediapipe/'
def create_vggish_frozen_graph():
"""Create the VGGish frozen graph."""
os.system('git clone https://github.com/tensorflow/models.git')
sys.path.append('models/research/audioset/vggish/')
import vggish_slim
os.system('curl -O https://storage.googleapis.com/audioset/vggish_model.ckpt')
ckpt_path = 'vggish_model.ckpt'
with tf.Graph().as_default(), tf.Session() as sess:
vggish_slim.define_vggish_slim(training=False)
vggish_slim.load_vggish_slim_checkpoint(sess, ckpt_path)
saver = tf.train.Saver(tf.all_variables())
freeze_graph.freeze_graph_with_def_protos(
sess.graph_def,
saver.as_saver_def(),
ckpt_path,
'vggish/fc2/BiasAdd',
restore_op_name=None,
filename_tensor_name=None,
output_graph='/tmp/mediapipe/vggish_new.pb',
clear_devices=True,
initializer_nodes=None)
os.system('rm -rf models/')
os.system('rm %s' % ckpt_path)
def main(argv):
if len(argv) > 1:
raise app.UsageError('Too many command-line arguments.')
create_vggish_frozen_graph()
if __name__ == '__main__':
app.run(main)

20
mediapipe/examples/ios/facedetectiongpu/Base.lproj/Main.storyboard
View File

@ -20,11 +20,24 @@
<view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="EfB-xq-knP">
<rect key="frame" x="0.0" y="20" width="375" height="647"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<<<<<<< HEAD
=======
<subviews>
<label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="emf-N5-sEd">
<rect key="frame" x="57" y="248" width="260" height="151"/>
<autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
<fontDescription key="fontDescription" type="system" pointSize="17"/>
<color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<nil key="highlightedColor"/>
</label>
</subviews>
>>>>>>> Project import generated by Copybara.
<color key="backgroundColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<accessibility key="accessibilityConfiguration" label="PreviewDisplayView">
<bool key="isElement" value="YES"/>
</accessibility>
</view>
<<<<<<< HEAD
<label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="YAx-c6-1kr">
<rect key="frame" x="57" y="258" width="260" height="151"/>
<autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
@ -32,13 +45,20 @@
<color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<nil key="highlightedColor"/>
</label>
=======
>>>>>>> Project import generated by Copybara.
</subviews>
<color key="backgroundColor" red="1" green="1" blue="1" alpha="1" colorSpace="custom" customColorSpace="sRGB"/>
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
</view>
<connections>
<<<<<<< HEAD
<outlet property="_liveView" destination="EfB-xq-knP" id="wac-VF-etz"/>
<outlet property="_noCameraLabel" destination="YAx-c6-1kr" id="Oiz-pZ-Vh7"/>
=======
<outlet property="_liveView" destination="EfB-xq-knP" id="JQp-2n-q9q"/>
<outlet property="_noCameraLabel" destination="emf-N5-sEd" id="91G-3Z-cU3"/>
>>>>>>> Project import generated by Copybara.
</connections>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>

17
mediapipe/examples/ios/handdetectiongpu/Base.lproj/Main.storyboard
View File

@ -20,6 +20,18 @@
<view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="EfB-xq-knP">
<rect key="frame" x="0.0" y="20" width="375" height="647"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<<<<<<< HEAD
=======
<subviews>
<label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="emf-N5-sEd">
<rect key="frame" x="57" y="248" width="260" height="151"/>
<autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
<fontDescription key="fontDescription" type="system" pointSize="17"/>
<color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<nil key="highlightedColor"/>
</label>
</subviews>
>>>>>>> Project import generated by Copybara.
<color key="backgroundColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<accessibility key="accessibilityConfiguration" label="PreviewDisplayView">
<bool key="isElement" value="YES"/>
@ -30,7 +42,12 @@
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
</view>
<connections>
<<<<<<< HEAD
<outlet property="_liveView" destination="EfB-xq-knP" id="wac-VF-etz"/>
=======
<outlet property="_liveView" destination="EfB-xq-knP" id="JQp-2n-q9q"/>
<outlet property="_noCameraLabel" destination="emf-N5-sEd" id="91G-3Z-cU3"/>
>>>>>>> Project import generated by Copybara.
</connections>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>

17
mediapipe/examples/ios/handtrackinggpu/Base.lproj/Main.storyboard
View File

@ -20,6 +20,18 @@
<view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="EfB-xq-knP">
<rect key="frame" x="0.0" y="20" width="375" height="647"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<<<<<<< HEAD
=======
<subviews>
<label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="emf-N5-sEd">
<rect key="frame" x="57" y="248" width="260" height="151"/>
<autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
<fontDescription key="fontDescription" type="system" pointSize="17"/>
<color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<nil key="highlightedColor"/>
</label>
</subviews>
>>>>>>> Project import generated by Copybara.
<color key="backgroundColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<accessibility key="accessibilityConfiguration" label="PreviewDisplayView">
<bool key="isElement" value="YES"/>
@ -30,7 +42,12 @@
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
</view>
<connections>
<<<<<<< HEAD
<outlet property="_liveView" destination="EfB-xq-knP" id="wac-VF-etz"/>
=======
<outlet property="_liveView" destination="EfB-xq-knP" id="JQp-2n-q9q"/>
<outlet property="_noCameraLabel" destination="emf-N5-sEd" id="91G-3Z-cU3"/>
>>>>>>> Project import generated by Copybara.
</connections>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>

17
mediapipe/examples/ios/objectdetectiongpu/Base.lproj/Main.storyboard
View File

@ -20,6 +20,18 @@
<view contentMode="scaleToFill" fixedFrame="YES" translatesAutoresizingMaskIntoConstraints="NO" id="EfB-xq-knP">
<rect key="frame" x="0.0" y="20" width="375" height="647"/>
<autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
<<<<<<< HEAD
=======
<subviews>
<label opaque="NO" userInteractionEnabled="NO" contentMode="left" horizontalHuggingPriority="251" verticalHuggingPriority="251" fixedFrame="YES" text="Camera access needed for this demo. Please enable camera access in the Settings app." textAlignment="center" lineBreakMode="tailTruncation" numberOfLines="0" baselineAdjustment="alignBaselines" adjustsFontSizeToFit="NO" translatesAutoresizingMaskIntoConstraints="NO" id="emf-N5-sEd">
<rect key="frame" x="57" y="248" width="260" height="151"/>
<autoresizingMask key="autoresizingMask" flexibleMinX="YES" flexibleMaxX="YES" flexibleMinY="YES" flexibleMaxY="YES"/>
<fontDescription key="fontDescription" type="system" pointSize="17"/>
<color key="textColor" white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<nil key="highlightedColor"/>
</label>
</subviews>
>>>>>>> Project import generated by Copybara.
<color key="backgroundColor" white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
<accessibility key="accessibilityConfiguration" label="PreviewDisplayView">
<bool key="isElement" value="YES"/>
@ -30,7 +42,12 @@
<viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
</view>
<connections>
<<<<<<< HEAD
<outlet property="_liveView" destination="EfB-xq-knP" id="wac-VF-etz"/>
=======
<outlet property="_liveView" destination="EfB-xq-knP" id="JQp-2n-q9q"/>
<outlet property="_noCameraLabel" destination="emf-N5-sEd" id="91G-3Z-cU3"/>
>>>>>>> Project import generated by Copybara.
</connections>
</viewController>
<placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>

8
mediapipe/framework/BUILD
View File

@ -139,7 +139,11 @@ mediapipe_cc_proto_library(
":stream_handler_cc_proto",
"@com_google_protobuf//:cc_wkt_protos",
],
<<<<<<< HEAD
visibility = ["//mediapipe:__subpackages__"],
=======
visibility = ["//visibility:public"],
>>>>>>> Project import generated by Copybara.
deps = [":calculator_proto"],
)
@ -155,7 +159,11 @@ mediapipe_cc_proto_library(
mediapipe_cc_proto_library(
name = "calculator_options_cc_proto",
srcs = ["calculator_options.proto"],
<<<<<<< HEAD
visibility = ["//mediapipe:__subpackages__"],
=======
visibility = ["//visibility:public"],
>>>>>>> Project import generated by Copybara.
deps = [":calculator_options_proto"],
)

19
mediapipe/framework/formats/BUILD
View File

@ -32,6 +32,15 @@ proto_library(
)
proto_library(
<<<<<<< HEAD
=======
name = "classification_proto",
srcs = ["classification.proto"],
visibility = ["//mediapipe:__subpackages__"],
)
proto_library(
>>>>>>> Project import generated by Copybara.
name = "image_format_proto",
srcs = ["image_format.proto"],
visibility = ["//mediapipe:__subpackages__"],
@ -64,6 +73,16 @@ mediapipe_cc_proto_library(
)
mediapipe_cc_proto_library(
<<<<<<< HEAD
=======
name = "classification_cc_proto",
srcs = ["classification.proto"],
visibility = ["//mediapipe:__subpackages__"],
deps = [":classification_proto"],
)
mediapipe_cc_proto_library(
>>>>>>> Project import generated by Copybara.
name = "image_format_cc_proto",
srcs = ["image_format.proto"],
visibility = [

35
mediapipe/framework/formats/classification.proto Normal file
View File

@ -0,0 +1,35 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// A protocol buffer encoding one or several classifications in an image. A
// classification is defined by label and corresponding score, representing the
// classifier confidence for the label.
syntax = "proto2";
package mediapipe;
message Classification {
// The index of the class in the corresponding label map.
optional int32 index = 1;
// The probability score for this class.
optional float score = 2;
// Label or name of the class.
optional string label = 3;
}
// Group of Classification protos.
message ClassificationList {
repeated Classification classification = 1;
}

12
mediapipe/framework/graph_validation_test.cc
View File

@ -32,7 +32,11 @@ namespace mediapipe {
namespace {
// Shows validation success for a graph and a subgraph.
<<<<<<< HEAD
TEST(ValidatedGraphConfigTest, InitilizeGraphFromProtos) {
=======
TEST(ValidatedGraphConfigTest, InitializeGraphFromProtos) {
>>>>>>> Project import generated by Copybara.
auto config_1 = ParseTextProtoOrDie<CalculatorGraphConfig>(R"(
type: "PassThroughGraph"
input_stream: "INPUT:stream_1"
@ -102,7 +106,11 @@ TEST(ValidatedGraphConfigTest, InitilizeGraphFromProtos) {
}
// Shows validation failure due to an unregistered subgraph.
<<<<<<< HEAD
TEST(ValidatedGraphConfigTest, InitilizeGraphFromLinker) {
=======
TEST(ValidatedGraphConfigTest, InitializeGraphFromLinker) {
>>>>>>> Project import generated by Copybara.
EXPECT_FALSE(SubgraphRegistry::IsRegistered("DubQuadTestSubgraph"));
ValidatedGraphConfig builder_1;
::mediapipe::Status status_1 =
@ -114,7 +122,11 @@ TEST(ValidatedGraphConfigTest, InitilizeGraphFromLinker) {
}
// Shows validation success for a graph and a template subgraph.
<<<<<<< HEAD
TEST(ValidatedGraphConfigTest, InitilizeTemplateFromProtos) {
=======
TEST(ValidatedGraphConfigTest, InitializeTemplateFromProtos) {
>>>>>>> Project import generated by Copybara.
mediapipe::tool::TemplateParser::Parser parser;
CalculatorGraphTemplate config_1;
CHECK(parser.ParseFromString(R"(

4
mediapipe/framework/profiler/graph_profiler.cc
View File

@ -520,7 +520,11 @@ class OstreamStream : public proto_ns::io::ZeroCopyOutputStream {
return WriteBuffer();
}
void BackUp(int count) override { buffer_used_ -= count; }
<<<<<<< HEAD
proto_int64 ByteCount() const override { return position_; }
=======
int64_t ByteCount() const override { return position_; }
>>>>>>> Project import generated by Copybara.
private:
// Writes the buffer to the ostream.

19
mediapipe/framework/stream_handler/fixed_size_input_stream_handler.cc
View File

@ -108,10 +108,25 @@ class FixedSizeInputStreamHandler : public DefaultInputStreamHandler {
}
// Returns the lowest timestamp of a packet ready to process.
<<<<<<< HEAD
Timestamp MinTimestampOrBound() {
Timestamp min_bound = Timestamp::Done();
for (const auto& stream : input_stream_managers_) {
min_bound = std::min(min_bound, stream->MinTimestampOrBound(nullptr));
=======
Timestamp MinTimestampToProcess() {
Timestamp min_bound = Timestamp::Done();
for (const auto& stream : input_stream_managers_) {
bool empty;
Timestamp stream_timestamp = stream->MinTimestampOrBound(&empty);
// If we're using the stream's *bound*, we only want to process up to the
// packet *before* the bound, because a packet may still arrive at that
// time.
if (empty) {
stream_timestamp = PreviousAllowedInStream(stream_timestamp);
}
min_bound = std::min(min_bound, stream_timestamp);
>>>>>>> Project import generated by Copybara.
}
return min_bound;
}
@ -199,7 +214,11 @@ class FixedSizeInputStreamHandler : public DefaultInputStreamHandler {
}
// input_timestamp is recalculated here to process the most recent packets.
EraseSurplusPackets(true);
<<<<<<< HEAD
input_timestamp = MinTimestampOrBound();
=======
input_timestamp = MinTimestampToProcess();
>>>>>>> Project import generated by Copybara.
DefaultInputStreamHandler::FillInputSet(input_timestamp, input_set);
pending_ = false;
}

43
mediapipe/gpu/metal_shader_base.h Normal file
View File

@ -0,0 +1,43 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_METAL_SHADER_BASE_H_
#define MEDIAPIPE_GPU_METAL_SHADER_BASE_H_
#include <simd/simd.h>
typedef struct {
// Vertex position in 2D clip space.
vector_float2 position;
// Corresponding texture coordinate.
vector_float2 texture_coordinate;
} MediaPipeTexturedVertex;
// Common buffer indices used in our Metal shaders.
typedef enum {
MediaPipeBufferIndexInputVertices = 0,
MediaPipeBufferIndexRgbWeights = 1,
MediaPipeBufferIndexPixelSize = 2,
MediaPipeBufferIndexOutputColor = 3,
} MediaPipeBufferIndex;
// Common texture indices used in our Metal shaders.
typedef enum {
MediaPipeTextureIndexInputColor = 0,
MediaPipeTextureIndexOutputColor = 1,
} MediaPipeTextureIndex;
typedef vector_float3 MetalRgbWeightPacketType;
#endif // MEDIAPIPE_GPU_METAL_SHADER_BASE_H_

46
mediapipe/graphs/youtube8m/BUILD Normal file
View File

@ -0,0 +1,46 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"]) # Apache 2.0
package(default_visibility = ["//visibility:public"])
cc_library(
name = "yt8m_calculators_deps",
deps = [
"//mediapipe/calculators/audio:audio_decoder_calculator",
"//mediapipe/calculators/audio:basic_time_series_calculators",
"//mediapipe/calculators/audio:mfcc_mel_calculators",
"//mediapipe/calculators/audio:rational_factor_resample_calculator",
"//mediapipe/calculators/audio:spectrogram_calculator",
"//mediapipe/calculators/audio:stabilized_log_calculator",
"//mediapipe/calculators/audio:time_series_framer_calculator",
"//mediapipe/calculators/core:add_header_calculator",
"//mediapipe/calculators/core:matrix_multiply_calculator",
"//mediapipe/calculators/core:matrix_subtract_calculator",
"//mediapipe/calculators/core:matrix_to_vector_calculator",
"//mediapipe/calculators/core:packet_cloner_calculator",
"//mediapipe/calculators/core:packet_resampler_calculator",
"//mediapipe/calculators/tensorflow:image_frame_to_tensor_calculator",
"//mediapipe/calculators/tensorflow:matrix_to_tensor_calculator",
"//mediapipe/calculators/tensorflow:pack_media_sequence_calculator",
"//mediapipe/calculators/tensorflow:string_to_sequence_example_calculator",
"//mediapipe/calculators/tensorflow:tensor_squeeze_dimensions_calculator",
"//mediapipe/calculators/tensorflow:tensor_to_matrix_calculator",
"//mediapipe/calculators/tensorflow:tensorflow_inference_calculator",
"//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator",
"//mediapipe/calculators/tensorflow:unpack_media_sequence_calculator",
"//mediapipe/calculators/video:opencv_video_decoder_calculator",
],
)

295
mediapipe/graphs/youtube8m/feature_extraction.pbtxt Normal file
View File

@ -0,0 +1,295 @@
input_side_packet: "input_sequence_example"
input_side_packet: "inception3_pca_mean_matrix"
input_side_packet: "inception3_pca_projection_matrix"
input_side_packet: "vggish_pca_mean_matrix"
input_side_packet: "vggish_pca_projection_matrix"
output_side_packet: "sequence_example_to_serialize"
node {
calculator: "StringToSequenceExampleCalculator"
input_side_packet: "STRING:input_sequence_example"
output_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
}
node {
calculator: "UnpackMediaSequenceCalculator"
input_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
output_side_packet: "DATA_PATH:input_file"
output_side_packet: "RESAMPLER_OPTIONS:packet_resampler_options"
node_options: {
[type.googleapis.com/mediapipe.UnpackMediaSequenceCalculatorOptions]: {
base_packet_resampler_options {
frame_rate: 1.0
base_timestamp: 0
}
}
}
}
# Decode the entire video.
node {
calculator: "OpenCvVideoDecoderCalculator"
input_side_packet: "INPUT_FILE_PATH:input_file"
output_stream: "VIDEO:decoded_frames"
}
# Extract the subset of frames we want to keep.
node {
calculator: "PacketResamplerCalculator"
input_stream: "decoded_frames"
output_stream: "sampled_decoded_frames"
input_side_packet: "OPTIONS:packet_resampler_options"
}
node {
calculator: "ImageFrameToTensorCalculator"
input_stream: "sampled_decoded_frames"
output_stream: "tensor_frame"
}
node {
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
output_side_packet: "SESSION:session"
node_options: {
[type.googleapis.com/mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions]: {
graph_proto_path: "/tmp/mediapipe/classify_image_graph_def.pb"
tag_to_tensor_names {
key: "IMG_UINT8"
value: "DecodeJpeg:0"
}
tag_to_tensor_names {
key: "INCEPTION_POOL3"
value: "pool_3/_reshape:0"
}
}
}
}
node {
calculator: "TensorFlowInferenceCalculator"
input_side_packet: "SESSION:session"
input_stream: "IMG_UINT8:tensor_frame"
output_stream: "INCEPTION_POOL3:inception3_hidden_activation_single_element_batch"
node_options: {
[type.googleapis.com/mediapipe.TensorFlowInferenceCalculatorOptions]: {
signature_name: ""
batch_size: 1
add_batch_dim_to_tensors: false
}
}
}
# Remove the batch dimension.
node: {
calculator: "TensorSqueezeDimensionsCalculator"
input_stream: "inception3_hidden_activation_single_element_batch"
output_stream: "inception3_hidden_activation"
node_options: {
[type.googleapis.com/mediapipe.TensorSqueezeDimensionsCalculatorOptions]: {
dim: 0
}
}
}
node {
calculator: "TensorToMatrixCalculator"
input_stream: "TENSOR:inception3_hidden_activation"
output_stream: "MATRIX:inception3_hidden_activation_matrix"
}
node {
calculator: "MatrixSubtractCalculator"
input_stream: "MINUEND:inception3_hidden_activation_matrix"
input_side_packet: "SUBTRAHEND:inception3_pca_mean_matrix"
output_stream: "mean_subtracted_inception3_matrix"
}
node {
calculator: "MatrixMultiplyCalculator"
input_stream: "mean_subtracted_inception3_matrix"
input_side_packet: "inception3_pca_projection_matrix"
output_stream: "pca_inception3_matrix"
}
node {
calculator: "MatrixToVectorCalculator"
input_stream: "pca_inception3_matrix"
output_stream: "pca_inception3_vf"
}
######################## END OF VISUAL ###########################
######################## BEGIN OF AUDIO ##########################
node {
calculator: "AudioDecoderCalculator"
input_side_packet: "INPUT_FILE_PATH:input_file"
output_stream: "AUDIO:audio"
output_stream: "AUDIO_HEADER:audio_header"
node_options: {
[type.googleapis.com/mediapipe.AudioDecoderOptions]: {
audio_stream { stream_index: 0 }
}
}
}
node {
calculator: "AddHeaderCalculator"
input_stream: "DATA:audio"
input_stream: "HEADER:audio_header"
output_stream: "media_audio"
}
# Always convert the audio to mono.
node {
calculator: "AverageTimeSeriesAcrossChannelsCalculator"
input_stream: "media_audio"
output_stream: "mono_waveform"
}
node {
calculator: "RationalFactorResampleCalculator"
input_stream: "mono_waveform"
output_stream: "resampled_waveform"
node_options: {
[type.googleapis.com/mediapipe.RationalFactorResampleCalculatorOptions] {
target_sample_rate: 16000.0
}
}
}
node {
calculator: "SpectrogramCalculator"
input_stream: "resampled_waveform"
output_stream: "spectrogram_squared_magnitude"
node_options: {
[type.googleapis.com/mediapipe.SpectrogramCalculatorOptions] {
frame_duration_seconds: 0.025
frame_overlap_seconds: 0.015
output_type: SQUARED_MAGNITUDE
}
}
}
node {
calculator: "MelSpectrumCalculator"
# MelSpectrumCalculator expects SQUARED_MAGNITUDE input, but its output is in
# linear magnitude units.
input_stream: "spectrogram_squared_magnitude"
output_stream: "mel_spectrum_magnitude"
node_options: {
[type.googleapis.com/mediapipe.MelSpectrumCalculatorOptions] {
# Follow the 'wideband' or '16kHz' speech convention.
channel_count: 64
min_frequency_hertz: 125.0
max_frequency_hertz: 7500.0
}
}
}
node {
calculator: "StabilizedLogCalculator"
input_stream: "mel_spectrum_magnitude"
output_stream: "log_mel_spectrum_magnitude"
node_options: {
[type.googleapis.com/mediapipe.StabilizedLogCalculatorOptions] {
stabilizer: 0.01
}
}
}
node {
calculator: "TimeSeriesFramerCalculator"
input_stream: "log_mel_spectrum_magnitude"
output_stream: "log_mel_spectrum_magnitude_with_context"
node_options: {
[type.googleapis.com/mediapipe.TimeSeriesFramerCalculatorOptions] {
frame_duration_seconds: 0.96
frame_overlap_seconds: -0.04
}
}
}
node {
calculator: "MatrixToTensorCalculator"
input_stream: "log_mel_spectrum_magnitude_with_context"
output_stream: "log_mel_spectrum_magnitude_tensor"
node_options: {
[type.googleapis.com/mediapipe.MatrixToTensorCalculatorOptions] {
transpose: true
}
}
}
node {
calculator: "TensorFlowSessionFromFrozenGraphCalculator"
output_side_packet: "SESSION:vggish_session"
node_options: {
[type.googleapis.com/mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions]: {
graph_proto_path: "/tmp/mediapipe/vggish_new.pb"
tag_to_tensor_names {
key: "INPUT"
value: "vggish/input_features:0"
}
tag_to_tensor_names {
key: "VGGISH"
value: "vggish/fc2/BiasAdd:0"
}
}
}
}
node {
calculator: "TensorFlowInferenceCalculator"
input_side_packet: "SESSION:vggish_session"
input_stream: "INPUT:log_mel_spectrum_magnitude_tensor"
output_stream: "VGGISH:vggish_tensor"
node_options: {
[type.googleapis.com/mediapipe.TensorFlowInferenceCalculatorOptions]: {
signature_name: ""
batch_size: 128
}
}
}
node {
calculator: "TensorToMatrixCalculator"
input_stream: "REFERENCE:log_mel_spectrum_magnitude_with_context"
input_stream: "TENSOR:vggish_tensor"
output_stream: "MATRIX:vggish_matrix"
node_options: {
[type.googleapis.com/mediapipe.TensorToMatrixCalculatorOptions] {
time_series_header_overrides {
num_channels: 128
num_samples: 1
}
}
}
}
node {
calculator: "MatrixSubtractCalculator"
input_stream: "MINUEND:vggish_matrix"
input_side_packet: "SUBTRAHEND:vggish_pca_mean_matrix"
output_stream: "mean_subtracted_vggish_matrix"
}
node {
calculator: "MatrixMultiplyCalculator"
input_stream: "mean_subtracted_vggish_matrix"
input_side_packet: "vggish_pca_projection_matrix"
output_stream: "pca_vggish_matrix"
}
node {
calculator: "MatrixToVectorCalculator"
input_stream: "pca_vggish_matrix"
output_stream: "pca_vggish_vf"
}
# Store the features in the SequenceExample.
node {
calculator: "PackMediaSequenceCalculator"
input_side_packet: "SEQUENCE_EXAMPLE:parsed_sequence_example"
output_side_packet: "SEQUENCE_EXAMPLE:sequence_example_to_serialize"
input_stream: "FLOAT_FEATURE_RGB:pca_inception3_vf"
input_stream: "FLOAT_FEATURE_AUDIO:pca_vggish_vf"
}
# Serialize the SequenceExample to a string for storage.
node {
calculator: "StringToSequenceExampleCalculator"
input_side_packet: "SEQUENCE_EXAMPLE:sequence_example_to_serialize"
output_side_packet: "STRING:output_sequence_example"
}

5
mediapipe/java/com/google/mediapipe/components/BUILD
View File

@ -64,6 +64,11 @@ android_library(
],
visibility = ["//visibility:public"],
deps = [
<<<<<<< HEAD
"//third_party/java/jsr305_annotations",
=======
"@com_google_code_findbugs//jar",
"@com_google_guava_android//jar",
>>>>>>> Project import generated by Copybara.
],
)

4
mediapipe/java/com/google/mediapipe/components/ExternalTextureConverter.java
View File

@ -88,7 +88,11 @@ public class ExternalTextureConverter implements TextureFrameProducer {
* Sets the input surface texture.
*
* <p>The provided width and height will be the size of the converted texture, so if the input
<<<<<<< HEAD
* surface texture is rotated (as expressed by its transfomration matrix) the provided width and
=======
* surface texture is rotated (as expressed by its transformation matrix) the provided width and
>>>>>>> Project import generated by Copybara.
* height should be swapped.
*/
// TODO: Clean up setSurfaceTexture methods.

13
mediapipe/util/audio_decoder.cc
View File

@ -655,6 +655,13 @@ AudioDecoder::~AudioDecoder() {
RETURN_IF_ERROR(processor->Open(stream_id, stream));
audio_processor_.emplace(stream_id, std::move(processor));
<<<<<<< HEAD
=======
CHECK(InsertIfNotPresent(
&stream_index_to_stream_id_,
options.audio_stream(*options_index_ptr).stream_index(),
stream_id));
>>>>>>> Project import generated by Copybara.
CHECK(InsertIfNotPresent(&stream_id_to_audio_options_index_,
stream_id, *options_index_ptr));
CHECK(InsertIfNotPresent(&audio_options_index_to_stream_id,
@ -752,8 +759,14 @@ AudioDecoder::~AudioDecoder() {
::mediapipe::Status AudioDecoder::FillAudioHeader(
const AudioStreamOptions& stream_option, TimeSeriesHeader* header) const {
<<<<<<< HEAD
const std::unique_ptr<AudioPacketProcessor>* processor_ptr_ =
FindOrNull(audio_processor_, stream_option.stream_index());
=======
const std::unique_ptr<AudioPacketProcessor>* processor_ptr_ = FindOrNull(
audio_processor_,
FindOrDie(stream_index_to_stream_id_, stream_option.stream_index()));
>>>>>>> Project import generated by Copybara.
RET_CHECK(processor_ptr_ && *processor_ptr_) << "audio stream is not open.";
RETURN_IF_ERROR((*processor_ptr_)->FillHeader(header));

8
mediapipe/util/audio_decoder.h
View File

@ -82,7 +82,11 @@ class BasePacketProcessor {
// Corrects the given PTS for MPEG PTS rollover. Assumed to be called with
// the PTS of each frame in decode order. We detect a rollover whenever the
<<<<<<< HEAD
// PTS timestamp changes by more than 2^33/2 (half the timstamp space). For
=======
// PTS timestamp changes by more than 2^33/2 (half the timestamp space). For
>>>>>>> Project import generated by Copybara.
// video this means every 26.5h with 1 PTS tick = 1/90000 of a second.
// Example timeline:
// CorrectPtsForRollover(0) -> 0
@ -209,6 +213,10 @@ class AudioDecoder {
::mediapipe::Status Flush();
std::map<int, int> stream_id_to_audio_options_index_;
<<<<<<< HEAD
=======
std::map<int, int> stream_index_to_stream_id_;
>>>>>>> Project import generated by Copybara.
std::map<int, std::unique_ptr<AudioPacketProcessor>> audio_processor_;
// Indexed by container stream index, true if the stream has not seen

8
mediapipe/util/sequence/README.md
View File

@ -438,6 +438,10 @@ tasks and tracking (or class) fields for tracking information.
|`region/class/string`|feature list bytes list|`add_bbox_class_string` / `AddBBoxClassString`|For each region, lists the string class. Multiple classes for one region require duplicating the region.|
|`region/class/confidence`|feature list float list|`add_bbox_class_confidence` / `AddBBoxClassConfidence`|For each region, lists the confidence or weight for the class. Multiple classes for one region require duplicating the region.|
|`region/embedding/float`|feature list float list|`add_bbox_embedding_floats` / `AddBBoxEmbeddingFloats`|For each region, provide an embedding as sequence of floats.|
<<<<<<< HEAD
=======
|`region/parts`|context bytes list|`set_bbox_parts` / `SetBBoxParts`|The list of region parts expected in this example.|
>>>>>>> Project import generated by Copybara.
|`region/embedding/ dimensions_per_region`|context int list|`set_bbox_embedding_dimensions_per_region` / `SetBBoxEmbeddingDimensionsPerRegion`|Provide the dimensions for each embedding.|
|`region/embedding/format`|context string|`set_bbox_embedding_format` / `SetBBoxEmbeddingFormat`|Provides the encoding format, if any, for region embeddings.|
|`region/embedding/encoded`|feature list bytes list|`add_bbox_embedding_encoded` / `AddBBoxEmbeddingEncoded`|For each region, provide an encoded embedding.|
@ -458,6 +462,10 @@ tasks and tracking (or class) fields for tracking information.
|`image/height`|context int|`set_image_height` / `SetImageHeight`|The height of the image in pixels.|
|`image/width`|context int|`set_image_width` / `SetImageWidth`|The width of the image in pixels.|
|`image/frame_rate`|context float|`set_image_frame_rate` / `SetImageFrameRate`|The rate of images in frames per second.|
<<<<<<< HEAD
=======
|`image/data_path`|context bytes|`set_image_data_path` / `SetImageDataPath`|The path of the image file if it did not come from a media clip.|
>>>>>>> Project import generated by Copybara.
### Keys related to image class segmentation
| key | type | python call / c++ call | description |

20
mediapipe/util/sequence/media_sequence.h
View File

@ -253,6 +253,11 @@ const char kRegionEmbeddingDimensionsPerRegionKey[] =
"region/embedding/dimensions_per_region";
// The format encoding embeddings as strings.
const char kRegionEmbeddingFormatKey[] = "region/embedding/format";
<<<<<<< HEAD
=======
// The list of region parts expected in this example.
const char kRegionPartsKey[] = "region/parts";
>>>>>>> Project import generated by Copybara.
// Feature list keys:
// The normalized coordinates of the bounding boxes are provided in four lists
@ -435,6 +440,11 @@ void ClearPoint(const std::string& prefix,
kRegionEmbeddingDimensionsPerRegionKey, prefix) \
FIXED_PREFIX_BYTES_CONTEXT_FEATURE(CONCAT_STR2(identifier, EmbeddingFormat), \
kRegionEmbeddingFormatKey, prefix) \
<<<<<<< HEAD
=======
FIXED_PREFIX_VECTOR_BYTES_CONTEXT_FEATURE(CONCAT_STR2(identifier, Parts), \
kRegionPartsKey, prefix) \
>>>>>>> Project import generated by Copybara.
FIXED_PREFIX_INT64_FEATURE_LIST(CONCAT_STR2(identifier, Timestamp), \
kRegionTimestampKey, prefix) \
FIXED_PREFIX_INT64_FEATURE_LIST( \
@ -471,6 +481,11 @@ const char kImageClassLabelIndexKey[] = "image/class/label/index";
const char kImageClassLabelStringKey[] = "image/class/label/string";
// The listing from discrete instance indices to class indices they embody.
const char kImageObjectClassIndexKey[] = "image/object/class/index";
<<<<<<< HEAD
=======
// The path of the image file if it did not come from a media clip.
const char kImageDataPathKey[] = "image/data_path";
>>>>>>> Project import generated by Copybara.
// Feature list keys:
// The encoded image frame.
@ -500,6 +515,11 @@ const char kImageLabelConfidenceKey[] = "image/label/confidence";
kImageFrameRateKey, prefix) \
FIXED_PREFIX_FLOAT_CONTEXT_FEATURE(CONCAT_STR2(identifier, Saturation), \
kImageSaturationKey, prefix) \
<<<<<<< HEAD
=======
FIXED_PREFIX_BYTES_CONTEXT_FEATURE(CONCAT_STR2(identifier, DataPath), \
kImageDataPathKey, prefix) \
>>>>>>> Project import generated by Copybara.
FIXED_PREFIX_VECTOR_INT64_CONTEXT_FEATURE( \
CONCAT_STR2(identifier, ClassLabelIndex), kImageClassLabelIndexKey, \
prefix) \

21
mediapipe/util/sequence/media_sequence.py
View File

@ -291,6 +291,11 @@ REGION_TIMESTAMP_KEY = "region/timestamp"
# If regions are aligned to image frames, this field preserves the original
# timestamps.
REGION_UNMODIFIED_TIMESTAMP_KEY = "region/unmodified_timestamp"
<<<<<<< HEAD
=======
# The list of region parts expected in this example.
REGION_PARTS_KEY = "region/parts"
>>>>>>> Project import generated by Copybara.
# The dimensions of each embedding per region / bounding box.
REGION_EMBEDDING_DIMENSIONS_PER_REGION_KEY = (
"region/embedding/dimensions_per_region")
@ -361,6 +366,12 @@ def _create_region_with_prefix(name, prefix):
prefix=prefix, module_dict=globals())
msu.create_float_list_feature_list(name + "_point_y", REGION_POINT_Y_KEY,
prefix=prefix, module_dict=globals())
<<<<<<< HEAD
=======
msu.create_bytes_list_context_feature(name + "_parts",
REGION_PARTS_KEY,
prefix=prefix, module_dict=globals())
>>>>>>> Project import generated by Copybara.
msu.create_float_list_context_feature(
name + "_embedding_dimensions_per_region",
REGION_EMBEDDING_DIMENSIONS_PER_REGION_KEY,
@ -451,6 +462,11 @@ IMAGE_TIMESTAMP_KEY = "image/timestamp"
IMAGE_LABEL_INDEX_KEY = "image/label/index"
IMAGE_LABEL_STRING_KEY = "image/label/string"
IMAGE_LABEL_CONFIDENCE_KEY = "image/label/confidence"
<<<<<<< HEAD
=======
# The path of the image file if it did not come from a media clip.
IMAGE_DATA_PATH_KEY = "image/data_path"
>>>>>>> Project import generated by Copybara.
def _create_image_with_prefix(name, prefix):
@ -478,6 +494,11 @@ def _create_image_with_prefix(name, prefix):
msu.create_int_list_context_feature(
name + "_object_class_index", IMAGE_OBJECT_CLASS_INDEX_KEY,
prefix=prefix, module_dict=globals())
<<<<<<< HEAD
=======
msu.create_bytes_context_feature(name + "_data_path", IMAGE_DATA_PATH_KEY,
prefix=prefix, module_dict=globals())
>>>>>>> Project import generated by Copybara.
msu.create_int_feature_list(name + "_timestamp", IMAGE_TIMESTAMP_KEY,
prefix=prefix, module_dict=globals())

22
mediapipe/util/sequence/media_sequence_test.cc
View File

@ -409,6 +409,18 @@ TEST(MediaSequenceTest, RoundTripBBoxPoint) {
}
}
<<<<<<< HEAD
=======
TEST(MediaSequenceTest, RoundTripRegionParts) {
tensorflow::SequenceExample sequence;
std::vector<std::string> parts = {"HEAD", "FEET"};
SetBBoxParts(parts, &sequence);
ASSERT_THAT(GetBBoxParts(sequence), testing::ElementsAreArray(parts));
ClearBBoxParts(&sequence);
EXPECT_EQ(GetBBoxPartsSize(sequence), 0);
}
>>>>>>> Project import generated by Copybara.
TEST(MediaSequenceTest, RoundTripPredictedBBox) {
tensorflow::SequenceExample sequence;
std::vector<std::vector<Location>> bboxes = {
@ -528,6 +540,16 @@ TEST(MediaSequenceTest, RoundTripImageFrameRate) {
ASSERT_EQ(GetImageFrameRate(sequence), frame_rate);
}
<<<<<<< HEAD
=======
TEST(MediaSequenceTest, RoundTripImageDataPath) {
tensorflow::SequenceExample sequence;
std::string data_path = "test";
SetImageDataPath(data_path, &sequence);
ASSERT_EQ(data_path, GetImageDataPath(sequence));
}
>>>>>>> Project import generated by Copybara.
TEST(MediaSequenceTest, RoundTripFeatureFloats) {
tensorflow::SequenceExample sequence;
int num_features = 3;

8
mediapipe/util/sequence/media_sequence_test.py
View File

@ -59,6 +59,10 @@ class MediaSequenceTest(tf.test.TestCase):
ms.set_image_height(47, example)
ms.set_image_width(47, example)
ms.set_image_frame_rate(0.47, example)
<<<<<<< HEAD
=======
ms.set_image_data_path(b"test", example)
>>>>>>> Project import generated by Copybara.
ms.set_forward_flow_format(b"test", example)
ms.set_forward_flow_channels(47, example)
ms.set_forward_flow_colorspace(b"test", example)
@ -74,6 +78,10 @@ class MediaSequenceTest(tf.test.TestCase):
ms.set_instance_segmentation_height(47, example)
ms.set_instance_segmentation_width(47, example)
ms.set_instance_segmentation_object_class_index((47, 49), example)
<<<<<<< HEAD
=======
ms.set_bbox_parts((b"HEAD", b"TOE"), example)
>>>>>>> Project import generated by Copybara.
# feature lists
ms.add_image_encoded(b"test", example)
ms.add_image_timestamp(47, example)

3
setup_android_sdk_and_ndk.sh
View File

@ -93,7 +93,10 @@ ndk_path_line=$((ndk_block+2))'i'
sdk_block=$(grep -n 'android_sdk_repository(' $workspace_file | awk -F ":" '{print $1}')
sdk_path_line=$((sdk_block+3))'i'
<<<<<<< HEAD
git checkout $workspace_file
=======
>>>>>>> Project import generated by Copybara.
if [ $platform == "darwin" ]; then
sed -i -e "$ndk_path_line\\
\ \ \ \ path = \"${android_ndk_path}/android-ndk-${ndk_version}\",

4
third_party/ffmpeg_linux.BUILD vendored
View File

@ -20,7 +20,11 @@ cc_library(
name = "libffmpeg",
srcs = glob(
[
<<<<<<< HEAD
"lib/x86_64-linux-gnu/libav*.so*",
=======
"lib/x86_64-linux-gnu/libav*.so",
>>>>>>> Project import generated by Copybara.
],
),
hdrs = glob(["include/x86_64-linux-gnu/libav*/*.h"]),

9
third_party/opencv_linux.BUILD vendored
View File

@ -13,12 +13,21 @@ cc_library(
name = "opencv",
srcs = glob(
[
<<<<<<< HEAD
"lib/x86_64-linux-gnu/libopencv_core.so*",
"lib/x86_64-linux-gnu/libopencv_highgui.so*",
"lib/x86_64-linux-gnu/libopencv_imgcodecs.so*",
"lib/x86_64-linux-gnu/libopencv_imgproc.so*",
"lib/x86_64-linux-gnu/libopencv_video.so*",
"lib/x86_64-linux-gnu/libopencv_videoio.so*",
=======
"lib/x86_64-linux-gnu/libopencv_core.so",
"lib/x86_64-linux-gnu/libopencv_highgui.so",
"lib/x86_64-linux-gnu/libopencv_imgcodecs.so",
"lib/x86_64-linux-gnu/libopencv_imgproc.so",
"lib/x86_64-linux-gnu/libopencv_video.so",
"lib/x86_64-linux-gnu/libopencv_videoio.so",
>>>>>>> Project import generated by Copybara.
],
),
hdrs = glob(["include/opencv2/**/*.h*"]),