From af676420555f60c71c481f597d5ed3aa3b4f6789 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 4 Sep 2019 18:19:36 -0700 Subject: [PATCH] Project import generated by Copybara. PiperOrigin-RevId: 267274408 --- Dockerfile | 8 + mediapipe/calculators/audio/BUILD | 60 ++++ .../audio/audio_decoder_calculator.cc | 4 + .../calculators/audio/mfcc_mel_calculators.cc | 22 ++ .../rational_factor_resample_calculator.cc | 5 + .../audio/spectrogram_calculator.cc | 17 + .../audio/spectrogram_calculator.proto | 4 + .../audio/stabilized_log_calculator.cc | 94 ++++++ .../audio/stabilized_log_calculator.proto | 37 ++ .../audio/stabilized_log_calculator_test.cc | 131 ++++++++ .../audio/time_series_framer_calculator.cc | 5 + mediapipe/calculators/core/BUILD | 46 +++ .../core/concatenate_vector_calculator.cc | 10 + .../core/matrix_to_vector_calculator.cc | 83 +++++ .../core/matrix_to_vector_calculator_test.cc | 88 +++++ .../core/split_vector_calculator.cc | 10 + .../image/image_transformation_calculator.cc | 4 + mediapipe/calculators/tensorflow/BUILD | 79 +++++ .../tensorflow_inference_calculator.cc | 19 ++ ...ow_session_from_frozen_graph_calculator.cc | 136 ++++++++ ...session_from_frozen_graph_calculator.proto | 72 ++++ ...ssion_from_frozen_graph_calculator_test.cc | 316 ++++++++++++++++++ mediapipe/calculators/tflite/BUILD | 77 +++++ .../calculators/tflite/testdata/labelmap.txt | 3 + .../tflite/tflite_inference_calculator.cc | 6 + ...te_tensors_to_classification_calculator.cc | 176 ++++++++++ ...tensors_to_classification_calculator.proto | 35 ++ ...nsors_to_classification_calculator_test.cc | 194 +++++++++++ .../video/opencv_video_decoder_calculator.cc | 15 + mediapipe/docs/cycles.md | 4 + mediapipe/docs/examples.md | 8 + mediapipe/docs/framework_concepts.md | 8 + mediapipe/docs/hello_world_android.md | 4 + mediapipe/docs/hello_world_ios.md | 4 + mediapipe/docs/help.md | 40 +++ mediapipe/docs/install.md | 170 ++++++++++ mediapipe/docs/media_sequence.md | 8 + mediapipe/docs/mediapipe_ios_setup.md | 16 + mediapipe/docs/troubleshooting.md | 4 + mediapipe/docs/youtube_8m.md | 83 +++++ .../apps/edgedetectiongpu/MainActivity.java | 4 + .../apps/facedetectioncpu/MainActivity.java | 4 + .../apps/facedetectiongpu/MainActivity.java | 4 + .../hairsegmentationgpu/MainActivity.java | 4 + .../apps/handdetectiongpu/MainActivity.java | 4 + .../apps/handtrackinggpu/MainActivity.java | 4 + .../apps/objectdetectioncpu/MainActivity.java | 4 + .../apps/objectdetectiongpu/MainActivity.java | 4 + mediapipe/examples/desktop/README.md | 3 + mediapipe/examples/desktop/youtube8m/BUILD | 35 ++ .../examples/desktop/youtube8m/README.md | 54 +++ .../examples/desktop/youtube8m/__init__.py | 14 + .../youtube8m/extract_yt8m_features.cc | 135 ++++++++ .../generate_input_sequence_example.py | 56 ++++ .../youtube8m/generate_vggish_frozen_graph.py | 70 ++++ .../Base.lproj/Main.storyboard | 20 ++ .../Base.lproj/Main.storyboard | 17 + .../Base.lproj/Main.storyboard | 17 + .../Base.lproj/Main.storyboard | 17 + mediapipe/framework/BUILD | 8 + mediapipe/framework/formats/BUILD | 19 ++ .../framework/formats/classification.proto | 35 ++ mediapipe/framework/graph_validation_test.cc | 12 + .../framework/profiler/graph_profiler.cc | 4 + .../fixed_size_input_stream_handler.cc | 19 ++ mediapipe/gpu/metal_shader_base.h | 43 +++ mediapipe/graphs/youtube8m/BUILD | 46 +++ .../graphs/youtube8m/feature_extraction.pbtxt | 295 ++++++++++++++++ .../com/google/mediapipe/components/BUILD | 5 + .../components/ExternalTextureConverter.java | 4 + mediapipe/util/audio_decoder.cc | 13 + mediapipe/util/audio_decoder.h | 8 + mediapipe/util/sequence/README.md | 8 + mediapipe/util/sequence/media_sequence.h | 20 ++ mediapipe/util/sequence/media_sequence.py | 21 ++ .../util/sequence/media_sequence_test.cc | 22 ++ .../util/sequence/media_sequence_test.py | 8 + setup_android_sdk_and_ndk.sh | 3 + third_party/ffmpeg_linux.BUILD | 4 + third_party/opencv_linux.BUILD | 9 + 80 files changed, 3181 insertions(+) create mode 100644 mediapipe/calculators/audio/stabilized_log_calculator.cc create mode 100644 mediapipe/calculators/audio/stabilized_log_calculator.proto create mode 100644 mediapipe/calculators/audio/stabilized_log_calculator_test.cc create mode 100644 mediapipe/calculators/core/matrix_to_vector_calculator.cc create mode 100644 mediapipe/calculators/core/matrix_to_vector_calculator_test.cc create mode 100644 mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc create mode 100644 mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.proto create mode 100644 mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc create mode 100644 mediapipe/calculators/tflite/testdata/labelmap.txt create mode 100644 mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc create mode 100644 mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto create mode 100644 mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator_test.cc create mode 100644 mediapipe/docs/youtube_8m.md create mode 100644 mediapipe/examples/desktop/youtube8m/BUILD create mode 100644 mediapipe/examples/desktop/youtube8m/README.md create mode 100644 mediapipe/examples/desktop/youtube8m/__init__.py create mode 100644 mediapipe/examples/desktop/youtube8m/extract_yt8m_features.cc create mode 100644 mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py create mode 100644 mediapipe/examples/desktop/youtube8m/generate_vggish_frozen_graph.py create mode 100644 mediapipe/framework/formats/classification.proto create mode 100644 mediapipe/gpu/metal_shader_base.h create mode 100644 mediapipe/graphs/youtube8m/BUILD create mode 100644 mediapipe/graphs/youtube8m/feature_extraction.pbtxt diff --git a/Dockerfile b/Dockerfile index 972198b52..ff0b059db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,6 +24,10 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ +<<<<<<< HEAD +======= + curl \ +>>>>>>> Project import generated by Copybara. git \ wget \ unzip \ @@ -35,7 +39,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libopencv-video-dev \ software-properties-common && \ add-apt-repository -y ppa:openjdk-r/ppa && \ +<<<<<<< HEAD apt-get update && apt-get install -y openjdk-11-jdk && \ +======= + apt-get update && apt-get install -y openjdk-8-jdk && \ +>>>>>>> Project import generated by Copybara. apt-get clean && \ rm -rf /var/lib/apt/lists/* diff --git a/mediapipe/calculators/audio/BUILD b/mediapipe/calculators/audio/BUILD index 0e845b20f..7306ea0ff 100644 --- a/mediapipe/calculators/audio/BUILD +++ b/mediapipe/calculators/audio/BUILD @@ -68,6 +68,26 @@ mediapipe_cc_proto_library( ) proto_library( +<<<<<<< HEAD +======= + name = "stabilized_log_calculator_proto", + srcs = ["stabilized_log_calculator.proto"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_proto", + ], +) + +mediapipe_cc_proto_library( + name = "stabilized_log_calculator_cc_proto", + srcs = ["stabilized_log_calculator.proto"], + cc_deps = ["//mediapipe/framework:calculator_cc_proto"], + visibility = ["//visibility:public"], + deps = [":stabilized_log_calculator_proto"], +) + +proto_library( +>>>>>>> Project import generated by Copybara. name = "time_series_framer_calculator_proto", srcs = ["time_series_framer_calculator.proto"], visibility = ["//visibility:public"], @@ -157,6 +177,25 @@ cc_library( ) cc_library( +<<<<<<< HEAD +======= + name = "stabilized_log_calculator", + srcs = ["stabilized_log_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + ":stabilized_log_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:matrix", + "//mediapipe/framework/formats:time_series_header_cc_proto", + "//mediapipe/framework/port:core_proto", + "//mediapipe/framework/port:status", + "//mediapipe/util:time_series_util", + ], + alwayslink = 1, +) + +cc_library( +>>>>>>> Project import generated by Copybara. name = "spectrogram_calculator", srcs = ["spectrogram_calculator.cc"], visibility = ["//visibility:public"], @@ -267,6 +306,27 @@ cc_test( ) cc_test( +<<<<<<< HEAD +======= + name = "stabilized_log_calculator_test", + srcs = ["stabilized_log_calculator_test.cc"], + deps = [ + ":stabilized_log_calculator", + ":stabilized_log_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:calculator_runner", + "//mediapipe/framework/formats:matrix", + "//mediapipe/framework/formats:time_series_header_cc_proto", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:status", + "//mediapipe/util:time_series_test_util", + "@eigen_archive//:eigen", + ], +) + +cc_test( +>>>>>>> Project import generated by Copybara. name = "time_series_framer_calculator_test", srcs = ["time_series_framer_calculator_test.cc"], deps = [ diff --git a/mediapipe/calculators/audio/audio_decoder_calculator.cc b/mediapipe/calculators/audio/audio_decoder_calculator.cc index 24dae2b44..4fffd4391 100644 --- a/mediapipe/calculators/audio/audio_decoder_calculator.cc +++ b/mediapipe/calculators/audio/audio_decoder_calculator.cc @@ -64,7 +64,11 @@ class AudioDecoderCalculator : public CalculatorBase { cc->Outputs().Tag("AUDIO").Set(); if (cc->Outputs().HasTag("AUDIO_HEADER")) { +<<<<<<< HEAD cc->Outputs().Tag("AUDIO_HEADER").Set(); +======= + cc->Outputs().Tag("AUDIO_HEADER").SetNone(); +>>>>>>> Project import generated by Copybara. } return ::mediapipe::OkStatus(); } diff --git a/mediapipe/calculators/audio/mfcc_mel_calculators.cc b/mediapipe/calculators/audio/mfcc_mel_calculators.cc index 3e6ebbe95..4d04414a5 100644 --- a/mediapipe/calculators/audio/mfcc_mel_calculators.cc +++ b/mediapipe/calculators/audio/mfcc_mel_calculators.cc @@ -90,8 +90,13 @@ class FramewiseTransformCalculatorBase : public CalculatorBase { private: // Takes header and options, and sets up state including calling // set_num_output_channels() on the base object. +<<<<<<< HEAD virtual ::mediapipe::Status ConfigureTransform( const TimeSeriesHeader& header, const CalculatorOptions& options) = 0; +======= + virtual ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header, + CalculatorContext* cc) = 0; +>>>>>>> Project import generated by Copybara. // Takes a vector corresponding to an input frame, and // perform the specific transformation to produce an output frame. @@ -108,7 +113,11 @@ class FramewiseTransformCalculatorBase : public CalculatorBase { RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid( cc->Inputs().Index(0).Header(), &input_header)); +<<<<<<< HEAD ::mediapipe::Status status = ConfigureTransform(input_header, cc->Options()); +======= + ::mediapipe::Status status = ConfigureTransform(input_header, cc); +>>>>>>> Project import generated by Copybara. auto output_header = new TimeSeriesHeader(input_header); output_header->set_num_channels(num_output_channels_); @@ -175,11 +184,17 @@ class MfccCalculator : public FramewiseTransformCalculatorBase { } private: +<<<<<<< HEAD ::mediapipe::Status ConfigureTransform( const TimeSeriesHeader& header, const CalculatorOptions& options) override { MfccCalculatorOptions mfcc_options; time_series_util::FillOptionsExtensionOrDie(options, &mfcc_options); +======= + ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header, + CalculatorContext* cc) override { + MfccCalculatorOptions mfcc_options = cc->Options(); +>>>>>>> Project import generated by Copybara. mfcc_.reset(new audio_dsp::Mfcc()); int input_length = header.num_channels(); // Set up the parameters to the Mfcc object. @@ -235,11 +250,18 @@ class MelSpectrumCalculator : public FramewiseTransformCalculatorBase { } private: +<<<<<<< HEAD ::mediapipe::Status ConfigureTransform( const TimeSeriesHeader& header, const CalculatorOptions& options) override { MelSpectrumCalculatorOptions mel_spectrum_options; time_series_util::FillOptionsExtensionOrDie(options, &mel_spectrum_options); +======= + ::mediapipe::Status ConfigureTransform(const TimeSeriesHeader& header, + CalculatorContext* cc) override { + MelSpectrumCalculatorOptions mel_spectrum_options = + cc->Options(); +>>>>>>> Project import generated by Copybara. mel_filterbank_.reset(new audio_dsp::MelFilterbank()); int input_length = header.num_channels(); set_num_output_channels(mel_spectrum_options.channel_count()); diff --git a/mediapipe/calculators/audio/rational_factor_resample_calculator.cc b/mediapipe/calculators/audio/rational_factor_resample_calculator.cc index 3a966f8f8..3c882db26 100644 --- a/mediapipe/calculators/audio/rational_factor_resample_calculator.cc +++ b/mediapipe/calculators/audio/rational_factor_resample_calculator.cc @@ -64,8 +64,13 @@ void CopyVectorToChannel(const std::vector& vec, Matrix* matrix, ::mediapipe::Status RationalFactorResampleCalculator::Open( CalculatorContext* cc) { +<<<<<<< HEAD RationalFactorResampleCalculatorOptions resample_options; time_series_util::FillOptionsExtensionOrDie(cc->Options(), &resample_options); +======= + RationalFactorResampleCalculatorOptions resample_options = + cc->Options(); +>>>>>>> Project import generated by Copybara. if (!resample_options.has_target_sample_rate()) { return tool::StatusInvalid( diff --git a/mediapipe/calculators/audio/spectrogram_calculator.cc b/mediapipe/calculators/audio/spectrogram_calculator.cc index 15f4c917e..fae9ae7cc 100644 --- a/mediapipe/calculators/audio/spectrogram_calculator.cc +++ b/mediapipe/calculators/audio/spectrogram_calculator.cc @@ -71,10 +71,15 @@ class SpectrogramCalculator : public CalculatorBase { // Input stream with TimeSeriesHeader. ); +<<<<<<< HEAD SpectrogramCalculatorOptions spectrogram_options; time_series_util::FillOptionsExtensionOrDie(cc->Options(), &spectrogram_options); +======= + SpectrogramCalculatorOptions spectrogram_options = + cc->Options(); +>>>>>>> Project import generated by Copybara. if (!spectrogram_options.allow_multichannel_input()) { if (spectrogram_options.output_type() == SpectrogramCalculatorOptions::COMPLEX) { @@ -172,9 +177,14 @@ REGISTER_CALCULATOR(SpectrogramCalculator); const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518; ::mediapipe::Status SpectrogramCalculator::Open(CalculatorContext* cc) { +<<<<<<< HEAD SpectrogramCalculatorOptions spectrogram_options; time_series_util::FillOptionsExtensionOrDie(cc->Options(), &spectrogram_options); +======= + SpectrogramCalculatorOptions spectrogram_options = + cc->Options(); +>>>>>>> Project import generated by Copybara. if (spectrogram_options.frame_duration_seconds() <= 0.0) { ::mediapipe::InvalidArgumentErrorBuilder(MEDIAPIPE_LOC) @@ -223,6 +233,13 @@ const float SpectrogramCalculator::kLnPowerToDb = 4.342944819032518; std::vector window; switch (spectrogram_options.window_type()) { +<<<<<<< HEAD +======= + case SpectrogramCalculatorOptions::COSINE: + audio_dsp::CosineWindow().GetPeriodicSamples(frame_duration_samples_, + &window); + break; +>>>>>>> Project import generated by Copybara. case SpectrogramCalculatorOptions::HANN: audio_dsp::HannWindow().GetPeriodicSamples(frame_duration_samples_, &window); diff --git a/mediapipe/calculators/audio/spectrogram_calculator.proto b/mediapipe/calculators/audio/spectrogram_calculator.proto index faef8d590..c29a6f206 100644 --- a/mediapipe/calculators/audio/spectrogram_calculator.proto +++ b/mediapipe/calculators/audio/spectrogram_calculator.proto @@ -58,6 +58,10 @@ message SpectrogramCalculatorOptions { enum WindowType { HANN = 0; HAMMING = 1; +<<<<<<< HEAD +======= + COSINE = 2; +>>>>>>> Project import generated by Copybara. } optional WindowType window_type = 6 [default = HANN]; diff --git a/mediapipe/calculators/audio/stabilized_log_calculator.cc b/mediapipe/calculators/audio/stabilized_log_calculator.cc new file mode 100644 index 000000000..817819c1c --- /dev/null +++ b/mediapipe/calculators/audio/stabilized_log_calculator.cc @@ -0,0 +1,94 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Defines StabilizedLogCalculator. + +#include +#include +#include + +#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/time_series_header.pb.h" +#include "mediapipe/framework/port/proto_ns.h" +#include "mediapipe/util/time_series_util.h" + +namespace mediapipe { + +// Example config: +// node { +// calculator: "StabilizedLogCalculator" +// input_stream: "input_time_series" +// output_stream: "stabilized_log_time_series" +// options { +// [mediapipe.StabilizedLogCalculatorOptions.ext] { +// stabilizer: .00001 +// check_nonnegativity: true +// } +// } +// } +class StabilizedLogCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + cc->Inputs().Index(0).Set( + // Input stream with TimeSeriesHeader. + ); + cc->Outputs().Index(0).Set( + // Output stabilized log stream with TimeSeriesHeader. + ); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override { + StabilizedLogCalculatorOptions stabilized_log_calculator_options = + cc->Options(); + + stabilizer_ = stabilized_log_calculator_options.stabilizer(); + output_scale_ = stabilized_log_calculator_options.output_scale(); + check_nonnegativity_ = + stabilized_log_calculator_options.check_nonnegativity(); + CHECK_GE(stabilizer_, 0.0) + << "stabilizer must be >= 0.0, received a value of " << stabilizer_; + + // If the input packets have a header, propagate the header to the output. + if (!cc->Inputs().Index(0).Header().IsEmpty()) { + TimeSeriesHeader input_header; + RETURN_IF_ERROR(time_series_util::FillTimeSeriesHeaderIfValid( + cc->Inputs().Index(0).Header(), &input_header)); + cc->Outputs().Index(0).SetHeader( + Adopt(new TimeSeriesHeader(input_header))); + } + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + auto input_matrix = cc->Inputs().Index(0).Get(); + if (check_nonnegativity_) { + CHECK_GE(input_matrix.minCoeff(), 0); + } + std::unique_ptr output_frame(new Matrix( + output_scale_ * (input_matrix.array() + stabilizer_).log().matrix())); + cc->Outputs().Index(0).Add(output_frame.release(), cc->InputTimestamp()); + return ::mediapipe::OkStatus(); + } + + private: + float stabilizer_; + bool check_nonnegativity_; + double output_scale_; +}; +REGISTER_CALCULATOR(StabilizedLogCalculator); + +} // namespace mediapipe diff --git a/mediapipe/calculators/audio/stabilized_log_calculator.proto b/mediapipe/calculators/audio/stabilized_log_calculator.proto new file mode 100644 index 000000000..ef6faa8e5 --- /dev/null +++ b/mediapipe/calculators/audio/stabilized_log_calculator.proto @@ -0,0 +1,37 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message StabilizedLogCalculatorOptions { + extend CalculatorOptions { + optional StabilizedLogCalculatorOptions ext = 101978339; + } + + // The calculator computes log(x + stabilizer). stabilizer must be >= + // 0, with 0 indicating a lack of stabilization. + optional float stabilizer = 1 [default = .00001]; + + // If true, CHECK that all input values in are >= 0. If false, the + // code will take the log of the potentially negative input values + // plus the stabilizer. + optional bool check_nonnegativity = 2 [default = true]; + + // Support a fixed multiplicative scaling of the output. + optional double output_scale = 3 [default = 1.0]; +} diff --git a/mediapipe/calculators/audio/stabilized_log_calculator_test.cc b/mediapipe/calculators/audio/stabilized_log_calculator_test.cc new file mode 100644 index 000000000..e7c6d64fe --- /dev/null +++ b/mediapipe/calculators/audio/stabilized_log_calculator_test.cc @@ -0,0 +1,131 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Eigen/Core" +#include "mediapipe/calculators/audio/stabilized_log_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_runner.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/formats/time_series_header.pb.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/integral_types.h" +#include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/util/time_series_test_util.h" + +namespace mediapipe { + +const float kStabilizer = 0.1; +const int kNumChannels = 3; +const int kNumSamples = 10; + +class StabilizedLogCalculatorTest + : public TimeSeriesCalculatorTest { + protected: + void SetUp() override { + calculator_name_ = "StabilizedLogCalculator"; + options_.set_stabilizer(kStabilizer); + + input_sample_rate_ = 8000.0; + num_input_channels_ = kNumChannels; + num_input_samples_ = kNumSamples; + } + + void RunGraphNoReturn() { MEDIAPIPE_ASSERT_OK(RunGraph()); } +}; + +TEST_F(StabilizedLogCalculatorTest, BasicOperation) { + const int kNumPackets = 5; + + InitializeGraph(); + FillInputHeader(); + + std::vector input_data_matrices; + for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) { + const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond; + Matrix input_data_matrix = + Matrix::Random(kNumChannels, kNumSamples).array().abs(); + input_data_matrices.push_back(input_data_matrix); + AppendInputPacket(new Matrix(input_data_matrix), timestamp); + } + + MEDIAPIPE_ASSERT_OK(RunGraph()); + ExpectOutputHeaderEqualsInputHeader(); + for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) { + ExpectApproximatelyEqual( + (input_data_matrices[output_packet].array() + kStabilizer).log(), + runner_->Outputs().Index(0).packets[output_packet].Get()); + } +} + +TEST_F(StabilizedLogCalculatorTest, OutputScaleWorks) { + const int kNumPackets = 5; + double output_scale = 2.5; + options_.set_output_scale(output_scale); + + InitializeGraph(); + FillInputHeader(); + + std::vector input_data_matrices; + for (int input_packet = 0; input_packet < kNumPackets; ++input_packet) { + const int64 timestamp = input_packet * Timestamp::kTimestampUnitsPerSecond; + Matrix input_data_matrix = + Matrix::Random(kNumChannels, kNumSamples).array().abs(); + input_data_matrices.push_back(input_data_matrix); + AppendInputPacket(new Matrix(input_data_matrix), timestamp); + } + + MEDIAPIPE_ASSERT_OK(RunGraph()); + ExpectOutputHeaderEqualsInputHeader(); + for (int output_packet = 0; output_packet < kNumPackets; ++output_packet) { + ExpectApproximatelyEqual( + output_scale * + ((input_data_matrices[output_packet].array() + kStabilizer).log()), + runner_->Outputs().Index(0).packets[output_packet].Get()); + } +} + +TEST_F(StabilizedLogCalculatorTest, ZerosAreStabilized) { + InitializeGraph(); + FillInputHeader(); + AppendInputPacket(new Matrix(Matrix::Zero(kNumChannels, kNumSamples)), + 0 /* timestamp */); + MEDIAPIPE_ASSERT_OK(RunGraph()); + ExpectOutputHeaderEqualsInputHeader(); + ExpectApproximatelyEqual( + Matrix::Constant(kNumChannels, kNumSamples, kStabilizer).array().log(), + runner_->Outputs().Index(0).packets[0].Get()); +} + +TEST_F(StabilizedLogCalculatorTest, NegativeValuesCheckFail) { + InitializeGraph(); + FillInputHeader(); + AppendInputPacket( + new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)), + 0 /* timestamp */); + ASSERT_DEATH(RunGraphNoReturn(), ""); +} + +TEST_F(StabilizedLogCalculatorTest, NegativeValuesDoNotCheckFailIfCheckIsOff) { + options_.set_check_nonnegativity(false); + InitializeGraph(); + FillInputHeader(); + AppendInputPacket( + new Matrix(Matrix::Constant(kNumChannels, kNumSamples, -1.0)), + 0 /* timestamp */); + MEDIAPIPE_ASSERT_OK(RunGraph()); + // Results are undefined. +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/audio/time_series_framer_calculator.cc b/mediapipe/calculators/audio/time_series_framer_calculator.cc index 60ab5d7b4..0879c89db 100644 --- a/mediapipe/calculators/audio/time_series_framer_calculator.cc +++ b/mediapipe/calculators/audio/time_series_framer_calculator.cc @@ -206,8 +206,13 @@ void TimeSeriesFramerCalculator::FrameOutput(CalculatorContext* cc) { } ::mediapipe::Status TimeSeriesFramerCalculator::Open(CalculatorContext* cc) { +<<<<<<< HEAD TimeSeriesFramerCalculatorOptions framer_options; time_series_util::FillOptionsExtensionOrDie(cc->Options(), &framer_options); +======= + TimeSeriesFramerCalculatorOptions framer_options = + cc->Options(); +>>>>>>> Project import generated by Copybara. RET_CHECK_GT(framer_options.frame_duration_seconds(), 0.0) << "Invalid or missing frame_duration_seconds. " diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index 4ecf8fe7b..4ede4bbb7 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -162,6 +162,10 @@ cc_library( deps = [ ":concatenate_vector_calculator_cc_proto", "//mediapipe/framework:calculator_framework", +<<<<<<< HEAD +======= + "//mediapipe/framework/formats:landmark_cc_proto", +>>>>>>> Project import generated by Copybara. "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@org_tensorflow//tensorflow/lite:framework", @@ -523,6 +527,10 @@ cc_library( deps = [ ":split_vector_calculator_cc_proto", "//mediapipe/framework:calculator_framework", +<<<<<<< HEAD +======= + "//mediapipe/framework/formats:landmark_cc_proto", +>>>>>>> Project import generated by Copybara. "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "//mediapipe/util:resource_util", @@ -628,6 +636,44 @@ cc_test( ) cc_library( +<<<<<<< HEAD +======= + name = "matrix_to_vector_calculator", + srcs = ["matrix_to_vector_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:matrix", + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:logging", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/tool:status_util", + "//mediapipe/util:time_series_util", + "@com_google_absl//absl/memory", + "@eigen_archive//:eigen", + ], + alwayslink = 1, +) + +cc_test( + name = "matrix_to_vector_calculator_test", + srcs = ["matrix_to_vector_calculator_test.cc"], + deps = [ + ":matrix_to_vector_calculator", + "//mediapipe/framework:calculator_runner", + "//mediapipe/framework/formats:matrix", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + "//mediapipe/framework/tool:validate_type", + "//mediapipe/util:time_series_test_util", + "//mediapipe/util:time_series_util", + ], +) + +cc_library( +>>>>>>> Project import generated by Copybara. name = "merge_calculator", srcs = ["merge_calculator.cc"], visibility = ["//visibility:public"], diff --git a/mediapipe/calculators/core/concatenate_vector_calculator.cc b/mediapipe/calculators/core/concatenate_vector_calculator.cc index 087a9b6fe..633236b65 100644 --- a/mediapipe/calculators/core/concatenate_vector_calculator.cc +++ b/mediapipe/calculators/core/concatenate_vector_calculator.cc @@ -16,6 +16,10 @@ #include +<<<<<<< HEAD +======= +#include "mediapipe/framework/formats/landmark.pb.h" +>>>>>>> Project import generated by Copybara. #include "tensorflow/lite/interpreter.h" namespace mediapipe { @@ -41,4 +45,10 @@ typedef ConcatenateVectorCalculator ConcatenateTfLiteTensorVectorCalculator; REGISTER_CALCULATOR(ConcatenateTfLiteTensorVectorCalculator); +<<<<<<< HEAD +======= +typedef ConcatenateVectorCalculator<::mediapipe::NormalizedLandmark> + ConcatenateLandmarkVectorCalculator; +REGISTER_CALCULATOR(ConcatenateLandmarkVectorCalculator); +>>>>>>> Project import generated by Copybara. } // namespace mediapipe diff --git a/mediapipe/calculators/core/matrix_to_vector_calculator.cc b/mediapipe/calculators/core/matrix_to_vector_calculator.cc new file mode 100644 index 000000000..b02fda77c --- /dev/null +++ b/mediapipe/calculators/core/matrix_to_vector_calculator.cc @@ -0,0 +1,83 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Defines MatrixToVectorCalculator. +#include + +#include +#include +#include + +#include "Eigen/Core" +#include "absl/memory/memory.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/port/integral_types.h" +#include "mediapipe/framework/port/logging.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/tool/status_util.h" +#include "mediapipe/util/time_series_util.h" + +namespace mediapipe { + +// A calculator that converts a Matrix M to a vector containing all the +// entries of M in column-major order. +// +// Example config: +// node { +// calculator: "MatrixToVectorCalculator" +// input_stream: "input_matrix" +// output_stream: "column_major_vector" +// } +class MatrixToVectorCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + cc->Inputs().Index(0).Set( + // Input Packet containing a Matrix. + ); + cc->Outputs().Index(0).Set>( + // Output Packet containing a vector, one for each input Packet. + ); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override; + + // Outputs a packet containing a vector for each input packet. + ::mediapipe::Status Process(CalculatorContext* cc) override; +}; +REGISTER_CALCULATOR(MatrixToVectorCalculator); + +::mediapipe::Status MatrixToVectorCalculator::Open(CalculatorContext* cc) { + // Inform the framework that we don't alter timestamps. + cc->SetOffset(mediapipe::TimestampDiff(0)); + return ::mediapipe::OkStatus(); +} + +::mediapipe::Status MatrixToVectorCalculator::Process(CalculatorContext* cc) { + const Matrix& input = cc->Inputs().Index(0).Get(); + auto output = absl::make_unique>(); + + // The following lines work to convert the Matrix to a vector because Matrix + // is an Eigen::MatrixXf and Eigen uses column-major layout by default. + output->resize(input.rows() * input.cols()); + auto output_as_matrix = + Eigen::Map(output->data(), input.rows(), input.cols()); + output_as_matrix = input; + + cc->Outputs().Index(0).Add(output.release(), cc->InputTimestamp()); + return ::mediapipe::OkStatus(); +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/core/matrix_to_vector_calculator_test.cc b/mediapipe/calculators/core/matrix_to_vector_calculator_test.cc new file mode 100644 index 000000000..08a548ad0 --- /dev/null +++ b/mediapipe/calculators/core/matrix_to_vector_calculator_test.cc @@ -0,0 +1,88 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "mediapipe/framework/calculator_runner.h" +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/integral_types.h" +#include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/framework/tool/validate_type.h" +#include "mediapipe/util/time_series_test_util.h" +#include "mediapipe/util/time_series_util.h" + +namespace mediapipe { +namespace { + +class MatrixToVectorCalculatorTest + : public mediapipe::TimeSeriesCalculatorTest { + protected: + void SetUp() override { calculator_name_ = "MatrixToVectorCalculator"; } + + void AppendInput(const std::vector& column_major_data, + int64 timestamp) { + ASSERT_EQ(num_input_samples_ * num_input_channels_, + column_major_data.size()); + Eigen::Map data_map(&column_major_data[0], + num_input_channels_, num_input_samples_); + AppendInputPacket(new Matrix(data_map), timestamp); + } + + void SetInputStreamParameters(int num_channels, int num_samples) { + num_input_channels_ = num_channels; + num_input_samples_ = num_samples; + input_sample_rate_ = 100; + input_packet_rate_ = 20.0; + } + + void SetInputHeader(int num_channels, int num_samples) { + SetInputStreamParameters(num_channels, num_samples); + FillInputHeader(); + } + + void CheckOutputPacket(int packet, std::vector expected_vector) { + const auto& actual_vector = + runner_->Outputs().Index(0).packets[packet].Get>(); + EXPECT_THAT(actual_vector, testing::ContainerEq(expected_vector)); + } +}; + +TEST_F(MatrixToVectorCalculatorTest, SingleRow) { + InitializeGraph(); + SetInputHeader(1, 4); // 1 channel x 4 samples + const std::vector& data_vector = {1.0, 2.0, 3.0, 4.0}; + AppendInput(data_vector, 0); + MEDIAPIPE_ASSERT_OK(RunGraph()); + CheckOutputPacket(0, data_vector); +} + +TEST_F(MatrixToVectorCalculatorTest, RegularMatrix) { + InitializeGraph(); + SetInputHeader(4, 2); // 4 channels x 2 samples + // Actual data matrix is the transpose of the appearance below. + const std::vector& data_vector = {1.0, 2.0, 3.0, 4.0, + 5.0, 6.0, 7.0, 8.0}; + AppendInput(data_vector, 0); + + MEDIAPIPE_ASSERT_OK(RunGraph()); + CheckOutputPacket(0, data_vector); +} + +} // namespace + +} // namespace mediapipe diff --git a/mediapipe/calculators/core/split_vector_calculator.cc b/mediapipe/calculators/core/split_vector_calculator.cc index 2e18a570e..cc09c5183 100644 --- a/mediapipe/calculators/core/split_vector_calculator.cc +++ b/mediapipe/calculators/core/split_vector_calculator.cc @@ -16,6 +16,10 @@ #include +<<<<<<< HEAD +======= +#include "mediapipe/framework/formats/landmark.pb.h" +>>>>>>> Project import generated by Copybara. #include "tensorflow/lite/interpreter.h" namespace mediapipe { @@ -37,4 +41,10 @@ namespace mediapipe { typedef SplitVectorCalculator SplitTfLiteTensorVectorCalculator; REGISTER_CALCULATOR(SplitTfLiteTensorVectorCalculator); +<<<<<<< HEAD +======= +typedef SplitVectorCalculator<::mediapipe::NormalizedLandmark> + SplitLandmarkVectorCalculator; +REGISTER_CALCULATOR(SplitLandmarkVectorCalculator); +>>>>>>> Project import generated by Copybara. } // namespace mediapipe diff --git a/mediapipe/calculators/image/image_transformation_calculator.cc b/mediapipe/calculators/image/image_transformation_calculator.cc index de33d4424..c3afadcbd 100644 --- a/mediapipe/calculators/image/image_transformation_calculator.cc +++ b/mediapipe/calculators/image/image_transformation_calculator.cc @@ -244,7 +244,11 @@ REGISTER_CALCULATOR(ImageTransformationCalculator); rotation_ = DegreesToRotationMode( cc->InputSidePackets().Tag("ROTATION_DEGREES").Get()); } else { +<<<<<<< HEAD rotation_ = DegreesToRotationMode(options_.rotation_mode()); +======= + rotation_ = options_.rotation_mode(); +>>>>>>> Project import generated by Copybara. } scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE); diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD index f88481cf7..a130f877d 100644 --- a/mediapipe/calculators/tensorflow/BUILD +++ b/mediapipe/calculators/tensorflow/BUILD @@ -188,6 +188,20 @@ mediapipe_cc_proto_library( ) mediapipe_cc_proto_library( +<<<<<<< HEAD +======= + name = "tensorflow_session_from_frozen_graph_calculator_cc_proto", + srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"], + cc_deps = [ + "//mediapipe/framework:calculator_cc_proto", + "@org_tensorflow//tensorflow/core:protos_all_cc", + ], + visibility = ["//mediapipe:__subpackages__"], + deps = [":tensorflow_session_from_frozen_graph_calculator_proto"], +) + +mediapipe_cc_proto_library( +>>>>>>> Project import generated by Copybara. name = "tensorflow_session_from_saved_model_generator_cc_proto", srcs = ["tensorflow_session_from_saved_model_generator.proto"], cc_deps = ["//mediapipe/framework:packet_generator_cc_proto"], @@ -445,6 +459,38 @@ cc_library( ) cc_library( +<<<<<<< HEAD +======= + name = "tensorflow_session_from_frozen_graph_calculator", + srcs = ["tensorflow_session_from_frozen_graph_calculator.cc"], + features = ["no_layering_check"], + visibility = ["//visibility:public"], + deps = [ + ":tensorflow_session", + "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/tool:status_util", + "//mediapipe/framework/port:status", + "//mediapipe/framework/port:ret_check", + ] + select({ + "//conditions:default": [ + "//mediapipe/framework/port:file_helpers", + "@org_tensorflow//tensorflow/core:core", + ], + "//mediapipe:android": [ + "@org_tensorflow//tensorflow/core:android_tensorflow_lib_lite_nortti_lite_protos", + "//mediapipe/android/file/base", + ], + "//mediapipe:ios": [ + "@org_tensorflow//tensorflow/core:ios_tensorflow_lib", + "//mediapipe/android/file/base", + ], + }), + alwayslink = 1, +) + +cc_library( +>>>>>>> Project import generated by Copybara. name = "tensorflow_session_from_frozen_graph_generator", srcs = ["tensorflow_session_from_frozen_graph_generator.cc"], features = ["no_layering_check"], @@ -738,6 +784,39 @@ cc_test( ) cc_test( +<<<<<<< HEAD +======= + name = "tensorflow_session_from_frozen_graph_calculator_test", + srcs = ["tensorflow_session_from_frozen_graph_calculator_test.cc"], + data = [":test_frozen_graph"], + linkstatic = 1, + deps = [ + ":tensorflow_inference_calculator", + ":tensorflow_session", + ":tensorflow_session_from_frozen_graph_calculator", + "//mediapipe/calculators/tensorflow:tensorflow_session_from_frozen_graph_calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:calculator_runner", + "//mediapipe/framework:packet", + "//mediapipe/framework/deps:file_path", + "//mediapipe/framework/port:file_helpers", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + "//mediapipe/framework/tool:tag_map_helper", + "//mediapipe/framework/tool:validate_type", + "@com_google_absl//absl/strings", + "@org_tensorflow//tensorflow/core:direct_session", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:testlib", + "@org_tensorflow//tensorflow/core/kernels:conv_ops", + "@org_tensorflow//tensorflow/core/kernels:math", + ], +) + +cc_test( +>>>>>>> Project import generated by Copybara. name = "tensorflow_session_from_frozen_graph_generator_test", srcs = ["tensorflow_session_from_frozen_graph_generator_test.cc"], data = [":test_frozen_graph"], diff --git a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc index 47c4f21b7..9e9c18f9c 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_inference_calculator.cc @@ -34,6 +34,13 @@ #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_util.h" +<<<<<<< HEAD +======= +#if !defined(__ANDROID__) && !defined(__APPLE__) +#include "tensorflow/core/profiler/lib/traceme.h" +#endif + +>>>>>>> Project import generated by Copybara. namespace tf = ::tensorflow; namespace mediapipe { @@ -435,9 +442,21 @@ class TensorFlowInferenceCalculator : public CalculatorBase { session_run_throttle->Acquire(1); } const int64 run_start_time = absl::ToUnixMicros(clock_->TimeNow()); +<<<<<<< HEAD const tf::Status tf_status = session_->Run(input_tensors, output_tensor_names, {} /* target_node_names */, &outputs); +======= + tf::Status tf_status; + { +#if !defined(__ANDROID__) && !defined(__APPLE__) + tensorflow::profiler::TraceMe trace(absl::string_view(cc->NodeName())); +#endif + tf_status = session_->Run(input_tensors, output_tensor_names, + {} /* target_node_names */, &outputs); + } + +>>>>>>> Project import generated by Copybara. if (session_run_throttle != nullptr) { session_run_throttle->Release(1); } diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc new file mode 100644 index 000000000..b6d678b6b --- /dev/null +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.cc @@ -0,0 +1,136 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Reads serialized GraphDef proto. There are three ways to load a model: +// 1. Specify the path to a graph.pb in the calculator options. +// 2. Specify the path to the graph.pb through the +// input_side_packet:STRING_MODEL_FILE_PATH +// 3. Provide a serialized GraphDef through input_side_packet:STRING_MODEL, +// typically provided by EmbeddingFilePacketFactory. +// +// Produces a SessionBundle that TensorFlowInferenceCalculator can use. + +#include + +#include "mediapipe/calculators/tensorflow/tensorflow_session.h" +#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/framework/port/status.h" +#include "mediapipe/framework/tool/status_util.h" +#include "tensorflow/core/public/session_options.h" + +#if defined(MEDIAPIPE_LITE) || defined(__ANDROID__) || \ + defined(__APPLE__) && !TARGET_OS_OSX +#include "mediapipe/util/android/file/base/helpers.h" +#else +#include "mediapipe/framework/port/file_helpers.h" +#endif + +namespace mediapipe { + +namespace tf = ::tensorflow; + +class TensorFlowSessionFromFrozenGraphCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + const auto& options = + cc->Options(); + bool has_exactly_one_model = + !options.graph_proto_path().empty() + ? !(cc->InputSidePackets().HasTag("STRING_MODEL") | + cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) + : (cc->InputSidePackets().HasTag("STRING_MODEL") ^ + cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")); + RET_CHECK(has_exactly_one_model) + << "Must have exactly one of graph_proto_path in options or " + "input_side_packets STRING_MODEL or STRING_MODEL_FILE_PATH"; + if (cc->InputSidePackets().HasTag("STRING_MODEL")) { + cc->InputSidePackets() + .Tag("STRING_MODEL") + .Set( + // String model from embedded path + ); + } else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) { + cc->InputSidePackets() + .Tag("STRING_MODEL_FILE_PATH") + .Set( + // Filename of std::string model. + ); + } + cc->OutputSidePackets().Tag("SESSION").Set( + // A TensorFlow model loaded and ready for use along with + // a map from tags to tensor names. + ); + RET_CHECK_GT(options.tag_to_tensor_names().size(), 0); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override { + const auto& options = + cc->Options(); + // Output bundle packet. + auto session = ::absl::make_unique(); + + tf::SessionOptions session_options; + session_options.config.CopyFrom(options.config()); + std::vector initialization_op_names; + initialization_op_names.reserve(options.initialization_op_names_size()); + for (int i = 0; i < options.initialization_op_names_size(); ++i) { + initialization_op_names.emplace_back(options.initialization_op_names(i)); + } + session->session.reset(tf::NewSession(session_options)); + + std::string graph_def_serialized; + if (cc->InputSidePackets().HasTag("STRING_MODEL")) { + graph_def_serialized = + cc->InputSidePackets().Tag("STRING_MODEL").Get(); + } else if (cc->InputSidePackets().HasTag("STRING_MODEL_FILE_PATH")) { + const std::string& frozen_graph = cc->InputSidePackets() + .Tag("STRING_MODEL_FILE_PATH") + .Get(); + RET_CHECK_OK( + mediapipe::file::GetContents(frozen_graph, &graph_def_serialized)); + } else { + RET_CHECK_OK(mediapipe::file::GetContents(options.graph_proto_path(), + &graph_def_serialized)); + } + tensorflow::GraphDef graph_def; + + RET_CHECK(graph_def.ParseFromString(graph_def_serialized)); + const tf::Status tf_status = session->session->Create(graph_def); + RET_CHECK(tf_status.ok()) << "Create failed: " << tf_status.error_message(); + + for (const auto& key_value : options.tag_to_tensor_names()) { + session->tag_to_tensor_map[key_value.first] = key_value.second; + } + if (!initialization_op_names.empty()) { + const tf::Status tf_status = + session->session->Run({}, {}, initialization_op_names, {}); + // RET_CHECK on the tf::Status object itself in order to print an + // informative error message. + RET_CHECK(tf_status.ok()) << "Run failed: " << tf_status.error_message(); + } + + cc->OutputSidePackets().Tag("SESSION").Set(Adopt(session.release())); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + return ::mediapipe::OkStatus(); + } +}; +REGISTER_CALCULATOR(TensorFlowSessionFromFrozenGraphCalculator); + +} // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.proto b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.proto new file mode 100644 index 000000000..3921d2016 --- /dev/null +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.proto @@ -0,0 +1,72 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; +import "tensorflow/core/protobuf/config.proto"; + +message TensorFlowSessionFromFrozenGraphCalculatorOptions { + extend mediapipe.CalculatorOptions { + optional TensorFlowSessionFromFrozenGraphCalculatorOptions ext = 266997877; + } + + // Path to file containing serialized proto of type tensorflow::GraphDef. + optional string graph_proto_path = 1; + + // To run inference with MediaPipe inputs MediaPipe streams need to be mapped + // to TensorFlow tensors. This map defines the which streams are fed into + // which tensors in the model. The MediaPipe tag of the stream is the map key. + // Tags must be capitalized, matching regex [A-Z0-9_]+. Examples: "JPG_STRING" + // and "SOFTMAX". Then, those tags can be used as the MediaPipe tags of + // input_stream or output_stream of the TensorflowInferenceCalculator + // consuming the packet produced by this calculator. The tensor names must + // match the tensor names in the graph that you want to feed or fetch into or + // out of. Examples: "DecodeJpeg/contents:0" or "softmax:0". For example, a + // mediapipe graph can include the nodes: + // + // node { + // calculator: "TensorFlowSessionFromFrozenGraphCalculator" + // output_side_packet: "SESSION:session" + // options { + // [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { + // graph_proto_path: "[PATH]" + // tag_to_tensor_names { + // key: "JPG_STRING" + // value: "input:0" + // } + // tag_to_tensor_names { + // key: "SOFTMAX" + // value: "softmax:0" + // } + // } + // } + // } + // node { + // calculator: "TensorflowInferenceCalculator" + // input_side_packet: "SESSION:graph_with_bindings" + // input_stream: "JPG_STRING:jpg_string_tensor" + // output_stream: "SOFTMAX:softmax_tensor" + // } + map tag_to_tensor_names = 2; + + // Tensorflow session config options. + optional tensorflow.ConfigProto config = 3; + + // Graph nodes to run to initialize the model. Any output of these ops is + // ignored. + repeated string initialization_op_names = 4; +} diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc new file mode 100644 index 000000000..e9c5c604b --- /dev/null +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator_test.cc @@ -0,0 +1,316 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/substitute.h" +#include "mediapipe/calculators/tensorflow/tensorflow_session.h" +#include "mediapipe/calculators/tensorflow/tensorflow_session_from_frozen_graph_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_runner.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/packet.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status.h" +#include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/framework/tool/tag_map_helper.h" +#include "mediapipe/framework/tool/validate_type.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/protobuf/config.pb.h" + +namespace mediapipe { + +namespace { + +namespace tf = ::tensorflow; + +std::string GetGraphDefPath() { + return mediapipe::file::JoinPath("./", + "mediapipe/calculators/tensorflow/" + "testdata/frozen_graph_def.pb"); +} + +// Helper function that creates Tensor INT32 matrix with size 1x3. +tf::Tensor TensorMatrix1x3(const int v1, const int v2, const int v3) { + tf::Tensor tensor(tf::DT_INT32, + tf::TensorShape(std::vector({1, 3}))); + auto matrix = tensor.matrix(); + matrix(0, 0) = v1; + matrix(0, 1) = v2; + matrix(0, 2) = v3; + return tensor; +} + +class TensorFlowSessionFromFrozenGraphCalculatorTest : public ::testing::Test { + protected: + void SetUp() override { + extendable_options_.Clear(); + calculator_options_ = extendable_options_.MutableExtension( + TensorFlowSessionFromFrozenGraphCalculatorOptions::ext); + calculator_options_->set_graph_proto_path(GetGraphDefPath()); + (*calculator_options_->mutable_tag_to_tensor_names())["MULTIPLIED"] = + "multiplied:0"; + (*calculator_options_->mutable_tag_to_tensor_names())["A"] = "a:0"; + (*calculator_options_->mutable_tag_to_tensor_names())["B"] = "b:0"; + calculator_options_->mutable_config()->set_intra_op_parallelism_threads(1); + calculator_options_->mutable_config()->set_inter_op_parallelism_threads(2); + } + + void VerifySignatureMap(const TensorFlowSession& session) { + // Session must be set. + ASSERT_NE(session.session, nullptr); + + // Bindings are inserted. + EXPECT_EQ(session.tag_to_tensor_map.size(), 3); + + // For some reason, EXPECT_EQ and EXPECT_NE are not working with iterators. + EXPECT_FALSE(session.tag_to_tensor_map.find("A") == + session.tag_to_tensor_map.end()); + EXPECT_FALSE(session.tag_to_tensor_map.find("B") == + session.tag_to_tensor_map.end()); + EXPECT_FALSE(session.tag_to_tensor_map.find("MULTIPLIED") == + session.tag_to_tensor_map.end()); + // Sanity: find() actually returns a reference to end() if element not + // found. + EXPECT_TRUE(session.tag_to_tensor_map.find("Z") == + session.tag_to_tensor_map.end()); + + EXPECT_EQ(session.tag_to_tensor_map.at("A"), "a:0"); + EXPECT_EQ(session.tag_to_tensor_map.at("B"), "b:0"); + EXPECT_EQ(session.tag_to_tensor_map.at("MULTIPLIED"), "multiplied:0"); + } + + CalculatorOptions extendable_options_; + TensorFlowSessionFromFrozenGraphCalculatorOptions* calculator_options_; +}; + +TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, + CreatesPacketWithGraphAndBindings) { + CalculatorRunner runner(absl::Substitute(R"( + calculator: "TensorFlowSessionFromFrozenGraphCalculator" + output_side_packet: "SESSION:tf_model" + options { + [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { + $0 + } + })", + calculator_options_->DebugString())); + + MEDIAPIPE_ASSERT_OK(runner.Run()); + const TensorFlowSession& session = + runner.OutputSidePackets().Tag("SESSION").Get(); + VerifySignatureMap(session); +} + +// Integration test. Verifies that TensorFlowInferenceCalculator correctly +// consumes the Packet emitted by this calculator. +TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, + ProducesPacketUsableByTensorFlowInferenceCalculator) { + CalculatorGraphConfig config = + ::mediapipe::ParseTextProtoOrDie( + absl::Substitute(R"( + node { + calculator: "TensorFlowInferenceCalculator" + input_side_packet: "SESSION:session" + input_stream: "A:a_tensor" + output_stream: "MULTIPLIED:multiplied_tensor" + options { + [mediapipe.TensorFlowInferenceCalculatorOptions.ext] { + batch_size: 5 + add_batch_dim_to_tensors: false + } + } + } + + node { + calculator: "TensorFlowSessionFromFrozenGraphCalculator" + output_side_packet: "SESSION:session" + options { + [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { + $0 + } + } + } + input_stream: "a_tensor" + )", + calculator_options_->DebugString())); + + CalculatorGraph graph; + MEDIAPIPE_ASSERT_OK(graph.Initialize(config)); + StatusOrPoller status_or_poller = + graph.AddOutputStreamPoller("multiplied_tensor"); + ASSERT_TRUE(status_or_poller.ok()); + OutputStreamPoller poller = std::move(status_or_poller.ValueOrDie()); + + MEDIAPIPE_ASSERT_OK(graph.StartRun({})); + MEDIAPIPE_ASSERT_OK(graph.AddPacketToInputStream( + "a_tensor", + Adopt(new auto(TensorMatrix1x3(1, -1, 10))).At(Timestamp(0)))); + MEDIAPIPE_ASSERT_OK(graph.CloseInputStream("a_tensor")); + + Packet packet; + ASSERT_TRUE(poller.Next(&packet)); + // input tensor gets multiplied by [[3, 2, 1]]. Expected output: + tf::Tensor expected_multiplication = TensorMatrix1x3(3, -2, 10); + EXPECT_EQ(expected_multiplication.DebugString(), + packet.Get().DebugString()); + + ASSERT_FALSE(poller.Next(&packet)); + MEDIAPIPE_ASSERT_OK(graph.WaitUntilDone()); +} + +TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, + CreatesPacketWithGraphAndBindingsFromInputSidePacket) { + calculator_options_->clear_graph_proto_path(); + CalculatorRunner runner(absl::Substitute(R"( + calculator: "TensorFlowSessionFromFrozenGraphCalculator" + input_side_packet: "STRING_MODEL:model" + output_side_packet: "SESSION:session" + options { + [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { + $0 + } + })", + calculator_options_->DebugString())); + + std::string serialized_graph_contents; + MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(), + &serialized_graph_contents)); + runner.MutableSidePackets()->Tag("STRING_MODEL") = + Adopt(new std::string(serialized_graph_contents)); + MEDIAPIPE_ASSERT_OK(runner.Run()); + + const TensorFlowSession& session = + runner.OutputSidePackets().Tag("SESSION").Get(); + VerifySignatureMap(session); +} + +TEST_F( + TensorFlowSessionFromFrozenGraphCalculatorTest, + CreatesPacketWithGraphAndBindingsFromInputSidePacketStringModelFilePath) { + calculator_options_->clear_graph_proto_path(); + CalculatorRunner runner(absl::Substitute(R"( + calculator: "TensorFlowSessionFromFrozenGraphCalculator" + input_side_packet: "STRING_MODEL_FILE_PATH:file_path" + output_side_packet: "SESSION:session" + options { + [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { + $0 + } + })", + calculator_options_->DebugString())); + runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") = + Adopt(new std::string(GetGraphDefPath())); + MEDIAPIPE_ASSERT_OK(runner.Run()); + + const TensorFlowSession& session = + runner.OutputSidePackets().Tag("SESSION").Get(); + VerifySignatureMap(session); +} + +TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, + CheckFailureForOptionsAndInputsProvideGraphDefProto) { + CalculatorRunner runner(absl::Substitute(R"( + calculator: "TensorFlowSessionFromFrozenGraphCalculator" + input_side_packet: "STRING_MODEL_FILE_PATH:file_path" + output_side_packet: "SESSION:session" + options { + [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { + $0 + } + })", + calculator_options_->DebugString())); + runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") = + Adopt(new std::string(GetGraphDefPath())); + auto run_status = runner.Run(); + EXPECT_THAT( + run_status.message(), + ::testing::HasSubstr("Must have exactly one of graph_proto_path")); +} + +TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, + CheckFailureForAllInputsProvideGraphDefProto) { + CalculatorRunner runner(absl::Substitute(R"( + calculator: "TensorFlowSessionFromFrozenGraphCalculator" + input_side_packet: "STRING_MODEL_FILE_PATH:file_path" + input_side_packet: "STRING_MODEL:model" + output_side_packet: "SESSION:session" + options { + [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { + $0 + } + })", + calculator_options_->DebugString())); + runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") = + Adopt(new std::string(GetGraphDefPath())); + std::string serialized_graph_contents; + MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(), + &serialized_graph_contents)); + runner.MutableSidePackets()->Tag("STRING_MODEL") = + Adopt(new std::string(serialized_graph_contents)); + auto run_status = runner.Run(); + EXPECT_THAT( + run_status.message(), + ::testing::HasSubstr("Must have exactly one of graph_proto_path")); +} + +TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, + CheckFailureForOnlyBothInputSidePacketsProvideGraphDefProto) { + calculator_options_->clear_graph_proto_path(); + CalculatorRunner runner(absl::Substitute(R"( + calculator: "TensorFlowSessionFromFrozenGraphCalculator" + input_side_packet: "STRING_MODEL_FILE_PATH:file_path" + input_side_packet: "STRING_MODEL:model" + output_side_packet: "SESSION:session" + options { + [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { + $0 + } + })", + calculator_options_->DebugString())); + runner.MutableSidePackets()->Tag("STRING_MODEL_FILE_PATH") = + Adopt(new std::string(GetGraphDefPath())); + std::string serialized_graph_contents; + MEDIAPIPE_EXPECT_OK(mediapipe::file::GetContents(GetGraphDefPath(), + &serialized_graph_contents)); + runner.MutableSidePackets()->Tag("STRING_MODEL") = + Adopt(new std::string(serialized_graph_contents)); + auto run_status = runner.Run(); + EXPECT_THAT( + run_status.message(), + ::testing::HasSubstr("Must have exactly one of graph_proto_path")); +} + +TEST_F(TensorFlowSessionFromFrozenGraphCalculatorTest, + CheckInitializationOpName) { + calculator_options_->add_initialization_op_names("multiplied:0"); + CalculatorRunner runner(absl::Substitute(R"( + calculator: "TensorFlowSessionFromFrozenGraphCalculator" + output_side_packet: "SESSION:session" + options { + [mediapipe.TensorFlowSessionFromFrozenGraphCalculatorOptions.ext]: { + $0 + } + })", + calculator_options_->DebugString())); + MEDIAPIPE_ASSERT_OK(runner.Run()); + + const TensorFlowSession& session = + runner.OutputSidePackets().Tag("SESSION").Get(); + VerifySignatureMap(session); +} + +} // namespace +} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/BUILD b/mediapipe/calculators/tflite/BUILD index 36d246eef..06c434be9 100644 --- a/mediapipe/calculators/tflite/BUILD +++ b/mediapipe/calculators/tflite/BUILD @@ -62,6 +62,16 @@ proto_library( ) proto_library( +<<<<<<< HEAD +======= + name = "tflite_tensors_to_classification_calculator_proto", + srcs = ["tflite_tensors_to_classification_calculator.proto"], + visibility = ["//visibility:public"], + deps = ["//mediapipe/framework:calculator_proto"], +) + +proto_library( +>>>>>>> Project import generated by Copybara. name = "tflite_tensors_to_landmarks_calculator_proto", srcs = ["tflite_tensors_to_landmarks_calculator.proto"], visibility = ["//visibility:public"], @@ -117,6 +127,17 @@ mediapipe_cc_proto_library( ) mediapipe_cc_proto_library( +<<<<<<< HEAD +======= + name = "tflite_tensors_to_classification_calculator_cc_proto", + srcs = ["tflite_tensors_to_classification_calculator.proto"], + cc_deps = ["//mediapipe/framework:calculator_cc_proto"], + visibility = ["//mediapipe:__subpackages__"], + deps = [":tflite_tensors_to_classification_calculator_proto"], +) + +mediapipe_cc_proto_library( +>>>>>>> Project import generated by Copybara. name = "tflite_tensors_to_landmarks_calculator_cc_proto", srcs = ["tflite_tensors_to_landmarks_calculator.proto"], cc_deps = ["//mediapipe/framework:calculator_cc_proto"], @@ -311,6 +332,28 @@ cc_library( alwayslink = 1, ) +<<<<<<< HEAD +======= +cc_test( + name = "tflite_tensors_to_classification_calculator_test", + srcs = ["tflite_tensors_to_classification_calculator_test.cc"], + data = ["testdata/labelmap.txt"], + deps = [ + ":tflite_tensors_to_classification_calculator", + ":tflite_tensors_to_classification_calculator_cc_proto", + "//mediapipe/framework:calculator_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework:calculator_runner", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:parse_text_proto", + "@com_google_absl//absl/memory", + "@com_google_googletest//:gtest_main", + "@org_tensorflow//tensorflow/lite:framework", + ], +) + +>>>>>>> Project import generated by Copybara. cc_library( name = "tflite_tensors_to_detections_calculator", srcs = ["tflite_tensors_to_detections_calculator.cc"], @@ -340,6 +383,40 @@ cc_library( ) cc_library( +<<<<<<< HEAD +======= + name = "tflite_tensors_to_classification_calculator", + srcs = ["tflite_tensors_to_classification_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + ":tflite_tensors_to_classification_calculator_cc_proto", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/types:span", + "//mediapipe/framework/formats:classification_cc_proto", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:location", + "//mediapipe/framework/port:ret_check", + "//mediapipe/util:resource_util", + "@org_tensorflow//tensorflow/lite:framework", + ] + select({ + "//mediapipe:android": [ + "//mediapipe/util/android/file/base", + ], + "//mediapipe:apple": [ + "//mediapipe/util/android/file/base", + ], + "//mediapipe:macos": [ + "//mediapipe/framework/port:file_helpers", + ], + "//conditions:default": [ + "//mediapipe/framework/port:file_helpers", + ], + }), + alwayslink = 1, +) + +cc_library( +>>>>>>> Project import generated by Copybara. name = "tflite_tensors_to_landmarks_calculator", srcs = ["tflite_tensors_to_landmarks_calculator.cc"], visibility = ["//visibility:public"], diff --git a/mediapipe/calculators/tflite/testdata/labelmap.txt b/mediapipe/calculators/tflite/testdata/labelmap.txt new file mode 100644 index 000000000..4291e3c6b --- /dev/null +++ b/mediapipe/calculators/tflite/testdata/labelmap.txt @@ -0,0 +1,3 @@ +classA +classB +classC diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator.cc b/mediapipe/calculators/tflite/tflite_inference_calculator.cc index 33a24993b..b7ef86a7a 100644 --- a/mediapipe/calculators/tflite/tflite_inference_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_inference_calculator.cc @@ -434,8 +434,14 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); use_quantized_tensors_ = false; } else { RET_CHECK_EQ(interpreter_->AllocateTensors(), kTfLiteOk); +<<<<<<< HEAD use_quantized_tensors_ = (interpreter_->tensor(0)->quantization.type == kTfLiteAffineQuantization); +======= + use_quantized_tensors_ = + (interpreter_->tensor(interpreter_->inputs()[0])->quantization.type == + kTfLiteAffineQuantization); +>>>>>>> Project import generated by Copybara. if (use_quantized_tensors_) gpu_inference_ = false; } diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc new file mode 100644 index 000000000..d786dafcc --- /dev/null +++ b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.cc @@ -0,0 +1,176 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "absl/strings/str_format.h" +#include "absl/types/span.h" +#include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/port/ret_check.h" +#include "mediapipe/util/resource_util.h" +#include "tensorflow/lite/interpreter.h" +#if defined(__ANDROID__) || (defined(__APPLE__) && !TARGET_OS_OSX) +#include "mediapipe/util/android/file/base/file.h" +#include "mediapipe/util/android/file/base/helpers.h" +#else +#include "mediapipe/framework/port/file_helpers.h" +#endif + +namespace mediapipe { + +// Convert result TFLite tensors from classification models into MediaPipe +// classifications. +// +// Input: +// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32 containing one +// tensor, the size of which must be (1, * num_classes). +// Output: +// CLASSIFICATIONS - Result MediaPipe ClassificationList. The score and index +// fields of each classification are set, while the label +// field is only set if label_map_path is provided. +// +// Usage example: +// node { +// calculator: "TfLiteTensorsToClassificationCalculator" +// input_stream: "TENSORS:tensors" +// output_stream: "CLASSIFICATIONS:classifications" +// options: { +// [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] { +// num_classes: 1024 +// min_score_threshold: 0.1 +// label_map_path: "labelmap.txt" +// } +// } +// } +class TfLiteTensorsToClassificationCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc); + + ::mediapipe::Status Open(CalculatorContext* cc) override; + ::mediapipe::Status Process(CalculatorContext* cc) override; + ::mediapipe::Status Close(CalculatorContext* cc) override; + + private: + int top_k_ = 0; + double min_score_threshold_ = 0; + std::unordered_map label_map_; + bool label_map_loaded_ = false; +}; +REGISTER_CALCULATOR(TfLiteTensorsToClassificationCalculator); + +::mediapipe::Status TfLiteTensorsToClassificationCalculator::GetContract( + CalculatorContract* cc) { + RET_CHECK(!cc->Inputs().GetTags().empty()); + RET_CHECK(!cc->Outputs().GetTags().empty()); + + if (cc->Inputs().HasTag("TENSORS")) { + cc->Inputs().Tag("TENSORS").Set>(); + } + + if (cc->Outputs().HasTag("CLASSIFICATIONS")) { + cc->Outputs().Tag("CLASSIFICATIONS").Set(); + } + + return ::mediapipe::OkStatus(); +} + +::mediapipe::Status TfLiteTensorsToClassificationCalculator::Open( + CalculatorContext* cc) { + cc->SetOffset(TimestampDiff(0)); + + auto options = cc->Options< + ::mediapipe::TfLiteTensorsToClassificationCalculatorOptions>(); + + top_k_ = options.top_k(); + min_score_threshold_ = options.min_score_threshold(); + if (options.has_label_map_path()) { + std::string string_path; + ASSIGN_OR_RETURN(string_path, + PathToResourceAsFile(options.label_map_path())); + std::string label_map_string; + RETURN_IF_ERROR(file::GetContents(string_path, &label_map_string)); + + std::istringstream stream(label_map_string); + std::string line; + int i = 0; + while (std::getline(stream, line)) { + label_map_[i++] = line; + } + label_map_loaded_ = true; + } + + return ::mediapipe::OkStatus(); +} + +::mediapipe::Status TfLiteTensorsToClassificationCalculator::Process( + CalculatorContext* cc) { + const auto& input_tensors = + cc->Inputs().Tag("TENSORS").Get>(); + + RET_CHECK_EQ(input_tensors.size(), 1); + + const TfLiteTensor* raw_score_tensor = &input_tensors[0]; + RET_CHECK_EQ(raw_score_tensor->dims->size, 2); + RET_CHECK_EQ(raw_score_tensor->dims->data[0], 1); + int num_classes = raw_score_tensor->dims->data[1]; + if (label_map_loaded_) { + RET_CHECK_EQ(num_classes, label_map_.size()); + } + const float* raw_scores = raw_score_tensor->data.f; + + auto classification_list = absl::make_unique(); + for (int i = 0; i < num_classes; ++i) { + if (raw_scores[i] < min_score_threshold_) { + continue; + } + Classification* classification = classification_list->add_classification(); + classification->set_index(i); + classification->set_score(raw_scores[i]); + if (label_map_loaded_) { + classification->set_label(label_map_[i]); + } + } + + // Note that partial_sort will raise error when top_k_ > + // classification_list->classification_size(). + auto raw_classification_list = classification_list->mutable_classification(); + if (top_k_ > 0 && classification_list->classification_size() >= top_k_) { + std::partial_sort(raw_classification_list->begin(), + raw_classification_list->begin() + top_k_, + raw_classification_list->end(), + [](const Classification a, const Classification b) { + return a.score() > b.score(); + }); + + // Resizes the underlying list to have only top_k_ classifications. + raw_classification_list->DeleteSubrange( + top_k_, raw_classification_list->size() - top_k_); + } + cc->Outputs() + .Tag("CLASSIFICATIONS") + .Add(classification_list.release(), cc->InputTimestamp()); + + return ::mediapipe::OkStatus(); +} + +::mediapipe::Status TfLiteTensorsToClassificationCalculator::Close( + CalculatorContext* cc) { + return ::mediapipe::OkStatus(); +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto new file mode 100644 index 000000000..a2b5dd224 --- /dev/null +++ b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.proto @@ -0,0 +1,35 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The option proto for the TfLiteTensorsToClassificationCalculator. + +syntax = "proto2"; + +package mediapipe; + +import "mediapipe/framework/calculator.proto"; + +message TfLiteTensorsToClassificationCalculatorOptions { + extend .mediapipe.CalculatorOptions { + optional TfLiteTensorsToClassificationCalculatorOptions ext = 266399463; + } + + // Score threshold for perserving the class. + optional float min_score_threshold = 1; + // Number of highest scoring labels to output. If top_k is not positive then + // all labels are used. + optional int32 top_k = 2; + // Path to a label map file for getting the actual name of class ids. + optional string label_map_path = 3; +} diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator_test.cc b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator_test.cc new file mode 100644 index 000000000..657fa8910 --- /dev/null +++ b/mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator_test.cc @@ -0,0 +1,194 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "absl/memory/memory.h" +#include "mediapipe/calculators/tflite/tflite_tensors_to_classification_calculator.pb.h" +#include "mediapipe/framework/calculator.pb.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_runner.h" +#include "mediapipe/framework/formats/classification.pb.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status_matchers.h" +#include "tensorflow/lite/interpreter.h" + +namespace mediapipe { + +using ::mediapipe::ParseTextProtoOrDie; +using ::tflite::Interpreter; +using Node = ::mediapipe::CalculatorGraphConfig::Node; + +class TfLiteTensorsToClassificationCalculatorTest : public ::testing::Test { + protected: + void BuildGraph(mediapipe::CalculatorRunner* runner, + const std::vector& scores) { + interpreter_ = absl::make_unique(); + + std::vector dims(2); + dims[0] = 1; + dims[1] = scores.size(); + + interpreter_->AddTensors(1); + interpreter_->SetInputs({0}); + interpreter_->SetTensorParametersReadWrite(0, kTfLiteFloat32, "", dims, + TfLiteQuantization()); + + int t = interpreter_->inputs()[0]; + TfLiteTensor* tensor = interpreter_->tensor(t); + interpreter_->ResizeInputTensor(t, dims); + interpreter_->AllocateTensors(); + + float* tensor_buffer = tensor->data.f; + ASSERT_NE(tensor_buffer, nullptr); + for (int i = 0; i < scores.size(); ++i) { + tensor_buffer[i] = scores[i]; + } + + auto tensors = absl::make_unique>(); + tensors->emplace_back(*tensor); + + int64 stream_timestamp = 0; + auto& input_stream_packets = + runner->MutableInputs()->Tag("TENSORS").packets; + + input_stream_packets.push_back( + mediapipe::Adopt(tensors.release()) + .At(mediapipe::Timestamp(stream_timestamp++))); + } + + std::unique_ptr interpreter_; +}; + +TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutput) { + mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"( + calculator: "TfLiteTensorsToClassificationCalculator" + input_stream: "TENSORS:tensors" + output_stream: "CLASSIFICATIONS:classifications" + options { + [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] {} + } + )")); + + BuildGraph(&runner, {0, 0.5, 1}); + MEDIAPIPE_ASSERT_OK(runner.Run()); + + const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; + + EXPECT_EQ(1, output_packets_.size()); + + const auto& classification_list = + output_packets_[0].Get(); + EXPECT_EQ(3, classification_list.classification_size()); + + // Verify that the label_id and score fields are set correctly. + for (int i = 0; i < classification_list.classification_size(); ++i) { + EXPECT_EQ(i, classification_list.classification(i).index()); + EXPECT_EQ(i * 0.5, classification_list.classification(i).score()); + ASSERT_FALSE(classification_list.classification(i).has_label()); + } +} + +TEST_F(TfLiteTensorsToClassificationCalculatorTest, + CorrectOutputWithLabelMapPath) { + mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"( + calculator: "TfLiteTensorsToClassificationCalculator" + input_stream: "TENSORS:tensors" + output_stream: "CLASSIFICATIONS:classifications" + options { + [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] { + label_map_path: "mediapipe/calculators/tflite/testdata/labelmap.txt" + } + } + )")); + + BuildGraph(&runner, {0, 0.5, 1}); + MEDIAPIPE_ASSERT_OK(runner.Run()); + + const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; + + EXPECT_EQ(1, output_packets_.size()); + + const auto& classification_list = + output_packets_[0].Get(); + EXPECT_EQ(3, classification_list.classification_size()); + + // Verify that the label field is set. + for (int i = 0; i < classification_list.classification_size(); ++i) { + EXPECT_EQ(i, classification_list.classification(i).index()); + EXPECT_EQ(i * 0.5, classification_list.classification(i).score()); + ASSERT_TRUE(classification_list.classification(i).has_label()); + } +} + +TEST_F(TfLiteTensorsToClassificationCalculatorTest, + CorrectOutputWithLabelMinScoreThreshold) { + mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"( + calculator: "TfLiteTensorsToClassificationCalculator" + input_stream: "TENSORS:tensors" + output_stream: "CLASSIFICATIONS:classifications" + options { + [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] { + min_score_threshold: 0.6 + } + } + )")); + + BuildGraph(&runner, {0, 0.5, 1}); + MEDIAPIPE_ASSERT_OK(runner.Run()); + + const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; + + EXPECT_EQ(1, output_packets_.size()); + + const auto& classification_list = + output_packets_[0].Get(); + + // Verify that the low score labels are filtered out. + EXPECT_EQ(1, classification_list.classification_size()); + EXPECT_EQ(1, classification_list.classification(0).score()); +} + +TEST_F(TfLiteTensorsToClassificationCalculatorTest, CorrectOutputWithTopK) { + mediapipe::CalculatorRunner runner(ParseTextProtoOrDie(R"( + calculator: "TfLiteTensorsToClassificationCalculator" + input_stream: "TENSORS:tensors" + output_stream: "CLASSIFICATIONS:classifications" + options { + [mediapipe.TfLiteTensorsToClassificationCalculatorOptions.ext] { + top_k: 2 + } + } + )")); + + BuildGraph(&runner, {0, 0.5, 1}); + MEDIAPIPE_ASSERT_OK(runner.Run()); + + const auto& output_packets_ = runner.Outputs().Tag("CLASSIFICATIONS").packets; + + EXPECT_EQ(1, output_packets_.size()); + + const auto& classification_list = + output_packets_[0].Get(); + + // Verify that the only top2 labels are left. + EXPECT_EQ(2, classification_list.classification_size()); + for (int i = 0; i < classification_list.classification_size(); ++i) { + EXPECT_EQ((classification_list.classification_size() - i) * 0.5, + classification_list.classification(i).score()); + } +} + +} // namespace mediapipe diff --git a/mediapipe/calculators/video/opencv_video_decoder_calculator.cc b/mediapipe/calculators/video/opencv_video_decoder_calculator.cc index 2b1f205c5..b4795b30c 100644 --- a/mediapipe/calculators/video/opencv_video_decoder_calculator.cc +++ b/mediapipe/calculators/video/opencv_video_decoder_calculator.cc @@ -154,8 +154,19 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase { cv::COLOR_BGRA2RGBA); } } +<<<<<<< HEAD cc->Outputs().Tag("VIDEO").Add(image_frame.release(), timestamp); decoded_frames_++; +======= + // If the timestamp of the current frame is not greater than the one of the + // previous frame, the new frame will be discarded. + if (prev_timestamp_ < timestamp) { + cc->Outputs().Tag("VIDEO").Add(image_frame.release(), timestamp); + prev_timestamp_ = timestamp; + decoded_frames_++; + } + +>>>>>>> Project import generated by Copybara. return ::mediapipe::OkStatus(); } @@ -178,6 +189,10 @@ class OpenCvVideoDecoderCalculator : public CalculatorBase { int frame_count_; int decoded_frames_ = 0; ImageFormat::Format format_; +<<<<<<< HEAD +======= + Timestamp prev_timestamp_ = Timestamp::Unset(); +>>>>>>> Project import generated by Copybara. }; REGISTER_CALCULATOR(OpenCvVideoDecoderCalculator); diff --git a/mediapipe/docs/cycles.md b/mediapipe/docs/cycles.md index ab5cfc184..89aa5428c 100644 --- a/mediapipe/docs/cycles.md +++ b/mediapipe/docs/cycles.md @@ -61,7 +61,11 @@ ordering and ignores packet timestamps, which will eliminate this inconvenience. By default, MediaPipe calls the `Close()` method of a non-source calculator when all of its input streams are done. In the example graph, we want to stop the adder node as soon as the integer source is done. This is accomplished by +<<<<<<< HEAD configuring the adder node with an alternative input stream hander, +======= +configuring the adder node with an alternative input stream handler, +>>>>>>> Project import generated by Copybara. `EarlyCloseInputStreamHandler`. ## Relevant Source Code diff --git a/mediapipe/docs/examples.md b/mediapipe/docs/examples.md index 8c5b7b491..aa503bbfc 100644 --- a/mediapipe/docs/examples.md +++ b/mediapipe/docs/examples.md @@ -85,6 +85,14 @@ and model details are described in the [Hello World for C++](./hello_world_desktop.md) shows how to run a simple graph using the MediaPipe C++ APIs. +<<<<<<< HEAD +======= +### Feature Extration for YouTube-8M Challenge + +[Feature Extration for YouTube-8M Challenge](./youtube_8m.md) shows how to use +MediaPipe to prepare training data for the YouTube-8M Challenge. + +>>>>>>> Project import generated by Copybara. ### Preparing Data Sets with MediaSequence [Preparing Data Sets with MediaSequence](./media_sequence.md) shows how to use diff --git a/mediapipe/docs/framework_concepts.md b/mediapipe/docs/framework_concepts.md index 69be7fed5..9c837514a 100644 --- a/mediapipe/docs/framework_concepts.md +++ b/mediapipe/docs/framework_concepts.md @@ -17,7 +17,11 @@ packets and produces zero or more output streams and/or side packets. ### CalculatorBase A calculator is created by defining a new sub-class of the +<<<<<<< HEAD [`CalculatorBase`](http://github.com/google/mediapipe/mediapipe/framework/calculator_base.cc) +======= +[`CalculatorBase`](https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator_base.cc) +>>>>>>> Project import generated by Copybara. class, implementing a number of methods, and registering the new sub-class with Mediapipe. At a minimum, a new calculator must implement the below four methods @@ -31,7 +35,11 @@ Mediapipe. At a minimum, a new calculator must implement the below four methods * After all calls to `Process()` finish or when all input streams close, the framework calls `Close()`. This function is always called if `Open()` was called and succeeded and even if the graph run terminated because of an error. No inputs are available via any input streams during `Close()`, but it still has access to input side packets and therefore may write outputs. After `Close()` returns, the calculator should be considered a dead node. The calculator object is destroyed as soon as the graph finishes running. The following are code snippets from +<<<<<<< HEAD [CalculatorBase.h](http://github.com/google/mediapipe/mediapipe/framework/calculator_base.h). +======= +[CalculatorBase.h](https://github.com/google/mediapipe/tree/master/mediapipe/framework/calculator_base.h). +>>>>>>> Project import generated by Copybara. ```c++ class CalculatorBase { diff --git a/mediapipe/docs/hello_world_android.md b/mediapipe/docs/hello_world_android.md index 5fc12e5df..d9feae7eb 100644 --- a/mediapipe/docs/hello_world_android.md +++ b/mediapipe/docs/hello_world_android.md @@ -643,7 +643,11 @@ Initialize the asset manager in `onCreate(Bundle)` before initializing `eglManager`: ``` +<<<<<<< HEAD // Initilize asset manager so that MediaPipe native libraries can access the app assets, e.g., +======= +// Initialize asset manager so that MediaPipe native libraries can access the app assets, e.g., +>>>>>>> Project import generated by Copybara. // binary graphs. AndroidAssetUtil.initializeNativeAssetManager(this); ``` diff --git a/mediapipe/docs/hello_world_ios.md b/mediapipe/docs/hello_world_ios.md index cc31aa247..3ed989da1 100644 --- a/mediapipe/docs/hello_world_ios.md +++ b/mediapipe/docs/hello_world_ios.md @@ -6,7 +6,11 @@ This codelab uses MediaPipe on an iOS device. ### What you will learn +<<<<<<< HEAD How to develop an Android application that uses MediaPipe and run a MediaPipe +======= +How to develop an iOS application that uses MediaPipe and run a MediaPipe +>>>>>>> Project import generated by Copybara. graph on iOS. ### What you will build diff --git a/mediapipe/docs/help.md b/mediapipe/docs/help.md index d1873a4ac..362f021de 100644 --- a/mediapipe/docs/help.md +++ b/mediapipe/docs/help.md @@ -1,9 +1,18 @@ +<<<<<<< HEAD ## Getting help - [Technical questions](#technical-questions) - [Bugs and Feature requests](#bugs-and-feature-requests) Below are the various ways to get help +======= +## Getting Help + +- [Technical questions](#technical-questions) +- [Bugs and feature requests](#bugs-and-feature-requests) + +Below are the various ways to get help: +>>>>>>> Project import generated by Copybara. ### Technical questions @@ -11,9 +20,40 @@ For help with technical or algorithmic questions, visit [Stack Overflow](https://stackoverflow.com/questions/tagged/mediapipe) to find answers and support from the MediaPipe community. +<<<<<<< HEAD ### Bugs and Feature requests To report bugs or make feature requests, [file an issue on GitHub](https://github.com/google/mediapipe/mediapipe/issues). Please choose the appropriate repository for the project from the [MediaPipe repo](https://github.com/google/mediapipe/mediapipe) +======= +### Bugs and feature requests + +To report bugs or make feature requests, +[file an issue on GitHub](https://github.com/google/mediapipe/issues). + +If you open a GitHub issue, here is our policy: + +1. It must be a bug, a feature request, or a significant problem with documentation (for small doc fixes please send a PR instead). +2. The form below must be filled out. + +**Here's why we have that policy**: MediaPipe developers respond to issues. We want to focus on work that benefits the whole community, e.g., fixing bugs and adding features. Support only helps individuals. GitHub also notifies thousands of people when issues are filed. We want them to see you communicating an interesting problem, rather than being redirected to Stack Overflow. + +------------------------ + +### System information +- **Have I written custom code**: +- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: +- **Mobile device (e.g. iPhone 8, Pixel 2, Samsung Galaxy) if the issue happens on mobile device**: +- **Bazel version**: +- **Android Studio, NDK, SDK versions (if issue is related to building in mobile dev enviroment)**: +- **Xcode & Tulsi version (if issue is related to building in mobile dev enviroment)**: +- **Exact steps to reproduce**: + +### Describe the problem +Describe the problem clearly here. Be sure to convey here why it's a bug in MediaPipe or a feature request. + +### Source code / logs +Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached instead of being pasted into the issue as text. +>>>>>>> Project import generated by Copybara. diff --git a/mediapipe/docs/install.md b/mediapipe/docs/install.md index cdaa119bb..a6aa2c11b 100644 --- a/mediapipe/docs/install.md +++ b/mediapipe/docs/install.md @@ -49,10 +49,17 @@ To build and run iOS apps: [documentation](https://docs.bazel.build/versions/master/install-ubuntu.html) to install any version of Bazel manually. +<<<<<<< HEAD 3. Install OpenCV. Option 1. Use package manager tool to install the pre-compiled OpenCV libraries. +======= +3. Install OpenCV and FFmpeg. + + Option 1. Use package manager tool to install the pre-compiled OpenCV + libraries. FFmpeg will be installed via libopencv-video-dev. +>>>>>>> Project import generated by Copybara. Note: Debian 9 and Ubuntu 16.04 provide OpenCV 2.4.9. You may want to take option 2 or 3 to install OpenCV 3 or above. @@ -83,6 +90,7 @@ To build and run iOS apps: ) cc_library( +<<<<<<< HEAD name = "opencv", srcs = glob( [ @@ -101,6 +109,24 @@ To build and run iOS apps: visibility = ["//visibility:public"], ) +======= + name = "opencv", + srcs = glob( + [ + "lib/libopencv_core.so", + "lib/libopencv_highgui.so", + "lib/libopencv_imgcodecs.so", + "lib/libopencv_imgproc.so", + "lib/libopencv_video.so", + "lib/libopencv_videoio.so", + ], + ), + hdrs = glob(["include/opencv4/**/*.h*"]), + includes = ["include/opencv4/"], + linkstatic = 1, + visibility = ["//visibility:public"], + ) +>>>>>>> Project import generated by Copybara. ``` 4. Run the [Hello World desktop example](./hello_world_desktop.md). @@ -168,6 +194,7 @@ To build and run iOS apps: ) cc_library( +<<<<<<< HEAD name = "opencv", srcs = glob( [ @@ -186,6 +213,24 @@ To build and run iOS apps: visibility = ["//visibility:public"], ) +======= + name = "opencv", + srcs = glob( + [ + "lib/libopencv_core.so", + "lib/libopencv_highgui.so", + "lib/libopencv_imgcodecs.so", + "lib/libopencv_imgproc.so", + "lib/libopencv_video.so", + "lib/libopencv_videoio.so", + ], + ), + hdrs = glob(["include/opencv4/**/*.h*"]), + includes = ["include/opencv4/"], + linkstatic = 1, + visibility = ["//visibility:public"], + ) +>>>>>>> Project import generated by Copybara. ``` 4. Run the [Hello World desktop example](./hello_world_desktop.md). @@ -239,10 +284,17 @@ To build and run iOS apps: [documentation](https://docs.bazel.build/versions/master/install-ubuntu.html) to install any version of Bazel manually. +<<<<<<< HEAD 4. Install OpenCV. Option 1. Use HomeBrew package manager tool to install the pre-compiled OpenCV libraries. +======= +4. Install OpenCV and FFmpeg. + + Option 1. Use HomeBrew package manager tool to install the pre-compiled + OpenCV libraries. FFmpeg will be installed via OpenCV. +>>>>>>> Project import generated by Copybara. ```bash $ brew install opencv @@ -254,6 +306,7 @@ To build and run iOS apps: $ port install opencv ``` +<<<<<<< HEAD Note: when using MacPorts, please edit the [`WORKSAPCE`] and [`opencv_linux.BUILD`] files like the following: @@ -281,6 +334,60 @@ To build and run iOS apps: linkstatic = 1, visibility = ["//visibility:public"], ) +======= + Note: when using MacPorts, please edit the [`WORKSAPCE`], + [`opencv_macos.BUILD`], and [`ffmpeg_macos.BUILD`] files like the following: + + ```bash + new_local_repository( + name = "macos_opencv", + build_file = "@//third_party:opencv_macos.BUILD", + path = "/opt", + ) + + new_local_repository( + name = "macos_ffmpeg", + build_file = "@//third_party:ffmpeg_macos.BUILD", + path = "/opt", + ) + + cc_library( + name = "opencv", + srcs = glob( + [ + "local/lib/libopencv_core.dylib", + "local/lib/libopencv_highgui.dylib", + "local/lib/libopencv_imgcodecs.dylib", + "local/lib/libopencv_imgproc.dylib", + "local/lib/libopencv_video.dylib", + "local/lib/libopencv_videoio.dylib", + ], + ), + hdrs = glob(["local/include/opencv2/**/*.h*"]), + includes = ["local/include/"], + linkstatic = 1, + visibility = ["//visibility:public"], + ) + + cc_library( + name = "libffmpeg", + srcs = glob( + [ + "local/lib/libav*.dylib", + ], + ), + hdrs = glob(["local/include/libav*/*.h"]), + includes = ["local/include/"], + linkopts = [ + "-lavcodec", + "-lavformat", + "-lavutil", + ], + linkstatic = 1, + visibility = ["//visibility:public"], + ) + +>>>>>>> Project import generated by Copybara. ``` 5. Run the [Hello World desktop example](./hello_world_desktop.md). @@ -350,10 +457,17 @@ To build and run iOS apps: username@DESKTOP-TMVLBJ1:~$ cd mediapipe ``` +<<<<<<< HEAD 7. Install OpenCV. Option 1. Use package manager tool to install the pre-compiled OpenCV libraries. +======= +7. Install OpenCV and FFmpeg. + + Option 1. Use package manager tool to install the pre-compiled OpenCV + libraries. FFmpeg will be installed via libopencv-video-dev. +>>>>>>> Project import generated by Copybara. ```bash username@DESKTOP-TMVLBJ1:~/mediapipe$ sudo apt-get install libopencv-core-dev libopencv-highgui-dev \ @@ -381,6 +495,7 @@ To build and run iOS apps: ) cc_library( +<<<<<<< HEAD name = "opencv", srcs = glob( [ @@ -399,6 +514,24 @@ To build and run iOS apps: visibility = ["//visibility:public"], ) +======= + name = "opencv", + srcs = glob( + [ + "lib/libopencv_core.so", + "lib/libopencv_highgui.so", + "lib/libopencv_imgcodecs.so", + "lib/libopencv_imgproc.so", + "lib/libopencv_video.so", + "lib/libopencv_videoio.so", + ], + ), + hdrs = glob(["include/opencv4/**/*.h*"]), + includes = ["include/opencv4/"], + linkstatic = 1, + visibility = ["//visibility:public"], + ) +>>>>>>> Project import generated by Copybara. ``` 8. Run the [Hello World desktop example](./hello_world_desktop.md). @@ -428,7 +561,11 @@ To build and run iOS apps: This will use a Docker image that will isolate mediapipe's installation from the rest of the system. 1. [Install Docker](https://docs.docker.com/install/#supported-platforms) on +<<<<<<< HEAD your host sytem. +======= + your host system. +>>>>>>> Project import generated by Copybara. 2. Build a docker image with tag "mediapipe". @@ -473,7 +610,35 @@ This will use a Docker image that will isolate mediapipe's installation from the # Hello World! ``` +<<<<<<< HEAD