diff --git a/README.md b/README.md index b458d5a4f..5191e6d63 100644 --- a/README.md +++ b/README.md @@ -8,15 +8,14 @@ nav_order: 1 -------------------------------------------------------------------------------- -## Cross-platform ML solutions made simple +## Live ML anywhere -[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers -and developers to build world-class ML solutions and applications for mobile, -desktop/cloud, web and IoT devices. +[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable +ML solutions for live and streaming media. ![accelerated.png](docs/images/accelerated_small.png) | ![cross_platform.png](docs/images/cross_platform_small.png) :------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------: -***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT* +***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT* ![ready_to_use.png](docs/images/ready_to_use_small.png) | ![open_source.png](docs/images/open_source_small.png) ***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable* @@ -93,7 +92,7 @@ run code search using ## Publications -* [MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog) +* [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html) in Google Developers Blog * [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html) in Google Developers Blog diff --git a/docs/index.md b/docs/index.md index 06fc76dd5..2e6f1f6da 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,15 +8,14 @@ nav_order: 1 -------------------------------------------------------------------------------- -## Cross-platform ML solutions made simple +## Live ML anywhere -[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers -and developers to build world-class ML solutions and applications for mobile, -desktop/cloud, web and IoT devices. +[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable +ML solutions for live and streaming media. ![accelerated.png](images/accelerated_small.png) | ![cross_platform.png](images/cross_platform_small.png) :------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------: -***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT* +***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT* ![ready_to_use.png](images/ready_to_use_small.png) | ![open_source.png](images/open_source_small.png) ***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable* @@ -93,7 +92,7 @@ run code search using ## Publications -* [MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog) +* [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html) in Google Developers Blog * [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html) in Google Developers Blog diff --git a/docs/solutions/face_mesh.md b/docs/solutions/face_mesh.md index 76726c208..d113318b0 100644 --- a/docs/solutions/face_mesh.md +++ b/docs/solutions/face_mesh.md @@ -277,7 +277,7 @@ only works for a single face. For visual reference, please refer to *Fig. 4*. * TensorFlow Blog: [Face and hand tracking in the browser with MediaPipe and TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html) * Google Developers Blog: - [MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog) + [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html) * Paper: [Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs](https://arxiv.org/abs/1907.06724) ([poster](https://docs.google.com/presentation/d/1-LWwOMO9TzEVdrZ1CS1ndJzciRHfYDJfbSxH_ke_JRg/present?slide=id.g5986dd4b4c_4_212)) diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index 7f9ffd7f8..9762f41ee 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -13,181 +13,120 @@ # limitations under the License. # -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") licenses(["notice"]) package(default_visibility = ["//visibility:private"]) -proto_library( +mediapipe_proto_library( name = "concatenate_vector_calculator_proto", srcs = ["concatenate_vector_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "dequantize_byte_array_calculator_proto", srcs = ["dequantize_byte_array_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "packet_cloner_calculator_proto", srcs = ["packet_cloner_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "packet_resampler_calculator_proto", srcs = ["packet_resampler_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "packet_thinner_calculator_proto", srcs = ["packet_thinner_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "split_vector_calculator_proto", srcs = ["split_vector_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "quantize_float_vector_calculator_proto", srcs = ["quantize_float_vector_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "sequence_shift_calculator_proto", srcs = ["sequence_shift_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "gate_calculator_proto", srcs = ["gate_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "constant_side_packet_calculator_proto", srcs = ["constant_side_packet_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "clip_vector_size_calculator_proto", srcs = ["clip_vector_size_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "packet_cloner_calculator_cc_proto", - srcs = ["packet_cloner_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":packet_cloner_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "packet_resampler_calculator_cc_proto", - srcs = ["packet_resampler_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":packet_resampler_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "packet_thinner_calculator_cc_proto", - srcs = ["packet_thinner_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":packet_thinner_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "split_vector_calculator_cc_proto", - srcs = ["split_vector_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":split_vector_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "concatenate_vector_calculator_cc_proto", - srcs = ["concatenate_vector_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":concatenate_vector_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "clip_vector_size_calculator_cc_proto", - srcs = ["clip_vector_size_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":clip_vector_size_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "dequantize_byte_array_calculator_cc_proto", - srcs = ["dequantize_byte_array_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":dequantize_byte_array_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "quantize_float_vector_calculator_cc_proto", - srcs = ["quantize_float_vector_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":quantize_float_vector_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "sequence_shift_calculator_cc_proto", - srcs = ["sequence_shift_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":sequence_shift_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "gate_calculator_cc_proto", - srcs = ["gate_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":gate_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "constant_side_packet_calculator_cc_proto", - srcs = ["constant_side_packet_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":constant_side_packet_calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) cc_library( diff --git a/mediapipe/calculators/core/gate_calculator.cc b/mediapipe/calculators/core/gate_calculator.cc index ea0f7b81b..8d05c9268 100644 --- a/mediapipe/calculators/core/gate_calculator.cc +++ b/mediapipe/calculators/core/gate_calculator.cc @@ -1,4 +1,4 @@ -// Copyright 2019 The MediaPipe Authors. +// Copyright 2019-2020 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -59,16 +59,30 @@ std::string ToString(GateState state) { // ALLOW or DISALLOW can also be specified as an input side packet. The rules // for evaluation remain the same as above. // +// If side_input_has_precedence isn't set in the calculator option, // ALLOW/DISALLOW inputs must be specified either using input stream or -// via input side packet but not both. +// via input side packet but not both. Otherwise, both input stream and input +// side packet can be specified and the calculator will take one signal over the +// other based on the value of the side_input_has_precedence field. // // Intended to be used with the default input stream handler, which synchronizes // all data input streams with the ALLOW/DISALLOW control input stream. // -// Example config: +// Example configs: // node { // calculator: "GateCalculator" +// input_stream: "input_stream0" +// input_stream: "input_stream1" +// input_stream: "input_streamN" // input_side_packet: "ALLOW:allow" or "DISALLOW:disallow" +// output_stream: "STATE_CHANGE:state_change" +// output_stream: "output_stream0" +// output_stream: "output_stream1" +// output_stream: "output_streamN" +// } +// +// node { +// calculator: "GateCalculator" // input_stream: "input_stream0" // input_stream: "input_stream1" // input_stream: "input_streamN" @@ -78,6 +92,25 @@ std::string ToString(GateState state) { // output_stream: "output_stream1" // output_stream: "output_streamN" // } +// +// With side_input_has_precedence: +// node { +// calculator: "GateCalculator" +// input_stream: "input_stream0" +// input_stream: "input_stream1" +// input_stream: "input_streamN" +// input_stream: "ALLOW:allow_stream" or "DISALLOW:disallow_stream" +// input_side_packet: "ALLOW:allow_packet" or "DISALLOW:disallow_packet" +// output_stream: "STATE_CHANGE:state_change" +// output_stream: "output_stream0" +// output_stream: "output_stream1" +// output_stream: "output_streamN" +// options: { +// [mediapipe.GateCalculatorOptions.ext] { +// side_input_has_precedence: true or false +// } +// } +// } class GateCalculator : public CalculatorBase { public: GateCalculator() {} @@ -88,9 +121,15 @@ class GateCalculator : public CalculatorBase { cc->InputSidePackets().HasTag("DISALLOW"); bool input_via_stream = cc->Inputs().HasTag("ALLOW") || cc->Inputs().HasTag("DISALLOW"); - // Only one of input_side_packet or input_stream may specify ALLOW/DISALLOW - // input. - RET_CHECK(input_via_side_packet ^ input_via_stream); + const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>(); + if (options.has_side_input_has_precedence()) { + RET_CHECK(input_via_side_packet && input_via_stream); + } else { + // Only one of input_side_packet or input_stream may specify + // ALLOW/DISALLOW input when side_input_has_precedence is not set + // in the options. + RET_CHECK(input_via_side_packet ^ input_via_stream); + } if (input_via_side_packet) { RET_CHECK(cc->InputSidePackets().HasTag("ALLOW") ^ @@ -101,7 +140,8 @@ class GateCalculator : public CalculatorBase { } else { cc->InputSidePackets().Tag("DISALLOW").Set(); } - } else { + } + if (input_via_stream) { RET_CHECK(cc->Inputs().HasTag("ALLOW") ^ cc->Inputs().HasTag("DISALLOW")); if (cc->Inputs().HasTag("ALLOW")) { @@ -134,13 +174,13 @@ class GateCalculator : public CalculatorBase { } ::mediapipe::Status Open(CalculatorContext* cc) final { - use_side_packet_for_allow_disallow_ = false; + bool use_side_packet_for_allow_disallow = false; if (cc->InputSidePackets().HasTag("ALLOW")) { - use_side_packet_for_allow_disallow_ = true; + use_side_packet_for_allow_disallow = true; allow_by_side_packet_decision_ = cc->InputSidePackets().Tag("ALLOW").Get(); } else if (cc->InputSidePackets().HasTag("DISALLOW")) { - use_side_packet_for_allow_disallow_ = true; + use_side_packet_for_allow_disallow = true; allow_by_side_packet_decision_ = !cc->InputSidePackets().Tag("DISALLOW").Get(); } @@ -152,23 +192,31 @@ class GateCalculator : public CalculatorBase { const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>(); empty_packets_as_allow_ = options.empty_packets_as_allow(); + if (!options.has_side_input_has_precedence()) { + side_input_has_precedence_ = use_side_packet_for_allow_disallow; + } else { + side_input_has_precedence_ = options.side_input_has_precedence(); + } return ::mediapipe::OkStatus(); } ::mediapipe::Status Process(CalculatorContext* cc) final { - bool allow = empty_packets_as_allow_; - if (use_side_packet_for_allow_disallow_) { - allow = allow_by_side_packet_decision_; + bool allow_by_stream = empty_packets_as_allow_; + if (cc->Inputs().HasTag("ALLOW") && !cc->Inputs().Tag("ALLOW").IsEmpty()) { + allow_by_stream = cc->Inputs().Tag("ALLOW").Get(); + } + if (cc->Inputs().HasTag("DISALLOW") && + !cc->Inputs().Tag("DISALLOW").IsEmpty()) { + allow_by_stream = !cc->Inputs().Tag("DISALLOW").Get(); + } + const bool allow_by_side_packet = + allow_by_side_packet_decision_ || empty_packets_as_allow_; + bool allow = false; + if (side_input_has_precedence_) { + allow = allow_by_side_packet; } else { - if (cc->Inputs().HasTag("ALLOW") && - !cc->Inputs().Tag("ALLOW").IsEmpty()) { - allow = cc->Inputs().Tag("ALLOW").Get(); - } - if (cc->Inputs().HasTag("DISALLOW") && - !cc->Inputs().Tag("DISALLOW").IsEmpty()) { - allow = !cc->Inputs().Tag("DISALLOW").Get(); - } + allow = allow_by_stream; } const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW; @@ -203,9 +251,9 @@ class GateCalculator : public CalculatorBase { private: GateState last_gate_state_ = GATE_UNINITIALIZED; int num_data_streams_; + bool allow_by_side_packet_decision_ = false; bool empty_packets_as_allow_; - bool use_side_packet_for_allow_disallow_; - bool allow_by_side_packet_decision_; + bool side_input_has_precedence_; }; REGISTER_CALCULATOR(GateCalculator); diff --git a/mediapipe/calculators/core/gate_calculator.proto b/mediapipe/calculators/core/gate_calculator.proto index 0ef2c3e1c..777b6bc85 100644 --- a/mediapipe/calculators/core/gate_calculator.proto +++ b/mediapipe/calculators/core/gate_calculator.proto @@ -1,4 +1,4 @@ -// Copyright 2019 The MediaPipe Authors. +// Copyright 2019-2020 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -27,4 +27,12 @@ message GateCalculatorOptions { // disallowing the corresponding packets in the data input streams. Setting // this option to true inverts that, allowing the data packets to go through. optional bool empty_packets_as_allow = 1; + + // Input side packet and input stream are allowed to coexist only if this + // field is set. When it's set to true, the input side packet has higher + // precedence and the input stream signal will be ignored. When it's set to + // false, the input stream signal always overrides the input side packet + // signal. + // + optional bool side_input_has_precedence = 2; } diff --git a/mediapipe/calculators/core/gate_calculator_test.cc b/mediapipe/calculators/core/gate_calculator_test.cc index fc34f6e97..07d20350c 100644 --- a/mediapipe/calculators/core/gate_calculator_test.cc +++ b/mediapipe/calculators/core/gate_calculator_test.cc @@ -1,4 +1,4 @@ -// Copyright 2019 The MediaPipe Authors. +// Copyright 2019-2020 The MediaPipe Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -330,5 +330,49 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) { ASSERT_EQ(0, output.size()); } +TEST_F(GateCalculatorTest, TestOverrideDecisionBySidePacketSignal) { + SetRunner(R"( + calculator: "GateCalculator" + input_stream: "test_input" + input_stream: "ALLOW:gating_stream" + input_side_packet: "ALLOW:gating_packet" + output_stream: "test_output" + options: { + [mediapipe.GateCalculatorOptions.ext] { + side_input_has_precedence: true + } + } + )"); + + constexpr int64 kTimestampValue0 = 42; + runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false)); + RunTimeStep(kTimestampValue0, "ALLOW", true); + + const std::vector& output = runner()->Outputs().Get("", 0).packets; + ASSERT_EQ(0, output.size()); +} + +TEST_F(GateCalculatorTest, TestOverrideDecisionByStreamSignal) { + SetRunner(R"( + calculator: "GateCalculator" + input_stream: "test_input" + input_stream: "ALLOW:gating_stream" + input_side_packet: "ALLOW:gating_packet" + output_stream: "test_output" + options: { + [mediapipe.GateCalculatorOptions.ext] { + side_input_has_precedence: false + } + } + )"); + + constexpr int64 kTimestampValue0 = 42; + runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false)); + RunTimeStep(kTimestampValue0, "ALLOW", true); + + const std::vector& output = runner()->Outputs().Get("", 0).packets; + ASSERT_EQ(1, output.size()); +} + } // namespace } // namespace mediapipe diff --git a/mediapipe/calculators/image/BUILD b/mediapipe/calculators/image/BUILD index a14bd31d1..6ad3be548 100644 --- a/mediapipe/calculators/image/BUILD +++ b/mediapipe/calculators/image/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") licenses(["notice"]) @@ -279,27 +279,17 @@ cc_library( alwayslink = 1, ) -proto_library( +mediapipe_proto_library( name = "image_transformation_calculator_proto", srcs = ["image_transformation_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", "//mediapipe/gpu:scale_mode_proto", ], ) -mediapipe_cc_proto_library( - name = "image_transformation_calculator_cc_proto", - srcs = ["image_transformation_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/gpu:scale_mode_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":image_transformation_calculator_proto"], -) - cc_library( name = "image_transformation_calculator", srcs = ["image_transformation_calculator.cc"], diff --git a/mediapipe/calculators/tflite/BUILD b/mediapipe/calculators/tflite/BUILD index ae56ae763..53775bed1 100644 --- a/mediapipe/calculators/tflite/BUILD +++ b/mediapipe/calculators/tflite/BUILD @@ -13,131 +13,91 @@ # limitations under the License. # -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") load("@bazel_skylib//lib:selects.bzl", "selects") licenses(["notice"]) package(default_visibility = ["//visibility:private"]) -proto_library( +mediapipe_proto_library( name = "ssd_anchors_calculator_proto", srcs = ["ssd_anchors_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "tflite_custom_op_resolver_calculator_proto", srcs = ["tflite_custom_op_resolver_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "tflite_inference_calculator_proto", srcs = ["tflite_inference_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "tflite_converter_calculator_proto", srcs = ["tflite_converter_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "tflite_tensors_to_segmentation_calculator_proto", srcs = ["tflite_tensors_to_segmentation_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "tflite_tensors_to_detections_calculator_proto", srcs = ["tflite_tensors_to_detections_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "tflite_tensors_to_classification_calculator_proto", srcs = ["tflite_tensors_to_classification_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "tflite_tensors_to_landmarks_calculator_proto", srcs = ["tflite_tensors_to_landmarks_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "ssd_anchors_calculator_cc_proto", - srcs = ["ssd_anchors_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":ssd_anchors_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tflite_custom_op_resolver_calculator_cc_proto", - srcs = ["tflite_custom_op_resolver_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":tflite_custom_op_resolver_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tflite_converter_calculator_cc_proto", - srcs = ["tflite_converter_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":tflite_converter_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tflite_tensors_to_segmentation_calculator_cc_proto", - srcs = ["tflite_tensors_to_segmentation_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":tflite_tensors_to_segmentation_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tflite_inference_calculator_cc_proto", - srcs = ["tflite_inference_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":tflite_inference_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tflite_tensors_to_detections_calculator_cc_proto", - srcs = ["tflite_tensors_to_detections_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":tflite_tensors_to_detections_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tflite_tensors_to_classification_calculator_cc_proto", - srcs = ["tflite_tensors_to_classification_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":tflite_tensors_to_classification_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "tflite_tensors_to_landmarks_calculator_cc_proto", - srcs = ["tflite_tensors_to_landmarks_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":tflite_tensors_to_landmarks_calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) cc_library( diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc index bc911efb6..8f3b8b35c 100644 --- a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc @@ -21,9 +21,9 @@ namespace mediapipe { // A calculator for converting TFLite tensors from regression models into -// landmarks. Note that if the landmarks in the tensor has more than 4 -// dimensions, only the first 4 dimensions will be converted to -// [x,y,z, visibility]. +// landmarks. Note that if the landmarks in the tensor has more than 5 +// dimensions, only the first 5 dimensions will be converted to +// [x,y,z, visibility, presence]. // // Input: // TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first @@ -209,6 +209,9 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator); if (num_dimensions > 3) { landmark->set_visibility(raw_landmarks[offset + 3]); } + if (num_dimensions > 4) { + landmark->set_presence(raw_landmarks[offset + 4]); + } } // Output normalized landmarks if required. @@ -223,6 +226,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator); norm_landmark->set_z(landmark.z() / options_.input_image_width() / options_.normalize_z()); norm_landmark->set_visibility(landmark.visibility()); + norm_landmark->set_presence(landmark.presence()); } cc->Outputs() .Tag("NORM_LANDMARKS") diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD index 5b4447f10..8ca63d7ee 100644 --- a/mediapipe/calculators/util/BUILD +++ b/mediapipe/calculators/util/BUILD @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library") +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") licenses(["notice"]) @@ -38,186 +38,95 @@ cc_library( alwayslink = 1, ) -proto_library( +mediapipe_proto_library( name = "annotation_overlay_calculator_proto", srcs = ["annotation_overlay_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", "//mediapipe/util:color_proto", ], ) -proto_library( +mediapipe_proto_library( name = "detection_label_id_to_text_calculator_proto", srcs = ["detection_label_id_to_text_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "timed_box_list_id_to_label_calculator_proto", srcs = ["timed_box_list_id_to_label_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "latency_proto", srcs = ["latency.proto"], ) -proto_library( +mediapipe_proto_library( name = "non_max_suppression_calculator_proto", srcs = ["non_max_suppression_calculator.proto"], visibility = ["//visibility:public"], - deps = ["//mediapipe/framework:calculator_proto"], + deps = [ + "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", + ], ) -proto_library( +mediapipe_proto_library( name = "packet_frequency_proto", srcs = ["packet_frequency.proto"], ) -proto_library( +mediapipe_proto_library( name = "packet_frequency_calculator_proto", srcs = ["packet_frequency_calculator.proto"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "packet_latency_calculator_proto", srcs = ["packet_latency_calculator.proto"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "collection_has_min_size_calculator_proto", srcs = ["collection_has_min_size_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "association_calculator_proto", srcs = ["association_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -mediapipe_cc_proto_library( - name = "annotation_overlay_calculator_cc_proto", - srcs = ["annotation_overlay_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util:color_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":annotation_overlay_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "detection_label_id_to_text_calculator_cc_proto", - srcs = ["detection_label_id_to_text_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [ - ":detection_label_id_to_text_calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "timed_box_list_id_to_label_calculator_cc_proto", - srcs = ["timed_box_list_id_to_label_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [ - ":timed_box_list_id_to_label_calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "latency_cc_proto", - srcs = ["latency.proto"], - visibility = ["//mediapipe:__subpackages__"], - deps = [":latency_proto"], -) - -mediapipe_cc_proto_library( - name = "non_max_suppression_calculator_cc_proto", - srcs = ["non_max_suppression_calculator.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":non_max_suppression_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "packet_frequency_cc_proto", - srcs = ["packet_frequency.proto"], - visibility = ["//mediapipe:__subpackages__"], - deps = [":packet_frequency_proto"], -) - -mediapipe_cc_proto_library( - name = "packet_frequency_calculator_cc_proto", - srcs = ["packet_frequency_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//mediapipe:__subpackages__"], - deps = [ - ":packet_frequency_calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "packet_latency_calculator_cc_proto", - srcs = ["packet_latency_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//mediapipe:__subpackages__"], - deps = [ - ":packet_latency_calculator_proto", - ], -) - -mediapipe_cc_proto_library( - name = "collection_has_min_size_calculator_cc_proto", - srcs = ["collection_has_min_size_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//mediapipe:__subpackages__"], - deps = [":collection_has_min_size_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "association_calculator_cc_proto", - srcs = ["association_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//mediapipe:__subpackages__"], - deps = [":association_calculator_proto"], -) - cc_library( name = "packet_frequency_calculator", srcs = ["packet_frequency_calculator.cc"], @@ -434,16 +343,6 @@ cc_library( alwayslink = 1, ) -mediapipe_cc_proto_library( - name = "thresholding_calculator_cc_proto", - srcs = ["thresholding_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":thresholding_calculator_proto"], -) - cc_library( name = "thresholding_calculator", srcs = ["thresholding_calculator.cc"], @@ -457,16 +356,6 @@ cc_library( alwayslink = 1, ) -mediapipe_cc_proto_library( - name = "landmarks_to_detection_calculator_cc_proto", - srcs = ["landmarks_to_detection_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":landmarks_to_detection_calculator_proto"], -) - cc_library( name = "landmarks_to_detection_calculator", srcs = ["landmarks_to_detection_calculator.cc"], @@ -483,46 +372,6 @@ cc_library( alwayslink = 1, ) -mediapipe_cc_proto_library( - name = "detections_to_rects_calculator_cc_proto", - srcs = ["detections_to_rects_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":detections_to_rects_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "landmark_projection_calculator_cc_proto", - srcs = ["landmark_projection_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":landmark_projection_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "landmarks_to_floats_calculator_cc_proto", - srcs = ["landmarks_to_floats_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":landmarks_to_floats_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "rect_transformation_calculator_cc_proto", - srcs = ["rect_transformation_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":rect_transformation_calculator_proto"], -) - cc_library( name = "detections_to_rects_calculator", srcs = [ @@ -593,162 +442,140 @@ cc_test( ], ) -proto_library( +mediapipe_proto_library( name = "rect_to_render_data_calculator_proto", srcs = ["rect_to_render_data_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", "//mediapipe/util:color_proto", "//mediapipe/util:render_data_proto", ], ) -proto_library( +mediapipe_proto_library( name = "rect_to_render_scale_calculator_proto", srcs = ["rect_to_render_scale_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "detections_to_render_data_calculator_proto", srcs = ["detections_to_render_data_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", "//mediapipe/util:color_proto", "//mediapipe/util:render_data_proto", ], ) -proto_library( +mediapipe_proto_library( name = "landmarks_to_render_data_calculator_proto", srcs = ["landmarks_to_render_data_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", "//mediapipe/util:color_proto", "//mediapipe/util:render_data_proto", ], ) -proto_library( +mediapipe_proto_library( name = "timed_box_list_to_render_data_calculator_proto", srcs = ["timed_box_list_to_render_data_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", "//mediapipe/util:color_proto", "//mediapipe/util:render_data_proto", ], ) -proto_library( +mediapipe_proto_library( name = "labels_to_render_data_calculator_proto", srcs = ["labels_to_render_data_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", "//mediapipe/util:color_proto", "//mediapipe/util:render_data_proto", ], ) -proto_library( +mediapipe_proto_library( name = "thresholding_calculator_proto", srcs = ["thresholding_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", "//mediapipe/util:color_proto", "//mediapipe/util:render_data_proto", ], ) -proto_library( +mediapipe_proto_library( name = "detections_to_rects_calculator_proto", srcs = ["detections_to_rects_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "landmark_projection_calculator_proto", srcs = ["landmark_projection_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "landmarks_to_floats_calculator_proto", srcs = ["landmarks_to_floats_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "rect_transformation_calculator_proto", srcs = ["rect_transformation_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "landmarks_to_detection_calculator_proto", srcs = ["landmarks_to_detection_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", "//mediapipe/util:color_proto", "//mediapipe/util:render_data_proto", ], ) -mediapipe_cc_proto_library( - name = "rect_to_render_data_calculator_cc_proto", - srcs = ["rect_to_render_data_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util:color_cc_proto", - "//mediapipe/util:render_data_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":rect_to_render_data_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "rect_to_render_scale_calculator_cc_proto", - srcs = ["rect_to_render_scale_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":rect_to_render_scale_calculator_proto"], -) - -mediapipe_cc_proto_library( - name = "detections_to_render_data_calculator_cc_proto", - srcs = ["detections_to_render_data_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util:color_cc_proto", - "//mediapipe/util:render_data_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":detections_to_render_data_calculator_proto"], -) - cc_library( name = "detections_to_render_data_calculator", srcs = ["detections_to_render_data_calculator.cc"], @@ -768,18 +595,6 @@ cc_library( alwayslink = 1, ) -mediapipe_cc_proto_library( - name = "landmarks_to_render_data_calculator_cc_proto", - srcs = ["landmarks_to_render_data_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util:color_cc_proto", - "//mediapipe/util:render_data_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":landmarks_to_render_data_calculator_proto"], -) - cc_library( name = "landmarks_to_render_data_calculator", srcs = ["landmarks_to_render_data_calculator.cc"], @@ -800,18 +615,6 @@ cc_library( alwayslink = 1, ) -mediapipe_cc_proto_library( - name = "timed_box_list_to_render_data_calculator_cc_proto", - srcs = ["timed_box_list_to_render_data_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util:color_cc_proto", - "//mediapipe/util:render_data_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":timed_box_list_to_render_data_calculator_proto"], -) - cc_library( name = "timed_box_list_to_render_data_calculator", srcs = ["timed_box_list_to_render_data_calculator.cc"], @@ -951,6 +754,7 @@ mediapipe_proto_library( visibility = ["//visibility:public"], deps = [ "//mediapipe/framework:calculator_options_proto", + "//mediapipe/framework:calculator_proto", ], ) @@ -1017,25 +821,16 @@ cc_test( ], ) -proto_library( +mediapipe_proto_library( name = "top_k_scores_calculator_proto", srcs = ["top_k_scores_calculator.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -mediapipe_cc_proto_library( - name = "top_k_scores_calculator_cc_proto", - srcs = ["top_k_scores_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":top_k_scores_calculator_proto"], -) - cc_library( name = "top_k_scores_calculator", srcs = ["top_k_scores_calculator.cc"], @@ -1081,17 +876,6 @@ cc_test( ], ) -mediapipe_cc_proto_library( - name = "labels_to_render_data_calculator_cc_proto", - srcs = ["labels_to_render_data_calculator.proto"], - cc_deps = [ - "//mediapipe/framework:calculator_cc_proto", - "//mediapipe/util:color_cc_proto", - ], - visibility = ["//visibility:public"], - deps = [":labels_to_render_data_calculator_proto"], -) - cc_library( name = "local_file_contents_calculator", srcs = ["local_file_contents_calculator.cc"], diff --git a/mediapipe/calculators/util/annotation_overlay_calculator.cc b/mediapipe/calculators/util/annotation_overlay_calculator.cc index 13dcabc7e..4936dbfb5 100644 --- a/mediapipe/calculators/util/annotation_overlay_calculator.cc +++ b/mediapipe/calculators/util/annotation_overlay_calculator.cc @@ -476,6 +476,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator); options_.canvas_height_px(), options_.canvas_width_px(), CV_8UC3, cv::Scalar(options_.canvas_color().r(), options_.canvas_color().g(), options_.canvas_color().b())); + *target_format = ImageFormat::SRGB; } return ::mediapipe::OkStatus(); diff --git a/mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc b/mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc index 925272230..737c44100 100644 --- a/mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc +++ b/mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc @@ -131,6 +131,8 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase { new_landmark->set_z(new_z); // Keep visibility as is. new_landmark->set_visibility(landmark.visibility()); + // Keep presence as is. + new_landmark->set_presence(landmark.presence()); } cc->Outputs().Get(output_id).AddPacket( diff --git a/mediapipe/calculators/util/landmark_projection_calculator.cc b/mediapipe/calculators/util/landmark_projection_calculator.cc index 0309c530a..dacb4ab50 100644 --- a/mediapipe/calculators/util/landmark_projection_calculator.cc +++ b/mediapipe/calculators/util/landmark_projection_calculator.cc @@ -131,6 +131,8 @@ class LandmarkProjectionCalculator : public CalculatorBase { new_landmark->set_z(new_z); // Keep visibility as is. new_landmark->set_visibility(landmark.visibility()); + // Keep presence as is. + new_landmark->set_presence(landmark.presence()); } cc->Outputs().Get(output_id).AddPacket( diff --git a/mediapipe/calculators/util/landmarks_smoothing_calculator.cc b/mediapipe/calculators/util/landmarks_smoothing_calculator.cc index be2bc24ad..751e730f6 100644 --- a/mediapipe/calculators/util/landmarks_smoothing_calculator.cc +++ b/mediapipe/calculators/util/landmarks_smoothing_calculator.cc @@ -137,6 +137,8 @@ class VelocityFilter : public LandmarksFilter { image_width); // Keep visibility as is. out_landmark->set_visibility(in_landmark.visibility()); + // Keep presence as is. + out_landmark->set_presence(in_landmark.presence()); } return ::mediapipe::OkStatus(); diff --git a/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc b/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc index fab2a06e8..c075f04e5 100644 --- a/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc +++ b/mediapipe/calculators/util/landmarks_to_render_data_calculator.cc @@ -33,7 +33,6 @@ constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS"; constexpr char kRenderScaleTag[] = "RENDER_SCALE"; constexpr char kRenderDataTag[] = "RENDER_DATA"; constexpr char kLandmarkLabel[] = "KEYPOINT"; -constexpr int kMaxLandmarkThickness = 18; inline void SetColor(RenderAnnotation* annotation, const Color& color) { annotation->mutable_color()->set_r(color.r()); @@ -59,15 +58,18 @@ inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min, } void SetColorSizeValueFromZ(float z, float z_min, float z_max, - RenderAnnotation* render_annotation) { + RenderAnnotation* render_annotation, + float min_depth_circle_thickness, + float max_depth_circle_thickness) { const int color_value = 255 - static_cast(Remap(z, z_min, z_max, 255)); ::mediapipe::Color color; color.set_r(color_value); color.set_g(color_value); color.set_b(color_value); SetColor(render_annotation, color); - const int thickness = static_cast((1.f - Remap(z, z_min, z_max, 1)) * - kMaxLandmarkThickness); + const float scale = max_depth_circle_thickness - min_depth_circle_thickness; + const int thickness = static_cast( + min_depth_circle_thickness + (1.f - Remap(z, z_min, z_max, 1)) * scale); render_annotation->set_thickness(thickness); } @@ -97,7 +99,8 @@ template void AddConnectionsWithDepth(const LandmarkListType& landmarks, const std::vector& landmark_connections, bool utilize_visibility, - float visibility_threshold, float thickness, + float visibility_threshold, bool utilize_presence, + float presence_threshold, float thickness, bool normalized, float min_z, float max_z, RenderData* render_data) { for (int i = 0; i < landmark_connections.size(); i += 2) { @@ -107,6 +110,10 @@ void AddConnectionsWithDepth(const LandmarkListType& landmarks, ld1.visibility() < visibility_threshold)) { continue; } + if (utilize_presence && (ld0.presence() < presence_threshold || + ld1.presence() < presence_threshold)) { + continue; + } const int gray_val1 = 255 - static_cast(Remap(ld0.z(), min_z, max_z, 255)); const int gray_val2 = @@ -136,6 +143,7 @@ template void AddConnections(const LandmarkListType& landmarks, const std::vector& landmark_connections, bool utilize_visibility, float visibility_threshold, + bool utilize_presence, float presence_threshold, const Color& connection_color, float thickness, bool normalized, RenderData* render_data) { for (int i = 0; i < landmark_connections.size(); i += 2) { @@ -145,6 +153,10 @@ void AddConnections(const LandmarkListType& landmarks, ld1.visibility() < visibility_threshold)) { continue; } + if (utilize_presence && (ld0.presence() < presence_threshold || + ld1.presence() < presence_threshold)) { + continue; + } AddConnectionToRenderData(ld0, ld1, connection_color, thickness, normalized, render_data); } @@ -238,13 +250,15 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color, if (visualize_depth) { AddConnectionsWithDepth( landmarks, landmark_connections_, options_.utilize_visibility(), - options_.visibility_threshold(), thickness, /*normalized=*/false, - z_min, z_max, render_data.get()); + options_.visibility_threshold(), options_.utilize_presence(), + options_.presence_threshold(), thickness, /*normalized=*/false, z_min, + z_max, render_data.get()); } else { AddConnections( landmarks, landmark_connections_, options_.utilize_visibility(), - options_.visibility_threshold(), options_.connection_color(), - thickness, /*normalized=*/false, render_data.get()); + options_.visibility_threshold(), options_.utilize_presence(), + options_.presence_threshold(), options_.connection_color(), thickness, + /*normalized=*/false, render_data.get()); } for (int i = 0; i < landmarks.landmark_size(); ++i) { const Landmark& landmark = landmarks.landmark(i); @@ -254,11 +268,17 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color, continue; } + if (options_.utilize_presence() && + landmark.presence() < options_.presence_threshold()) { + continue; + } + auto* landmark_data_render = AddPointRenderData( options_.landmark_color(), thickness, render_data.get()); if (visualize_depth) { - SetColorSizeValueFromZ(landmark.z(), z_min, z_max, - landmark_data_render); + SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render, + options_.min_depth_circle_thickness(), + options_.max_depth_circle_thickness()); } auto* landmark_data = landmark_data_render->mutable_point(); landmark_data->set_normalized(false); @@ -279,13 +299,15 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color, if (visualize_depth) { AddConnectionsWithDepth( landmarks, landmark_connections_, options_.utilize_visibility(), - options_.visibility_threshold(), thickness, /*normalized=*/true, - z_min, z_max, render_data.get()); + options_.visibility_threshold(), options_.utilize_presence(), + options_.presence_threshold(), thickness, /*normalized=*/true, z_min, + z_max, render_data.get()); } else { AddConnections( landmarks, landmark_connections_, options_.utilize_visibility(), - options_.visibility_threshold(), options_.connection_color(), - thickness, /*normalized=*/true, render_data.get()); + options_.visibility_threshold(), options_.utilize_presence(), + options_.presence_threshold(), options_.connection_color(), thickness, + /*normalized=*/true, render_data.get()); } for (int i = 0; i < landmarks.landmark_size(); ++i) { const NormalizedLandmark& landmark = landmarks.landmark(i); @@ -294,12 +316,17 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color, landmark.visibility() < options_.visibility_threshold()) { continue; } + if (options_.utilize_presence() && + landmark.presence() < options_.presence_threshold()) { + continue; + } auto* landmark_data_render = AddPointRenderData( options_.landmark_color(), thickness, render_data.get()); if (visualize_depth) { - SetColorSizeValueFromZ(landmark.z(), z_min, z_max, - landmark_data_render); + SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render, + options_.min_depth_circle_thickness(), + options_.max_depth_circle_thickness()); } auto* landmark_data = landmark_data_render->mutable_point(); landmark_data->set_normalized(true); diff --git a/mediapipe/calculators/util/landmarks_to_render_data_calculator.proto b/mediapipe/calculators/util/landmarks_to_render_data_calculator.proto index ff1e8fdc5..34f073f26 100644 --- a/mediapipe/calculators/util/landmarks_to_render_data_calculator.proto +++ b/mediapipe/calculators/util/landmarks_to_render_data_calculator.proto @@ -49,4 +49,19 @@ message LandmarksToRenderDataCalculatorOptions { // Threshold to determine visibility of the landmark. Landmark with visibility // greater or equal than threshold is considered visible. optional double visibility_threshold = 7 [default = 0.0]; + + // Use landmarks presence while rendering landmarks and connections. If + // landmark is not present, neither it nor adjacent connections will be + // rendered. + optional bool utilize_presence = 8 [default = false]; + + // Threshold to determine presence of the landmark. Landmark with presence + // greater or equal than threshold is considered present. + optional double presence_threshold = 9 [default = 0.0]; + + // Min thickness of the drawing for landmark circle. + optional double min_depth_circle_thickness = 10 [default = 0.0]; + + // Max thickness of the drawing for landmark circle. + optional double max_depth_circle_thickness = 11 [default = 18.0]; } diff --git a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc index bf122f0fe..3c22951dc 100644 --- a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc +++ b/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc @@ -244,6 +244,9 @@ void MakeStaticFeatures(const int top_border, const int bottom_border, frame_width_ = cc->Inputs().Tag(kVideoFrame).Get().Width(); frame_height_ = cc->Inputs().Tag(kVideoFrame).Get().Height(); } else if (cc->Inputs().HasTag(kVideoSize)) { + if (cc->Inputs().Tag(kVideoSize).IsEmpty()) { + return ::mediapipe::OkStatus(); + } frame_width_ = cc->Inputs().Tag(kVideoSize).Get>().first; frame_height_ = @@ -302,6 +305,14 @@ void MakeStaticFeatures(const int top_border, const int bottom_border, } if (cc->Inputs().HasTag(kDetections)) { + if (cc->Inputs().Tag(kDetections).IsEmpty()) { + auto default_rect = absl::make_unique(); + default_rect->set_width(frame_width_); + default_rect->set_height(frame_height_); + cc->Outputs().Tag(kCropRect).Add(default_rect.release(), + Timestamp(cc->InputTimestamp())); + return ::mediapipe::OkStatus(); + } auto raw_detections = cc->Inputs().Tag(kDetections).Get>(); for (const auto& detection : raw_detections) { diff --git a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc b/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc index 3477e4cc9..0d2f77993 100644 --- a/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc +++ b/mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc @@ -418,6 +418,25 @@ TEST(ContentZoomingCalculatorTest, ShiftOutsideBounds) { runner->Outputs().Tag("CROP_RECT").packets); } +TEST(ContentZoomingCalculatorTest, EmptySize) { + auto config = ParseTextProtoOrDie(kConfigD); + auto runner = ::absl::make_unique(config); + MP_ASSERT_OK(runner->Run()); + ASSERT_EQ(runner->Outputs().Tag("CROP_RECT").packets.size(), 0); +} + +TEST(ContentZoomingCalculatorTest, EmptyDetections) { + auto config = ParseTextProtoOrDie(kConfigD); + auto runner = ::absl::make_unique(config); + auto input_size = ::absl::make_unique>(1000, 1000); + runner->MutableInputs() + ->Tag("VIDEO_SIZE") + .packets.push_back(Adopt(input_size.release()).At(Timestamp(0))); + MP_ASSERT_OK(runner->Run()); + CheckCropRect(0, 0, 1000, 1000, 0, + runner->Outputs().Tag("CROP_RECT").packets); +} + } // namespace } // namespace autoflip diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index 8b092bcae..266a05956 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -14,6 +14,7 @@ # load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") +load("//mediapipe/framework:mediapipe_register_type.bzl", "mediapipe_register_type") package( default_visibility = ["//visibility:private"], @@ -221,6 +222,19 @@ mediapipe_proto_library( ], ) +mediapipe_register_type( + base_name = "landmark", + include_headers = ["mediapipe/framework/formats/landmark.pb.h"], + types = [ + "::mediapipe::Landmark", + "::mediapipe::NormalizedLandmark", + "::mediapipe::NormalizedLandmarkList", + "::std::vector<::mediapipe::Landmark>", + "::std::vector<::mediapipe::NormalizedLandmark>", + ], + deps = [":landmark_cc_proto"], +) + # Expose the proto source files for building mediapipe AAR. filegroup( name = "protos_src", diff --git a/mediapipe/framework/formats/landmark.proto b/mediapipe/framework/formats/landmark.proto index bc3821609..0b4106ef8 100644 --- a/mediapipe/framework/formats/landmark.proto +++ b/mediapipe/framework/formats/landmark.proto @@ -28,9 +28,17 @@ message Landmark { optional float z = 3; // Landmark visibility. Float score of whether landmark is visible or occluded - // by other obects. Depending on the model, visibility value is either a + // by other objects. Landmark considered as invisible also if it is + // not present on the screen (out of scene bounds). + // Depending on the model, visibility value is either a // sigmoid or an argument of sigmoid. optional float visibility = 4; + + // Landmark presence. Float score of whether landmark is present on the scene + // (located within scene bounds). + // Depending on the model, presence value is either a result of sigmoid + // or an argument of sigmoid function to get landmark presence probability. + optional float presence = 5; } // Group of Landmark protos. @@ -45,6 +53,7 @@ message NormalizedLandmark { optional float y = 2; optional float z = 3; optional float visibility = 4; + optional float presence = 5; } // Group of NormalizedLandmark protos. diff --git a/mediapipe/objc/MPPTimestampConverter.mm b/mediapipe/objc/MPPTimestampConverter.mm index c7b66bd5a..e53758d71 100644 --- a/mediapipe/objc/MPPTimestampConverter.mm +++ b/mediapipe/objc/MPPTimestampConverter.mm @@ -36,7 +36,7 @@ } - (mediapipe::Timestamp)timestampForMediaTime:(CMTime)mediaTime { - float sampleSeconds = CMTIME_IS_VALID(mediaTime) ? CMTimeGetSeconds(mediaTime) : 0; + Float64 sampleSeconds = CMTIME_IS_VALID(mediaTime) ? CMTimeGetSeconds(mediaTime) : 0; const int64 sampleUsec = sampleSeconds * mediapipe::Timestamp::kTimestampUnitsPerSecond; _mediapipeTimestamp = mediapipe::Timestamp(sampleUsec) + _timestampOffset; if (_mediapipeTimestamp <= _lastTimestamp) { diff --git a/mediapipe/python/image_frame_test.py b/mediapipe/python/image_frame_test.py index 800e46172..b099e0402 100644 --- a/mediapipe/python/image_frame_test.py +++ b/mediapipe/python/image_frame_test.py @@ -124,7 +124,7 @@ class ImageFrameTest(absltest.TestCase): cv2.COLOR_RGB2GRAY) image_frame = mp.ImageFrame( image_format=mp.ImageFormat.GRAY8, - data=mat[offset:-offset, offset:-offset]) + data=np.ascontiguousarray(mat[offset:-offset, offset:-offset])) self.assertTrue( np.array_equal(mat[offset:-offset, offset:-offset], image_frame.numpy_view())) @@ -137,7 +137,7 @@ class ImageFrameTest(absltest.TestCase): cv2.COLOR_RGB2BGR) image_frame = mp.ImageFrame( image_format=mp.ImageFormat.SRGB, - data=mat[offset:-offset, offset:-offset, :]) + data=np.ascontiguousarray(mat[offset:-offset, offset:-offset, :])) self.assertTrue( np.array_equal(mat[offset:-offset, offset:-offset, :], image_frame.numpy_view())) diff --git a/mediapipe/util/BUILD b/mediapipe/util/BUILD index f662c5ed4..9347fc009 100644 --- a/mediapipe/util/BUILD +++ b/mediapipe/util/BUILD @@ -12,58 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. # +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") licenses(["notice"]) package(default_visibility = ["//visibility:private"]) -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library") - -proto_library( +mediapipe_proto_library( name = "audio_decoder_proto", srcs = ["audio_decoder.proto"], visibility = ["//visibility:public"], deps = [ + "//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_proto", ], ) -proto_library( +mediapipe_proto_library( name = "color_proto", srcs = ["color.proto"], visibility = ["//visibility:public"], ) -proto_library( +mediapipe_proto_library( name = "render_data_proto", srcs = ["render_data.proto"], visibility = ["//visibility:public"], deps = ["//mediapipe/util:color_proto"], ) -mediapipe_cc_proto_library( - name = "color_cc_proto", - srcs = ["color.proto"], - visibility = ["//visibility:public"], - deps = [":color_proto"], -) - -mediapipe_cc_proto_library( - name = "render_data_cc_proto", - srcs = ["render_data.proto"], - cc_deps = [":color_cc_proto"], - visibility = ["//visibility:public"], - deps = [":render_data_proto"], -) - -mediapipe_cc_proto_library( - name = "audio_decoder_cc_proto", - srcs = ["audio_decoder.proto"], - cc_deps = ["//mediapipe/framework:calculator_cc_proto"], - visibility = ["//visibility:public"], - deps = [":audio_decoder_proto"], -) - cc_library( name = "audio_decoder", srcs = ["audio_decoder.cc"], diff --git a/mediapipe/util/annotation_renderer.cc b/mediapipe/util/annotation_renderer.cc index 52f372e70..7d0673908 100644 --- a/mediapipe/util/annotation_renderer.cc +++ b/mediapipe/util/annotation_renderer.cc @@ -513,6 +513,17 @@ void AnnotationRenderer::DrawText(const RenderAnnotation& annotation) { const int font_face = text.font_face(); const double font_scale = ComputeFontScale(font_face, font_size, thickness); + int text_baseline = 0; + cv::Size text_size = cv::getTextSize(text.display_text(), font_face, + font_scale, thickness, &text_baseline); + + if (text.center_horizontally()) { + origin.x -= text_size.width / 2; + } + if (text.center_vertically()) { + origin.y += text_size.height / 2; + } + cv::putText(mat_image_, text.display_text(), origin, font_face, font_scale, color, thickness, /*lineType=*/8, /*bottomLeftOrigin=*/flip_text_vertically_); diff --git a/mediapipe/util/render_data.proto b/mediapipe/util/render_data.proto index e98df974c..54a2d65b8 100644 --- a/mediapipe/util/render_data.proto +++ b/mediapipe/util/render_data.proto @@ -160,6 +160,12 @@ message RenderAnnotation { // cv::FONT_HERSHEY_SCRIPT_SIMPLEX (6) // cv::FONT_HERSHEY_SCRIPT_COMPLEX (7) optional int32 font_face = 6 [default = 0]; + + // Options to center text around the anchor point (left, baseline) by + // taking into account font shape, size and text length (e.g., + // [left, baseline] represent [center_x, center_y]. + optional bool center_horizontally = 7 [default = false]; + optional bool center_vertically = 8 [default = false]; } // The RenderAnnotation can be one of the below formats.