Project import generated by Copybara.

GitOrigin-RevId: a67c2c19ade0ba122816a250ecee3aab490ef16b
This commit is contained in:
MediaPipe Team 2020-09-28 00:50:43 -04:00 committed by chuoling
parent 8f69af91fe
commit cccf6244d3
26 changed files with 428 additions and 557 deletions

View File

@ -8,15 +8,14 @@ nav_order: 1
--------------------------------------------------------------------------------
## Cross-platform ML solutions made simple
## Live ML anywhere
[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers
and developers to build world-class ML solutions and applications for mobile,
desktop/cloud, web and IoT devices.
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
ML solutions for live and streaming media.
![accelerated.png](docs/images/accelerated_small.png) | ![cross_platform.png](docs/images/cross_platform_small.png)
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
![ready_to_use.png](docs/images/ready_to_use_small.png) | ![open_source.png](docs/images/open_source_small.png)
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
@ -93,7 +92,7 @@ run code search using
## Publications
* [MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
* [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
in Google Developers Blog
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
in Google Developers Blog

View File

@ -8,15 +8,14 @@ nav_order: 1
--------------------------------------------------------------------------------
## Cross-platform ML solutions made simple
## Live ML anywhere
[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers
and developers to build world-class ML solutions and applications for mobile,
desktop/cloud, web and IoT devices.
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
ML solutions for live and streaming media.
![accelerated.png](images/accelerated_small.png) | ![cross_platform.png](images/cross_platform_small.png)
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
![ready_to_use.png](images/ready_to_use_small.png) | ![open_source.png](images/open_source_small.png)
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
@ -93,7 +92,7 @@ run code search using
## Publications
* [MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
* [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
in Google Developers Blog
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
in Google Developers Blog

View File

@ -277,7 +277,7 @@ only works for a single face. For visual reference, please refer to *Fig. 4*.
* TensorFlow Blog:
[Face and hand tracking in the browser with MediaPipe and TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
* Google Developers Blog:
[MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
[MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
* Paper:
[Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs](https://arxiv.org/abs/1907.06724)
([poster](https://docs.google.com/presentation/d/1-LWwOMO9TzEVdrZ1CS1ndJzciRHfYDJfbSxH_ke_JRg/present?slide=id.g5986dd4b4c_4_212))

View File

@ -13,181 +13,120 @@
# limitations under the License.
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"])
package(default_visibility = ["//visibility:private"])
proto_library(
mediapipe_proto_library(
name = "concatenate_vector_calculator_proto",
srcs = ["concatenate_vector_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "dequantize_byte_array_calculator_proto",
srcs = ["dequantize_byte_array_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "packet_cloner_calculator_proto",
srcs = ["packet_cloner_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "packet_resampler_calculator_proto",
srcs = ["packet_resampler_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "packet_thinner_calculator_proto",
srcs = ["packet_thinner_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "split_vector_calculator_proto",
srcs = ["split_vector_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "quantize_float_vector_calculator_proto",
srcs = ["quantize_float_vector_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "sequence_shift_calculator_proto",
srcs = ["sequence_shift_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "gate_calculator_proto",
srcs = ["gate_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "constant_side_packet_calculator_proto",
srcs = ["constant_side_packet_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "clip_vector_size_calculator_proto",
srcs = ["clip_vector_size_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
)
mediapipe_cc_proto_library(
name = "packet_cloner_calculator_cc_proto",
srcs = ["packet_cloner_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":packet_cloner_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "packet_resampler_calculator_cc_proto",
srcs = ["packet_resampler_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":packet_resampler_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "packet_thinner_calculator_cc_proto",
srcs = ["packet_thinner_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":packet_thinner_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "split_vector_calculator_cc_proto",
srcs = ["split_vector_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":split_vector_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "concatenate_vector_calculator_cc_proto",
srcs = ["concatenate_vector_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":concatenate_vector_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "clip_vector_size_calculator_cc_proto",
srcs = ["clip_vector_size_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":clip_vector_size_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "dequantize_byte_array_calculator_cc_proto",
srcs = ["dequantize_byte_array_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":dequantize_byte_array_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "quantize_float_vector_calculator_cc_proto",
srcs = ["quantize_float_vector_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":quantize_float_vector_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "sequence_shift_calculator_cc_proto",
srcs = ["sequence_shift_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":sequence_shift_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "gate_calculator_cc_proto",
srcs = ["gate_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":gate_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "constant_side_packet_calculator_cc_proto",
srcs = ["constant_side_packet_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":constant_side_packet_calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(

View File

@ -1,4 +1,4 @@
// Copyright 2019 The MediaPipe Authors.
// Copyright 2019-2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -59,16 +59,30 @@ std::string ToString(GateState state) {
// ALLOW or DISALLOW can also be specified as an input side packet. The rules
// for evaluation remain the same as above.
//
// If side_input_has_precedence isn't set in the calculator option,
// ALLOW/DISALLOW inputs must be specified either using input stream or
// via input side packet but not both.
// via input side packet but not both. Otherwise, both input stream and input
// side packet can be specified and the calculator will take one signal over the
// other based on the value of the side_input_has_precedence field.
//
// Intended to be used with the default input stream handler, which synchronizes
// all data input streams with the ALLOW/DISALLOW control input stream.
//
// Example config:
// Example configs:
// node {
// calculator: "GateCalculator"
// input_stream: "input_stream0"
// input_stream: "input_stream1"
// input_stream: "input_streamN"
// input_side_packet: "ALLOW:allow" or "DISALLOW:disallow"
// output_stream: "STATE_CHANGE:state_change"
// output_stream: "output_stream0"
// output_stream: "output_stream1"
// output_stream: "output_streamN"
// }
//
// node {
// calculator: "GateCalculator"
// input_stream: "input_stream0"
// input_stream: "input_stream1"
// input_stream: "input_streamN"
@ -78,6 +92,25 @@ std::string ToString(GateState state) {
// output_stream: "output_stream1"
// output_stream: "output_streamN"
// }
//
// With side_input_has_precedence:
// node {
// calculator: "GateCalculator"
// input_stream: "input_stream0"
// input_stream: "input_stream1"
// input_stream: "input_streamN"
// input_stream: "ALLOW:allow_stream" or "DISALLOW:disallow_stream"
// input_side_packet: "ALLOW:allow_packet" or "DISALLOW:disallow_packet"
// output_stream: "STATE_CHANGE:state_change"
// output_stream: "output_stream0"
// output_stream: "output_stream1"
// output_stream: "output_streamN"
// options: {
// [mediapipe.GateCalculatorOptions.ext] {
// side_input_has_precedence: true or false
// }
// }
// }
class GateCalculator : public CalculatorBase {
public:
GateCalculator() {}
@ -88,9 +121,15 @@ class GateCalculator : public CalculatorBase {
cc->InputSidePackets().HasTag("DISALLOW");
bool input_via_stream =
cc->Inputs().HasTag("ALLOW") || cc->Inputs().HasTag("DISALLOW");
// Only one of input_side_packet or input_stream may specify ALLOW/DISALLOW
// input.
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
if (options.has_side_input_has_precedence()) {
RET_CHECK(input_via_side_packet && input_via_stream);
} else {
// Only one of input_side_packet or input_stream may specify
// ALLOW/DISALLOW input when side_input_has_precedence is not set
// in the options.
RET_CHECK(input_via_side_packet ^ input_via_stream);
}
if (input_via_side_packet) {
RET_CHECK(cc->InputSidePackets().HasTag("ALLOW") ^
@ -101,7 +140,8 @@ class GateCalculator : public CalculatorBase {
} else {
cc->InputSidePackets().Tag("DISALLOW").Set<bool>();
}
} else {
}
if (input_via_stream) {
RET_CHECK(cc->Inputs().HasTag("ALLOW") ^ cc->Inputs().HasTag("DISALLOW"));
if (cc->Inputs().HasTag("ALLOW")) {
@ -134,13 +174,13 @@ class GateCalculator : public CalculatorBase {
}
::mediapipe::Status Open(CalculatorContext* cc) final {
use_side_packet_for_allow_disallow_ = false;
bool use_side_packet_for_allow_disallow = false;
if (cc->InputSidePackets().HasTag("ALLOW")) {
use_side_packet_for_allow_disallow_ = true;
use_side_packet_for_allow_disallow = true;
allow_by_side_packet_decision_ =
cc->InputSidePackets().Tag("ALLOW").Get<bool>();
} else if (cc->InputSidePackets().HasTag("DISALLOW")) {
use_side_packet_for_allow_disallow_ = true;
use_side_packet_for_allow_disallow = true;
allow_by_side_packet_decision_ =
!cc->InputSidePackets().Tag("DISALLOW").Get<bool>();
}
@ -152,23 +192,31 @@ class GateCalculator : public CalculatorBase {
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
empty_packets_as_allow_ = options.empty_packets_as_allow();
if (!options.has_side_input_has_precedence()) {
side_input_has_precedence_ = use_side_packet_for_allow_disallow;
} else {
side_input_has_precedence_ = options.side_input_has_precedence();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
bool allow = empty_packets_as_allow_;
if (use_side_packet_for_allow_disallow_) {
allow = allow_by_side_packet_decision_;
} else {
if (cc->Inputs().HasTag("ALLOW") &&
!cc->Inputs().Tag("ALLOW").IsEmpty()) {
allow = cc->Inputs().Tag("ALLOW").Get<bool>();
bool allow_by_stream = empty_packets_as_allow_;
if (cc->Inputs().HasTag("ALLOW") && !cc->Inputs().Tag("ALLOW").IsEmpty()) {
allow_by_stream = cc->Inputs().Tag("ALLOW").Get<bool>();
}
if (cc->Inputs().HasTag("DISALLOW") &&
!cc->Inputs().Tag("DISALLOW").IsEmpty()) {
allow = !cc->Inputs().Tag("DISALLOW").Get<bool>();
allow_by_stream = !cc->Inputs().Tag("DISALLOW").Get<bool>();
}
const bool allow_by_side_packet =
allow_by_side_packet_decision_ || empty_packets_as_allow_;
bool allow = false;
if (side_input_has_precedence_) {
allow = allow_by_side_packet;
} else {
allow = allow_by_stream;
}
const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
@ -203,9 +251,9 @@ class GateCalculator : public CalculatorBase {
private:
GateState last_gate_state_ = GATE_UNINITIALIZED;
int num_data_streams_;
bool allow_by_side_packet_decision_ = false;
bool empty_packets_as_allow_;
bool use_side_packet_for_allow_disallow_;
bool allow_by_side_packet_decision_;
bool side_input_has_precedence_;
};
REGISTER_CALCULATOR(GateCalculator);

View File

@ -1,4 +1,4 @@
// Copyright 2019 The MediaPipe Authors.
// Copyright 2019-2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -27,4 +27,12 @@ message GateCalculatorOptions {
// disallowing the corresponding packets in the data input streams. Setting
// this option to true inverts that, allowing the data packets to go through.
optional bool empty_packets_as_allow = 1;
// Input side packet and input stream are allowed to coexist only if this
// field is set. When it's set to true, the input side packet has higher
// precedence and the input stream signal will be ignored. When it's set to
// false, the input stream signal always overrides the input side packet
// signal.
//
optional bool side_input_has_precedence = 2;
}

View File

@ -1,4 +1,4 @@
// Copyright 2019 The MediaPipe Authors.
// Copyright 2019-2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -330,5 +330,49 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
ASSERT_EQ(0, output.size());
}
TEST_F(GateCalculatorTest, TestOverrideDecisionBySidePacketSignal) {
SetRunner(R"(
calculator: "GateCalculator"
input_stream: "test_input"
input_stream: "ALLOW:gating_stream"
input_side_packet: "ALLOW:gating_packet"
output_stream: "test_output"
options: {
[mediapipe.GateCalculatorOptions.ext] {
side_input_has_precedence: true
}
}
)");
constexpr int64 kTimestampValue0 = 42;
runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false));
RunTimeStep(kTimestampValue0, "ALLOW", true);
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
ASSERT_EQ(0, output.size());
}
TEST_F(GateCalculatorTest, TestOverrideDecisionByStreamSignal) {
SetRunner(R"(
calculator: "GateCalculator"
input_stream: "test_input"
input_stream: "ALLOW:gating_stream"
input_side_packet: "ALLOW:gating_packet"
output_stream: "test_output"
options: {
[mediapipe.GateCalculatorOptions.ext] {
side_input_has_precedence: false
}
}
)");
constexpr int64 kTimestampValue0 = 42;
runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false));
RunTimeStep(kTimestampValue0, "ALLOW", true);
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
ASSERT_EQ(1, output.size());
}
} // namespace
} // namespace mediapipe

View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
licenses(["notice"])
@ -279,27 +279,17 @@ cc_library(
alwayslink = 1,
)
proto_library(
mediapipe_proto_library(
name = "image_transformation_calculator_proto",
srcs = ["image_transformation_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/gpu:scale_mode_proto",
],
)
mediapipe_cc_proto_library(
name = "image_transformation_calculator_cc_proto",
srcs = ["image_transformation_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/gpu:scale_mode_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":image_transformation_calculator_proto"],
)
cc_library(
name = "image_transformation_calculator",
srcs = ["image_transformation_calculator.cc"],

View File

@ -13,131 +13,91 @@
# limitations under the License.
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
load("@bazel_skylib//lib:selects.bzl", "selects")
licenses(["notice"])
package(default_visibility = ["//visibility:private"])
proto_library(
mediapipe_proto_library(
name = "ssd_anchors_calculator_proto",
srcs = ["ssd_anchors_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_custom_op_resolver_calculator_proto",
srcs = ["tflite_custom_op_resolver_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_inference_calculator_proto",
srcs = ["tflite_inference_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_converter_calculator_proto",
srcs = ["tflite_converter_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_tensors_to_segmentation_calculator_proto",
srcs = ["tflite_tensors_to_segmentation_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_tensors_to_detections_calculator_proto",
srcs = ["tflite_tensors_to_detections_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_tensors_to_classification_calculator_proto",
srcs = ["tflite_tensors_to_classification_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_tensors_to_landmarks_calculator_proto",
srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
)
mediapipe_cc_proto_library(
name = "ssd_anchors_calculator_cc_proto",
srcs = ["ssd_anchors_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":ssd_anchors_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_custom_op_resolver_calculator_cc_proto",
srcs = ["tflite_custom_op_resolver_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_custom_op_resolver_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_converter_calculator_cc_proto",
srcs = ["tflite_converter_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_converter_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_tensors_to_segmentation_calculator_cc_proto",
srcs = ["tflite_tensors_to_segmentation_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_tensors_to_segmentation_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_inference_calculator_cc_proto",
srcs = ["tflite_inference_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_inference_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_tensors_to_detections_calculator_cc_proto",
srcs = ["tflite_tensors_to_detections_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_tensors_to_detections_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_tensors_to_classification_calculator_cc_proto",
srcs = ["tflite_tensors_to_classification_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_tensors_to_classification_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_tensors_to_landmarks_calculator_cc_proto",
srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_tensors_to_landmarks_calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(

View File

@ -21,9 +21,9 @@
namespace mediapipe {
// A calculator for converting TFLite tensors from regression models into
// landmarks. Note that if the landmarks in the tensor has more than 4
// dimensions, only the first 4 dimensions will be converted to
// [x,y,z, visibility].
// landmarks. Note that if the landmarks in the tensor has more than 5
// dimensions, only the first 5 dimensions will be converted to
// [x,y,z, visibility, presence].
//
// Input:
// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first
@ -209,6 +209,9 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
if (num_dimensions > 3) {
landmark->set_visibility(raw_landmarks[offset + 3]);
}
if (num_dimensions > 4) {
landmark->set_presence(raw_landmarks[offset + 4]);
}
}
// Output normalized landmarks if required.
@ -223,6 +226,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
norm_landmark->set_z(landmark.z() / options_.input_image_width() /
options_.normalize_z());
norm_landmark->set_visibility(landmark.visibility());
norm_landmark->set_presence(landmark.presence());
}
cc->Outputs()
.Tag("NORM_LANDMARKS")

View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"])
@ -38,186 +38,95 @@ cc_library(
alwayslink = 1,
)
proto_library(
mediapipe_proto_library(
name = "annotation_overlay_calculator_proto",
srcs = ["annotation_overlay_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "detection_label_id_to_text_calculator_proto",
srcs = ["detection_label_id_to_text_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "timed_box_list_id_to_label_calculator_proto",
srcs = ["timed_box_list_id_to_label_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "latency_proto",
srcs = ["latency.proto"],
)
proto_library(
mediapipe_proto_library(
name = "non_max_suppression_calculator_proto",
srcs = ["non_max_suppression_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "packet_frequency_proto",
srcs = ["packet_frequency.proto"],
)
proto_library(
mediapipe_proto_library(
name = "packet_frequency_calculator_proto",
srcs = ["packet_frequency_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "packet_latency_calculator_proto",
srcs = ["packet_latency_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "collection_has_min_size_calculator_proto",
srcs = ["collection_has_min_size_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "association_calculator_proto",
srcs = ["association_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "annotation_overlay_calculator_cc_proto",
srcs = ["annotation_overlay_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":annotation_overlay_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "detection_label_id_to_text_calculator_cc_proto",
srcs = ["detection_label_id_to_text_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [
":detection_label_id_to_text_calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "timed_box_list_id_to_label_calculator_cc_proto",
srcs = ["timed_box_list_id_to_label_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [
":timed_box_list_id_to_label_calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "latency_cc_proto",
srcs = ["latency.proto"],
visibility = ["//mediapipe:__subpackages__"],
deps = [":latency_proto"],
)
mediapipe_cc_proto_library(
name = "non_max_suppression_calculator_cc_proto",
srcs = ["non_max_suppression_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":non_max_suppression_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "packet_frequency_cc_proto",
srcs = ["packet_frequency.proto"],
visibility = ["//mediapipe:__subpackages__"],
deps = [":packet_frequency_proto"],
)
mediapipe_cc_proto_library(
name = "packet_frequency_calculator_cc_proto",
srcs = ["packet_frequency_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe:__subpackages__"],
deps = [
":packet_frequency_calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "packet_latency_calculator_cc_proto",
srcs = ["packet_latency_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe:__subpackages__"],
deps = [
":packet_latency_calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "collection_has_min_size_calculator_cc_proto",
srcs = ["collection_has_min_size_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe:__subpackages__"],
deps = [":collection_has_min_size_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "association_calculator_cc_proto",
srcs = ["association_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe:__subpackages__"],
deps = [":association_calculator_proto"],
)
cc_library(
name = "packet_frequency_calculator",
srcs = ["packet_frequency_calculator.cc"],
@ -434,16 +343,6 @@ cc_library(
alwayslink = 1,
)
mediapipe_cc_proto_library(
name = "thresholding_calculator_cc_proto",
srcs = ["thresholding_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":thresholding_calculator_proto"],
)
cc_library(
name = "thresholding_calculator",
srcs = ["thresholding_calculator.cc"],
@ -457,16 +356,6 @@ cc_library(
alwayslink = 1,
)
mediapipe_cc_proto_library(
name = "landmarks_to_detection_calculator_cc_proto",
srcs = ["landmarks_to_detection_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":landmarks_to_detection_calculator_proto"],
)
cc_library(
name = "landmarks_to_detection_calculator",
srcs = ["landmarks_to_detection_calculator.cc"],
@ -483,46 +372,6 @@ cc_library(
alwayslink = 1,
)
mediapipe_cc_proto_library(
name = "detections_to_rects_calculator_cc_proto",
srcs = ["detections_to_rects_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":detections_to_rects_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "landmark_projection_calculator_cc_proto",
srcs = ["landmark_projection_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":landmark_projection_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "landmarks_to_floats_calculator_cc_proto",
srcs = ["landmarks_to_floats_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":landmarks_to_floats_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "rect_transformation_calculator_cc_proto",
srcs = ["rect_transformation_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":rect_transformation_calculator_proto"],
)
cc_library(
name = "detections_to_rects_calculator",
srcs = [
@ -593,162 +442,140 @@ cc_test(
],
)
proto_library(
mediapipe_proto_library(
name = "rect_to_render_data_calculator_proto",
srcs = ["rect_to_render_data_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "rect_to_render_scale_calculator_proto",
srcs = ["rect_to_render_scale_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "detections_to_render_data_calculator_proto",
srcs = ["detections_to_render_data_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "landmarks_to_render_data_calculator_proto",
srcs = ["landmarks_to_render_data_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "timed_box_list_to_render_data_calculator_proto",
srcs = ["timed_box_list_to_render_data_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "labels_to_render_data_calculator_proto",
srcs = ["labels_to_render_data_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "thresholding_calculator_proto",
srcs = ["thresholding_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "detections_to_rects_calculator_proto",
srcs = ["detections_to_rects_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "landmark_projection_calculator_proto",
srcs = ["landmark_projection_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "landmarks_to_floats_calculator_proto",
srcs = ["landmarks_to_floats_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "rect_transformation_calculator_proto",
srcs = ["rect_transformation_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "landmarks_to_detection_calculator_proto",
srcs = ["landmarks_to_detection_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
mediapipe_cc_proto_library(
name = "rect_to_render_data_calculator_cc_proto",
srcs = ["rect_to_render_data_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":rect_to_render_data_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "rect_to_render_scale_calculator_cc_proto",
srcs = ["rect_to_render_scale_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":rect_to_render_scale_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "detections_to_render_data_calculator_cc_proto",
srcs = ["detections_to_render_data_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":detections_to_render_data_calculator_proto"],
)
cc_library(
name = "detections_to_render_data_calculator",
srcs = ["detections_to_render_data_calculator.cc"],
@ -768,18 +595,6 @@ cc_library(
alwayslink = 1,
)
mediapipe_cc_proto_library(
name = "landmarks_to_render_data_calculator_cc_proto",
srcs = ["landmarks_to_render_data_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":landmarks_to_render_data_calculator_proto"],
)
cc_library(
name = "landmarks_to_render_data_calculator",
srcs = ["landmarks_to_render_data_calculator.cc"],
@ -800,18 +615,6 @@ cc_library(
alwayslink = 1,
)
mediapipe_cc_proto_library(
name = "timed_box_list_to_render_data_calculator_cc_proto",
srcs = ["timed_box_list_to_render_data_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":timed_box_list_to_render_data_calculator_proto"],
)
cc_library(
name = "timed_box_list_to_render_data_calculator",
srcs = ["timed_box_list_to_render_data_calculator.cc"],
@ -951,6 +754,7 @@ mediapipe_proto_library(
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
@ -1017,25 +821,16 @@ cc_test(
],
)
proto_library(
mediapipe_proto_library(
name = "top_k_scores_calculator_proto",
srcs = ["top_k_scores_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "top_k_scores_calculator_cc_proto",
srcs = ["top_k_scores_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":top_k_scores_calculator_proto"],
)
cc_library(
name = "top_k_scores_calculator",
srcs = ["top_k_scores_calculator.cc"],
@ -1081,17 +876,6 @@ cc_test(
],
)
mediapipe_cc_proto_library(
name = "labels_to_render_data_calculator_cc_proto",
srcs = ["labels_to_render_data_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":labels_to_render_data_calculator_proto"],
)
cc_library(
name = "local_file_contents_calculator",
srcs = ["local_file_contents_calculator.cc"],

View File

@ -476,6 +476,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
options_.canvas_height_px(), options_.canvas_width_px(), CV_8UC3,
cv::Scalar(options_.canvas_color().r(), options_.canvas_color().g(),
options_.canvas_color().b()));
*target_format = ImageFormat::SRGB;
}
return ::mediapipe::OkStatus();

View File

@ -131,6 +131,8 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
new_landmark->set_z(new_z);
// Keep visibility as is.
new_landmark->set_visibility(landmark.visibility());
// Keep presence as is.
new_landmark->set_presence(landmark.presence());
}
cc->Outputs().Get(output_id).AddPacket(

View File

@ -131,6 +131,8 @@ class LandmarkProjectionCalculator : public CalculatorBase {
new_landmark->set_z(new_z);
// Keep visibility as is.
new_landmark->set_visibility(landmark.visibility());
// Keep presence as is.
new_landmark->set_presence(landmark.presence());
}
cc->Outputs().Get(output_id).AddPacket(

View File

@ -137,6 +137,8 @@ class VelocityFilter : public LandmarksFilter {
image_width);
// Keep visibility as is.
out_landmark->set_visibility(in_landmark.visibility());
// Keep presence as is.
out_landmark->set_presence(in_landmark.presence());
}
return ::mediapipe::OkStatus();

View File

@ -33,7 +33,6 @@ constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS";
constexpr char kRenderScaleTag[] = "RENDER_SCALE";
constexpr char kRenderDataTag[] = "RENDER_DATA";
constexpr char kLandmarkLabel[] = "KEYPOINT";
constexpr int kMaxLandmarkThickness = 18;
inline void SetColor(RenderAnnotation* annotation, const Color& color) {
annotation->mutable_color()->set_r(color.r());
@ -59,15 +58,18 @@ inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min,
}
void SetColorSizeValueFromZ(float z, float z_min, float z_max,
RenderAnnotation* render_annotation) {
RenderAnnotation* render_annotation,
float min_depth_circle_thickness,
float max_depth_circle_thickness) {
const int color_value = 255 - static_cast<int>(Remap(z, z_min, z_max, 255));
::mediapipe::Color color;
color.set_r(color_value);
color.set_g(color_value);
color.set_b(color_value);
SetColor(render_annotation, color);
const int thickness = static_cast<int>((1.f - Remap(z, z_min, z_max, 1)) *
kMaxLandmarkThickness);
const float scale = max_depth_circle_thickness - min_depth_circle_thickness;
const int thickness = static_cast<int>(
min_depth_circle_thickness + (1.f - Remap(z, z_min, z_max, 1)) * scale);
render_annotation->set_thickness(thickness);
}
@ -97,7 +99,8 @@ template <class LandmarkListType, class LandmarkType>
void AddConnectionsWithDepth(const LandmarkListType& landmarks,
const std::vector<int>& landmark_connections,
bool utilize_visibility,
float visibility_threshold, float thickness,
float visibility_threshold, bool utilize_presence,
float presence_threshold, float thickness,
bool normalized, float min_z, float max_z,
RenderData* render_data) {
for (int i = 0; i < landmark_connections.size(); i += 2) {
@ -107,6 +110,10 @@ void AddConnectionsWithDepth(const LandmarkListType& landmarks,
ld1.visibility() < visibility_threshold)) {
continue;
}
if (utilize_presence && (ld0.presence() < presence_threshold ||
ld1.presence() < presence_threshold)) {
continue;
}
const int gray_val1 =
255 - static_cast<int>(Remap(ld0.z(), min_z, max_z, 255));
const int gray_val2 =
@ -136,6 +143,7 @@ template <class LandmarkListType, class LandmarkType>
void AddConnections(const LandmarkListType& landmarks,
const std::vector<int>& landmark_connections,
bool utilize_visibility, float visibility_threshold,
bool utilize_presence, float presence_threshold,
const Color& connection_color, float thickness,
bool normalized, RenderData* render_data) {
for (int i = 0; i < landmark_connections.size(); i += 2) {
@ -145,6 +153,10 @@ void AddConnections(const LandmarkListType& landmarks,
ld1.visibility() < visibility_threshold)) {
continue;
}
if (utilize_presence && (ld0.presence() < presence_threshold ||
ld1.presence() < presence_threshold)) {
continue;
}
AddConnectionToRenderData<LandmarkType>(ld0, ld1, connection_color,
thickness, normalized, render_data);
}
@ -238,13 +250,15 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
if (visualize_depth) {
AddConnectionsWithDepth<LandmarkList, Landmark>(
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), thickness, /*normalized=*/false,
z_min, z_max, render_data.get());
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold(), thickness, /*normalized=*/false, z_min,
z_max, render_data.get());
} else {
AddConnections<LandmarkList, Landmark>(
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), options_.connection_color(),
thickness, /*normalized=*/false, render_data.get());
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold(), options_.connection_color(), thickness,
/*normalized=*/false, render_data.get());
}
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const Landmark& landmark = landmarks.landmark(i);
@ -254,11 +268,17 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
continue;
}
if (options_.utilize_presence() &&
landmark.presence() < options_.presence_threshold()) {
continue;
}
auto* landmark_data_render = AddPointRenderData(
options_.landmark_color(), thickness, render_data.get());
if (visualize_depth) {
SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
landmark_data_render);
SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
options_.min_depth_circle_thickness(),
options_.max_depth_circle_thickness());
}
auto* landmark_data = landmark_data_render->mutable_point();
landmark_data->set_normalized(false);
@ -279,13 +299,15 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
if (visualize_depth) {
AddConnectionsWithDepth<NormalizedLandmarkList, NormalizedLandmark>(
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), thickness, /*normalized=*/true,
z_min, z_max, render_data.get());
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold(), thickness, /*normalized=*/true, z_min,
z_max, render_data.get());
} else {
AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), options_.connection_color(),
thickness, /*normalized=*/true, render_data.get());
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold(), options_.connection_color(), thickness,
/*normalized=*/true, render_data.get());
}
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = landmarks.landmark(i);
@ -294,12 +316,17 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
landmark.visibility() < options_.visibility_threshold()) {
continue;
}
if (options_.utilize_presence() &&
landmark.presence() < options_.presence_threshold()) {
continue;
}
auto* landmark_data_render = AddPointRenderData(
options_.landmark_color(), thickness, render_data.get());
if (visualize_depth) {
SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
landmark_data_render);
SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
options_.min_depth_circle_thickness(),
options_.max_depth_circle_thickness());
}
auto* landmark_data = landmark_data_render->mutable_point();
landmark_data->set_normalized(true);

View File

@ -49,4 +49,19 @@ message LandmarksToRenderDataCalculatorOptions {
// Threshold to determine visibility of the landmark. Landmark with visibility
// greater or equal than threshold is considered visible.
optional double visibility_threshold = 7 [default = 0.0];
// Use landmarks presence while rendering landmarks and connections. If
// landmark is not present, neither it nor adjacent connections will be
// rendered.
optional bool utilize_presence = 8 [default = false];
// Threshold to determine presence of the landmark. Landmark with presence
// greater or equal than threshold is considered present.
optional double presence_threshold = 9 [default = 0.0];
// Min thickness of the drawing for landmark circle.
optional double min_depth_circle_thickness = 10 [default = 0.0];
// Max thickness of the drawing for landmark circle.
optional double max_depth_circle_thickness = 11 [default = 18.0];
}

View File

@ -244,6 +244,9 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
frame_width_ = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Width();
frame_height_ = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Height();
} else if (cc->Inputs().HasTag(kVideoSize)) {
if (cc->Inputs().Tag(kVideoSize).IsEmpty()) {
return ::mediapipe::OkStatus();
}
frame_width_ =
cc->Inputs().Tag(kVideoSize).Get<std::pair<int, int>>().first;
frame_height_ =
@ -302,6 +305,14 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
}
if (cc->Inputs().HasTag(kDetections)) {
if (cc->Inputs().Tag(kDetections).IsEmpty()) {
auto default_rect = absl::make_unique<mediapipe::Rect>();
default_rect->set_width(frame_width_);
default_rect->set_height(frame_height_);
cc->Outputs().Tag(kCropRect).Add(default_rect.release(),
Timestamp(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}
auto raw_detections =
cc->Inputs().Tag(kDetections).Get<std::vector<mediapipe::Detection>>();
for (const auto& detection : raw_detections) {

View File

@ -418,6 +418,25 @@ TEST(ContentZoomingCalculatorTest, ShiftOutsideBounds) {
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, EmptySize) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
MP_ASSERT_OK(runner->Run());
ASSERT_EQ(runner->Outputs().Tag("CROP_RECT").packets.size(), 0);
}
TEST(ContentZoomingCalculatorTest, EmptyDetections) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
auto input_size = ::absl::make_unique<std::pair<int, int>>(1000, 1000);
runner->MutableInputs()
->Tag("VIDEO_SIZE")
.packets.push_back(Adopt(input_size.release()).At(Timestamp(0)));
MP_ASSERT_OK(runner->Run());
CheckCropRect(0, 0, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
} // namespace
} // namespace autoflip

View File

@ -14,6 +14,7 @@
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
load("//mediapipe/framework:mediapipe_register_type.bzl", "mediapipe_register_type")
package(
default_visibility = ["//visibility:private"],
@ -221,6 +222,19 @@ mediapipe_proto_library(
],
)
mediapipe_register_type(
base_name = "landmark",
include_headers = ["mediapipe/framework/formats/landmark.pb.h"],
types = [
"::mediapipe::Landmark",
"::mediapipe::NormalizedLandmark",
"::mediapipe::NormalizedLandmarkList",
"::std::vector<::mediapipe::Landmark>",
"::std::vector<::mediapipe::NormalizedLandmark>",
],
deps = [":landmark_cc_proto"],
)
# Expose the proto source files for building mediapipe AAR.
filegroup(
name = "protos_src",

View File

@ -28,9 +28,17 @@ message Landmark {
optional float z = 3;
// Landmark visibility. Float score of whether landmark is visible or occluded
// by other obects. Depending on the model, visibility value is either a
// by other objects. Landmark considered as invisible also if it is
// not present on the screen (out of scene bounds).
// Depending on the model, visibility value is either a
// sigmoid or an argument of sigmoid.
optional float visibility = 4;
// Landmark presence. Float score of whether landmark is present on the scene
// (located within scene bounds).
// Depending on the model, presence value is either a result of sigmoid
// or an argument of sigmoid function to get landmark presence probability.
optional float presence = 5;
}
// Group of Landmark protos.
@ -45,6 +53,7 @@ message NormalizedLandmark {
optional float y = 2;
optional float z = 3;
optional float visibility = 4;
optional float presence = 5;
}
// Group of NormalizedLandmark protos.

View File

@ -36,7 +36,7 @@
}
- (mediapipe::Timestamp)timestampForMediaTime:(CMTime)mediaTime {
float sampleSeconds = CMTIME_IS_VALID(mediaTime) ? CMTimeGetSeconds(mediaTime) : 0;
Float64 sampleSeconds = CMTIME_IS_VALID(mediaTime) ? CMTimeGetSeconds(mediaTime) : 0;
const int64 sampleUsec = sampleSeconds * mediapipe::Timestamp::kTimestampUnitsPerSecond;
_mediapipeTimestamp = mediapipe::Timestamp(sampleUsec) + _timestampOffset;
if (_mediapipeTimestamp <= _lastTimestamp) {

View File

@ -124,7 +124,7 @@ class ImageFrameTest(absltest.TestCase):
cv2.COLOR_RGB2GRAY)
image_frame = mp.ImageFrame(
image_format=mp.ImageFormat.GRAY8,
data=mat[offset:-offset, offset:-offset])
data=np.ascontiguousarray(mat[offset:-offset, offset:-offset]))
self.assertTrue(
np.array_equal(mat[offset:-offset, offset:-offset],
image_frame.numpy_view()))
@ -137,7 +137,7 @@ class ImageFrameTest(absltest.TestCase):
cv2.COLOR_RGB2BGR)
image_frame = mp.ImageFrame(
image_format=mp.ImageFormat.SRGB,
data=mat[offset:-offset, offset:-offset, :])
data=np.ascontiguousarray(mat[offset:-offset, offset:-offset, :]))
self.assertTrue(
np.array_equal(mat[offset:-offset, offset:-offset, :],
image_frame.numpy_view()))

View File

@ -12,58 +12,35 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"])
package(default_visibility = ["//visibility:private"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
proto_library(
mediapipe_proto_library(
name = "audio_decoder_proto",
srcs = ["audio_decoder.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "color_proto",
srcs = ["color.proto"],
visibility = ["//visibility:public"],
)
proto_library(
mediapipe_proto_library(
name = "render_data_proto",
srcs = ["render_data.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/util:color_proto"],
)
mediapipe_cc_proto_library(
name = "color_cc_proto",
srcs = ["color.proto"],
visibility = ["//visibility:public"],
deps = [":color_proto"],
)
mediapipe_cc_proto_library(
name = "render_data_cc_proto",
srcs = ["render_data.proto"],
cc_deps = [":color_cc_proto"],
visibility = ["//visibility:public"],
deps = [":render_data_proto"],
)
mediapipe_cc_proto_library(
name = "audio_decoder_cc_proto",
srcs = ["audio_decoder.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":audio_decoder_proto"],
)
cc_library(
name = "audio_decoder",
srcs = ["audio_decoder.cc"],

View File

@ -513,6 +513,17 @@ void AnnotationRenderer::DrawText(const RenderAnnotation& annotation) {
const int font_face = text.font_face();
const double font_scale = ComputeFontScale(font_face, font_size, thickness);
int text_baseline = 0;
cv::Size text_size = cv::getTextSize(text.display_text(), font_face,
font_scale, thickness, &text_baseline);
if (text.center_horizontally()) {
origin.x -= text_size.width / 2;
}
if (text.center_vertically()) {
origin.y += text_size.height / 2;
}
cv::putText(mat_image_, text.display_text(), origin, font_face, font_scale,
color, thickness, /*lineType=*/8,
/*bottomLeftOrigin=*/flip_text_vertically_);

View File

@ -160,6 +160,12 @@ message RenderAnnotation {
// cv::FONT_HERSHEY_SCRIPT_SIMPLEX (6)
// cv::FONT_HERSHEY_SCRIPT_COMPLEX (7)
optional int32 font_face = 6 [default = 0];
// Options to center text around the anchor point (left, baseline) by
// taking into account font shape, size and text length (e.g.,
// [left, baseline] represent [center_x, center_y].
optional bool center_horizontally = 7 [default = false];
optional bool center_vertically = 8 [default = false];
}
// The RenderAnnotation can be one of the below formats.