cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Project import generated by Copybara.

Browse Source 
GitOrigin-RevId: a67c2c19ade0ba122816a250ecee3aab490ef16b
This commit is contained in:
MediaPipe Team 2020-09-28 00:50:43 -04:00 committed by chuoling
parent 8f69af91fe
commit cccf6244d3
26 changed files with 428 additions and 557 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@ -8,15 +8,14 @@ nav_order: 1
--------------------------------------------------------------------------------
## Cross-platform ML solutions made simple
## Live ML anywhere
[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers
and developers to build world-class ML solutions and applications for mobile,
desktop/cloud, web and IoT devices.
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
ML solutions for live and streaming media.
![accelerated.png](docs/images/accelerated_small.png) | ![cross_platform.png](docs/images/cross_platform_small.png)
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
![ready_to_use.png](docs/images/ready_to_use_small.png) | ![open_source.png](docs/images/open_source_small.png)
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
@ -93,7 +92,7 @@ run code search using
## Publications
* [MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
* [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
in Google Developers Blog
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
in Google Developers Blog

11
docs/index.md
View File

@ -8,15 +8,14 @@ nav_order: 1
--------------------------------------------------------------------------------
## Cross-platform ML solutions made simple
## Live ML anywhere
[MediaPipe](https://google.github.io/mediapipe/) is the simplest way for researchers
and developers to build world-class ML solutions and applications for mobile,
desktop/cloud, web and IoT devices.
[MediaPipe](https://google.github.io/mediapipe/) offers cross-platform, customizable
ML solutions for live and streaming media.
![accelerated.png](images/accelerated_small.png) | ![cross_platform.png](images/cross_platform_small.png)
:------------------------------------------------------------------------------------------------------------: | :----------------------------------------------------:
***End-to-End acceleration***: *built-in fast ML inference and processing accelerated even on common hardware* | ***Build one, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
***End-to-End acceleration***: *Built-in fast ML inference and processing accelerated even on common hardware* | ***Build once, deploy anywhere***: *Unified solution works across Android, iOS, desktop/cloud, web and IoT*
![ready_to_use.png](images/ready_to_use_small.png) | ![open_source.png](images/open_source_small.png)
***Ready-to-use solutions***: *Cutting-edge ML solutions demonstrating full power of the framework* | ***Free and open source***: *Framework and solutions both under Apache 2.0, fully extensible and customizable*
@ -93,7 +92,7 @@ run code search using
## Publications
* [MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
* [MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
in Google Developers Blog
* [Instant Motion Tracking With MediaPipe](https://developers.googleblog.com/2020/08/instant-motion-tracking-with-mediapipe.html)
in Google Developers Blog

2
docs/solutions/face_mesh.md
View File

@ -277,7 +277,7 @@ only works for a single face. For visual reference, please refer to *Fig. 4*.
* TensorFlow Blog:
[Face and hand tracking in the browser with MediaPipe and TensorFlow.js](https://blog.tensorflow.org/2020/03/face-and-hand-tracking-in-browser-with-mediapipe-and-tensorflowjs.html)
* Google Developers Blog:
[MediaPipe 3D Face Transform](https://mediapipe.page.link/face-geometry-blog)
[MediaPipe 3D Face Transform](https://developers.googleblog.com/2020/09/mediapipe-3d-face-transform.html)
* Paper:
[Real-time Facial Surface Geometry from Monocular Video on Mobile GPUs](https://arxiv.org/abs/1907.06724)
([poster](https://docs.google.com/presentation/d/1-LWwOMO9TzEVdrZ1CS1ndJzciRHfYDJfbSxH_ke_JRg/present?slide=id.g5986dd4b4c_4_212))

155
mediapipe/calculators/core/BUILD
View File

@ -13,181 +13,120 @@
# limitations under the License.
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"])
package(default_visibility = ["//visibility:private"])
proto_library(
mediapipe_proto_library(
name = "concatenate_vector_calculator_proto",
srcs = ["concatenate_vector_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "dequantize_byte_array_calculator_proto",
srcs = ["dequantize_byte_array_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "packet_cloner_calculator_proto",
srcs = ["packet_cloner_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "packet_resampler_calculator_proto",
srcs = ["packet_resampler_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "packet_thinner_calculator_proto",
srcs = ["packet_thinner_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "split_vector_calculator_proto",
srcs = ["split_vector_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "quantize_float_vector_calculator_proto",
srcs = ["quantize_float_vector_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "sequence_shift_calculator_proto",
srcs = ["sequence_shift_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "gate_calculator_proto",
srcs = ["gate_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "constant_side_packet_calculator_proto",
srcs = ["constant_side_packet_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "clip_vector_size_calculator_proto",
srcs = ["clip_vector_size_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
)
mediapipe_cc_proto_library(
name = "packet_cloner_calculator_cc_proto",
srcs = ["packet_cloner_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":packet_cloner_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "packet_resampler_calculator_cc_proto",
srcs = ["packet_resampler_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":packet_resampler_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "packet_thinner_calculator_cc_proto",
srcs = ["packet_thinner_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":packet_thinner_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "split_vector_calculator_cc_proto",
srcs = ["split_vector_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":split_vector_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "concatenate_vector_calculator_cc_proto",
srcs = ["concatenate_vector_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":concatenate_vector_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "clip_vector_size_calculator_cc_proto",
srcs = ["clip_vector_size_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":clip_vector_size_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "dequantize_byte_array_calculator_cc_proto",
srcs = ["dequantize_byte_array_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":dequantize_byte_array_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "quantize_float_vector_calculator_cc_proto",
srcs = ["quantize_float_vector_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":quantize_float_vector_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "sequence_shift_calculator_cc_proto",
srcs = ["sequence_shift_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":sequence_shift_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "gate_calculator_cc_proto",
srcs = ["gate_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":gate_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "constant_side_packet_calculator_cc_proto",
srcs = ["constant_side_packet_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":constant_side_packet_calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(

94
mediapipe/calculators/core/gate_calculator.cc
View File

@ -1,4 +1,4 @@
// Copyright 2019 The MediaPipe Authors.
// Copyright 2019-2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -59,16 +59,30 @@ std::string ToString(GateState state) {
// ALLOW or DISALLOW can also be specified as an input side packet. The rules
// for evaluation remain the same as above.
//
// If side_input_has_precedence isn't set in the calculator option,
// ALLOW/DISALLOW inputs must be specified either using input stream or
// via input side packet but not both.
// via input side packet but not both. Otherwise, both input stream and input
// side packet can be specified and the calculator will take one signal over the
// other based on the value of the side_input_has_precedence field.
//
// Intended to be used with the default input stream handler, which synchronizes
// all data input streams with the ALLOW/DISALLOW control input stream.
//
// Example config:
// Example configs:
// node {
// calculator: "GateCalculator"
// input_stream: "input_stream0"
// input_stream: "input_stream1"
// input_stream: "input_streamN"
// input_side_packet: "ALLOW:allow" or "DISALLOW:disallow"
// output_stream: "STATE_CHANGE:state_change"
// output_stream: "output_stream0"
// output_stream: "output_stream1"
// output_stream: "output_streamN"
// }
//
// node {
// calculator: "GateCalculator"
// input_stream: "input_stream0"
// input_stream: "input_stream1"
// input_stream: "input_streamN"
@ -78,6 +92,25 @@ std::string ToString(GateState state) {
// output_stream: "output_stream1"
// output_stream: "output_streamN"
// }
//
// With side_input_has_precedence:
// node {
// calculator: "GateCalculator"
// input_stream: "input_stream0"
// input_stream: "input_stream1"
// input_stream: "input_streamN"
// input_stream: "ALLOW:allow_stream" or "DISALLOW:disallow_stream"
// input_side_packet: "ALLOW:allow_packet" or "DISALLOW:disallow_packet"
// output_stream: "STATE_CHANGE:state_change"
// output_stream: "output_stream0"
// output_stream: "output_stream1"
// output_stream: "output_streamN"
// options: {
// [mediapipe.GateCalculatorOptions.ext] {
// side_input_has_precedence: true or false
// }
// }
// }
class GateCalculator : public CalculatorBase {
public:
GateCalculator() {}
@ -88,9 +121,15 @@ class GateCalculator : public CalculatorBase {
cc->InputSidePackets().HasTag("DISALLOW");
bool input_via_stream =
cc->Inputs().HasTag("ALLOW") || cc->Inputs().HasTag("DISALLOW");
// Only one of input_side_packet or input_stream may specify ALLOW/DISALLOW
// input.
RET_CHECK(input_via_side_packet ^ input_via_stream);
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
if (options.has_side_input_has_precedence()) {
RET_CHECK(input_via_side_packet && input_via_stream);
} else {
// Only one of input_side_packet or input_stream may specify
// ALLOW/DISALLOW input when side_input_has_precedence is not set
// in the options.
RET_CHECK(input_via_side_packet ^ input_via_stream);
}
if (input_via_side_packet) {
RET_CHECK(cc->InputSidePackets().HasTag("ALLOW") ^
@ -101,7 +140,8 @@ class GateCalculator : public CalculatorBase {
} else {
cc->InputSidePackets().Tag("DISALLOW").Set<bool>();
}
} else {
}
if (input_via_stream) {
RET_CHECK(cc->Inputs().HasTag("ALLOW") ^ cc->Inputs().HasTag("DISALLOW"));
if (cc->Inputs().HasTag("ALLOW")) {
@ -134,13 +174,13 @@ class GateCalculator : public CalculatorBase {
}
::mediapipe::Status Open(CalculatorContext* cc) final {
use_side_packet_for_allow_disallow_ = false;
bool use_side_packet_for_allow_disallow = false;
if (cc->InputSidePackets().HasTag("ALLOW")) {
use_side_packet_for_allow_disallow_ = true;
use_side_packet_for_allow_disallow = true;
allow_by_side_packet_decision_ =
cc->InputSidePackets().Tag("ALLOW").Get<bool>();
} else if (cc->InputSidePackets().HasTag("DISALLOW")) {
use_side_packet_for_allow_disallow_ = true;
use_side_packet_for_allow_disallow = true;
allow_by_side_packet_decision_ =
!cc->InputSidePackets().Tag("DISALLOW").Get<bool>();
}
@ -152,23 +192,31 @@ class GateCalculator : public CalculatorBase {
const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>();
empty_packets_as_allow_ = options.empty_packets_as_allow();
if (!options.has_side_input_has_precedence()) {
side_input_has_precedence_ = use_side_packet_for_allow_disallow;
} else {
side_input_has_precedence_ = options.side_input_has_precedence();
}
return ::mediapipe::OkStatus();
}
::mediapipe::Status Process(CalculatorContext* cc) final {
bool allow = empty_packets_as_allow_;
if (use_side_packet_for_allow_disallow_) {
allow = allow_by_side_packet_decision_;
bool allow_by_stream = empty_packets_as_allow_;
if (cc->Inputs().HasTag("ALLOW") && !cc->Inputs().Tag("ALLOW").IsEmpty()) {
allow_by_stream = cc->Inputs().Tag("ALLOW").Get<bool>();
}
if (cc->Inputs().HasTag("DISALLOW") &&
!cc->Inputs().Tag("DISALLOW").IsEmpty()) {
allow_by_stream = !cc->Inputs().Tag("DISALLOW").Get<bool>();
}
const bool allow_by_side_packet =
allow_by_side_packet_decision_ || empty_packets_as_allow_;
bool allow = false;
if (side_input_has_precedence_) {
allow = allow_by_side_packet;
} else {
if (cc->Inputs().HasTag("ALLOW") &&
!cc->Inputs().Tag("ALLOW").IsEmpty()) {
allow = cc->Inputs().Tag("ALLOW").Get<bool>();
}
if (cc->Inputs().HasTag("DISALLOW") &&
!cc->Inputs().Tag("DISALLOW").IsEmpty()) {
allow = !cc->Inputs().Tag("DISALLOW").Get<bool>();
}
allow = allow_by_stream;
}
const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW;
@ -203,9 +251,9 @@ class GateCalculator : public CalculatorBase {
private:
GateState last_gate_state_ = GATE_UNINITIALIZED;
int num_data_streams_;
bool allow_by_side_packet_decision_ = false;
bool empty_packets_as_allow_;
bool use_side_packet_for_allow_disallow_;
bool allow_by_side_packet_decision_;
bool side_input_has_precedence_;
};
REGISTER_CALCULATOR(GateCalculator);

10
mediapipe/calculators/core/gate_calculator.proto
View File

@ -1,4 +1,4 @@
// Copyright 2019 The MediaPipe Authors.
// Copyright 2019-2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -27,4 +27,12 @@ message GateCalculatorOptions {
// disallowing the corresponding packets in the data input streams. Setting
// this option to true inverts that, allowing the data packets to go through.
optional bool empty_packets_as_allow = 1;
// Input side packet and input stream are allowed to coexist only if this
// field is set. When it's set to true, the input side packet has higher
// precedence and the input stream signal will be ignored. When it's set to
// false, the input stream signal always overrides the input side packet
// signal.
//
optional bool side_input_has_precedence = 2;
}

46
mediapipe/calculators/core/gate_calculator_test.cc
View File

@ -1,4 +1,4 @@
// Copyright 2019 The MediaPipe Authors.
// Copyright 2019-2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -330,5 +330,49 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) {
ASSERT_EQ(0, output.size());
}
TEST_F(GateCalculatorTest, TestOverrideDecisionBySidePacketSignal) {
SetRunner(R"(
calculator: "GateCalculator"
input_stream: "test_input"
input_stream: "ALLOW:gating_stream"
input_side_packet: "ALLOW:gating_packet"
output_stream: "test_output"
options: {
[mediapipe.GateCalculatorOptions.ext] {
side_input_has_precedence: true
}
}
)");
constexpr int64 kTimestampValue0 = 42;
runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false));
RunTimeStep(kTimestampValue0, "ALLOW", true);
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
ASSERT_EQ(0, output.size());
}
TEST_F(GateCalculatorTest, TestOverrideDecisionByStreamSignal) {
SetRunner(R"(
calculator: "GateCalculator"
input_stream: "test_input"
input_stream: "ALLOW:gating_stream"
input_side_packet: "ALLOW:gating_packet"
output_stream: "test_output"
options: {
[mediapipe.GateCalculatorOptions.ext] {
side_input_has_precedence: false
}
}
)");
constexpr int64 kTimestampValue0 = 42;
runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false));
RunTimeStep(kTimestampValue0, "ALLOW", true);
const std::vector<Packet>& output = runner()->Outputs().Get("", 0).packets;
ASSERT_EQ(1, output.size());
}
} // namespace
} // namespace mediapipe

16
mediapipe/calculators/image/BUILD
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
licenses(["notice"])
@ -279,27 +279,17 @@ cc_library(
alwayslink = 1,
)
proto_library(
mediapipe_proto_library(
name = "image_transformation_calculator_proto",
srcs = ["image_transformation_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/gpu:scale_mode_proto",
],
)
mediapipe_cc_proto_library(
name = "image_transformation_calculator_cc_proto",
srcs = ["image_transformation_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/gpu:scale_mode_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":image_transformation_calculator_proto"],
)
cc_library(
name = "image_transformation_calculator",
srcs = ["image_transformation_calculator.cc"],

122
mediapipe/calculators/tflite/BUILD
View File

@ -13,131 +13,91 @@
# limitations under the License.
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
load("@bazel_skylib//lib:selects.bzl", "selects")
licenses(["notice"])
package(default_visibility = ["//visibility:private"])
proto_library(
mediapipe_proto_library(
name = "ssd_anchors_calculator_proto",
srcs = ["ssd_anchors_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_custom_op_resolver_calculator_proto",
srcs = ["tflite_custom_op_resolver_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_inference_calculator_proto",
srcs = ["tflite_inference_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_converter_calculator_proto",
srcs = ["tflite_converter_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_tensors_to_segmentation_calculator_proto",
srcs = ["tflite_tensors_to_segmentation_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_tensors_to_detections_calculator_proto",
srcs = ["tflite_tensors_to_detections_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_tensors_to_classification_calculator_proto",
srcs = ["tflite_tensors_to_classification_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "tflite_tensors_to_landmarks_calculator_proto",
srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
)
mediapipe_cc_proto_library(
name = "ssd_anchors_calculator_cc_proto",
srcs = ["ssd_anchors_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":ssd_anchors_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_custom_op_resolver_calculator_cc_proto",
srcs = ["tflite_custom_op_resolver_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_custom_op_resolver_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_converter_calculator_cc_proto",
srcs = ["tflite_converter_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_converter_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_tensors_to_segmentation_calculator_cc_proto",
srcs = ["tflite_tensors_to_segmentation_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_tensors_to_segmentation_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_inference_calculator_cc_proto",
srcs = ["tflite_inference_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_inference_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_tensors_to_detections_calculator_cc_proto",
srcs = ["tflite_tensors_to_detections_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_tensors_to_detections_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_tensors_to_classification_calculator_cc_proto",
srcs = ["tflite_tensors_to_classification_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_tensors_to_classification_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "tflite_tensors_to_landmarks_calculator_cc_proto",
srcs = ["tflite_tensors_to_landmarks_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":tflite_tensors_to_landmarks_calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(

10
mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc
View File

@ -21,9 +21,9 @@
namespace mediapipe {
// A calculator for converting TFLite tensors from regression models into
// landmarks. Note that if the landmarks in the tensor has more than 4
// dimensions, only the first 4 dimensions will be converted to
// [x,y,z, visibility].
// landmarks. Note that if the landmarks in the tensor has more than 5
// dimensions, only the first 5 dimensions will be converted to
// [x,y,z, visibility, presence].
//
// Input:
// TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first
@ -209,6 +209,9 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
if (num_dimensions > 3) {
landmark->set_visibility(raw_landmarks[offset + 3]);
}
if (num_dimensions > 4) {
landmark->set_presence(raw_landmarks[offset + 4]);
}
}
// Output normalized landmarks if required.
@ -223,6 +226,7 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator);
norm_landmark->set_z(landmark.z() / options_.input_image_width() /
options_.normalize_z());
norm_landmark->set_visibility(landmark.visibility());
norm_landmark->set_presence(landmark.presence());
}
cc->Outputs()
.Tag("NORM_LANDMARKS")

314
mediapipe/calculators/util/BUILD
View File

@ -12,7 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library", "mediapipe_proto_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"])
@ -38,186 +38,95 @@ cc_library(
alwayslink = 1,
)
proto_library(
mediapipe_proto_library(
name = "annotation_overlay_calculator_proto",
srcs = ["annotation_overlay_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "detection_label_id_to_text_calculator_proto",
srcs = ["detection_label_id_to_text_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "timed_box_list_id_to_label_calculator_proto",
srcs = ["timed_box_list_id_to_label_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "latency_proto",
srcs = ["latency.proto"],
)
proto_library(
mediapipe_proto_library(
name = "non_max_suppression_calculator_proto",
srcs = ["non_max_suppression_calculator.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/framework:calculator_proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "packet_frequency_proto",
srcs = ["packet_frequency.proto"],
)
proto_library(
mediapipe_proto_library(
name = "packet_frequency_calculator_proto",
srcs = ["packet_frequency_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "packet_latency_calculator_proto",
srcs = ["packet_latency_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "collection_has_min_size_calculator_proto",
srcs = ["collection_has_min_size_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "association_calculator_proto",
srcs = ["association_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "annotation_overlay_calculator_cc_proto",
srcs = ["annotation_overlay_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":annotation_overlay_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "detection_label_id_to_text_calculator_cc_proto",
srcs = ["detection_label_id_to_text_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [
":detection_label_id_to_text_calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "timed_box_list_id_to_label_calculator_cc_proto",
srcs = ["timed_box_list_id_to_label_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [
":timed_box_list_id_to_label_calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "latency_cc_proto",
srcs = ["latency.proto"],
visibility = ["//mediapipe:__subpackages__"],
deps = [":latency_proto"],
)
mediapipe_cc_proto_library(
name = "non_max_suppression_calculator_cc_proto",
srcs = ["non_max_suppression_calculator.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":non_max_suppression_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "packet_frequency_cc_proto",
srcs = ["packet_frequency.proto"],
visibility = ["//mediapipe:__subpackages__"],
deps = [":packet_frequency_proto"],
)
mediapipe_cc_proto_library(
name = "packet_frequency_calculator_cc_proto",
srcs = ["packet_frequency_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe:__subpackages__"],
deps = [
":packet_frequency_calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "packet_latency_calculator_cc_proto",
srcs = ["packet_latency_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe:__subpackages__"],
deps = [
":packet_latency_calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "collection_has_min_size_calculator_cc_proto",
srcs = ["collection_has_min_size_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe:__subpackages__"],
deps = [":collection_has_min_size_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "association_calculator_cc_proto",
srcs = ["association_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//mediapipe:__subpackages__"],
deps = [":association_calculator_proto"],
)
cc_library(
name = "packet_frequency_calculator",
srcs = ["packet_frequency_calculator.cc"],
@ -434,16 +343,6 @@ cc_library(
alwayslink = 1,
)
mediapipe_cc_proto_library(
name = "thresholding_calculator_cc_proto",
srcs = ["thresholding_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":thresholding_calculator_proto"],
)
cc_library(
name = "thresholding_calculator",
srcs = ["thresholding_calculator.cc"],
@ -457,16 +356,6 @@ cc_library(
alwayslink = 1,
)
mediapipe_cc_proto_library(
name = "landmarks_to_detection_calculator_cc_proto",
srcs = ["landmarks_to_detection_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":landmarks_to_detection_calculator_proto"],
)
cc_library(
name = "landmarks_to_detection_calculator",
srcs = ["landmarks_to_detection_calculator.cc"],
@ -483,46 +372,6 @@ cc_library(
alwayslink = 1,
)
mediapipe_cc_proto_library(
name = "detections_to_rects_calculator_cc_proto",
srcs = ["detections_to_rects_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":detections_to_rects_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "landmark_projection_calculator_cc_proto",
srcs = ["landmark_projection_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":landmark_projection_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "landmarks_to_floats_calculator_cc_proto",
srcs = ["landmarks_to_floats_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":landmarks_to_floats_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "rect_transformation_calculator_cc_proto",
srcs = ["rect_transformation_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":rect_transformation_calculator_proto"],
)
cc_library(
name = "detections_to_rects_calculator",
srcs = [
@ -593,162 +442,140 @@ cc_test(
],
)
proto_library(
mediapipe_proto_library(
name = "rect_to_render_data_calculator_proto",
srcs = ["rect_to_render_data_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "rect_to_render_scale_calculator_proto",
srcs = ["rect_to_render_scale_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "detections_to_render_data_calculator_proto",
srcs = ["detections_to_render_data_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "landmarks_to_render_data_calculator_proto",
srcs = ["landmarks_to_render_data_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "timed_box_list_to_render_data_calculator_proto",
srcs = ["timed_box_list_to_render_data_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "labels_to_render_data_calculator_proto",
srcs = ["labels_to_render_data_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "thresholding_calculator_proto",
srcs = ["thresholding_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "detections_to_rects_calculator_proto",
srcs = ["detections_to_rects_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "landmark_projection_calculator_proto",
srcs = ["landmark_projection_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "landmarks_to_floats_calculator_proto",
srcs = ["landmarks_to_floats_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "rect_transformation_calculator_proto",
srcs = ["rect_transformation_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "landmarks_to_detection_calculator_proto",
srcs = ["landmarks_to_detection_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/util:color_proto",
"//mediapipe/util:render_data_proto",
],
)
mediapipe_cc_proto_library(
name = "rect_to_render_data_calculator_cc_proto",
srcs = ["rect_to_render_data_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":rect_to_render_data_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "rect_to_render_scale_calculator_cc_proto",
srcs = ["rect_to_render_scale_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":rect_to_render_scale_calculator_proto"],
)
mediapipe_cc_proto_library(
name = "detections_to_render_data_calculator_cc_proto",
srcs = ["detections_to_render_data_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":detections_to_render_data_calculator_proto"],
)
cc_library(
name = "detections_to_render_data_calculator",
srcs = ["detections_to_render_data_calculator.cc"],
@ -768,18 +595,6 @@ cc_library(
alwayslink = 1,
)
mediapipe_cc_proto_library(
name = "landmarks_to_render_data_calculator_cc_proto",
srcs = ["landmarks_to_render_data_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":landmarks_to_render_data_calculator_proto"],
)
cc_library(
name = "landmarks_to_render_data_calculator",
srcs = ["landmarks_to_render_data_calculator.cc"],
@ -800,18 +615,6 @@ cc_library(
alwayslink = 1,
)
mediapipe_cc_proto_library(
name = "timed_box_list_to_render_data_calculator_cc_proto",
srcs = ["timed_box_list_to_render_data_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
"//mediapipe/util:render_data_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":timed_box_list_to_render_data_calculator_proto"],
)
cc_library(
name = "timed_box_list_to_render_data_calculator",
srcs = ["timed_box_list_to_render_data_calculator.cc"],
@ -951,6 +754,7 @@ mediapipe_proto_library(
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
@ -1017,25 +821,16 @@ cc_test(
],
)
proto_library(
mediapipe_proto_library(
name = "top_k_scores_calculator_proto",
srcs = ["top_k_scores_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
mediapipe_cc_proto_library(
name = "top_k_scores_calculator_cc_proto",
srcs = ["top_k_scores_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":top_k_scores_calculator_proto"],
)
cc_library(
name = "top_k_scores_calculator",
srcs = ["top_k_scores_calculator.cc"],
@ -1081,17 +876,6 @@ cc_test(
],
)
mediapipe_cc_proto_library(
name = "labels_to_render_data_calculator_cc_proto",
srcs = ["labels_to_render_data_calculator.proto"],
cc_deps = [
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/util:color_cc_proto",
],
visibility = ["//visibility:public"],
deps = [":labels_to_render_data_calculator_proto"],
)
cc_library(
name = "local_file_contents_calculator",
srcs = ["local_file_contents_calculator.cc"],

1
mediapipe/calculators/util/annotation_overlay_calculator.cc
View File

@ -476,6 +476,7 @@ REGISTER_CALCULATOR(AnnotationOverlayCalculator);
options_.canvas_height_px(), options_.canvas_width_px(), CV_8UC3,
cv::Scalar(options_.canvas_color().r(), options_.canvas_color().g(),
options_.canvas_color().b()));
*target_format = ImageFormat::SRGB;
}
return ::mediapipe::OkStatus();

2
mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc
View File

@ -131,6 +131,8 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase {
new_landmark->set_z(new_z);
// Keep visibility as is.
new_landmark->set_visibility(landmark.visibility());
// Keep presence as is.
new_landmark->set_presence(landmark.presence());
}
cc->Outputs().Get(output_id).AddPacket(

2
mediapipe/calculators/util/landmark_projection_calculator.cc
View File

@ -131,6 +131,8 @@ class LandmarkProjectionCalculator : public CalculatorBase {
new_landmark->set_z(new_z);
// Keep visibility as is.
new_landmark->set_visibility(landmark.visibility());
// Keep presence as is.
new_landmark->set_presence(landmark.presence());
}
cc->Outputs().Get(output_id).AddPacket(

2
mediapipe/calculators/util/landmarks_smoothing_calculator.cc
View File

@ -137,6 +137,8 @@ class VelocityFilter : public LandmarksFilter {
image_width);
// Keep visibility as is.
out_landmark->set_visibility(in_landmark.visibility());
// Keep presence as is.
out_landmark->set_presence(in_landmark.presence());
}
return ::mediapipe::OkStatus();

61
mediapipe/calculators/util/landmarks_to_render_data_calculator.cc
View File

@ -33,7 +33,6 @@ constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS";
constexpr char kRenderScaleTag[] = "RENDER_SCALE";
constexpr char kRenderDataTag[] = "RENDER_DATA";
constexpr char kLandmarkLabel[] = "KEYPOINT";
constexpr int kMaxLandmarkThickness = 18;
inline void SetColor(RenderAnnotation* annotation, const Color& color) {
annotation->mutable_color()->set_r(color.r());
@ -59,15 +58,18 @@ inline void GetMinMaxZ(const LandmarkListType& landmarks, float* z_min,
}
void SetColorSizeValueFromZ(float z, float z_min, float z_max,
RenderAnnotation* render_annotation) {
RenderAnnotation* render_annotation,
float min_depth_circle_thickness,
float max_depth_circle_thickness) {
const int color_value = 255 - static_cast<int>(Remap(z, z_min, z_max, 255));
::mediapipe::Color color;
color.set_r(color_value);
color.set_g(color_value);
color.set_b(color_value);
SetColor(render_annotation, color);
const int thickness = static_cast<int>((1.f - Remap(z, z_min, z_max, 1)) *
kMaxLandmarkThickness);
const float scale = max_depth_circle_thickness - min_depth_circle_thickness;
const int thickness = static_cast<int>(
min_depth_circle_thickness + (1.f - Remap(z, z_min, z_max, 1)) * scale);
render_annotation->set_thickness(thickness);
}
@ -97,7 +99,8 @@ template <class LandmarkListType, class LandmarkType>
void AddConnectionsWithDepth(const LandmarkListType& landmarks,
const std::vector<int>& landmark_connections,
bool utilize_visibility,
float visibility_threshold, float thickness,
float visibility_threshold, bool utilize_presence,
float presence_threshold, float thickness,
bool normalized, float min_z, float max_z,
RenderData* render_data) {
for (int i = 0; i < landmark_connections.size(); i += 2) {
@ -107,6 +110,10 @@ void AddConnectionsWithDepth(const LandmarkListType& landmarks,
ld1.visibility() < visibility_threshold)) {
continue;
}
if (utilize_presence && (ld0.presence() < presence_threshold ||
ld1.presence() < presence_threshold)) {
continue;
}
const int gray_val1 =
255 - static_cast<int>(Remap(ld0.z(), min_z, max_z, 255));
const int gray_val2 =
@ -136,6 +143,7 @@ template <class LandmarkListType, class LandmarkType>
void AddConnections(const LandmarkListType& landmarks,
const std::vector<int>& landmark_connections,
bool utilize_visibility, float visibility_threshold,
bool utilize_presence, float presence_threshold,
const Color& connection_color, float thickness,
bool normalized, RenderData* render_data) {
for (int i = 0; i < landmark_connections.size(); i += 2) {
@ -145,6 +153,10 @@ void AddConnections(const LandmarkListType& landmarks,
ld1.visibility() < visibility_threshold)) {
continue;
}
if (utilize_presence && (ld0.presence() < presence_threshold ||
ld1.presence() < presence_threshold)) {
continue;
}
AddConnectionToRenderData<LandmarkType>(ld0, ld1, connection_color,
thickness, normalized, render_data);
}
@ -238,13 +250,15 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
if (visualize_depth) {
AddConnectionsWithDepth<LandmarkList, Landmark>(
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), thickness, /*normalized=*/false,
z_min, z_max, render_data.get());
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold(), thickness, /*normalized=*/false, z_min,
z_max, render_data.get());
} else {
AddConnections<LandmarkList, Landmark>(
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), options_.connection_color(),
thickness, /*normalized=*/false, render_data.get());
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold(), options_.connection_color(), thickness,
/*normalized=*/false, render_data.get());
}
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const Landmark& landmark = landmarks.landmark(i);
@ -254,11 +268,17 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
continue;
}
if (options_.utilize_presence() &&
landmark.presence() < options_.presence_threshold()) {
continue;
}
auto* landmark_data_render = AddPointRenderData(
options_.landmark_color(), thickness, render_data.get());
if (visualize_depth) {
SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
landmark_data_render);
SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
options_.min_depth_circle_thickness(),
options_.max_depth_circle_thickness());
}
auto* landmark_data = landmark_data_render->mutable_point();
landmark_data->set_normalized(false);
@ -279,13 +299,15 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
if (visualize_depth) {
AddConnectionsWithDepth<NormalizedLandmarkList, NormalizedLandmark>(
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), thickness, /*normalized=*/true,
z_min, z_max, render_data.get());
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold(), thickness, /*normalized=*/true, z_min,
z_max, render_data.get());
} else {
AddConnections<NormalizedLandmarkList, NormalizedLandmark>(
landmarks, landmark_connections_, options_.utilize_visibility(),
options_.visibility_threshold(), options_.connection_color(),
thickness, /*normalized=*/true, render_data.get());
options_.visibility_threshold(), options_.utilize_presence(),
options_.presence_threshold(), options_.connection_color(), thickness,
/*normalized=*/true, render_data.get());
}
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const NormalizedLandmark& landmark = landmarks.landmark(i);
@ -294,12 +316,17 @@ RenderAnnotation* AddPointRenderData(const Color& landmark_color,
landmark.visibility() < options_.visibility_threshold()) {
continue;
}
if (options_.utilize_presence() &&
landmark.presence() < options_.presence_threshold()) {
continue;
}
auto* landmark_data_render = AddPointRenderData(
options_.landmark_color(), thickness, render_data.get());
if (visualize_depth) {
SetColorSizeValueFromZ(landmark.z(), z_min, z_max,
landmark_data_render);
SetColorSizeValueFromZ(landmark.z(), z_min, z_max, landmark_data_render,
options_.min_depth_circle_thickness(),
options_.max_depth_circle_thickness());
}
auto* landmark_data = landmark_data_render->mutable_point();
landmark_data->set_normalized(true);

15
mediapipe/calculators/util/landmarks_to_render_data_calculator.proto
View File

@ -49,4 +49,19 @@ message LandmarksToRenderDataCalculatorOptions {
// Threshold to determine visibility of the landmark. Landmark with visibility
// greater or equal than threshold is considered visible.
optional double visibility_threshold = 7 [default = 0.0];
// Use landmarks presence while rendering landmarks and connections. If
// landmark is not present, neither it nor adjacent connections will be
// rendered.
optional bool utilize_presence = 8 [default = false];
// Threshold to determine presence of the landmark. Landmark with presence
// greater or equal than threshold is considered present.
optional double presence_threshold = 9 [default = 0.0];
// Min thickness of the drawing for landmark circle.
optional double min_depth_circle_thickness = 10 [default = 0.0];
// Max thickness of the drawing for landmark circle.
optional double max_depth_circle_thickness = 11 [default = 18.0];
}

11
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator.cc
View File

@ -244,6 +244,9 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
frame_width_ = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Width();
frame_height_ = cc->Inputs().Tag(kVideoFrame).Get<ImageFrame>().Height();
} else if (cc->Inputs().HasTag(kVideoSize)) {
if (cc->Inputs().Tag(kVideoSize).IsEmpty()) {
return ::mediapipe::OkStatus();
}
frame_width_ =
cc->Inputs().Tag(kVideoSize).Get<std::pair<int, int>>().first;
frame_height_ =
@ -302,6 +305,14 @@ void MakeStaticFeatures(const int top_border, const int bottom_border,
}
if (cc->Inputs().HasTag(kDetections)) {
if (cc->Inputs().Tag(kDetections).IsEmpty()) {
auto default_rect = absl::make_unique<mediapipe::Rect>();
default_rect->set_width(frame_width_);
default_rect->set_height(frame_height_);
cc->Outputs().Tag(kCropRect).Add(default_rect.release(),
Timestamp(cc->InputTimestamp()));
return ::mediapipe::OkStatus();
}
auto raw_detections =
cc->Inputs().Tag(kDetections).Get<std::vector<mediapipe::Detection>>();
for (const auto& detection : raw_detections) {

19
mediapipe/examples/desktop/autoflip/calculators/content_zooming_calculator_test.cc
View File

@ -418,6 +418,25 @@ TEST(ContentZoomingCalculatorTest, ShiftOutsideBounds) {
runner->Outputs().Tag("CROP_RECT").packets);
}
TEST(ContentZoomingCalculatorTest, EmptySize) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
MP_ASSERT_OK(runner->Run());
ASSERT_EQ(runner->Outputs().Tag("CROP_RECT").packets.size(), 0);
}
TEST(ContentZoomingCalculatorTest, EmptyDetections) {
auto config = ParseTextProtoOrDie<CalculatorGraphConfig::Node>(kConfigD);
auto runner = ::absl::make_unique<CalculatorRunner>(config);
auto input_size = ::absl::make_unique<std::pair<int, int>>(1000, 1000);
runner->MutableInputs()
->Tag("VIDEO_SIZE")
.packets.push_back(Adopt(input_size.release()).At(Timestamp(0)));
MP_ASSERT_OK(runner->Run());
CheckCropRect(0, 0, 1000, 1000, 0,
runner->Outputs().Tag("CROP_RECT").packets);
}
} // namespace
} // namespace autoflip

14
mediapipe/framework/formats/BUILD
View File

@ -14,6 +14,7 @@
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
load("//mediapipe/framework:mediapipe_register_type.bzl", "mediapipe_register_type")
package(
default_visibility = ["//visibility:private"],
@ -221,6 +222,19 @@ mediapipe_proto_library(
],
)
mediapipe_register_type(
base_name = "landmark",
include_headers = ["mediapipe/framework/formats/landmark.pb.h"],
types = [
"::mediapipe::Landmark",
"::mediapipe::NormalizedLandmark",
"::mediapipe::NormalizedLandmarkList",
"::std::vector<::mediapipe::Landmark>",
"::std::vector<::mediapipe::NormalizedLandmark>",
],
deps = [":landmark_cc_proto"],
)
# Expose the proto source files for building mediapipe AAR.
filegroup(
name = "protos_src",

11
mediapipe/framework/formats/landmark.proto
View File

@ -28,9 +28,17 @@ message Landmark {
optional float z = 3;
// Landmark visibility. Float score of whether landmark is visible or occluded
// by other obects. Depending on the model, visibility value is either a
// by other objects. Landmark considered as invisible also if it is
// not present on the screen (out of scene bounds).
// Depending on the model, visibility value is either a
// sigmoid or an argument of sigmoid.
optional float visibility = 4;
// Landmark presence. Float score of whether landmark is present on the scene
// (located within scene bounds).
// Depending on the model, presence value is either a result of sigmoid
// or an argument of sigmoid function to get landmark presence probability.
optional float presence = 5;
}
// Group of Landmark protos.
@ -45,6 +53,7 @@ message NormalizedLandmark {
optional float y = 2;
optional float z = 3;
optional float visibility = 4;
optional float presence = 5;
}
// Group of NormalizedLandmark protos.

2
mediapipe/objc/MPPTimestampConverter.mm
View File

@ -36,7 +36,7 @@
}
- (mediapipe::Timestamp)timestampForMediaTime:(CMTime)mediaTime {
float sampleSeconds = CMTIME_IS_VALID(mediaTime) ? CMTimeGetSeconds(mediaTime) : 0;
Float64 sampleSeconds = CMTIME_IS_VALID(mediaTime) ? CMTimeGetSeconds(mediaTime) : 0;
const int64 sampleUsec = sampleSeconds * mediapipe::Timestamp::kTimestampUnitsPerSecond;
_mediapipeTimestamp = mediapipe::Timestamp(sampleUsec) + _timestampOffset;
if (_mediapipeTimestamp <= _lastTimestamp) {

4
mediapipe/python/image_frame_test.py
View File

@ -124,7 +124,7 @@ class ImageFrameTest(absltest.TestCase):
cv2.COLOR_RGB2GRAY)
image_frame = mp.ImageFrame(
image_format=mp.ImageFormat.GRAY8,
data=mat[offset:-offset, offset:-offset])
data=np.ascontiguousarray(mat[offset:-offset, offset:-offset]))
self.assertTrue(
np.array_equal(mat[offset:-offset, offset:-offset],
image_frame.numpy_view()))
@ -137,7 +137,7 @@ class ImageFrameTest(absltest.TestCase):
cv2.COLOR_RGB2BGR)
image_frame = mp.ImageFrame(
image_format=mp.ImageFormat.SRGB,
data=mat[offset:-offset, offset:-offset, :])
data=np.ascontiguousarray(mat[offset:-offset, offset:-offset, :]))
self.assertTrue(
np.array_equal(mat[offset:-offset, offset:-offset, :],
image_frame.numpy_view()))

33
mediapipe/util/BUILD
View File

@ -12,58 +12,35 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"])
package(default_visibility = ["//visibility:private"])
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_cc_proto_library")
proto_library(
mediapipe_proto_library(
name = "audio_decoder_proto",
srcs = ["audio_decoder.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
proto_library(
mediapipe_proto_library(
name = "color_proto",
srcs = ["color.proto"],
visibility = ["//visibility:public"],
)
proto_library(
mediapipe_proto_library(
name = "render_data_proto",
srcs = ["render_data.proto"],
visibility = ["//visibility:public"],
deps = ["//mediapipe/util:color_proto"],
)
mediapipe_cc_proto_library(
name = "color_cc_proto",
srcs = ["color.proto"],
visibility = ["//visibility:public"],
deps = [":color_proto"],
)
mediapipe_cc_proto_library(
name = "render_data_cc_proto",
srcs = ["render_data.proto"],
cc_deps = [":color_cc_proto"],
visibility = ["//visibility:public"],
deps = [":render_data_proto"],
)
mediapipe_cc_proto_library(
name = "audio_decoder_cc_proto",
srcs = ["audio_decoder.proto"],
cc_deps = ["//mediapipe/framework:calculator_cc_proto"],
visibility = ["//visibility:public"],
deps = [":audio_decoder_proto"],
)
cc_library(
name = "audio_decoder",
srcs = ["audio_decoder.cc"],

11
mediapipe/util/annotation_renderer.cc
View File

@ -513,6 +513,17 @@ void AnnotationRenderer::DrawText(const RenderAnnotation& annotation) {
const int font_face = text.font_face();
const double font_scale = ComputeFontScale(font_face, font_size, thickness);
int text_baseline = 0;
cv::Size text_size = cv::getTextSize(text.display_text(), font_face,
font_scale, thickness, &text_baseline);
if (text.center_horizontally()) {
origin.x -= text_size.width / 2;
}
if (text.center_vertically()) {
origin.y += text_size.height / 2;
}
cv::putText(mat_image_, text.display_text(), origin, font_face, font_scale,
color, thickness, /*lineType=*/8,
/*bottomLeftOrigin=*/flip_text_vertically_);

6
mediapipe/util/render_data.proto
View File

@ -160,6 +160,12 @@ message RenderAnnotation {
// cv::FONT_HERSHEY_SCRIPT_SIMPLEX (6)
// cv::FONT_HERSHEY_SCRIPT_COMPLEX (7)
optional int32 font_face = 6 [default = 0];
// Options to center text around the anchor point (left, baseline) by
// taking into account font shape, size and text length (e.g.,
// [left, baseline] represent [center_x, center_y].
optional bool center_horizontally = 7 [default = false];
optional bool center_vertically = 8 [default = false];
}
// The RenderAnnotation can be one of the below formats.