From 252a5713c7996e14c9bcff2caf0b2e1afca1e6f7 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Fri, 28 Feb 2020 20:44:27 -0800 Subject: [PATCH] Project import generated by Copybara. GitOrigin-RevId: 6f964e58d874e47fb6207aa97d060a4cd6428527 --- WORKSPACE | 21 +- mediapipe/calculators/core/BUILD | 16 + .../core/side_packet_to_stream_calculator.cc | 114 +- .../side_packet_to_stream_calculator_test.cc | 275 + mediapipe/calculators/image/BUILD | 4 +- .../image/color_convert_calculator.cc | 5 + .../image/image_cropping_calculator.cc | 62 +- .../image/image_cropping_calculator.h | 5 + .../image/image_cropping_calculator.proto | 11 + .../image/image_cropping_calculator_test.cc | 64 +- .../image/image_transformation_calculator.cc | 86 +- .../image/scale_image_calculator.cc | 24 +- .../image/scale_image_calculator.proto | 24 +- .../calculators/image/scale_image_utils.cc | 36 +- .../calculators/image/scale_image_utils.h | 20 +- .../image/scale_image_utils_test.cc | 86 +- mediapipe/calculators/tensorflow/BUILD | 52 +- ...low_session_from_saved_model_calculator.cc | 4 +- ...ession_from_saved_model_calculator_test.cc | 2 +- ...flow_session_from_saved_model_generator.cc | 4 +- ...session_from_saved_model_generator_test.cc | 2 +- mediapipe/calculators/tflite/BUILD | 1 + .../tflite/tflite_inference_calculator.cc | 105 +- .../tflite/tflite_inference_calculator.proto | 34 +- .../tflite_inference_calculator_test.cc | 47 +- .../tflite_tensors_to_landmarks_calculator.cc | 62 +- mediapipe/calculators/util/BUILD | 14 + .../util/annotation_overlay_calculator.cc | 20 +- .../util/detections_to_rects_calculator.cc | 80 +- .../util/detections_to_rects_calculator.h | 19 +- .../landmark_letterbox_removal_calculator.cc | 86 +- .../util/landmark_projection_calculator.cc | 100 +- .../local_file_pattern_contents_calculator.cc | 75 + .../util/rect_to_render_data_calculator.cc | 16 +- mediapipe/calculators/video/BUILD | 1 + mediapipe/docs/face_detection_desktop.md | 4 +- mediapipe/docs/face_detection_mobile_cpu.md | 4 +- mediapipe/docs/face_detection_mobile_gpu.md | 4 +- mediapipe/docs/hand_detection_mobile_gpu.md | 4 +- mediapipe/docs/hand_tracking_mobile_gpu.md | 4 +- mediapipe/docs/install.md | 22 +- .../docs/multi_hand_tracking_mobile_gpu.md | 4 +- mediapipe/docs/object_detection_desktop.md | 8 +- mediapipe/docs/object_detection_mobile_cpu.md | 4 +- mediapipe/docs/object_detection_mobile_gpu.md | 4 +- mediapipe/docs/object_tracking_mobile_gpu.md | 7 +- .../graphs/face_detection_desktop_live.pbtxt | 4 +- .../object_detection_desktop_live.pbtxt | 4 +- mediapipe/examples/desktop/BUILD | 2 +- .../media_sequence/charades_dataset.py | 2 +- .../desktop/media_sequence/demo_dataset.py | 2 +- .../media_sequence/kinetics_dataset.py | 2 +- .../generate_input_sequence_example.py | 3 +- mediapipe/framework/calculator_graph.cc | 2 +- mediapipe/framework/deps/BUILD | 1 + mediapipe/framework/deps/file_helpers.cc | 24 + mediapipe/framework/deps/file_helpers.h | 4 + mediapipe/framework/deps/status.cc | 97 - mediapipe/framework/deps/status.h | 118 +- mediapipe/framework/deps/status_builder.cc | 6 +- .../framework/deps/status_builder_test.cc | 9 +- mediapipe/framework/deps/status_test.cc | 4 +- mediapipe/framework/deps/statusor_test.cc | 12 +- mediapipe/framework/encode_binary_proto.bzl | 5 + mediapipe/framework/formats/BUILD | 2 +- mediapipe/framework/formats/rect.proto | 7 +- mediapipe/framework/graph_validation_test.cc | 65 +- mediapipe/framework/packet.cc | 7 + mediapipe/framework/packet.h | 17 + mediapipe/framework/port.h | 30 + mediapipe/framework/port/BUILD | 1 + .../framework/profiler/graph_tracer_test.cc | 4 +- .../profiler/profiler_resource_util_apple.cc | 2 +- mediapipe/framework/profiler/testdata/BUILD | 24 + .../profiler/testdata/profile_opencv_0.pbtxt | 8814 +++++++++++++++++ .../profiler/testdata/profile_opencv_1.pbtxt | 4957 +++++++++ mediapipe/framework/tool/BUILD | 2 +- mediapipe/framework/tool/status_util_test.cc | 15 +- .../framework/tool/subgraph_expansion.cc | 7 +- .../face_detection_desktop_live.pbtxt | 4 +- .../face_detection_mobile_cpu.pbtxt | 4 +- .../face_detection_mobile_gpu.pbtxt | 4 +- mediapipe/graphs/hand_tracking/BUILD | 4 + .../hand_detection_desktop.pbtxt | 4 +- .../hand_detection_desktop_live.pbtxt | 4 +- .../hand_tracking/hand_detection_mobile.pbtxt | 4 +- .../subgraphs/hand_landmark_cpu.pbtxt | 15 +- .../subgraphs/hand_landmark_gpu.pbtxt | 5 + .../subgraphs/multi_hand_renderer_cpu.pbtxt | 4 +- .../subgraphs/multi_hand_renderer_gpu.pbtxt | 4 +- .../subgraphs/renderer_cpu.pbtxt | 4 +- .../subgraphs/renderer_gpu.pbtxt | 4 +- .../object_detection_desktop_live.pbtxt | 4 +- ...t_detection_desktop_tensorflow_graph.pbtxt | 4 +- ...bject_detection_desktop_tflite_graph.pbtxt | 4 +- .../object_detection_mobile_cpu.pbtxt | 4 +- .../object_detection_mobile_gpu.pbtxt | 4 +- .../tracking/subgraphs/renderer_cpu.pbtxt | 4 +- .../tracking/subgraphs/renderer_gpu.pbtxt | 4 +- .../local_video_model_inference.pbtxt | 4 +- .../mediapipe/components/FrameProcessor.java | 149 +- .../components/MicrophoneHelper.java | 248 +- .../java/com/google/mediapipe/framework/BUILD | 21 +- .../mediapipe/framework/PacketCreator.java | 16 + .../com/google/mediapipe/framework/jni/BUILD | 37 + .../mediapipe/framework/jni/class_registry.cc | 52 + .../mediapipe/framework/jni/class_registry.h | 67 + .../google/mediapipe/framework/jni/graph.cc | 34 +- .../google/mediapipe/framework/jni/graph.h | 4 - .../mediapipe/framework/jni/graph_jni.cc | 1 + .../mediapipe/framework/jni/jni_util.cc | 14 +- .../framework/jni/packet_context_jni.cc | 15 +- .../framework/jni/packet_creator_jni.cc | 21 + .../framework/jni/packet_creator_jni.h | 4 + .../framework/jni/packet_getter_jni.cc | 25 +- .../framework/jni/register_natives.cc | 233 + .../framework/jni/register_natives.h | 30 + .../framework/proguard_allowobfuscation.pgcfg | 29 + .../java/com/google/mediapipe/glutil/BUILD | 2 +- .../mediapipe/glutil/TextureRenderer.java | 24 +- mediapipe/objc/MPPGraphTests.mm | 4 +- mediapipe/objc/NSError+util_status.mm | 4 +- mediapipe/util/android/BUILD | 35 +- mediapipe/util/android/asset_manager_util.cc | 88 +- mediapipe/util/android/asset_manager_util.h | 18 + mediapipe/util/android/file/base/helpers.cc | 32 +- mediapipe/util/android/file/base/helpers.h | 7 + mediapipe/util/android/jni_helper.cc | 115 + mediapipe/util/android/jni_helper.h | 69 + mediapipe/util/android/logging.h | 39 + mediapipe/util/resource_util_android.cc | 13 + mediapipe/util/sequence/BUILD | 8 +- ...e22eae8bf3206189a066032c66e9e5c9b4a61.diff | 65 + ...7bdbebb99352351a19e2e403136166aa52934.diff | 19 - 134 files changed, 16665 insertions(+), 874 deletions(-) create mode 100644 mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc create mode 100644 mediapipe/calculators/util/local_file_pattern_contents_calculator.cc create mode 100644 mediapipe/framework/profiler/testdata/BUILD create mode 100644 mediapipe/framework/profiler/testdata/profile_opencv_0.pbtxt create mode 100644 mediapipe/framework/profiler/testdata/profile_opencv_1.pbtxt create mode 100644 mediapipe/java/com/google/mediapipe/framework/jni/class_registry.cc create mode 100644 mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h create mode 100644 mediapipe/java/com/google/mediapipe/framework/jni/register_natives.cc create mode 100644 mediapipe/java/com/google/mediapipe/framework/jni/register_natives.h create mode 100644 mediapipe/java/com/google/mediapipe/framework/proguard_allowobfuscation.pgcfg create mode 100644 mediapipe/util/android/jni_helper.cc create mode 100644 mediapipe/util/android/jni_helper.h create mode 100644 mediapipe/util/android/logging.h create mode 100644 third_party/org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff delete mode 100644 third_party/org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff diff --git a/WORKSPACE b/WORKSPACE index ca858a88f..411f21b94 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -10,15 +10,15 @@ http_archive( sha256 = "2ef429f5d7ce7111263289644d233707dba35e39696377ebab8b0bc701f7818e", ) load("@bazel_skylib//lib:versions.bzl", "versions") -versions.check(minimum_bazel_version = "0.24.1", +versions.check(minimum_bazel_version = "1.0.0", maximum_bazel_version = "1.2.1") -# ABSL cpp library lts_2019_08_08. +# ABSL cpp library lts_2020_02_25 http_archive( name = "com_google_absl", urls = [ - "https://github.com/abseil/abseil-cpp/archive/20190808.tar.gz", + "https://github.com/abseil/abseil-cpp/archive/20200225.tar.gz", ], # Remove after https://github.com/abseil/abseil-cpp/issues/326 is solved. patches = [ @@ -27,8 +27,8 @@ http_archive( patch_args = [ "-p1", ], - strip_prefix = "abseil-cpp-20190808", - sha256 = "8100085dada279bf3ee00cd064d43b5f55e5d913be0dfe2906f06f8f28d5b37e" + strip_prefix = "abseil-cpp-20200225", + sha256 = "728a813291bdec2aa46eab8356ace9f75ac2ed9dfe2df5ab603c4e6c09f1c353" ) http_archive( @@ -117,18 +117,19 @@ http_archive( ], ) -# 2019-11-21 -_TENSORFLOW_GIT_COMMIT = "f482488b481a799ca07e7e2d153cf47b8e91a60c" -_TENSORFLOW_SHA256= "8d9118c2ce186c7e1403f04b96982fe72c184060c7f7a93e30a28dca358694f0" +# 2020-02-12 +# The last commit before TensorFlow switched to Bazel 2.0 +_TENSORFLOW_GIT_COMMIT = "77e9ffb9b2bfb1a4f7056e62d84039626923e328" +_TENSORFLOW_SHA256= "176ccd82f7dd17c5e117b50d353603b129c7a6ccbfebd522ca47cc2a40f33f13" http_archive( name = "org_tensorflow", urls = [ "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT, "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % _TENSORFLOW_GIT_COMMIT, ], - # Patch https://github.com/tensorflow/tensorflow/commit/e3a7bdbebb99352351a19e2e403136166aa52934 + # A compatibility patch patches = [ - "@//third_party:org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff" + "@//third_party:org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff" ], patch_args = [ "-p1", diff --git a/mediapipe/calculators/core/BUILD b/mediapipe/calculators/core/BUILD index 353e487d9..c3074c2c8 100644 --- a/mediapipe/calculators/core/BUILD +++ b/mediapipe/calculators/core/BUILD @@ -610,6 +610,22 @@ cc_library( alwayslink = 1, ) +cc_test( + name = "side_packet_to_stream_calculator_test", + srcs = ["side_packet_to_stream_calculator_test.cc"], + deps = [ + ":side_packet_to_stream_calculator", + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/port:gtest_main", + "//mediapipe/framework/port:integral_types", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + "//mediapipe/framework/tool:options_util", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + ], +) + cc_test( name = "immediate_mux_calculator_test", srcs = ["immediate_mux_calculator_test.cc"], diff --git a/mediapipe/calculators/core/side_packet_to_stream_calculator.cc b/mediapipe/calculators/core/side_packet_to_stream_calculator.cc index 043c91f32..d7df7530b 100644 --- a/mediapipe/calculators/core/side_packet_to_stream_calculator.cc +++ b/mediapipe/calculators/core/side_packet_to_stream_calculator.cc @@ -28,55 +28,133 @@ using mediapipe::PacketTypeSet; using mediapipe::Timestamp; namespace { + +constexpr char kTagAtPreStream[] = "AT_PRESTREAM"; +constexpr char kTagAtPostStream[] = "AT_POSTSTREAM"; +constexpr char kTagAtZero[] = "AT_ZERO"; +constexpr char kTagAtTick[] = "AT_TICK"; +constexpr char kTagTick[] = "TICK"; + static std::map* kTimestampMap = []() { auto* res = new std::map(); - res->emplace("AT_PRESTREAM", Timestamp::PreStream()); - res->emplace("AT_POSTSTREAM", Timestamp::PostStream()); - res->emplace("AT_ZERO", Timestamp(0)); + res->emplace(kTagAtPreStream, Timestamp::PreStream()); + res->emplace(kTagAtPostStream, Timestamp::PostStream()); + res->emplace(kTagAtZero, Timestamp(0)); + res->emplace(kTagAtTick, Timestamp::Unset()); return res; }(); +template +std::string GetOutputTag(const CC& cc) { + // Single output tag only is required by contract. + return *cc.Outputs().GetTags().begin(); +} + } // namespace -// Outputs the single input_side_packet at the timestamp specified in the -// output_stream tag. Valid tags are AT_PRESTREAM, AT_POSTSTREAM and AT_ZERO. +// Outputs side packet(s) in corresponding output stream(s) with a particular +// timestamp, depending on the tag used to define output stream(s). (One tag can +// be used only.) +// +// Valid tags are AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK and +// corresponding timestamps are Timestamp::PreStream(), Timestamp::PostStream(), +// Timestamp(0) and timestamp of a packet received in TICK input. +// +// Examples: +// node { +// calculator: "SidePacketToStreamCalculator" +// input_side_packet: "side_packet" +// output_stream: "AT_PRESTREAM:packet" +// } +// +// node { +// calculator: "SidePacketToStreamCalculator" +// input_stream: "TICK:tick" +// input_side_packet: "side_packet" +// output_stream: "AT_TICK:packet" +// } class SidePacketToStreamCalculator : public CalculatorBase { public: SidePacketToStreamCalculator() = default; ~SidePacketToStreamCalculator() override = default; static ::mediapipe::Status GetContract(CalculatorContract* cc); + ::mediapipe::Status Open(CalculatorContext* cc) override; ::mediapipe::Status Process(CalculatorContext* cc) override; ::mediapipe::Status Close(CalculatorContext* cc) override; + + private: + bool is_tick_processing_ = false; + std::string output_tag_; }; REGISTER_CALCULATOR(SidePacketToStreamCalculator); ::mediapipe::Status SidePacketToStreamCalculator::GetContract( CalculatorContract* cc) { - cc->InputSidePackets().Index(0).SetAny(); + const auto& tags = cc->Outputs().GetTags(); + RET_CHECK(tags.size() == 1 && kTimestampMap->count(*tags.begin()) == 1) + << "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK tags is " + "allowed and required to specify output stream(s)."; + RET_CHECK( + (cc->Outputs().HasTag(kTagAtTick) && cc->Inputs().HasTag(kTagTick)) || + (!cc->Outputs().HasTag(kTagAtTick) && !cc->Inputs().HasTag(kTagTick))) + << "Either both of TICK and AT_TICK should be used or none of them."; + const std::string output_tag = GetOutputTag(*cc); + const int num_entries = cc->Outputs().NumEntries(output_tag); + RET_CHECK_EQ(num_entries, cc->InputSidePackets().NumEntries()) + << "Same number of input side packets and output streams is required."; + for (int i = 0; i < num_entries; ++i) { + cc->InputSidePackets().Index(i).SetAny(); + cc->Outputs() + .Get(output_tag, i) + .SetSameAs(cc->InputSidePackets().Index(i).GetSameAs()); + } - std::set tags = cc->Outputs().GetTags(); - RET_CHECK_EQ(tags.size(), 1); + if (cc->Inputs().HasTag(kTagTick)) { + cc->Inputs().Tag(kTagTick).SetAny(); + } - RET_CHECK_EQ(kTimestampMap->count(*tags.begin()), 1); - cc->Outputs().Tag(*tags.begin()).SetAny(); + return ::mediapipe::OkStatus(); +} +::mediapipe::Status SidePacketToStreamCalculator::Open(CalculatorContext* cc) { + output_tag_ = GetOutputTag(*cc); + if (cc->Inputs().HasTag(kTagTick)) { + is_tick_processing_ = true; + // Set offset, so output timestamp bounds are updated in response to TICK + // timestamp bound update. + cc->SetOffset(TimestampDiff(0)); + } return ::mediapipe::OkStatus(); } ::mediapipe::Status SidePacketToStreamCalculator::Process( CalculatorContext* cc) { - return mediapipe::tool::StatusStop(); + if (is_tick_processing_) { + // TICK input is guaranteed to be non-empty, as it's the only input stream + // for this calculator. + const auto& timestamp = cc->Inputs().Tag(kTagTick).Value().Timestamp(); + for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) { + cc->Outputs() + .Get(output_tag_, i) + .AddPacket(cc->InputSidePackets().Index(i).At(timestamp)); + } + + return ::mediapipe::OkStatus(); + } + + return ::mediapipe::tool::StatusStop(); } ::mediapipe::Status SidePacketToStreamCalculator::Close(CalculatorContext* cc) { - std::set tags = cc->Outputs().GetTags(); - RET_CHECK_EQ(tags.size(), 1); - const std::string& tag = *tags.begin(); - RET_CHECK_EQ(kTimestampMap->count(tag), 1); - cc->Outputs().Tag(tag).AddPacket( - cc->InputSidePackets().Index(0).At(kTimestampMap->at(tag))); - + if (!cc->Outputs().HasTag(kTagAtTick)) { + const auto& timestamp = kTimestampMap->at(output_tag_); + for (int i = 0; i < cc->Outputs().NumEntries(output_tag_); ++i) { + cc->Outputs() + .Get(output_tag_, i) + .AddPacket(cc->InputSidePackets().Index(i).At(timestamp)); + } + } return ::mediapipe::OkStatus(); } diff --git a/mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc b/mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc new file mode 100644 index 000000000..078055f07 --- /dev/null +++ b/mediapipe/calculators/core/side_packet_to_stream_calculator_test.cc @@ -0,0 +1,275 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "absl/memory/memory.h" +#include "absl/strings/match.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/integral_types.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status.h" +#include "mediapipe/framework/port/status_matchers.h" +#include "mediapipe/framework/tool/options_util.h" + +namespace mediapipe { +namespace { + +TEST(SidePacketToStreamCalculator, WrongConfig_MissingTick) { + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie( + R"( + input_stream: "tick" + input_side_packet: "side_packet" + output_stream: "packet" + node { + calculator: "SidePacketToStreamCalculator" + input_side_packet: "side_packet" + output_stream: "AT_TICK:packet" + } + )"); + CalculatorGraph graph; + auto status = graph.Initialize(graph_config); + EXPECT_FALSE(status.ok()); + EXPECT_PRED2( + absl::StrContains, status.message(), + "Either both of TICK and AT_TICK should be used or none of them."); +} + +TEST(SidePacketToStreamCalculator, WrongConfig_NonExistentTag) { + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie( + R"( + input_stream: "tick" + input_side_packet: "side_packet" + output_stream: "packet" + node { + calculator: "SidePacketToStreamCalculator" + input_side_packet: "side_packet" + output_stream: "DOES_NOT_EXIST:packet" + } + )"); + CalculatorGraph graph; + auto status = graph.Initialize(graph_config); + EXPECT_FALSE(status.ok()); + EXPECT_PRED2(absl::StrContains, status.message(), + "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK " + "tags is allowed and required to specify output stream(s)."); +} + +TEST(SidePacketToStreamCalculator, WrongConfig_MixedTags) { + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie( + R"( + input_stream: "tick" + input_side_packet: "side_packet0" + input_side_packet: "side_packet1" + node { + calculator: "SidePacketToStreamCalculator" + input_side_packet: "side_packet0" + input_side_packet: "side_packet1" + output_stream: "AT_TICK:packet0" + output_stream: "AT_PRE_STREAM:packet1" + } + )"); + CalculatorGraph graph; + auto status = graph.Initialize(graph_config); + EXPECT_FALSE(status.ok()); + EXPECT_PRED2(absl::StrContains, status.message(), + "Only one of AT_PRESTREAM, AT_POSTSTREAM, AT_ZERO and AT_TICK " + "tags is allowed and required to specify output stream(s)."); +} + +TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughSidePackets) { + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie( + R"( + input_side_packet: "side_packet0" + input_side_packet: "side_packet1" + node { + calculator: "SidePacketToStreamCalculator" + input_side_packet: "side_packet0" + output_stream: "AT_PRESTREAM:0:packet0" + output_stream: "AT_PRESTREAM:1:packet1" + } + )"); + CalculatorGraph graph; + auto status = graph.Initialize(graph_config); + EXPECT_FALSE(status.ok()); + EXPECT_PRED2( + absl::StrContains, status.message(), + "Same number of input side packets and output streams is required."); +} + +TEST(SidePacketToStreamCalculator, WrongConfig_NotEnoughOutputStreams) { + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie( + R"( + input_side_packet: "side_packet0" + input_side_packet: "side_packet1" + node { + calculator: "SidePacketToStreamCalculator" + input_side_packet: "side_packet0" + input_side_packet: "side_packet1" + output_stream: "AT_PRESTREAM:packet0" + } + )"); + CalculatorGraph graph; + auto status = graph.Initialize(graph_config); + EXPECT_FALSE(status.ok()); + EXPECT_PRED2( + absl::StrContains, status.message(), + "Same number of input side packets and output streams is required."); +} + +void DoTestNonAtTickOutputTag(absl::string_view tag, + Timestamp expected_timestamp) { + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie(absl::StrReplaceAll( + R"( + input_side_packet: "side_packet" + output_stream: "packet" + node { + calculator: "SidePacketToStreamCalculator" + input_side_packet: "side_packet" + output_stream: "$tag:packet" + } + )", + {{"$tag", tag}})); + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config)); + const int expected_value = 10; + std::vector output_packets; + MP_ASSERT_OK(graph.ObserveOutputStream( + "packet", [&output_packets](const Packet& packet) { + output_packets.push_back(packet); + return ::mediapipe::OkStatus(); + })); + MP_ASSERT_OK( + graph.StartRun({{"side_packet", MakePacket(expected_value)}})); + MP_ASSERT_OK(graph.WaitForObservedOutput()); + + ASSERT_FALSE(output_packets.empty()); + EXPECT_EQ(expected_timestamp, output_packets.back().Timestamp()); + EXPECT_EQ(expected_value, output_packets.back().Get()); +} + +TEST(SidePacketToStreamCalculator, NoAtTickOutputTags) { + DoTestNonAtTickOutputTag("AT_PRESTREAM", Timestamp::PreStream()); + DoTestNonAtTickOutputTag("AT_POSTSTREAM", Timestamp::PostStream()); + DoTestNonAtTickOutputTag("AT_ZERO", Timestamp(0)); +} + +TEST(SidePacketToStreamCalculator, AtTick) { + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie( + R"( + input_stream: "tick" + input_side_packet: "side_packet" + output_stream: "packet" + node { + calculator: "SidePacketToStreamCalculator" + input_stream: "TICK:tick" + input_side_packet: "side_packet" + output_stream: "AT_TICK:packet" + } + )"); + std::vector output_packets; + tool::AddVectorSink("packet", &graph_config, &output_packets); + CalculatorGraph graph; + + MP_ASSERT_OK(graph.Initialize(graph_config)); + const int expected_value = 20; + MP_ASSERT_OK( + graph.StartRun({{"side_packet", MakePacket(expected_value)}})); + + auto tick_and_verify = [&graph, &output_packets, + expected_value](int at_timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "tick", + MakePacket(/*doesn't matter*/ 1).At(Timestamp(at_timestamp)))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + + ASSERT_FALSE(output_packets.empty()); + EXPECT_EQ(Timestamp(at_timestamp), output_packets.back().Timestamp()); + EXPECT_EQ(expected_value, output_packets.back().Get()); + }; + + tick_and_verify(/*at_timestamp=*/0); + tick_and_verify(/*at_timestamp=*/1); + tick_and_verify(/*at_timestamp=*/128); + tick_and_verify(/*at_timestamp=*/1024); + tick_and_verify(/*at_timestamp=*/1025); +} + +TEST(SidePacketToStreamCalculator, AtTick_MultipleSidePackets) { + CalculatorGraphConfig graph_config = + ParseTextProtoOrDie( + R"( + input_stream: "tick" + input_side_packet: "side_packet0" + input_side_packet: "side_packet1" + output_stream: "packet0" + output_stream: "packet1" + node { + calculator: "SidePacketToStreamCalculator" + input_stream: "TICK:tick" + input_side_packet: "side_packet0" + input_side_packet: "side_packet1" + output_stream: "AT_TICK:0:packet0" + output_stream: "AT_TICK:1:packet1" + } + )"); + std::vector output_packets0; + tool::AddVectorSink("packet0", &graph_config, &output_packets0); + std::vector output_packets1; + tool::AddVectorSink("packet1", &graph_config, &output_packets1); + CalculatorGraph graph; + + MP_ASSERT_OK(graph.Initialize(graph_config)); + const int expected_value0 = 20; + const int expected_value1 = 128; + MP_ASSERT_OK( + graph.StartRun({{"side_packet0", MakePacket(expected_value0)}, + {"side_packet1", MakePacket(expected_value1)}})); + + auto tick_and_verify = [&graph, &output_packets0, &output_packets1, + expected_value0, expected_value1](int at_timestamp) { + MP_ASSERT_OK(graph.AddPacketToInputStream( + "tick", + MakePacket(/*doesn't matter*/ 1).At(Timestamp(at_timestamp)))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + + ASSERT_FALSE(output_packets0.empty()); + ASSERT_FALSE(output_packets1.empty()); + + EXPECT_EQ(Timestamp(at_timestamp), output_packets0.back().Timestamp()); + EXPECT_EQ(expected_value0, output_packets0.back().Get()); + EXPECT_EQ(Timestamp(at_timestamp), output_packets1.back().Timestamp()); + EXPECT_EQ(expected_value1, output_packets1.back().Get()); + }; + + tick_and_verify(/*at_timestamp=*/0); + tick_and_verify(/*at_timestamp=*/1); + tick_and_verify(/*at_timestamp=*/128); + tick_and_verify(/*at_timestamp=*/1024); + tick_and_verify(/*at_timestamp=*/1025); +} + +} // namespace +} // namespace mediapipe diff --git a/mediapipe/calculators/image/BUILD b/mediapipe/calculators/image/BUILD index 1a8c1219f..a6159b554 100644 --- a/mediapipe/calculators/image/BUILD +++ b/mediapipe/calculators/image/BUILD @@ -346,9 +346,7 @@ cc_library( ], "//conditions:default": [], }), - visibility = [ - "//visibility:public", - ], + visibility = ["//visibility:public"], deps = [ ":image_cropping_calculator_cc_proto", "//mediapipe/framework:calculator_framework", diff --git a/mediapipe/calculators/image/color_convert_calculator.cc b/mediapipe/calculators/image/color_convert_calculator.cc index 3ff1a738f..aa1b51c51 100644 --- a/mediapipe/calculators/image/color_convert_calculator.cc +++ b/mediapipe/calculators/image/color_convert_calculator.cc @@ -75,6 +75,11 @@ class ColorConvertCalculator : public CalculatorBase { static ::mediapipe::Status GetContract(CalculatorContract* cc); ::mediapipe::Status Process(CalculatorContext* cc) override; + ::mediapipe::Status Open(CalculatorContext* cc) override { + cc->SetOffset(TimestampDiff(0)); + return ::mediapipe::OkStatus(); + } + private: // Wrangles the appropriate inputs and outputs to perform the color // conversion. The ImageFrame on input_tag is converted using the diff --git a/mediapipe/calculators/image/image_cropping_calculator.cc b/mediapipe/calculators/image/image_cropping_calculator.cc index beb131b3b..a9c7ae657 100644 --- a/mediapipe/calculators/image/image_cropping_calculator.cc +++ b/mediapipe/calculators/image/image_cropping_calculator.cc @@ -119,6 +119,13 @@ REGISTER_CALCULATOR(ImageCroppingCalculator); #endif // !MEDIAPIPE_DISABLE_GPU } + // Validate border mode. + if (use_gpu_) { + MP_RETURN_IF_ERROR(ValidateBorderModeForGPU(cc)); + } else { + MP_RETURN_IF_ERROR(ValidateBorderModeForCPU(cc)); + } + return ::mediapipe::OkStatus(); } @@ -162,6 +169,32 @@ REGISTER_CALCULATOR(ImageCroppingCalculator); return ::mediapipe::OkStatus(); } +::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForCPU( + CalculatorContext* cc) { + int border_mode; + return GetBorderModeForOpenCV(cc, &border_mode); +} + +::mediapipe::Status ImageCroppingCalculator::ValidateBorderModeForGPU( + CalculatorContext* cc) { + mediapipe::ImageCroppingCalculatorOptions options = + cc->Options(); + + switch (options.border_mode()) { + case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO: + LOG(WARNING) << "BORDER_ZERO mode is not supported by GPU " + << "implementation and will fall back into BORDER_REPLICATE"; + break; + case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE: + break; + default: + RET_CHECK_FAIL() << "Unsupported border mode for GPU: " + << options.border_mode(); + } + + return ::mediapipe::OkStatus(); +} + ::mediapipe::Status ImageCroppingCalculator::RenderCpu(CalculatorContext* cc) { if (cc->Inputs().Tag(kImageTag).IsEmpty()) { return ::mediapipe::OkStatus(); @@ -172,6 +205,10 @@ REGISTER_CALCULATOR(ImageCroppingCalculator); auto [target_width, target_height, rect_center_x, rect_center_y, rotation] = GetCropSpecs(cc, input_img.Width(), input_img.Height()); + // Get border mode and value for OpenCV. + int border_mode; + MP_RETURN_IF_ERROR(GetBorderModeForOpenCV(cc, &border_mode)); + const cv::RotatedRect min_rect(cv::Point2f(rect_center_x, rect_center_y), cv::Size2f(target_width, target_height), rotation * 180.f / M_PI); @@ -191,7 +228,9 @@ REGISTER_CALCULATOR(ImageCroppingCalculator); cv::getPerspectiveTransform(src_points, dst_points); cv::Mat cropped_image; cv::warpPerspective(input_mat, cropped_image, projection_matrix, - cv::Size(min_rect.size.width, min_rect.size.height)); + cv::Size(min_rect.size.width, min_rect.size.height), + /* flags = */ 0, + /* borderMode = */ border_mode); std::unique_ptr output_frame(new ImageFrame( input_img.Format(), cropped_image.cols, cropped_image.rows)); @@ -453,6 +492,7 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc, rotation = options.rotation(); } } + return { .width = crop_width, .height = crop_height, @@ -462,4 +502,24 @@ RectSpec ImageCroppingCalculator::GetCropSpecs(const CalculatorContext* cc, }; } +::mediapipe::Status ImageCroppingCalculator::GetBorderModeForOpenCV( + CalculatorContext* cc, int* border_mode) { + mediapipe::ImageCroppingCalculatorOptions options = + cc->Options(); + + switch (options.border_mode()) { + case mediapipe::ImageCroppingCalculatorOptions::BORDER_ZERO: + *border_mode = cv::BORDER_CONSTANT; + break; + case mediapipe::ImageCroppingCalculatorOptions::BORDER_REPLICATE: + *border_mode = cv::BORDER_REPLICATE; + break; + default: + RET_CHECK_FAIL() << "Unsupported border mode for CPU: " + << options.border_mode(); + } + + return ::mediapipe::OkStatus(); +} + } // namespace mediapipe diff --git a/mediapipe/calculators/image/image_cropping_calculator.h b/mediapipe/calculators/image/image_cropping_calculator.h index 005180250..38938c223 100644 --- a/mediapipe/calculators/image/image_cropping_calculator.h +++ b/mediapipe/calculators/image/image_cropping_calculator.h @@ -36,6 +36,7 @@ // Note: input_stream values take precedence over options defined in the graph. // namespace mediapipe { + struct RectSpec { int width; int height; @@ -63,12 +64,16 @@ class ImageCroppingCalculator : public CalculatorBase { int src_height); private: + ::mediapipe::Status ValidateBorderModeForCPU(CalculatorContext* cc); + ::mediapipe::Status ValidateBorderModeForGPU(CalculatorContext* cc); ::mediapipe::Status RenderCpu(CalculatorContext* cc); ::mediapipe::Status RenderGpu(CalculatorContext* cc); ::mediapipe::Status InitGpu(CalculatorContext* cc); void GlRender(); void GetOutputDimensions(CalculatorContext* cc, int src_width, int src_height, int* dst_width, int* dst_height); + ::mediapipe::Status GetBorderModeForOpenCV(CalculatorContext* cc, + int* border_mode); mediapipe::ImageCroppingCalculatorOptions options_; diff --git a/mediapipe/calculators/image/image_cropping_calculator.proto b/mediapipe/calculators/image/image_cropping_calculator.proto index f02dfcbf7..83dce6ae7 100644 --- a/mediapipe/calculators/image/image_cropping_calculator.proto +++ b/mediapipe/calculators/image/image_cropping_calculator.proto @@ -40,4 +40,15 @@ message ImageCroppingCalculatorOptions { // The (0, 0) point is at the (top, left) corner. optional float norm_center_x = 6 [default = 0]; optional float norm_center_y = 7 [default = 0]; + + enum BorderMode { + // First unspecified value is required by the guideline. See details here: + // https://developers.google.com/protocol-buffers/docs/style#enums + BORDER_UNSPECIFIED = 0; + BORDER_ZERO = 1; + BORDER_REPLICATE = 2; + } + + // Specifies behaviour for crops that go beyond image borders. + optional BorderMode border_mode = 8 [default = BORDER_ZERO]; } diff --git a/mediapipe/calculators/image/image_cropping_calculator_test.cc b/mediapipe/calculators/image/image_cropping_calculator_test.cc index 9a5877292..c511014aa 100644 --- a/mediapipe/calculators/image/image_cropping_calculator_test.cc +++ b/mediapipe/calculators/image/image_cropping_calculator_test.cc @@ -56,11 +56,11 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) { } )"); - auto calculator_state = - CalculatorState("Node", 0, "Calculator", calculator_node, nullptr); - auto cc = - CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(), - tool::CreateTagMap({}).ValueOrDie()); + auto calculator_state = absl::make_unique( + "Node", 0, "Calculator", calculator_node, nullptr); + auto cc = absl::make_unique( + calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(), + tool::CreateTagMap({}).ValueOrDie()); RectSpec expectRect = { .width = 60, @@ -69,9 +69,9 @@ TEST(ImageCroppingCalculatorTest, GetCroppingDimensionsNormal) { .center_y = 50, .rotation = 0.3, }; - EXPECT_EQ( - ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height), - expectRect); + EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width, + input_height), + expectRect); } // TEST // Test when (width height) + (norm_width norm_height) are set in options. @@ -96,11 +96,11 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) { } )"); - auto calculator_state = - CalculatorState("Node", 0, "Calculator", calculator_node, nullptr); - auto cc = - CalculatorContext(&calculator_state, tool::CreateTagMap({}).ValueOrDie(), - tool::CreateTagMap({}).ValueOrDie()); + auto calculator_state = absl::make_unique( + "Node", 0, "Calculator", calculator_node, nullptr); + auto cc = absl::make_unique( + calculator_state.get(), tool::CreateTagMap({}).ValueOrDie(), + tool::CreateTagMap({}).ValueOrDie()); RectSpec expectRect = { .width = 50, .height = 50, @@ -108,9 +108,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecInOptions) { .center_y = 50, .rotation = 0.3, }; - EXPECT_EQ( - ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height), - expectRect); + EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width, + input_height), + expectRect); } // TEST // Test when WIDTH HEIGHT are set from input stream, @@ -138,16 +138,16 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) { } )"); - auto calculator_state = - CalculatorState("Node", 0, "Calculator", calculator_node, nullptr); + auto calculator_state = absl::make_unique( + "Node", 0, "Calculator", calculator_node, nullptr); auto inputTags = tool::CreateTagMap({ "HEIGHT:0:crop_height", "WIDTH:0:crop_width", }) .ValueOrDie(); - auto cc = CalculatorContext(&calculator_state, inputTags, - tool::CreateTagMap({}).ValueOrDie()); - auto& inputs = cc.Inputs(); + auto cc = absl::make_unique( + calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie()); + auto& inputs = cc->Inputs(); inputs.Tag(kHeightTag).Value() = MakePacket(1); inputs.Tag(kWidthTag).Value() = MakePacket(1); RectSpec expectRect = { @@ -157,9 +157,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpectWithInputStream) { .center_y = 50, .rotation = 0.3, }; - EXPECT_EQ( - ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height), - expectRect); + EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width, + input_height), + expectRect); } // TEST // Test when RECT is set from input stream, @@ -186,15 +186,15 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) { } )"); - auto calculator_state = - CalculatorState("Node", 0, "Calculator", calculator_node, nullptr); + auto calculator_state = absl::make_unique( + "Node", 0, "Calculator", calculator_node, nullptr); auto inputTags = tool::CreateTagMap({ "RECT:0:rect", }) .ValueOrDie(); - auto cc = CalculatorContext(&calculator_state, inputTags, - tool::CreateTagMap({}).ValueOrDie()); - auto& inputs = cc.Inputs(); + auto cc = absl::make_unique( + calculator_state.get(), inputTags, tool::CreateTagMap({}).ValueOrDie()); + auto& inputs = cc->Inputs(); mediapipe::Rect rect = ParseTextProtoOrDie( R"( width: 1 height: 1 x_center: 40 y_center: 40 rotation: 0.5 @@ -207,9 +207,9 @@ TEST(ImageCroppingCalculatorTest, RedundantSpecWithInputStream) { .center_y = 40, .rotation = 0.5, }; - EXPECT_EQ( - ImageCroppingCalculator::GetCropSpecs(&cc, input_width, input_height), - expectRect); + EXPECT_EQ(ImageCroppingCalculator::GetCropSpecs(cc.get(), input_width, + input_height), + expectRect); } // TEST } // namespace diff --git a/mediapipe/calculators/image/image_transformation_calculator.cc b/mediapipe/calculators/image/image_transformation_calculator.cc index bb1f6756e..683e82511 100644 --- a/mediapipe/calculators/image/image_transformation_calculator.cc +++ b/mediapipe/calculators/image/image_transformation_calculator.cc @@ -104,6 +104,14 @@ mediapipe::ScaleMode_Mode ParseScaleMode( // to be a multiple of 90 degrees. If provided, it overrides the // ROTATION_DEGREES input side packet. // +// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not. If +// provided, it overrides the FLIP_HORIZONTALLY input side packet and/or +// corresponding field in the calculator options. +// +// FLIP_VERTICALLY (optional): Whether to flip image vertically or not. If +// provided, it overrides the FLIP_VERTICALLY input side packet and/or +// corresponding field in the calculator options. +// // Output: // One of the following two tags: // IMAGE - ImageFrame representing the output image. @@ -129,6 +137,12 @@ mediapipe::ScaleMode_Mode ParseScaleMode( // degrees. It has to be a multiple of 90 degrees. It overrides the // corresponding field in the calculator options. // +// FLIP_HORIZONTALLY (optional): Whether to flip image horizontally or not. +// It overrides the corresponding field in the calculator options. +// +// FLIP_VERTICALLY (optional): Whether to flip image vertically or not. +// It overrides the corresponding field in the calculator options. +// // Calculator options (see image_transformation_calculator.proto): // output_width, output_height - (optional) Desired scaled image size. // rotation_mode - (optional) Rotation in multiples of 90 degrees. @@ -167,6 +181,8 @@ class ImageTransformationCalculator : public CalculatorBase { int output_height_ = 0; mediapipe::RotationMode_Mode rotation_; mediapipe::ScaleMode_Mode scale_mode_; + bool flip_horizontally_ = false; + bool flip_vertically_ = false; bool use_gpu_ = false; #if !defined(MEDIAPIPE_DISABLE_GPU) @@ -203,6 +219,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator); if (cc->Inputs().HasTag("ROTATION_DEGREES")) { cc->Inputs().Tag("ROTATION_DEGREES").Set(); } + if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) { + cc->Inputs().Tag("FLIP_HORIZONTALLY").Set(); + } + if (cc->Inputs().HasTag("FLIP_VERTICALLY")) { + cc->Inputs().Tag("FLIP_VERTICALLY").Set(); + } if (cc->InputSidePackets().HasTag("OUTPUT_DIMENSIONS")) { cc->InputSidePackets().Tag("OUTPUT_DIMENSIONS").Set(); @@ -210,6 +232,12 @@ REGISTER_CALCULATOR(ImageTransformationCalculator); if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) { cc->InputSidePackets().Tag("ROTATION_DEGREES").Set(); } + if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) { + cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set(); + } + if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) { + cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set(); + } if (cc->Outputs().HasTag("LETTERBOX_PADDING")) { cc->Outputs().Tag("LETTERBOX_PADDING").Set>(); @@ -245,6 +273,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator); output_width_ = options_.output_width(); output_height_ = options_.output_height(); } + if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) { rotation_ = DegreesToRotationMode( cc->InputSidePackets().Tag("ROTATION_DEGREES").Get()); @@ -252,6 +281,20 @@ REGISTER_CALCULATOR(ImageTransformationCalculator); rotation_ = options_.rotation_mode(); } + if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) { + flip_horizontally_ = + cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get(); + } else { + flip_horizontally_ = options_.flip_horizontally(); + } + + if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) { + flip_vertically_ = + cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get(); + } else { + flip_vertically_ = options_.flip_vertically(); + } + scale_mode_ = ParseScaleMode(options_.scale_mode(), DEFAULT_SCALE_MODE); if (use_gpu_) { @@ -268,12 +311,37 @@ REGISTER_CALCULATOR(ImageTransformationCalculator); ::mediapipe::Status ImageTransformationCalculator::Process( CalculatorContext* cc) { + // Override values if specified so. + if (cc->Inputs().HasTag("ROTATION_DEGREES") && + !cc->Inputs().Tag("ROTATION_DEGREES").IsEmpty()) { + rotation_ = + DegreesToRotationMode(cc->Inputs().Tag("ROTATION_DEGREES").Get()); + } + if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") && + !cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) { + flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get(); + } + if (cc->Inputs().HasTag("FLIP_VERTICALLY") && + !cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) { + flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get(); + } + if (use_gpu_) { #if !defined(MEDIAPIPE_DISABLE_GPU) + if (cc->Inputs().Tag("IMAGE_GPU").IsEmpty()) { + // Image is missing, hence no way to produce output image. (Timestamp + // bound will be updated automatically.) + return ::mediapipe::OkStatus(); + } return helper_.RunInGlContext( [this, cc]() -> ::mediapipe::Status { return RenderGpu(cc); }); #endif // !MEDIAPIPE_DISABLE_GPU } else { + if (cc->Inputs().Tag("IMAGE").IsEmpty()) { + // Image is missing, hence no way to produce output image. (Timestamp + // bound will be updated automatically.) + return ::mediapipe::OkStatus(); + } return RenderCpu(cc); } return ::mediapipe::OkStatus(); @@ -360,11 +428,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator); .Add(padding.release(), cc->InputTimestamp()); } - if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) { - rotation_ = DegreesToRotationMode( - cc->InputSidePackets().Tag("ROTATION_DEGREES").Get()); - } - cv::Mat rotated_mat; const int angle = RotationModeToDegrees(rotation_); cv::Point2f src_center(scaled_mat.cols / 2.0, scaled_mat.rows / 2.0); @@ -372,11 +435,9 @@ REGISTER_CALCULATOR(ImageTransformationCalculator); cv::warpAffine(scaled_mat, rotated_mat, rotation_mat, scaled_mat.size()); cv::Mat flipped_mat; - if (options_.flip_horizontally() || options_.flip_vertically()) { + if (flip_horizontally_ || flip_vertically_) { const int flip_code = - options_.flip_horizontally() && options_.flip_vertically() - ? -1 - : options_.flip_horizontally(); + flip_horizontally_ && flip_vertically_ ? -1 : flip_horizontally_; cv::flip(rotated_mat, flipped_mat, flip_code); } else { flipped_mat = rotated_mat; @@ -450,11 +511,6 @@ REGISTER_CALCULATOR(ImageTransformationCalculator); } RET_CHECK(renderer) << "Unsupported input texture type"; - if (cc->InputSidePackets().HasTag("ROTATION_DEGREES")) { - rotation_ = DegreesToRotationMode( - cc->InputSidePackets().Tag("ROTATION_DEGREES").Get()); - } - mediapipe::FrameScaleMode scale_mode = mediapipe::FrameScaleModeFromProto( scale_mode_, mediapipe::FrameScaleMode::kStretch); mediapipe::FrameRotation rotation = @@ -469,7 +525,7 @@ REGISTER_CALCULATOR(ImageTransformationCalculator); MP_RETURN_IF_ERROR(renderer->GlRender( src1.width(), src1.height(), dst.width(), dst.height(), scale_mode, - rotation, options_.flip_horizontally(), options_.flip_vertically(), + rotation, flip_horizontally_, flip_vertically_, /*flip_texture=*/false)); glActiveTexture(GL_TEXTURE1); diff --git a/mediapipe/calculators/image/scale_image_calculator.cc b/mediapipe/calculators/image/scale_image_calculator.cc index fc14c5d0a..ac441689e 100644 --- a/mediapipe/calculators/image/scale_image_calculator.cc +++ b/mediapipe/calculators/image/scale_image_calculator.cc @@ -260,11 +260,11 @@ ScaleImageCalculator::~ScaleImageCalculator() {} &crop_width_, &crop_height_, // &col_start_, &row_start_)); MP_RETURN_IF_ERROR( - scale_image::FindOutputDimensions(crop_width_, crop_height_, // - options_.target_width(), // - options_.target_height(), // - options_.preserve_aspect_ratio(), // - options_.scale_to_multiple_of_two(), // + scale_image::FindOutputDimensions(crop_width_, crop_height_, // + options_.target_width(), // + options_.target_height(), // + options_.preserve_aspect_ratio(), // + options_.scale_to_multiple_of(), // &output_width_, &output_height_)); MP_RETURN_IF_ERROR(FindInterpolationAlgorithm(options_.algorithm(), &interpolation_algorithm_)); @@ -361,17 +361,21 @@ ScaleImageCalculator::~ScaleImageCalculator() {} output_format_ = input_format_; } + const bool is_positive_and_even = + (options_.scale_to_multiple_of() >= 1) && + (options_.scale_to_multiple_of() % 2 == 0); + if (output_format_ == ImageFormat::YCBCR420P) { - RET_CHECK(options_.scale_to_multiple_of_two()) + RET_CHECK(is_positive_and_even) << "ScaleImageCalculator always outputs width and height that are " "divisible by 2 when output format is YCbCr420P. To scale to " "width and height of odd numbers, the output format must be SRGB."; } else if (options_.preserve_aspect_ratio()) { - RET_CHECK(options_.scale_to_multiple_of_two()) + RET_CHECK(options_.scale_to_multiple_of() == 2) << "ScaleImageCalculator always outputs width and height that are " - "divisible by 2 when perserving aspect ratio. To scale to width " - "and height of odd numbers, please set " - "preserve_aspect_ratio to false."; + "divisible by 2 when preserving aspect ratio. If you'd like to " + "set scale_to_multiple_of to something other than 2, please " + "set preserve_aspect_ratio to false."; } if (input_width_ > 0 && input_height_ > 0 && diff --git a/mediapipe/calculators/image/scale_image_calculator.proto b/mediapipe/calculators/image/scale_image_calculator.proto index 0f0d5aae4..e51ccafaa 100644 --- a/mediapipe/calculators/image/scale_image_calculator.proto +++ b/mediapipe/calculators/image/scale_image_calculator.proto @@ -11,9 +11,10 @@ import "mediapipe/framework/formats/image_format.proto"; // 2) Scale and convert the image to fit inside target_width x target_height // using the specified scaling algorithm. (maintaining the aspect // ratio if preserve_aspect_ratio is true). -// The output width and height will be divisible by 2. It is possible to output -// width and height that are odd number when the output format is SRGB and not -// perserving the aspect ratio. See scale_to_multiple_of_two option for details. +// The output width and height will be divisible by 2, by default. It is +// possible to output width and height that are odd numbers when the output +// format is SRGB and the aspect ratio is left unpreserved. See +// scale_to_multiple_of for details. message ScaleImageCalculatorOptions { extend CalculatorOptions { optional ScaleImageCalculatorOptions ext = 66237115; @@ -23,7 +24,7 @@ message ScaleImageCalculatorOptions { // depending on the other options below. If unset, use the same width // or height as the input. If only one is set then determine the other // from the aspect ratio (after cropping). The output width and height - // will be divisible by 2. + // will be divisible by 2, by default. optional int32 target_width = 1; optional int32 target_height = 2; @@ -31,7 +32,8 @@ message ScaleImageCalculatorOptions { // fits inside the box represented by target_width and target_height. // Otherwise it is scaled to fit target_width and target_height // completely. In any case, the aspect ratio that is preserved is - // that after cropping to the minimum/maximum aspect ratio. + // that after cropping to the minimum/maximum aspect ratio. Additionally, if + // true, the output width and height will be divisible by 2. optional bool preserve_aspect_ratio = 3 [default = true]; // If ratio is positive, crop the image to this minimum and maximum @@ -95,11 +97,13 @@ message ScaleImageCalculatorOptions { // SRGB or YCBCR420P. optional ImageFormat.Format input_format = 12; - // If true, the output width and height will be divisible by 2. Otherwise it - // will use the exact specified output width and height, which is only - // supported when the output format is SRGB and preserve_aspect_ratio option - // is set to false. - optional bool scale_to_multiple_of_two = 13 [default = true]; + // If set to 2, the target width and height will be rounded-down + // to the nearest even number. If set to any positive value other than 2, + // preserve_aspect_ratio must be false and the target width and height will be + // rounded-down to multiples of the given value. If set to any value less than + // 1, it will be treated like 1. + // NOTE: If set to an odd number, the output format must be SRGB. + optional int32 scale_to_multiple_of = 13 [default = 2]; // If true, assume the input YUV is BT.709 (this is the HDTV standard, so most // content is likely using it). If false use the previous assumption of BT.601 diff --git a/mediapipe/calculators/image/scale_image_utils.cc b/mediapipe/calculators/image/scale_image_utils.cc index db55774ad..3225521a5 100644 --- a/mediapipe/calculators/image/scale_image_utils.cc +++ b/mediapipe/calculators/image/scale_image_utils.cc @@ -88,17 +88,27 @@ double ParseRational(const std::string& rational) { return ::mediapipe::OkStatus(); } -::mediapipe::Status FindOutputDimensions(int input_width, // - int input_height, // - int target_width, // - int target_height, // - bool preserve_aspect_ratio, // - bool scale_to_multiple_of_two, // +::mediapipe::Status FindOutputDimensions(int input_width, // + int input_height, // + int target_width, // + int target_height, // + bool preserve_aspect_ratio, // + int scale_to_multiple_of, // int* output_width, int* output_height) { CHECK(output_width); CHECK(output_height); + if (preserve_aspect_ratio) { + RET_CHECK(scale_to_multiple_of == 2) + << "FindOutputDimensions always outputs width and height that are " + "divisible by 2 when preserving aspect ratio. If you'd like to " + "set scale_to_multiple_of to something other than 2, please " + "set preserve_aspect_ratio to false."; + } + + if (scale_to_multiple_of < 1) scale_to_multiple_of = 1; + if (!preserve_aspect_ratio || (target_width <= 0 && target_height <= 0)) { if (target_width <= 0) { target_width = input_width; @@ -106,13 +116,13 @@ double ParseRational(const std::string& rational) { if (target_height <= 0) { target_height = input_height; } - if (scale_to_multiple_of_two) { - *output_width = (target_width / 2) * 2; - *output_height = (target_height / 2) * 2; - } else { - *output_width = target_width; - *output_height = target_height; - } + + target_width -= target_width % scale_to_multiple_of; + target_height -= target_height % scale_to_multiple_of; + + *output_width = target_width; + *output_height = target_height; + return ::mediapipe::OkStatus(); } diff --git a/mediapipe/calculators/image/scale_image_utils.h b/mediapipe/calculators/image/scale_image_utils.h index 450b81d23..ea9dd3f0f 100644 --- a/mediapipe/calculators/image/scale_image_utils.h +++ b/mediapipe/calculators/image/scale_image_utils.h @@ -35,17 +35,19 @@ namespace scale_image { int* col_start, int* row_start); // Given an input width and height, a target width and height, whether to -// preserve the aspect ratio, and whether to round down to a multiple of 2, -// determine the output width and height. If target_width or target_height is -// non-positive, then they will be set to the input_width and input_height -// respectively. The output_width and output_height will be reduced as necessary -// to preserve_aspect_ratio and to scale_to_multipe_of_two if these options are -// specified. +// preserve the aspect ratio, and whether to round-down to the multiple of a +// given number nearest to the targets, determine the output width and height. +// If target_width or target_height is non-positive, then they will be set to +// the input_width and input_height respectively. If scale_to_multiple_of is +// less than 1, it will be treated like 1. The output_width and +// output_height will be reduced as necessary to preserve_aspect_ratio if the +// option is specified. If preserving the aspect ratio is desired, you must set +// scale_to_multiple_of to 2. ::mediapipe::Status FindOutputDimensions(int input_width, int input_height, // int target_width, - int target_height, // - bool preserve_aspect_ratio, // - bool scale_to_multiple_of_two, // + int target_height, // + bool preserve_aspect_ratio, // + int scale_to_multiple_of, // int* output_width, int* output_height); } // namespace scale_image diff --git a/mediapipe/calculators/image/scale_image_utils_test.cc b/mediapipe/calculators/image/scale_image_utils_test.cc index d6421631b..14a58e762 100644 --- a/mediapipe/calculators/image/scale_image_utils_test.cc +++ b/mediapipe/calculators/image/scale_image_utils_test.cc @@ -79,49 +79,49 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsPreserveRatio) { int output_width; int output_height; // Not scale. - MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, true, &output_width, + MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, -1, true, 2, &output_width, &output_height)); EXPECT_EQ(200, output_width); EXPECT_EQ(100, output_height); // Not scale with odd input size. - MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, false, - &output_width, &output_height)); + MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, -1, false, 1, &output_width, + &output_height)); EXPECT_EQ(201, output_width); EXPECT_EQ(101, output_height); // Scale down by 1/2. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, true, - &output_width, &output_height)); + MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, true, 2, &output_width, + &output_height)); EXPECT_EQ(100, output_width); EXPECT_EQ(50, output_height); // Scale up, doubling dimensions. - MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, true, - &output_width, &output_height)); + MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, true, 2, &output_width, + &output_height)); EXPECT_EQ(400, output_width); EXPECT_EQ(200, output_height); // Fits a 2:1 image into a 150 x 150 box. Output dimensions are always // visible by 2. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, true, - &output_width, &output_height)); + MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 150, true, 2, &output_width, + &output_height)); EXPECT_EQ(150, output_width); EXPECT_EQ(74, output_height); // Fits a 2:1 image into a 400 x 50 box. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, true, - &output_width, &output_height)); + MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 50, true, 2, &output_width, + &output_height)); EXPECT_EQ(100, output_width); EXPECT_EQ(50, output_height); // Scale to multiple number with odd targe size. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, true, - &output_width, &output_height)); + MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width, + &output_height)); EXPECT_EQ(100, output_width); EXPECT_EQ(50, output_height); // Scale to multiple number with odd targe size. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, false, - &output_width, &output_height)); + MP_ASSERT_OK(FindOutputDimensions(200, 100, 101, -1, true, 2, &output_width, + &output_height)); EXPECT_EQ(100, output_width); EXPECT_EQ(50, output_height); // Scale to odd size. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, false, - &output_width, &output_height)); + MP_ASSERT_OK(FindOutputDimensions(200, 100, 151, 101, false, 1, &output_width, + &output_height)); EXPECT_EQ(151, output_width); EXPECT_EQ(101, output_height); } @@ -131,22 +131,62 @@ TEST(ScaleImageUtilsTest, FindOutputDimensionsNoAspectRatio) { int output_width; int output_height; // Scale width only. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, true, - &output_width, &output_height)); + MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 2, &output_width, + &output_height)); EXPECT_EQ(100, output_width); EXPECT_EQ(100, output_height); // Scale height only. - MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, true, - &output_width, &output_height)); + MP_ASSERT_OK(FindOutputDimensions(200, 100, -1, 200, false, 2, &output_width, + &output_height)); EXPECT_EQ(200, output_width); EXPECT_EQ(200, output_height); // Scale both dimensions. - MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, true, - &output_width, &output_height)); + MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 2, &output_width, + &output_height)); EXPECT_EQ(150, output_width); EXPECT_EQ(200, output_height); } +// Tests scale_to_multiple_of. +TEST(ScaleImageUtilsTest, FindOutputDimensionsDownScaleToMultipleOf) { + int output_width; + int output_height; + // Set no targets, downscale to a multiple of 8. + MP_ASSERT_OK(FindOutputDimensions(100, 100, -1, -1, false, 8, &output_width, + &output_height)); + EXPECT_EQ(96, output_width); + EXPECT_EQ(96, output_height); + // Set width target, downscale to a multiple of 8. + MP_ASSERT_OK(FindOutputDimensions(200, 100, 100, -1, false, 8, &output_width, + &output_height)); + EXPECT_EQ(96, output_width); + EXPECT_EQ(96, output_height); + // Set height target, downscale to a multiple of 8. + MP_ASSERT_OK(FindOutputDimensions(201, 101, -1, 201, false, 8, &output_width, + &output_height)); + EXPECT_EQ(200, output_width); + EXPECT_EQ(200, output_height); + // Set both targets, downscale to a multiple of 8. + MP_ASSERT_OK(FindOutputDimensions(200, 100, 150, 200, false, 8, &output_width, + &output_height)); + EXPECT_EQ(144, output_width); + EXPECT_EQ(200, output_height); + // Doesn't throw error if keep aspect is true and downscale multiple is 2. + MP_ASSERT_OK(FindOutputDimensions(200, 100, 400, 200, true, 2, &output_width, + &output_height)); + EXPECT_EQ(400, output_width); + EXPECT_EQ(200, output_height); + // Throws error if keep aspect is true, but downscale multiple is not 2. + ASSERT_THAT(FindOutputDimensions(200, 100, 400, 200, true, 4, &output_width, + &output_height), + testing::Not(testing::status::IsOk())); + // Downscaling to multiple ignored if multiple is less than 2. + MP_ASSERT_OK(FindOutputDimensions(200, 100, 401, 201, false, 1, &output_width, + &output_height)); + EXPECT_EQ(401, output_width); + EXPECT_EQ(201, output_height); +} + } // namespace } // namespace scale_image } // namespace mediapipe diff --git a/mediapipe/calculators/tensorflow/BUILD b/mediapipe/calculators/tensorflow/BUILD index 8c9d0df84..93c4f751e 100644 --- a/mediapipe/calculators/tensorflow/BUILD +++ b/mediapipe/calculators/tensorflow/BUILD @@ -138,7 +138,7 @@ mediapipe_cc_proto_library( srcs = ["image_frame_to_tensor_calculator.proto"], cc_deps = [ "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], visibility = ["//visibility:public"], deps = [":image_frame_to_tensor_calculator_proto"], @@ -173,7 +173,7 @@ mediapipe_cc_proto_library( srcs = ["pack_media_sequence_calculator.proto"], cc_deps = [ "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], visibility = ["//visibility:public"], deps = [":pack_media_sequence_calculator_proto"], @@ -192,7 +192,7 @@ mediapipe_cc_proto_library( srcs = ["tensorflow_session_from_frozen_graph_generator.proto"], cc_deps = [ "//mediapipe/framework:packet_generator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], visibility = ["//visibility:public"], deps = [":tensorflow_session_from_frozen_graph_generator_proto"], @@ -203,7 +203,7 @@ mediapipe_cc_proto_library( srcs = ["tensorflow_session_from_frozen_graph_calculator.proto"], cc_deps = [ "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], visibility = ["//visibility:public"], deps = [":tensorflow_session_from_frozen_graph_calculator_proto"], @@ -277,7 +277,7 @@ mediapipe_cc_proto_library( srcs = ["vector_int_to_tensor_calculator_options.proto"], cc_deps = [ "//mediapipe/framework:calculator_cc_proto", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], visibility = ["//visibility:public"], deps = [":vector_int_to_tensor_calculator_options_proto"], @@ -408,7 +408,7 @@ cc_library( "//mediapipe/util/sequence:media_sequence", "//mediapipe/util/sequence:media_sequence_util", "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], alwayslink = 1, ) @@ -423,7 +423,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], alwayslink = 1, ) @@ -654,7 +654,7 @@ cc_library( "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], alwayslink = 1, ) @@ -695,7 +695,7 @@ cc_library( "//mediapipe/util:audio_decoder_cc_proto", "//mediapipe/util/sequence:media_sequence", "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], alwayslink = 1, ) @@ -737,7 +737,7 @@ cc_library( "//mediapipe/framework:calculator_framework", "//mediapipe/framework:packet", "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], alwayslink = 1, ) @@ -745,6 +745,7 @@ cc_library( cc_test( name = "graph_tensors_packet_generator_test", srcs = ["graph_tensors_packet_generator_test.cc"], + linkstatic = 1, deps = [ ":graph_tensors_packet_generator", ":graph_tensors_packet_generator_cc_proto", @@ -761,6 +762,7 @@ cc_test( name = "image_frame_to_tensor_calculator_test", size = "small", srcs = ["image_frame_to_tensor_calculator_test.cc"], + linkstatic = 1, deps = [ ":image_frame_to_tensor_calculator", "//mediapipe/framework:calculator_framework", @@ -777,6 +779,7 @@ cc_test( name = "matrix_to_tensor_calculator_test", size = "small", srcs = ["matrix_to_tensor_calculator_test.cc"], + linkstatic = 1, deps = [ ":matrix_to_tensor_calculator", ":matrix_to_tensor_calculator_options_cc_proto", @@ -793,6 +796,7 @@ cc_test( name = "lapped_tensor_buffer_calculator_test", size = "small", srcs = ["lapped_tensor_buffer_calculator_test.cc"], + linkstatic = 1, deps = [ ":lapped_tensor_buffer_calculator", ":lapped_tensor_buffer_calculator_cc_proto", @@ -801,7 +805,7 @@ cc_test( "//mediapipe/framework/port:gtest_main", "@com_google_absl//absl/memory", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -840,7 +844,7 @@ cc_test( "//mediapipe/util/sequence:media_sequence", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -867,7 +871,7 @@ cc_test( "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/core:direct_session", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:testlib", "@org_tensorflow//tensorflow/core/kernels:conv_ops", "@org_tensorflow//tensorflow/core/kernels:math", @@ -897,7 +901,7 @@ cc_test( "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/core:direct_session", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:testlib", "@org_tensorflow//tensorflow/core/kernels:conv_ops", "@org_tensorflow//tensorflow/core/kernels:math", @@ -956,6 +960,7 @@ cc_test( cc_test( name = "tensor_squeeze_dimensions_calculator_test", srcs = ["tensor_squeeze_dimensions_calculator_test.cc"], + linkstatic = 1, deps = [ ":tensor_squeeze_dimensions_calculator", ":tensor_squeeze_dimensions_calculator_cc_proto", @@ -963,7 +968,7 @@ cc_test( "//mediapipe/framework:calculator_runner", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -971,6 +976,7 @@ cc_test( name = "tensor_to_image_frame_calculator_test", size = "small", srcs = ["tensor_to_image_frame_calculator_test.cc"], + linkstatic = 1, deps = [ ":tensor_to_image_frame_calculator", ":tensor_to_image_frame_calculator_cc_proto", @@ -979,7 +985,7 @@ cc_test( "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -987,6 +993,7 @@ cc_test( name = "tensor_to_matrix_calculator_test", size = "small", srcs = ["tensor_to_matrix_calculator_test.cc"], + linkstatic = 1, deps = [ ":tensor_to_matrix_calculator", ":tensor_to_matrix_calculator_cc_proto", @@ -996,13 +1003,14 @@ cc_test( "//mediapipe/framework/formats:time_series_header_cc_proto", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) cc_test( name = "tensor_to_vector_float_calculator_test", srcs = ["tensor_to_vector_float_calculator_test.cc"], + linkstatic = 1, deps = [ ":tensor_to_vector_float_calculator", ":tensor_to_vector_float_calculator_options_cc_proto", @@ -1010,7 +1018,7 @@ cc_test( "//mediapipe/framework:calculator_runner", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -1030,13 +1038,14 @@ cc_test( "//mediapipe/util/sequence:media_sequence", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) cc_test( name = "vector_int_to_tensor_calculator_test", srcs = ["vector_int_to_tensor_calculator_test.cc"], + linkstatic = 1, deps = [ ":vector_int_to_tensor_calculator", ":vector_int_to_tensor_calculator_options_cc_proto", @@ -1044,13 +1053,14 @@ cc_test( "//mediapipe/framework:calculator_runner", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) cc_test( name = "vector_float_to_tensor_calculator_test", srcs = ["vector_float_to_tensor_calculator_test.cc"], + linkstatic = 1, deps = [ ":vector_float_to_tensor_calculator", ":vector_float_to_tensor_calculator_options_cc_proto", @@ -1058,7 +1068,7 @@ cc_test( "//mediapipe/framework:calculator_runner", "//mediapipe/framework/port:gtest_main", "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc index 1d116b8c2..03ce08f7b 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.cc @@ -17,7 +17,7 @@ #if !defined(__ANDROID__) #include "mediapipe/framework/port/file_helpers.h" #endif -#include "absl/strings/substitute.h" +#include "absl/strings/str_replace.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" @@ -63,7 +63,7 @@ const std::string MaybeConvertSignatureToTag( output.resize(name.length()); std::transform(name.begin(), name.end(), output.begin(), [](unsigned char c) { return std::toupper(c); }); - output = absl::Substitute(output, "/", "_"); + output = absl::StrReplaceAll(output, {{"/", "_"}}); return output; } else { return name; diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc index a202308db..fee0da0fb 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/strings/substitute.h" +#include "absl/strings/str_replace.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_calculator.pb.h" #include "mediapipe/framework/calculator.pb.h" diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc index 64764eb36..a2aaa1e9a 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.cc @@ -17,7 +17,7 @@ #if !defined(__ANDROID__) #include "mediapipe/framework/port/file_helpers.h" #endif -#include "absl/strings/substitute.h" +#include "absl/strings/str_replace.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h" #include "mediapipe/framework/deps/file_path.h" @@ -65,7 +65,7 @@ const std::string MaybeConvertSignatureToTag( output.resize(name.length()); std::transform(name.begin(), name.end(), output.begin(), [](unsigned char c) { return std::toupper(c); }); - output = absl::Substitute(output, "/", "_"); + output = absl::StrReplaceAll(output, {{"/", "_"}}); return output; } else { return name; diff --git a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc index 24603d050..d12fee12a 100644 --- a/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc +++ b/mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/strings/substitute.h" +#include "absl/strings/str_replace.h" #include "mediapipe/calculators/tensorflow/tensorflow_session.h" #include "mediapipe/calculators/tensorflow/tensorflow_session_from_saved_model_generator.pb.h" #include "mediapipe/framework/calculator_framework.h" diff --git a/mediapipe/calculators/tflite/BUILD b/mediapipe/calculators/tflite/BUILD index 50531a58b..7c711e842 100644 --- a/mediapipe/calculators/tflite/BUILD +++ b/mediapipe/calculators/tflite/BUILD @@ -485,6 +485,7 @@ cc_test( "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/framework/tool:validate_type", + "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/lite:framework", "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", ], diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator.cc b/mediapipe/calculators/tflite/tflite_inference_calculator.cc index ddbb70ea1..7634fe251 100644 --- a/mediapipe/calculators/tflite/tflite_inference_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_inference_calculator.cc @@ -148,7 +148,7 @@ struct GPUData { // options: { // [mediapipe.TfLiteInferenceCalculatorOptions.ext] { // model_path: "modelname.tflite" -// use_gpu: true +// delegate { gpu {} } // } // } // } @@ -163,6 +163,9 @@ struct GPUData { // class TfLiteInferenceCalculator : public CalculatorBase { public: + using TfLiteDelegatePtr = + std::unique_ptr>; + static ::mediapipe::Status GetContract(CalculatorContract* cc); ::mediapipe::Status Open(CalculatorContext* cc) override; @@ -176,7 +179,7 @@ class TfLiteInferenceCalculator : public CalculatorBase { std::unique_ptr interpreter_; std::unique_ptr model_; - TfLiteDelegate* delegate_ = nullptr; + TfLiteDelegatePtr delegate_; #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) mediapipe::GlCalculatorHelper gpu_helper_; @@ -212,12 +215,18 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); RET_CHECK(cc->Outputs().HasTag("TENSORS") ^ cc->Outputs().HasTag("TENSORS_GPU")); - bool use_gpu = false; + const auto& options = + cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>(); + bool use_gpu = + options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu(); if (cc->Inputs().HasTag("TENSORS")) cc->Inputs().Tag("TENSORS").Set>(); #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) if (cc->Inputs().HasTag("TENSORS_GPU")) { + RET_CHECK(!options.has_delegate() || options.delegate().has_gpu()) + << "GPU input is compatible with GPU delegate only."; + cc->Inputs().Tag("TENSORS_GPU").Set>(); use_gpu |= true; } @@ -227,6 +236,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); cc->Outputs().Tag("TENSORS").Set>(); #if !defined(MEDIAPIPE_DISABLE_GPU) && !defined(__EMSCRIPTEN__) if (cc->Outputs().HasTag("TENSORS_GPU")) { + RET_CHECK(!options.has_delegate() || options.delegate().has_gpu()) + << "GPU output is compatible with GPU delegate only."; + cc->Outputs().Tag("TENSORS_GPU").Set>(); use_gpu |= true; } @@ -238,10 +250,6 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); .Set(); } - const auto& options = - cc->Options<::mediapipe::TfLiteInferenceCalculatorOptions>(); - use_gpu |= options.use_gpu(); - if (use_gpu) { #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); @@ -454,7 +462,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); if (gpu_inference_) { #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> Status { - TfLiteGpuDelegateDelete(delegate_); + delegate_ = nullptr; for (int i = 0; i < gpu_data_in_.size(); ++i) { gpu_data_in_[i].reset(); } @@ -464,7 +472,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); return ::mediapipe::OkStatus(); })); #elif defined(MEDIAPIPE_IOS) - TFLGpuDelegateDelete(delegate_); + delegate_ = nullptr; for (int i = 0; i < gpu_data_in_.size(); ++i) { gpu_data_in_[i].reset(); } @@ -472,8 +480,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); gpu_data_out_[i].reset(); } #endif + } else { + delegate_ = nullptr; } - delegate_ = nullptr; } #if defined(MEDIAPIPE_EDGE_TPU) edgetpu_context_.reset(); @@ -501,7 +510,8 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); } // Get execution modes. - gpu_inference_ = options.use_gpu(); + gpu_inference_ = + options.has_delegate() ? options.delegate().has_gpu() : options.use_gpu(); return ::mediapipe::OkStatus(); } @@ -526,8 +536,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); RET_CHECK(interpreter_); -#if defined(__EMSCRIPTEN__) +#if defined(__EMSCRIPTEN__) || defined(MEDIAPIPE_EDGE_TPU) interpreter_->SetNumThreads(1); +#else + interpreter_->SetNumThreads( + cc->Options() + .cpu_num_thread()); #endif // __EMSCRIPTEN__ if (gpu_output_) { @@ -545,20 +559,37 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); ::mediapipe::Status TfLiteInferenceCalculator::LoadDelegate( CalculatorContext* cc) { -#if defined(MEDIAPIPE_ANDROID) + const auto& calculator_opts = + cc->Options(); + if (calculator_opts.has_delegate() && + calculator_opts.delegate().has_tflite()) { + // Default tflite inference requeqsted - no need to modify graph. + return ::mediapipe::OkStatus(); + } + if (!gpu_inference_) { - if (cc->Options() - .use_nnapi()) { +#if defined(MEDIAPIPE_ANDROID) + const bool nnapi_requested = calculator_opts.has_delegate() + ? calculator_opts.delegate().has_nnapi() + : calculator_opts.use_nnapi(); + if (nnapi_requested) { // Attempt to use NNAPI. // If not supported, the default CPU delegate will be created and used. interpreter_->SetAllowFp16PrecisionForFp32(1); - delegate_ = tflite::NnApiDelegate(); - RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk); + delegate_ = + TfLiteDelegatePtr(tflite::NnApiDelegate(), [](TfLiteDelegate*) { + // No need to free according to tflite::NnApiDelegate() + // documentation. + }); + RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), + kTfLiteOk); + return ::mediapipe::OkStatus(); } +#endif // MEDIAPIPE_ANDROID + // Return, no need for GPU delegate below. return ::mediapipe::OkStatus(); } -#endif // ANDROID #if !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) // Configure and create the delegate. @@ -568,7 +599,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); TFLITE_GL_OBJECT_TYPE_FASTEST; options.compile_options.dynamic_batch_enabled = 0; options.compile_options.inline_parameters = 1; - if (!delegate_) delegate_ = TfLiteGpuDelegateCreate(&options); + if (!delegate_) + delegate_ = TfLiteDelegatePtr(TfLiteGpuDelegateCreate(&options), + &TfLiteGpuDelegateDelete); if (gpu_input_) { // Get input image sizes. @@ -586,7 +619,7 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); ::tflite::gpu::gl::CreateReadWriteShaderStorageBuffer( gpu_data_in_[i]->elements, &gpu_data_in_[i]->buffer)); RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor( - delegate_, gpu_data_in_[i]->buffer.id(), + delegate_.get(), gpu_data_in_[i]->buffer.id(), interpreter_->inputs()[i]), kTfLiteOk); } @@ -609,15 +642,16 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); for (int i = 0; i < gpu_data_out_.size(); ++i) { RET_CHECK_CALL(CreateReadWriteShaderStorageBuffer( gpu_data_out_[i]->elements, &gpu_data_out_[i]->buffer)); - RET_CHECK_EQ( - TfLiteGpuDelegateBindBufferToTensor( - delegate_, gpu_data_out_[i]->buffer.id(), output_indices[i]), - kTfLiteOk); + RET_CHECK_EQ(TfLiteGpuDelegateBindBufferToTensor( + delegate_.get(), gpu_data_out_[i]->buffer.id(), + output_indices[i]), + kTfLiteOk); } } // Must call this last. - RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk); + RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), + kTfLiteOk); #endif // OpenGL #if defined(MEDIAPIPE_IOS) @@ -626,7 +660,9 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); TFLGpuDelegateOptions options; options.allow_precision_loss = true; options.wait_type = TFLGpuDelegateWaitType::TFLGpuDelegateWaitTypePassive; - if (!delegate_) delegate_ = TFLGpuDelegateCreate(&options); + if (!delegate_) + delegate_ = TfLiteDelegatePtr(TFLGpuDelegateCreate(&options), + &TFLGpuDelegateDelete); id device = gpu_helper_.mtlDevice; if (gpu_input_) { @@ -678,10 +714,12 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); gpu_data_in_[i]->buffer = [device newBufferWithLength:gpu_data_in_[i]->elements * kHalfSize options:MTLResourceStorageModeShared]; - RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_), kTfLiteOk); - RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor( - delegate_, input_indices[i], gpu_data_in_[i]->buffer), - true); + RET_CHECK_EQ(interpreter_->ModifyGraphWithDelegate(delegate_.get()), + kTfLiteOk); + RET_CHECK_EQ( + TFLGpuDelegateBindMetalBufferToTensor( + delegate_.get(), input_indices[i], gpu_data_in_[i]->buffer), + true); } } if (gpu_output_) { @@ -725,9 +763,10 @@ REGISTER_CALCULATOR(TfLiteInferenceCalculator); gpu_data_out_[i]->buffer = [device newBufferWithLength:gpu_data_out_[i]->elements * kHalfSize options:MTLResourceStorageModeShared]; - RET_CHECK_EQ(TFLGpuDelegateBindMetalBufferToTensor( - delegate_, output_indices[i], gpu_data_out_[i]->buffer), - true); + RET_CHECK_EQ( + TFLGpuDelegateBindMetalBufferToTensor( + delegate_.get(), output_indices[i], gpu_data_out_[i]->buffer), + true); } // Create converter for GPU output. diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator.proto b/mediapipe/calculators/tflite/tflite_inference_calculator.proto index 8a862f3df..893574b6a 100644 --- a/mediapipe/calculators/tflite/tflite_inference_calculator.proto +++ b/mediapipe/calculators/tflite/tflite_inference_calculator.proto @@ -27,7 +27,7 @@ import "mediapipe/framework/calculator.proto"; // options { // [mediapipe.TfLiteInferenceCalculatorOptions.ext] { // model_path: "model.tflite" -// use_gpu: true +// delegate { gpu {} } // } // } // } @@ -37,6 +37,22 @@ message TfLiteInferenceCalculatorOptions { optional TfLiteInferenceCalculatorOptions ext = 233867213; } + message Delegate { + // Default inference provided by tflite. + message TfLite {} + // Delegate to run GPU inference depending on the device. + // (Can use OpenGl, OpenCl, Metal depending on the device.) + message Gpu {} + // Android only. + message Nnapi {} + + oneof delegate { + TfLite tflite = 1; + Gpu gpu = 2; + Nnapi nnapi = 3; + } + } + // Path to the TF Lite model (ex: /path/to/modelname.tflite). // On mobile, this is generally just modelname.tflite. optional string model_path = 1; @@ -44,10 +60,22 @@ message TfLiteInferenceCalculatorOptions { // Whether the TF Lite GPU or CPU backend should be used. Effective only when // input tensors are on CPU. For input tensors on GPU, GPU backend is always // used. - optional bool use_gpu = 2 [default = false]; + // DEPRECATED: configure "delegate" instead. + optional bool use_gpu = 2 [deprecated = true, default = false]; // Android only. When true, an NNAPI delegate will be used for inference. // If NNAPI is not available, then the default CPU delegate will be used // automatically. - optional bool use_nnapi = 3 [default = false]; + // DEPRECATED: configure "delegate" instead. + optional bool use_nnapi = 3 [deprecated = true, default = false]; + + // The number of threads available to the interpreter. Effective only when + // input tensors are on CPU and 'use_gpu' is false. + optional int32 cpu_num_thread = 4 [default = -1]; + + // TfLite delegate to run inference. + // NOTE: calculator is free to choose delegate if not specified explicitly. + // NOTE: use_gpu/use_nnapi are ignored if specified. (Delegate takes + // precedence over use_* deprecated options.) + optional Delegate delegate = 5; } diff --git a/mediapipe/calculators/tflite/tflite_inference_calculator_test.cc b/mediapipe/calculators/tflite/tflite_inference_calculator_test.cc index ae9ac144d..9529a8ecb 100644 --- a/mediapipe/calculators/tflite/tflite_inference_calculator_test.cc +++ b/mediapipe/calculators/tflite/tflite_inference_calculator_test.cc @@ -16,6 +16,8 @@ #include #include +#include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" #include "mediapipe/calculators/tflite/tflite_inference_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_runner.h" @@ -39,13 +41,7 @@ namespace mediapipe { using ::tflite::Interpreter; -class TfLiteInferenceCalculatorTest : public ::testing::Test { - protected: - std::unique_ptr runner_ = nullptr; -}; - -// Tests a simple add model that adds an input tensor to itself. -TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) { +void DoSmokeTest(absl::string_view delegate) { const int width = 8; const int height = 8; const int channels = 3; @@ -73,23 +69,24 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) { auto input_vec = absl::make_unique>(); input_vec->emplace_back(*tensor); + std::string graph_proto = R"( + input_stream: "tensor_in" + node { + calculator: "TfLiteInferenceCalculator" + input_stream: "TENSORS:tensor_in" + output_stream: "TENSORS:tensor_out" + options { + [mediapipe.TfLiteInferenceCalculatorOptions.ext] { + model_path: "mediapipe/calculators/tflite/testdata/add.bin" + $delegate + } + } + } + )"; + ASSERT_EQ(absl::StrReplaceAll({{"$delegate", delegate}}, &graph_proto), 1); // Prepare single calculator graph to and wait for packets. CalculatorGraphConfig graph_config = - ::mediapipe::ParseTextProtoOrDie( - R"( - input_stream: "tensor_in" - node { - calculator: "TfLiteInferenceCalculator" - input_stream: "TENSORS:tensor_in" - output_stream: "TENSORS:tensor_out" - options { - [mediapipe.TfLiteInferenceCalculatorOptions.ext] { - use_gpu: false - model_path: "mediapipe/calculators/tflite/testdata/add.bin" - } - } - } - )"); + ::mediapipe::ParseTextProtoOrDie(graph_proto); std::vector output_packets; tool::AddVectorSink("tensor_out", &graph_config, &output_packets); CalculatorGraph graph(graph_config); @@ -120,4 +117,10 @@ TEST_F(TfLiteInferenceCalculatorTest, SmokeTest) { MP_ASSERT_OK(graph.WaitUntilDone()); } +// Tests a simple add model that adds an input tensor to itself. +TEST(TfLiteInferenceCalculatorTest, SmokeTest) { + DoSmokeTest(/*delegate=*/""); + DoSmokeTest(/*delegate=*/"delegate { tflite {} }"); +} + } // namespace mediapipe diff --git a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc index f6cffee40..bc4199969 100644 --- a/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc +++ b/mediapipe/calculators/tflite/tflite_tensors_to_landmarks_calculator.cc @@ -28,6 +28,21 @@ namespace mediapipe { // TENSORS - Vector of TfLiteTensor of type kTfLiteFloat32. Only the first // tensor will be used. The size of the values must be // (num_dimension x num_landmarks). +// +// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or +// not. Overrides corresponding side packet and/or field in the calculator +// options. +// +// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not. +// Overrides corresponding side packet and/or field in the calculator options. +// +// Input side packet: +// FLIP_HORIZONTALLY (optional): Whether to flip landmarks horizontally or +// not. Overrides the corresponding field in the calculator options. +// +// FLIP_VERTICALLY (optional): Whether to flip landmarks vertically or not. +// Overrides the corresponding field in the calculator options. +// // Output: // LANDMARKS(optional) - Result MediaPipe landmarks. // NORM_LANDMARKS(optional) - Result MediaPipe normalized landmarks. @@ -61,6 +76,8 @@ class TfLiteTensorsToLandmarksCalculator : public CalculatorBase { private: ::mediapipe::Status LoadOptions(CalculatorContext* cc); int num_landmarks_ = 0; + bool flip_vertically_ = false; + bool flip_horizontally_ = false; ::mediapipe::TfLiteTensorsToLandmarksCalculatorOptions options_; }; @@ -75,6 +92,22 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator); cc->Inputs().Tag("TENSORS").Set>(); } + if (cc->Inputs().HasTag("FLIP_HORIZONTALLY")) { + cc->Inputs().Tag("FLIP_HORIZONTALLY").Set(); + } + + if (cc->Inputs().HasTag("FLIP_VERTICALLY")) { + cc->Inputs().Tag("FLIP_VERTICALLY").Set(); + } + + if (cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY")) { + cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Set(); + } + + if (cc->InputSidePackets().HasTag("FLIP_VERTICALLY")) { + cc->InputSidePackets().Tag("FLIP_VERTICALLY").Set(); + } + if (cc->Outputs().HasTag("LANDMARKS")) { cc->Outputs().Tag("LANDMARKS").Set(); } @@ -98,17 +131,40 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator); << "Must provide input with/height for getting normalized landmarks."; } if (cc->Outputs().HasTag("LANDMARKS") && - (options_.flip_vertically() || options_.flip_horizontally())) { + (options_.flip_vertically() || options_.flip_horizontally() || + cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY") || + cc->InputSidePackets().HasTag("FLIP_VERTICALLY"))) { RET_CHECK(options_.has_input_image_height() && options_.has_input_image_width()) << "Must provide input with/height for using flip_vertically option " "when outputing landmarks in absolute coordinates."; } + + flip_horizontally_ = + cc->InputSidePackets().HasTag("FLIP_HORIZONTALLY") + ? cc->InputSidePackets().Tag("FLIP_HORIZONTALLY").Get() + : options_.flip_horizontally(); + + flip_horizontally_ = + cc->InputSidePackets().HasTag("FLIP_VERTICALLY") + ? cc->InputSidePackets().Tag("FLIP_VERTICALLY").Get() + : options_.flip_vertically(); + return ::mediapipe::OkStatus(); } ::mediapipe::Status TfLiteTensorsToLandmarksCalculator::Process( CalculatorContext* cc) { + // Override values if specified so. + if (cc->Inputs().HasTag("FLIP_HORIZONTALLY") && + !cc->Inputs().Tag("FLIP_HORIZONTALLY").IsEmpty()) { + flip_horizontally_ = cc->Inputs().Tag("FLIP_HORIZONTALLY").Get(); + } + if (cc->Inputs().HasTag("FLIP_VERTICALLY") && + !cc->Inputs().Tag("FLIP_VERTICALLY").IsEmpty()) { + flip_vertically_ = cc->Inputs().Tag("FLIP_VERTICALLY").Get(); + } + if (cc->Inputs().Tag("TENSORS").IsEmpty()) { return ::mediapipe::OkStatus(); } @@ -133,13 +189,13 @@ REGISTER_CALCULATOR(TfLiteTensorsToLandmarksCalculator); const int offset = ld * num_dimensions; Landmark* landmark = output_landmarks.add_landmark(); - if (options_.flip_horizontally()) { + if (flip_horizontally_) { landmark->set_x(options_.input_image_width() - raw_landmarks[offset]); } else { landmark->set_x(raw_landmarks[offset]); } if (num_dimensions > 1) { - if (options_.flip_vertically()) { + if (flip_vertically_) { landmark->set_y(options_.input_image_height() - raw_landmarks[offset + 1]); } else { diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD index 615057ce7..6f7064e40 100644 --- a/mediapipe/calculators/util/BUILD +++ b/mediapipe/calculators/util/BUILD @@ -437,6 +437,7 @@ cc_library( "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:status", + "@com_google_absl//absl/types:optional", ], alwayslink = 1, ) @@ -928,6 +929,19 @@ cc_library( alwayslink = 1, ) +cc_library( + name = "local_file_pattern_contents_calculator", + srcs = ["local_file_pattern_contents_calculator.cc"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/port:file_helpers", + "//mediapipe/framework/port:ret_check", + "//mediapipe/framework/port:status", + ], + alwayslink = 1, +) + cc_library( name = "filter_collection_calculator", srcs = ["filter_collection_calculator.cc"], diff --git a/mediapipe/calculators/util/annotation_overlay_calculator.cc b/mediapipe/calculators/util/annotation_overlay_calculator.cc index 35341656f..ead18a76d 100644 --- a/mediapipe/calculators/util/annotation_overlay_calculator.cc +++ b/mediapipe/calculators/util/annotation_overlay_calculator.cc @@ -39,13 +39,13 @@ namespace mediapipe { namespace { -constexpr char kInputFrameTag[] = "INPUT_FRAME"; -constexpr char kOutputFrameTag[] = "OUTPUT_FRAME"; +constexpr char kInputFrameTag[] = "IMAGE"; +constexpr char kOutputFrameTag[] = "IMAGE"; constexpr char kInputVectorTag[] = "VECTOR"; -constexpr char kInputFrameTagGpu[] = "INPUT_FRAME_GPU"; -constexpr char kOutputFrameTagGpu[] = "OUTPUT_FRAME_GPU"; +constexpr char kInputFrameTagGpu[] = "IMAGE_GPU"; +constexpr char kOutputFrameTagGpu[] = "IMAGE_GPU"; enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES }; @@ -61,7 +61,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102}; // A calculator for rendering data on images. // // Inputs: -// 1. INPUT_FRAME or INPUT_FRAME_GPU (optional): An ImageFrame (or GpuBuffer) +// 1. IMAGE or IMAGE_GPU (optional): An ImageFrame (or GpuBuffer) // containing the input image. // If output is CPU, and input isn't provided, the renderer creates a // blank canvas with the width, height and color provided in the options. @@ -73,7 +73,7 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102}; // input vector items. These input streams are tagged with "VECTOR". // // Output: -// 1. OUTPUT_FRAME or OUTPUT_FRAME_GPU: A rendered ImageFrame (or GpuBuffer). +// 1. IMAGE or IMAGE_GPU: A rendered ImageFrame (or GpuBuffer). // // For CPU input frames, only SRGBA, SRGB and GRAY8 format are supported. The // output format is the same as input except for GRAY8 where the output is in @@ -87,13 +87,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102}; // Example config (CPU): // node { // calculator: "AnnotationOverlayCalculator" -// input_stream: "INPUT_FRAME:image_frames" +// input_stream: "IMAGE:image_frames" // input_stream: "render_data_1" // input_stream: "render_data_2" // input_stream: "render_data_3" // input_stream: "VECTOR:0:render_data_vec_0" // input_stream: "VECTOR:1:render_data_vec_1" -// output_stream: "OUTPUT_FRAME:decorated_frames" +// output_stream: "IMAGE:decorated_frames" // options { // [mediapipe.AnnotationOverlayCalculatorOptions.ext] { // } @@ -103,13 +103,13 @@ constexpr int kAnnotationBackgroundColor[] = {100, 101, 102}; // Example config (GPU): // node { // calculator: "AnnotationOverlayCalculator" -// input_stream: "INPUT_FRAME_GPU:image_frames" +// input_stream: "IMAGE_GPU:image_frames" // input_stream: "render_data_1" // input_stream: "render_data_2" // input_stream: "render_data_3" // input_stream: "VECTOR:0:render_data_vec_0" // input_stream: "VECTOR:1:render_data_vec_1" -// output_stream: "OUTPUT_FRAME_GPU:decorated_frames" +// output_stream: "IMAGE_GPU:decorated_frames" // options { // [mediapipe.AnnotationOverlayCalculatorOptions.ext] { // } diff --git a/mediapipe/calculators/util/detections_to_rects_calculator.cc b/mediapipe/calculators/util/detections_to_rects_calculator.cc index 91a400ca1..52ba9dd7a 100644 --- a/mediapipe/calculators/util/detections_to_rects_calculator.cc +++ b/mediapipe/calculators/util/detections_to_rects_calculator.cc @@ -39,7 +39,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS"; } // namespace ::mediapipe::Status DetectionsToRectsCalculator::DetectionToRect( - const Detection& detection, Rect* rect) { + const Detection& detection, const DetectionSpec& detection_spec, + Rect* rect) { const LocationData location_data = detection.location_data(); RET_CHECK(location_data.format() == LocationData::BOUNDING_BOX) << "Only Detection with formats of BOUNDING_BOX can be converted to Rect"; @@ -52,7 +53,8 @@ constexpr char kNormRectsTag[] = "NORM_RECTS"; } ::mediapipe::Status DetectionsToRectsCalculator::DetectionToNormalizedRect( - const Detection& detection, NormalizedRect* rect) { + const Detection& detection, const DetectionSpec& detection_spec, + NormalizedRect* rect) { const LocationData location_data = detection.location_data(); RET_CHECK(location_data.format() == LocationData::RELATIVE_BOUNDING_BOX) << "Only Detection with formats of RELATIVE_BOUNDING_BOX can be " @@ -174,27 +176,31 @@ constexpr char kNormRectsTag[] = "NORM_RECTS"; } } - std::pair image_size; - if (rotate_) { - RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty()); - image_size = cc->Inputs().Tag(kImageSizeTag).Get>(); - } + // Get dynamic calculator options (e.g. `image_size`). + const DetectionSpec detection_spec = GetDetectionSpec(cc); if (cc->Outputs().HasTag(kRectTag)) { auto output_rect = absl::make_unique(); - MP_RETURN_IF_ERROR(DetectionToRect(detections[0], output_rect.get())); + MP_RETURN_IF_ERROR( + DetectionToRect(detections[0], detection_spec, output_rect.get())); if (rotate_) { - output_rect->set_rotation(ComputeRotation(detections[0], image_size)); + float rotation; + MP_RETURN_IF_ERROR( + ComputeRotation(detections[0], detection_spec, &rotation)); + output_rect->set_rotation(rotation); } cc->Outputs().Tag(kRectTag).Add(output_rect.release(), cc->InputTimestamp()); } if (cc->Outputs().HasTag(kNormRectTag)) { auto output_rect = absl::make_unique(); - MP_RETURN_IF_ERROR( - DetectionToNormalizedRect(detections[0], output_rect.get())); + MP_RETURN_IF_ERROR(DetectionToNormalizedRect(detections[0], detection_spec, + output_rect.get())); if (rotate_) { - output_rect->set_rotation(ComputeRotation(detections[0], image_size)); + float rotation; + MP_RETURN_IF_ERROR( + ComputeRotation(detections[0], detection_spec, &rotation)); + output_rect->set_rotation(rotation); } cc->Outputs() .Tag(kNormRectTag) @@ -203,11 +209,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS"; if (cc->Outputs().HasTag(kRectsTag)) { auto output_rects = absl::make_unique>(detections.size()); for (int i = 0; i < detections.size(); ++i) { - MP_RETURN_IF_ERROR( - DetectionToRect(detections[i], &(output_rects->at(i)))); + MP_RETURN_IF_ERROR(DetectionToRect(detections[i], detection_spec, + &(output_rects->at(i)))); if (rotate_) { - output_rects->at(i).set_rotation( - ComputeRotation(detections[i], image_size)); + float rotation; + MP_RETURN_IF_ERROR( + ComputeRotation(detections[i], detection_spec, &rotation)); + output_rects->at(i).set_rotation(rotation); } } cc->Outputs().Tag(kRectsTag).Add(output_rects.release(), @@ -217,11 +225,13 @@ constexpr char kNormRectsTag[] = "NORM_RECTS"; auto output_rects = absl::make_unique>(detections.size()); for (int i = 0; i < detections.size(); ++i) { - MP_RETURN_IF_ERROR( - DetectionToNormalizedRect(detections[i], &(output_rects->at(i)))); + MP_RETURN_IF_ERROR(DetectionToNormalizedRect( + detections[i], detection_spec, &(output_rects->at(i)))); if (rotate_) { - output_rects->at(i).set_rotation( - ComputeRotation(detections[i], image_size)); + float rotation; + MP_RETURN_IF_ERROR( + ComputeRotation(detections[i], detection_spec, &rotation)); + output_rects->at(i).set_rotation(rotation); } } cc->Outputs() @@ -232,21 +242,35 @@ constexpr char kNormRectsTag[] = "NORM_RECTS"; return ::mediapipe::OkStatus(); } -float DetectionsToRectsCalculator::ComputeRotation( - const Detection& detection, const std::pair image_size) { +::mediapipe::Status DetectionsToRectsCalculator::ComputeRotation( + const Detection& detection, const DetectionSpec& detection_spec, + float* rotation) { const auto& location_data = detection.location_data(); + const auto& image_size = detection_spec.image_size; + RET_CHECK(image_size) << "Image size is required to calculate rotation"; + const float x0 = location_data.relative_keypoints(start_keypoint_index_).x() * - image_size.first; + image_size->first; const float y0 = location_data.relative_keypoints(start_keypoint_index_).y() * - image_size.second; + image_size->second; const float x1 = location_data.relative_keypoints(end_keypoint_index_).x() * - image_size.first; + image_size->first; const float y1 = location_data.relative_keypoints(end_keypoint_index_).y() * - image_size.second; + image_size->second; - float rotation = target_angle_ - std::atan2(-(y1 - y0), x1 - x0); + *rotation = NormalizeRadians(target_angle_ - std::atan2(-(y1 - y0), x1 - x0)); - return NormalizeRadians(rotation); + return ::mediapipe::OkStatus(); +} + +DetectionSpec DetectionsToRectsCalculator::GetDetectionSpec( + const CalculatorContext* cc) { + absl::optional> image_size; + if (cc->Inputs().HasTag(kImageSizeTag)) { + image_size = cc->Inputs().Tag(kImageSizeTag).Get>(); + } + + return {image_size}; } REGISTER_CALCULATOR(DetectionsToRectsCalculator); diff --git a/mediapipe/calculators/util/detections_to_rects_calculator.h b/mediapipe/calculators/util/detections_to_rects_calculator.h index 82b9f7bcc..7fb26895e 100644 --- a/mediapipe/calculators/util/detections_to_rects_calculator.h +++ b/mediapipe/calculators/util/detections_to_rects_calculator.h @@ -16,6 +16,7 @@ #include +#include "absl/types/optional.h" #include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_options.pb.h" @@ -27,6 +28,13 @@ namespace mediapipe { +// Dynamic options passed as calculator `input_stream` that can be used for +// calculation of rectangle or rotation for given detection. Does not include +// static calculator options which are available via private fields. +struct DetectionSpec { + absl::optional> image_size; +}; + // A calculator that converts Detection proto to Rect proto. // // Detection is the format for encoding one or more detections in an image. @@ -81,13 +89,16 @@ class DetectionsToRectsCalculator : public CalculatorBase { ::mediapipe::Status Process(CalculatorContext* cc) override; protected: - virtual float ComputeRotation(const ::mediapipe::Detection& detection, - const std::pair image_size); virtual ::mediapipe::Status DetectionToRect( - const ::mediapipe::Detection& detection, ::mediapipe::Rect* rect); + const ::mediapipe::Detection& detection, + const DetectionSpec& detection_spec, ::mediapipe::Rect* rect); virtual ::mediapipe::Status DetectionToNormalizedRect( const ::mediapipe::Detection& detection, - ::mediapipe::NormalizedRect* rect); + const DetectionSpec& detection_spec, ::mediapipe::NormalizedRect* rect); + virtual ::mediapipe::Status ComputeRotation( + const ::mediapipe::Detection& detection, + const DetectionSpec& detection_spec, float* rotation); + virtual DetectionSpec GetDetectionSpec(const CalculatorContext* cc); static inline float NormalizeRadians(float angle) { return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI)); diff --git a/mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc b/mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc index aca312c30..b0f7cc0fd 100644 --- a/mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc +++ b/mediapipe/calculators/util/landmark_letterbox_removal_calculator.cc @@ -12,20 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - #include #include @@ -67,6 +53,15 @@ constexpr char kLetterboxPaddingTag[] = "LETTERBOX_PADDING"; // input_stream: "LETTERBOX_PADDING:letterbox_padding" // output_stream: "LANDMARKS:adjusted_landmarks" // } +// +// node { +// calculator: "LandmarkLetterboxRemovalCalculator" +// input_stream: "LANDMARKS:0:landmarks_0" +// input_stream: "LANDMARKS:1:landmarks_1" +// input_stream: "LETTERBOX_PADDING:letterbox_padding" +// output_stream: "LANDMARKS:0:adjusted_landmarks_0" +// output_stream: "LANDMARKS:1:adjusted_landmarks_1" +// } class LandmarkLetterboxRemovalCalculator : public CalculatorBase { public: static ::mediapipe::Status GetContract(CalculatorContract* cc) { @@ -74,10 +69,20 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase { cc->Inputs().HasTag(kLetterboxPaddingTag)) << "Missing one or more input streams."; - cc->Inputs().Tag(kLandmarksTag).Set(); + RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag), + cc->Outputs().NumEntries(kLandmarksTag)) + << "Same number of input and output landmarks is required."; + + for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag); + id != cc->Inputs().EndId(kLandmarksTag); ++id) { + cc->Inputs().Get(id).Set(); + } cc->Inputs().Tag(kLetterboxPaddingTag).Set>(); - cc->Outputs().Tag(kLandmarksTag).Set(); + for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag); + id != cc->Outputs().EndId(kLandmarksTag); ++id) { + cc->Outputs().Get(id).Set(); + } return ::mediapipe::OkStatus(); } @@ -89,38 +94,45 @@ class LandmarkLetterboxRemovalCalculator : public CalculatorBase { } ::mediapipe::Status Process(CalculatorContext* cc) override { - // Only process if there's input landmarks. - if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) { + if (cc->Inputs().Tag(kLetterboxPaddingTag).IsEmpty()) { return ::mediapipe::OkStatus(); } - - const NormalizedLandmarkList& input_landmarks = - cc->Inputs().Tag(kLandmarksTag).Get(); const auto& letterbox_padding = cc->Inputs().Tag(kLetterboxPaddingTag).Get>(); - const float left = letterbox_padding[0]; const float top = letterbox_padding[1]; const float left_and_right = letterbox_padding[0] + letterbox_padding[2]; const float top_and_bottom = letterbox_padding[1] + letterbox_padding[3]; - NormalizedLandmarkList output_landmarks; - for (int i = 0; i < input_landmarks.landmark_size(); ++i) { - const NormalizedLandmark& landmark = input_landmarks.landmark(i); - NormalizedLandmark* new_landmark = output_landmarks.add_landmark(); - const float new_x = (landmark.x() - left) / (1.0f - left_and_right); - const float new_y = (landmark.y() - top) / (1.0f - top_and_bottom); + CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag); + CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag); + // Number of inputs and outpus is the same according to the contract. + for (; input_id != cc->Inputs().EndId(kLandmarksTag); + ++input_id, ++output_id) { + const auto& input_packet = cc->Inputs().Get(input_id); + if (input_packet.IsEmpty()) { + continue; + } - new_landmark->set_x(new_x); - new_landmark->set_y(new_y); - // Keep z-coord as is. - new_landmark->set_z(landmark.z()); + const NormalizedLandmarkList& input_landmarks = + input_packet.Get(); + NormalizedLandmarkList output_landmarks; + for (int i = 0; i < input_landmarks.landmark_size(); ++i) { + const NormalizedLandmark& landmark = input_landmarks.landmark(i); + NormalizedLandmark* new_landmark = output_landmarks.add_landmark(); + const float new_x = (landmark.x() - left) / (1.0f - left_and_right); + const float new_y = (landmark.y() - top) / (1.0f - top_and_bottom); + + new_landmark->set_x(new_x); + new_landmark->set_y(new_y); + // Keep z-coord as is. + new_landmark->set_z(landmark.z()); + } + + cc->Outputs().Get(output_id).AddPacket( + MakePacket(output_landmarks) + .At(cc->InputTimestamp())); } - - cc->Outputs() - .Tag(kLandmarksTag) - .AddPacket(MakePacket(output_landmarks) - .At(cc->InputTimestamp())); return ::mediapipe::OkStatus(); } }; diff --git a/mediapipe/calculators/util/landmark_projection_calculator.cc b/mediapipe/calculators/util/landmark_projection_calculator.cc index 9c868fd50..0ab22ad97 100644 --- a/mediapipe/calculators/util/landmark_projection_calculator.cc +++ b/mediapipe/calculators/util/landmark_projection_calculator.cc @@ -12,20 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Copyright 2019 The MediaPipe Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - #include #include @@ -63,6 +49,15 @@ constexpr char kRectTag[] = "NORM_RECT"; // input_stream: "NORM_RECT:rect" // output_stream: "NORM_LANDMARKS:projected_landmarks" // } +// +// node { +// calculator: "LandmarkProjectionCalculator" +// input_stream: "NORM_LANDMARKS:0:landmarks_0" +// input_stream: "NORM_LANDMARKS:1:landmarks_1" +// input_stream: "NORM_RECT:rect" +// output_stream: "NORM_LANDMARKS:0:projected_landmarks_0" +// output_stream: "NORM_LANDMARKS:1:projected_landmarks_1" +// } class LandmarkProjectionCalculator : public CalculatorBase { public: static ::mediapipe::Status GetContract(CalculatorContract* cc) { @@ -70,10 +65,20 @@ class LandmarkProjectionCalculator : public CalculatorBase { cc->Inputs().HasTag(kRectTag)) << "Missing one or more input streams."; - cc->Inputs().Tag(kLandmarksTag).Set(); + RET_CHECK_EQ(cc->Inputs().NumEntries(kLandmarksTag), + cc->Outputs().NumEntries(kLandmarksTag)) + << "Same number of input and output landmarks is required."; + + for (CollectionItemId id = cc->Inputs().BeginId(kLandmarksTag); + id != cc->Inputs().EndId(kLandmarksTag); ++id) { + cc->Inputs().Get(id).Set(); + } cc->Inputs().Tag(kRectTag).Set(); - cc->Outputs().Tag(kLandmarksTag).Set(); + for (CollectionItemId id = cc->Outputs().BeginId(kLandmarksTag); + id != cc->Outputs().EndId(kLandmarksTag); ++id) { + cc->Outputs().Get(id).Set(); + } return ::mediapipe::OkStatus(); } @@ -85,41 +90,50 @@ class LandmarkProjectionCalculator : public CalculatorBase { } ::mediapipe::Status Process(CalculatorContext* cc) override { - const auto& options = - cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>(); - // Only process if there's input landmarks. - if (cc->Inputs().Tag(kLandmarksTag).IsEmpty()) { + if (cc->Inputs().Tag(kRectTag).IsEmpty()) { return ::mediapipe::OkStatus(); } - - const NormalizedLandmarkList& input_landmarks = - cc->Inputs().Tag(kLandmarksTag).Get(); const auto& input_rect = cc->Inputs().Tag(kRectTag).Get(); - NormalizedLandmarkList output_landmarks; - for (int i = 0; i < input_landmarks.landmark_size(); ++i) { - const NormalizedLandmark& landmark = input_landmarks.landmark(i); - NormalizedLandmark* new_landmark = output_landmarks.add_landmark(); + const auto& options = + cc->Options<::mediapipe::LandmarkProjectionCalculatorOptions>(); - const float x = landmark.x() - 0.5f; - const float y = landmark.y() - 0.5f; - const float angle = options.ignore_rotation() ? 0 : input_rect.rotation(); - float new_x = std::cos(angle) * x - std::sin(angle) * y; - float new_y = std::sin(angle) * x + std::cos(angle) * y; + CollectionItemId input_id = cc->Inputs().BeginId(kLandmarksTag); + CollectionItemId output_id = cc->Outputs().BeginId(kLandmarksTag); + // Number of inputs and outpus is the same according to the contract. + for (; input_id != cc->Inputs().EndId(kLandmarksTag); + ++input_id, ++output_id) { + const auto& input_packet = cc->Inputs().Get(input_id); + if (input_packet.IsEmpty()) { + continue; + } - new_x = new_x * input_rect.width() + input_rect.x_center(); - new_y = new_y * input_rect.height() + input_rect.y_center(); + const auto& input_landmarks = input_packet.Get(); + NormalizedLandmarkList output_landmarks; + for (int i = 0; i < input_landmarks.landmark_size(); ++i) { + const NormalizedLandmark& landmark = input_landmarks.landmark(i); + NormalizedLandmark* new_landmark = output_landmarks.add_landmark(); - new_landmark->set_x(new_x); - new_landmark->set_y(new_y); - // Keep z-coord as is. - new_landmark->set_z(landmark.z()); + const float x = landmark.x() - 0.5f; + const float y = landmark.y() - 0.5f; + const float angle = + options.ignore_rotation() ? 0 : input_rect.rotation(); + float new_x = std::cos(angle) * x - std::sin(angle) * y; + float new_y = std::sin(angle) * x + std::cos(angle) * y; + + new_x = new_x * input_rect.width() + input_rect.x_center(); + new_y = new_y * input_rect.height() + input_rect.y_center(); + + new_landmark->set_x(new_x); + new_landmark->set_y(new_y); + // Keep z-coord as is. + new_landmark->set_z(landmark.z()); + } + + cc->Outputs().Get(output_id).AddPacket( + MakePacket(output_landmarks) + .At(cc->InputTimestamp())); } - - cc->Outputs() - .Tag(kLandmarksTag) - .AddPacket(MakePacket(output_landmarks) - .At(cc->InputTimestamp())); return ::mediapipe::OkStatus(); } }; diff --git a/mediapipe/calculators/util/local_file_pattern_contents_calculator.cc b/mediapipe/calculators/util/local_file_pattern_contents_calculator.cc new file mode 100644 index 000000000..04fe3ac1c --- /dev/null +++ b/mediapipe/calculators/util/local_file_pattern_contents_calculator.cc @@ -0,0 +1,75 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/status.h" + +namespace mediapipe { +// The calculator takes the path to local directory and desired file suffix to +// mach as input side packets, and outputs the contents of those files that +// match the pattern. Those matched files will be sent sequentially through the +// output stream with incremental timestamp difference by 1. +// +// Example config: +// node { +// calculator: "LocalFilePatternContentsCalculator" +// input_side_packet: "FILE_DIRECTORY:file_directory" +// input_side_packet: "FILE_SUFFIX:file_suffix" +// output_stream: "CONTENTS:contents" +// } +class LocalFilePatternContentsCalculator : public CalculatorBase { + public: + static ::mediapipe::Status GetContract(CalculatorContract* cc) { + cc->InputSidePackets().Tag("FILE_DIRECTORY").Set(); + cc->InputSidePackets().Tag("FILE_SUFFIX").Set(); + cc->Outputs().Tag("CONTENTS").Set(); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Open(CalculatorContext* cc) override { + MP_RETURN_IF_ERROR(::mediapipe::file::MatchFileTypeInDirectory( + cc->InputSidePackets().Tag("FILE_DIRECTORY").Get(), + cc->InputSidePackets().Tag("FILE_SUFFIX").Get(), + &filenames_)); + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext* cc) override { + if (current_output_ < filenames_.size()) { + auto contents = absl::make_unique(); + LOG(INFO) << filenames_[current_output_]; + MP_RETURN_IF_ERROR(mediapipe::file::GetContents( + filenames_[current_output_], contents.get())); + ++current_output_; + cc->Outputs() + .Tag("CONTENTS") + .Add(contents.release(), Timestamp(current_output_)); + } else { + return tool::StatusStop(); + } + return ::mediapipe::OkStatus(); + } + + private: + std::vector filenames_; + int current_output_ = 0; +}; + +REGISTER_CALCULATOR(LocalFilePatternContentsCalculator); + +} // namespace mediapipe diff --git a/mediapipe/calculators/util/rect_to_render_data_calculator.cc b/mediapipe/calculators/util/rect_to_render_data_calculator.cc index 24db78a88..365d364dc 100644 --- a/mediapipe/calculators/util/rect_to_render_data_calculator.cc +++ b/mediapipe/calculators/util/rect_to_render_data_calculator.cc @@ -45,15 +45,17 @@ RenderAnnotation::Rectangle* NewRect( void SetRect(bool normalized, double xmin, double ymin, double width, double height, double rotation, RenderAnnotation::Rectangle* rect) { - if (xmin + width < 0.0 || ymin + height < 0.0) return; - if (normalized) { - if (xmin > 1.0 || ymin > 1.0) return; + if (rotation == 0.0) { + if (xmin + width < 0.0 || ymin + height < 0.0) return; + if (normalized) { + if (xmin > 1.0 || ymin > 1.0) return; + } } rect->set_normalized(normalized); - rect->set_left(normalized ? std::max(xmin, 0.0) : xmin); - rect->set_top(normalized ? std::max(ymin, 0.0) : ymin); - rect->set_right(normalized ? std::min(xmin + width, 1.0) : xmin + width); - rect->set_bottom(normalized ? std::min(ymin + height, 1.0) : ymin + height); + rect->set_left(xmin); + rect->set_top(ymin); + rect->set_right(xmin + width); + rect->set_bottom(ymin + height); rect->set_rotation(rotation); } diff --git a/mediapipe/calculators/video/BUILD b/mediapipe/calculators/video/BUILD index 292c43359..da2bc7fbd 100644 --- a/mediapipe/calculators/video/BUILD +++ b/mediapipe/calculators/video/BUILD @@ -368,6 +368,7 @@ cc_test( cc_test( name = "tvl1_optical_flow_calculator_test", srcs = ["tvl1_optical_flow_calculator_test.cc"], + linkstatic = 1, deps = [ ":tvl1_optical_flow_calculator", "//mediapipe/framework:calculator_framework", diff --git a/mediapipe/docs/face_detection_desktop.md b/mediapipe/docs/face_detection_desktop.md index ca35140a7..b6be25b3f 100644 --- a/mediapipe/docs/face_detection_desktop.md +++ b/mediapipe/docs/face_detection_desktop.md @@ -259,9 +259,9 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:throttled_input_video" + input_stream: "IMAGE:throttled_input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } ``` diff --git a/mediapipe/docs/face_detection_mobile_cpu.md b/mediapipe/docs/face_detection_mobile_cpu.md index bcef66a48..e6b1d91b8 100644 --- a/mediapipe/docs/face_detection_mobile_cpu.md +++ b/mediapipe/docs/face_detection_mobile_cpu.md @@ -229,9 +229,9 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_video_cpu" + input_stream: "IMAGE:input_video_cpu" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video_cpu" + output_stream: "IMAGE:output_video_cpu" } # Transfers the annotated image from CPU back to GPU memory, to be sent out of diff --git a/mediapipe/docs/face_detection_mobile_gpu.md b/mediapipe/docs/face_detection_mobile_gpu.md index 9c5ba5b2d..7a1b6152c 100644 --- a/mediapipe/docs/face_detection_mobile_gpu.md +++ b/mediapipe/docs/face_detection_mobile_gpu.md @@ -221,8 +221,8 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:throttled_input_video" + input_stream: "IMAGE_GPU:throttled_input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME_GPU:output_video" + output_stream: "IMAGE_GPU:output_video" } ``` diff --git a/mediapipe/docs/hand_detection_mobile_gpu.md b/mediapipe/docs/hand_detection_mobile_gpu.md index 5c4a41bbd..53b4d9905 100644 --- a/mediapipe/docs/hand_detection_mobile_gpu.md +++ b/mediapipe/docs/hand_detection_mobile_gpu.md @@ -136,10 +136,10 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:throttled_input_video" + input_stream: "IMAGE_GPU:throttled_input_video" input_stream: "detection_render_data" input_stream: "rect_render_data" - output_stream: "OUTPUT_FRAME_GPU:output_video" + output_stream: "IMAGE_GPU:output_video" } ``` diff --git a/mediapipe/docs/hand_tracking_mobile_gpu.md b/mediapipe/docs/hand_tracking_mobile_gpu.md index e097dc9d5..edc50f9c2 100644 --- a/mediapipe/docs/hand_tracking_mobile_gpu.md +++ b/mediapipe/docs/hand_tracking_mobile_gpu.md @@ -716,10 +716,10 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:input_image" + input_stream: "IMAGE_GPU:input_image" input_stream: "detection_render_data" input_stream: "landmark_render_data" input_stream: "rect_render_data" - output_stream: "OUTPUT_FRAME_GPU:output_image" + output_stream: "IMAGE_GPU:output_image" } ``` diff --git a/mediapipe/docs/install.md b/mediapipe/docs/install.md index e1bf28709..899f6a5aa 100644 --- a/mediapipe/docs/install.md +++ b/mediapipe/docs/install.md @@ -40,7 +40,7 @@ To build and run iOS apps: $ cd mediapipe ``` -2. Install Bazel (version between 0.24.1 and 1.2.1). +2. Install Bazel (version between 1.0.0 and 1.2.1). Follow the official [Bazel documentation](https://docs.bazel.build/versions/master/install-ubuntu.html) @@ -152,7 +152,7 @@ To build and run iOS apps: $ cd mediapipe ``` -2. Install Bazel (version between 0.24.1 and 1.2.1). +2. Install Bazel (version between 1.0.0 and 1.2.1). Follow the official [Bazel documentation](https://docs.bazel.build/versions/master/install-redhat.html) @@ -241,7 +241,7 @@ To build and run iOS apps: $ cd mediapipe ``` -3. Install Bazel (version between 0.24.1 and 1.1.0). +3. Install Bazel (version between 1.0.0 and 1.1.0). Option 1. Use package manager tool to install Bazel 1.1.0 @@ -389,18 +389,18 @@ a video file as input. username@DESKTOP-TMVLBJ1:~$ sudo apt-get update && sudo apt-get install -y build-essential git python zip adb openjdk-8-jdk ``` -5. Install Bazel (version between 0.24.1 and 1.2.1). +5. Install Bazel (version between 1.0.0 and 1.2.1). ```bash username@DESKTOP-TMVLBJ1:~$ curl -sLO --retry 5 --retry-max-time 10 \ - https://storage.googleapis.com/bazel/0.27.0/release/bazel-0.27.0-installer-linux-x86_64.sh && \ - sudo mkdir -p /usr/local/bazel/0.27.0 && \ - chmod 755 bazel-0.27.0-installer-linux-x86_64.sh && \ - sudo ./bazel-0.27.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/0.27.0 && \ - source /usr/local/bazel/0.27.0/lib/bazel/bin/bazel-complete.bash + https://storage.googleapis.com/bazel/1.0.0/release/bazel-1.0.0-installer-linux-x86_64.sh && \ + sudo mkdir -p /usr/local/bazel/1.0.0 && \ + chmod 755 bazel-1.0.0-installer-linux-x86_64.sh && \ + sudo ./bazel-1.0.0-installer-linux-x86_64.sh --prefix=/usr/local/bazel/1.0.0 && \ + source /usr/local/bazel/1.0.0/lib/bazel/bin/bazel-complete.bash - username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/0.27.0/lib/bazel/bin/bazel version && \ - alias bazel='/usr/local/bazel/0.27.0/lib/bazel/bin/bazel' + username@DESKTOP-TMVLBJ1:~$ /usr/local/bazel/1.0.0/lib/bazel/bin/bazel version && \ + alias bazel='/usr/local/bazel/1.0.0/lib/bazel/bin/bazel' ``` 6. Checkout MediaPipe repository. diff --git a/mediapipe/docs/multi_hand_tracking_mobile_gpu.md b/mediapipe/docs/multi_hand_tracking_mobile_gpu.md index 111cd894d..58ae8c38b 100644 --- a/mediapipe/docs/multi_hand_tracking_mobile_gpu.md +++ b/mediapipe/docs/multi_hand_tracking_mobile_gpu.md @@ -745,11 +745,11 @@ node { # a vector of RenderData objects and draws each of them on the input frame. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:input_image" + input_stream: "IMAGE_GPU:input_image" input_stream: "detection_render_data" input_stream: "multi_hand_rects_render_data" input_stream: "multi_palm_rects_render_data" input_stream: "VECTOR:0:multi_hand_landmarks_render_data" - output_stream: "OUTPUT_FRAME_GPU:output_image" + output_stream: "IMAGE_GPU:output_image" } ``` diff --git a/mediapipe/docs/object_detection_desktop.md b/mediapipe/docs/object_detection_desktop.md index f988941fc..efd23ef86 100644 --- a/mediapipe/docs/object_detection_desktop.md +++ b/mediapipe/docs/object_detection_desktop.md @@ -163,9 +163,9 @@ node { # the graph. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_video" + input_stream: "IMAGE:input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } # Encodes the annotated images into a video file, adopting properties specified @@ -396,9 +396,9 @@ node { # the graph. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_video" + input_stream: "IMAGE:input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } # Encodes the annotated images into a video file, adopting properties specified diff --git a/mediapipe/docs/object_detection_mobile_cpu.md b/mediapipe/docs/object_detection_mobile_cpu.md index b8fa5d35c..cce007414 100644 --- a/mediapipe/docs/object_detection_mobile_cpu.md +++ b/mediapipe/docs/object_detection_mobile_cpu.md @@ -230,9 +230,9 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:throttled_input_video_cpu" + input_stream: "IMAGE:throttled_input_video_cpu" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video_cpu" + output_stream: "IMAGE:output_video_cpu" } # Transfers the annotated image from CPU back to GPU memory, to be sent out of diff --git a/mediapipe/docs/object_detection_mobile_gpu.md b/mediapipe/docs/object_detection_mobile_gpu.md index 86e499c05..031b7a02e 100644 --- a/mediapipe/docs/object_detection_mobile_gpu.md +++ b/mediapipe/docs/object_detection_mobile_gpu.md @@ -212,8 +212,8 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:throttled_input_video" + input_stream: "IMAGE_GPU:throttled_input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME_GPU:output_video" + output_stream: "IMAGE_GPU:output_video" } ``` diff --git a/mediapipe/docs/object_tracking_mobile_gpu.md b/mediapipe/docs/object_tracking_mobile_gpu.md index ea83dd3fe..aaac36386 100644 --- a/mediapipe/docs/object_tracking_mobile_gpu.md +++ b/mediapipe/docs/object_tracking_mobile_gpu.md @@ -467,9 +467,9 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:input_image" + input_stream: "IMAGE_GPU:input_image" input_stream: "detections_render_data" - output_stream: "OUTPUT_FRAME_GPU:output_image" + output_stream: "IMAGE_GPU:output_image" } ``` @@ -484,7 +484,8 @@ CPU. To build and run the app: ```bash -bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu +bazel build -c opt mediapipe/examples/desktop/object_tracking:object_tracking_cpu \ + --define MEDIAPIPE_DISABLE_GPU=1 bazel-bin/mediapipe/examples/desktop/object_tracking/object_tracking_cpu \ --calculator_graph_config_file=mediapipe/graphs/tracking/object_detection_tracking_desktop_live.pbtxt diff --git a/mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt b/mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt index e3cddfc96..553212868 100644 --- a/mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt +++ b/mediapipe/examples/coral/graphs/face_detection_desktop_live.pbtxt @@ -182,8 +182,8 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:throttled_input_video" + input_stream: "IMAGE:throttled_input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } diff --git a/mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt b/mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt index c47c7d19f..03bc9e18f 100644 --- a/mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt +++ b/mediapipe/examples/coral/graphs/object_detection_desktop_live.pbtxt @@ -173,7 +173,7 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:throttled_input_video" + input_stream: "IMAGE:throttled_input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } diff --git a/mediapipe/examples/desktop/BUILD b/mediapipe/examples/desktop/BUILD index f579c49e5..0e0335157 100644 --- a/mediapipe/examples/desktop/BUILD +++ b/mediapipe/examples/desktop/BUILD @@ -53,7 +53,7 @@ cc_library( # Linux only. # Must have a GPU with EGL support: -# ex: sudo aptitude install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev +# ex: sudo apt-get install mesa-common-dev libegl1-mesa-dev libgles2-mesa-dev # (or similar nvidia/amd equivalent) cc_library( name = "demo_run_graph_main_gpu", diff --git a/mediapipe/examples/desktop/media_sequence/charades_dataset.py b/mediapipe/examples/desktop/media_sequence/charades_dataset.py index c0176c6cb..0a3f005f4 100644 --- a/mediapipe/examples/desktop/media_sequence/charades_dataset.py +++ b/mediapipe/examples/desktop/media_sequence/charades_dataset.py @@ -68,7 +68,7 @@ import zipfile from absl import app from absl import flags from absl import logging -import tensorflow as tf +import tensorflow.compat.v1 as tf from mediapipe.util.sequence import media_sequence as ms diff --git a/mediapipe/examples/desktop/media_sequence/demo_dataset.py b/mediapipe/examples/desktop/media_sequence/demo_dataset.py index 79a6bacfe..8279f9bfb 100644 --- a/mediapipe/examples/desktop/media_sequence/demo_dataset.py +++ b/mediapipe/examples/desktop/media_sequence/demo_dataset.py @@ -62,7 +62,7 @@ import urllib from absl import app from absl import flags from absl import logging -import tensorflow as tf +import tensorflow.compat.v1 as tf from mediapipe.util.sequence import media_sequence as ms diff --git a/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py b/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py index 284cb8b21..04e6c59d1 100644 --- a/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py +++ b/mediapipe/examples/desktop/media_sequence/kinetics_dataset.py @@ -78,7 +78,7 @@ import urllib from absl import app from absl import flags from absl import logging -import tensorflow as tf +import tensorflow.compat.v1 as tf from mediapipe.util.sequence import media_sequence as ms diff --git a/mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py b/mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py index 2d075e223..a639e1056 100644 --- a/mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py +++ b/mediapipe/examples/desktop/youtube8m/generate_input_sequence_example.py @@ -21,6 +21,7 @@ import sys from absl import app from absl import flags +import six import tensorflow.compat.v1 as tf from mediapipe.util.sequence import media_sequence as ms @@ -54,7 +55,7 @@ def main(argv): ms.set_clip_end_timestamp( flags.FLAGS.clip_end_time_sec * SECONDS_TO_MICROSECONDS, metadata) with open('/tmp/mediapipe/metadata.pb', 'wb') as writer: - writer.write(metadata.SerializeToString()) + writer.write(six.ensure_binary(metadata.SerializeToString())) if __name__ == '__main__': diff --git a/mediapipe/framework/calculator_graph.cc b/mediapipe/framework/calculator_graph.cc index ab0dd650f..47abdaf5b 100644 --- a/mediapipe/framework/calculator_graph.cc +++ b/mediapipe/framework/calculator_graph.cc @@ -1301,7 +1301,7 @@ void PrintTimingToInfo(const std::string& label, int64 timer_value) { "%02lld days, %02lld:%02lld:%02lld.%03lld (total seconds: " "%lld.%06lld)", days, hours, minutes, seconds, milliseconds, total_seconds, - timer_value % 1000000ll); + timer_value % int64{1000000}); } bool MetricElementComparator(const std::pair& e1, diff --git a/mediapipe/framework/deps/BUILD b/mediapipe/framework/deps/BUILD index 949ca3358..40aab9ab4 100644 --- a/mediapipe/framework/deps/BUILD +++ b/mediapipe/framework/deps/BUILD @@ -251,6 +251,7 @@ cc_library( "//mediapipe/framework/port:logging", "@com_google_absl//absl/base:core_headers", "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/framework/deps/file_helpers.cc b/mediapipe/framework/deps/file_helpers.cc index 259e4a654..d094e5b72 100644 --- a/mediapipe/framework/deps/file_helpers.cc +++ b/mediapipe/framework/deps/file_helpers.cc @@ -105,6 +105,30 @@ namespace file { return ::mediapipe::OkStatus(); } +::mediapipe::Status MatchFileTypeInDirectory( + const std::string& directory, const std::string& file_suffix, + std::vector* results) { + DIR* dir = opendir(directory.c_str()); + CHECK(dir); + // Iterates through the direcotry. + while (true) { + struct dirent* dir_ent = readdir(dir); + if (dir_ent == nullptr) { + break; + } + if (std::string(dir_ent->d_name) == "." || + std::string(dir_ent->d_name) == "..") { + continue; + } + + if (absl::EndsWith(std::string(dir_ent->d_name), file_suffix)) { + results->push_back(JoinPath(directory, std::string(dir_ent->d_name))); + } + } + closedir(dir); + return ::mediapipe::OkStatus(); +} + ::mediapipe::Status Exists(absl::string_view file_name) { struct stat buffer; int status; diff --git a/mediapipe/framework/deps/file_helpers.h b/mediapipe/framework/deps/file_helpers.h index 516447d6b..08a7f1f99 100644 --- a/mediapipe/framework/deps/file_helpers.h +++ b/mediapipe/framework/deps/file_helpers.h @@ -30,6 +30,10 @@ namespace file { const std::string& parent_directory, const std::string& file_name, std::vector* results); +::mediapipe::Status MatchFileTypeInDirectory(const std::string& directory, + const std::string& file_suffix, + std::vector* results); + ::mediapipe::Status Exists(absl::string_view file_name); } // namespace file diff --git a/mediapipe/framework/deps/status.cc b/mediapipe/framework/deps/status.cc index c0a02ee3d..43f4f03ea 100644 --- a/mediapipe/framework/deps/status.cc +++ b/mediapipe/framework/deps/status.cc @@ -18,103 +18,6 @@ namespace mediapipe { -Status::Status(::mediapipe::StatusCode code, absl::string_view msg) { - state_ = std::unique_ptr(new State); - state_->code = code; - state_->msg = std::string(msg); -} - -void Status::Update(const Status& new_status) { - if (ok()) { - *this = new_status; - } -} - -void Status::SlowCopyFrom(const State* src) { - if (src == nullptr) { - state_ = nullptr; - } else { - state_ = std::unique_ptr(new State(*src)); - } -} - -const std::string& Status::empty_string() { - static std::string* empty = new std::string; - return *empty; -} - -std::string Status::ToString() const { - if (state_ == nullptr) { - return "OK"; - } else { - char tmp[30]; - const char* type; - switch (code()) { - case ::mediapipe::StatusCode::kCancelled: - type = "Cancelled"; - break; - case ::mediapipe::StatusCode::kUnknown: - type = "Unknown"; - break; - case ::mediapipe::StatusCode::kInvalidArgument: - type = "Invalid argument"; - break; - case ::mediapipe::StatusCode::kDeadlineExceeded: - type = "Deadline exceeded"; - break; - case ::mediapipe::StatusCode::kNotFound: - type = "Not found"; - break; - case ::mediapipe::StatusCode::kAlreadyExists: - type = "Already exists"; - break; - case ::mediapipe::StatusCode::kPermissionDenied: - type = "Permission denied"; - break; - case ::mediapipe::StatusCode::kUnauthenticated: - type = "Unauthenticated"; - break; - case ::mediapipe::StatusCode::kResourceExhausted: - type = "Resource exhausted"; - break; - case ::mediapipe::StatusCode::kFailedPrecondition: - type = "Failed precondition"; - break; - case ::mediapipe::StatusCode::kAborted: - type = "Aborted"; - break; - case ::mediapipe::StatusCode::kOutOfRange: - type = "Out of range"; - break; - case ::mediapipe::StatusCode::kUnimplemented: - type = "Unimplemented"; - break; - case ::mediapipe::StatusCode::kInternal: - type = "Internal"; - break; - case ::mediapipe::StatusCode::kUnavailable: - type = "Unavailable"; - break; - case ::mediapipe::StatusCode::kDataLoss: - type = "Data loss"; - break; - default: - snprintf(tmp, sizeof(tmp), "Unknown code(%d)", - static_cast(code())); - type = tmp; - break; - } - std::string result(type); - result += ": "; - result += state_->msg; - return result; - } -} - -void Status::IgnoreError() const { - // no-op -} - std::ostream& operator<<(std::ostream& os, const Status& x) { os << x.ToString(); return os; diff --git a/mediapipe/framework/deps/status.h b/mediapipe/framework/deps/status.h index 8f5e3bfc0..9d878779b 100644 --- a/mediapipe/framework/deps/status.h +++ b/mediapipe/framework/deps/status.h @@ -20,126 +20,16 @@ #include #include +#include "absl/status/status.h" #include "absl/strings/string_view.h" #include "mediapipe/framework/port/logging.h" namespace mediapipe { -enum class StatusCode { - kOk = 0, - kCancelled = 1, - kUnknown = 2, - kInvalidArgument = 3, - kDeadlineExceeded = 4, - kNotFound = 5, - kAlreadyExists = 6, - kPermissionDenied = 7, - kResourceExhausted = 8, - kFailedPrecondition = 9, - kAborted = 10, - kOutOfRange = 11, - kUnimplemented = 12, - kInternal = 13, - kUnavailable = 14, - kDataLoss = 15, - kUnauthenticated = 16, - kDoNotUseReservedForFutureExpansionUseDefaultInSwitchInstead_ = 20 -}; +using Status = absl::Status; +using StatusCode = absl::StatusCode; -#if defined(__clang__) -// Only clang supports warn_unused_result as a type annotation. -class ABSL_MUST_USE_RESULT Status; -#endif - -// Denotes success or failure of a call in MediaPipe. -class Status { - public: - // Creates a success status. - Status() {} - - // Creates a status with the specified error code and msg as a - // human-readable std::string containing more detailed information. - Status(::mediapipe::StatusCode code, absl::string_view msg); - - // Copies the specified status. - Status(const Status& s); - void operator=(const Status& s); - - // Returns true iff the status indicates success. - bool ok() const { - return (state_ == NULL) || (state_->code == ::mediapipe::StatusCode::kOk); - } - - ::mediapipe::StatusCode code() const { - return ok() ? ::mediapipe::StatusCode::kOk : state_->code; - } - - const std::string& error_message() const { - return ok() ? empty_string() : state_->msg; - } - - absl::string_view message() const { - return absl::string_view(error_message()); - } - - bool operator==(const Status& x) const; - bool operator!=(const Status& x) const; - - // If `ok()`, stores `new_status` into `*this`. If `!ok()`, - // preserves the current status, but may augment with additional - // information about `new_status`. - // - // Convenient way of keeping track of the first error encountered. - // Instead of: - // `if (overall_status.ok()) overall_status = new_status` - // Use: - // `overall_status.Update(new_status);` - void Update(const Status& new_status); - - // Returns a std::string representation of this status suitable for - // printing. Returns the std::string `"OK"` for success. - std::string ToString() const; - - // Ignores any errors. This method does nothing except potentially suppress - // complaints from any tools that are checking that errors are not dropped on - // the floor. - void IgnoreError() const; - - private: - static const std::string& empty_string(); - struct State { - ::mediapipe::StatusCode code; - std::string msg; - }; - // OK status has a `NULL` state_. Otherwise, `state_` points to - // a `State` structure containing the error code and message(s) - std::unique_ptr state_; - - void SlowCopyFrom(const State* src); -}; - -inline Status::Status(const Status& s) - : state_((s.state_ == NULL) ? NULL : new State(*s.state_)) {} - -inline void Status::operator=(const Status& s) { - // The following condition catches both aliasing (when this == &s), - // and the common case where both s and *this are ok. - if (state_ != s.state_) { - SlowCopyFrom(s.state_.get()); - } -} - -inline bool Status::operator==(const Status& x) const { - return (this->state_ == x.state_) || (ToString() == x.ToString()); -} - -inline bool Status::operator!=(const Status& x) const { return !(*this == x); } - -inline Status OkStatus() { return Status(); } - -std::ostream& operator<<(std::ostream& os, const Status& x); - -typedef std::function StatusCallback; +inline ::mediapipe::Status OkStatus() { return absl::OkStatus(); } extern std::string* MediaPipeCheckOpHelperOutOfLine( const ::mediapipe::Status& v, const char* msg); diff --git a/mediapipe/framework/deps/status_builder.cc b/mediapipe/framework/deps/status_builder.cc index 2e66af296..21b80638e 100644 --- a/mediapipe/framework/deps/status_builder.cc +++ b/mediapipe/framework/deps/status_builder.cc @@ -72,12 +72,12 @@ StatusBuilder::operator Status() && { std::string message; if (join_style_ == MessageJoinStyle::kAnnotate) { if (!status_.ok()) { - message = absl::StrCat(status_.error_message(), "; ", stream_->str()); + message = absl::StrCat(status_.message(), "; ", stream_->str()); } } else { message = join_style_ == MessageJoinStyle::kPrepend - ? absl::StrCat(stream_->str(), status_.error_message()) - : absl::StrCat(status_.error_message(), stream_->str()); + ? absl::StrCat(stream_->str(), status_.message()) + : absl::StrCat(status_.message(), stream_->str()); } return Status(status_.code(), message); } diff --git a/mediapipe/framework/deps/status_builder_test.cc b/mediapipe/framework/deps/status_builder_test.cc index 76c92eed5..fbb59fc44 100644 --- a/mediapipe/framework/deps/status_builder_test.cc +++ b/mediapipe/framework/deps/status_builder_test.cc @@ -27,7 +27,7 @@ TEST(StatusBuilder, AnnotateMode) { << "annotated message2"; ASSERT_FALSE(status.ok()); EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kNotFound); - EXPECT_EQ(status.error_message(), + EXPECT_EQ(status.message(), "original message; annotated message1 annotated message2"); } @@ -42,7 +42,7 @@ TEST(StatusBuilder, PrependMode) { << "prepended message2 "; ASSERT_FALSE(status.ok()); EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kInvalidArgument); - EXPECT_EQ(status.error_message(), + EXPECT_EQ(status.message(), "prepended message1 prepended message2 original message"); } @@ -56,8 +56,7 @@ TEST(StatusBuilder, AppendMode) { << " extra message2"; ASSERT_FALSE(status.ok()); EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kInternal); - EXPECT_EQ(status.error_message(), - "original message extra message1 extra message2"); + EXPECT_EQ(status.message(), "original message extra message1 extra message2"); } TEST(StatusBuilder, NoLoggingMode) { @@ -69,7 +68,7 @@ TEST(StatusBuilder, NoLoggingMode) { << " extra message"; ASSERT_FALSE(status.ok()); EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kUnavailable); - EXPECT_EQ(status.error_message(), "original message"); + EXPECT_EQ(status.message(), "original message"); } } // namespace mediapipe diff --git a/mediapipe/framework/deps/status_test.cc b/mediapipe/framework/deps/status_test.cc index 61dd88ecf..10a70c0f7 100644 --- a/mediapipe/framework/deps/status_test.cc +++ b/mediapipe/framework/deps/status_test.cc @@ -21,7 +21,7 @@ namespace mediapipe { TEST(Status, OK) { EXPECT_EQ(OkStatus().code(), ::mediapipe::StatusCode::kOk); - EXPECT_EQ(OkStatus().error_message(), ""); + EXPECT_EQ(OkStatus().message(), ""); MP_EXPECT_OK(OkStatus()); MP_ASSERT_OK(OkStatus()); EXPECT_EQ(OkStatus(), Status()); @@ -38,7 +38,7 @@ TEST(Status, Set) { Status status; status = Status(::mediapipe::StatusCode::kCancelled, "Error message"); EXPECT_EQ(status.code(), ::mediapipe::StatusCode::kCancelled); - EXPECT_EQ(status.error_message(), "Error message"); + EXPECT_EQ(status.message(), "Error message"); } TEST(Status, Copy) { diff --git a/mediapipe/framework/deps/statusor_test.cc b/mediapipe/framework/deps/statusor_test.cc index dde69a424..dae6390d7 100644 --- a/mediapipe/framework/deps/statusor_test.cc +++ b/mediapipe/framework/deps/statusor_test.cc @@ -158,12 +158,12 @@ TEST(StatusOr, TestMoveWithValuesAndErrors) { // Overwrite the value in status_or with an error. status_or = std::move(error1); ASSERT_FALSE(status_or.ok()); - EXPECT_EQ("error1", status_or.status().error_message()); + EXPECT_EQ("error1", status_or.status().message()); // Overwrite the error in status_or with another error. status_or = std::move(error2); ASSERT_FALSE(status_or.ok()); - EXPECT_EQ("error2", status_or.status().error_message()); + EXPECT_EQ("error2", status_or.status().message()); // Overwrite the error with a value. status_or = std::move(value2); @@ -191,12 +191,12 @@ TEST(StatusOr, TestCopyWithValuesAndErrors) { // Overwrite the value in status_or with an error. status_or = error1; ASSERT_FALSE(status_or.ok()); - EXPECT_EQ("error1", status_or.status().error_message()); + EXPECT_EQ("error1", status_or.status().message()); // Overwrite the error in status_or with another error. status_or = error2; ASSERT_FALSE(status_or.ok()); - EXPECT_EQ("error2", status_or.status().error_message()); + EXPECT_EQ("error2", status_or.status().message()); // Overwrite the error with a value. status_or = value2; @@ -205,8 +205,8 @@ TEST(StatusOr, TestCopyWithValuesAndErrors) { // Verify original values unchanged. EXPECT_EQ(std::string(1000, '1'), value1.ValueOrDie()); - EXPECT_EQ("error1", error1.status().error_message()); - EXPECT_EQ("error2", error2.status().error_message()); + EXPECT_EQ("error1", error1.status().message()); + EXPECT_EQ("error2", error2.status().message()); EXPECT_EQ(std::string(1000, '2'), value2.ValueOrDie()); } diff --git a/mediapipe/framework/encode_binary_proto.bzl b/mediapipe/framework/encode_binary_proto.bzl index 55e2f0554..6bbfc33e9 100644 --- a/mediapipe/framework/encode_binary_proto.bzl +++ b/mediapipe/framework/encode_binary_proto.bzl @@ -36,6 +36,11 @@ def _canonicalize_proto_path_oss(all_protos, genfile_path): for s in all_protos.to_list(): if s.path.startswith(genfile_path): repo_name, _, file_name = s.path[len(genfile_path + "/external/"):].partition("/") + + # handle virtual imports + if file_name.startswith("_virtual_imports"): + repo_name = repo_name + "/" + "/".join(file_name.split("/", 2)[:2]) + file_name = file_name.split("/", 2)[-1] proto_paths.append(genfile_path + "/external/" + repo_name) proto_file_names.append(file_name) else: diff --git a/mediapipe/framework/formats/BUILD b/mediapipe/framework/formats/BUILD index 14b4066ac..97e651f9e 100644 --- a/mediapipe/framework/formats/BUILD +++ b/mediapipe/framework/formats/BUILD @@ -268,7 +268,7 @@ java_lite_proto_library( visibility = [ "//mediapipe:__subpackages__", ], - deps = [":landmark_proto"], + deps = [":rect_proto"], ) proto_library( diff --git a/mediapipe/framework/formats/rect.proto b/mediapipe/framework/formats/rect.proto index f58647ccc..6b57a4a40 100644 --- a/mediapipe/framework/formats/rect.proto +++ b/mediapipe/framework/formats/rect.proto @@ -16,6 +16,9 @@ syntax = "proto2"; package mediapipe; +option java_package = "com.google.mediapipe.formats.proto"; +option java_outer_classname = "RectProto"; + // A rectangle with rotation in image coordinates. message Rect { // Location of the center of the rectangle in image coordinates. @@ -27,7 +30,7 @@ message Rect { required int32 height = 3; required int32 width = 4; - // Rotation angle is counter-clockwise in radian. + // Rotation angle is clockwise in radians. optional float rotation = 5 [default = 0.0]; // Optional unique id to help associate different Rects to each other. @@ -46,7 +49,7 @@ message NormalizedRect { required float height = 3; required float width = 4; - // Rotation angle is counter-clockwise in radian. + // Rotation angle is clockwise in radians. optional float rotation = 5 [default = 0.0]; // Optional unique id to help associate different NormalizedRects to each diff --git a/mediapipe/framework/graph_validation_test.cc b/mediapipe/framework/graph_validation_test.cc index 727e594a3..632ecac4c 100644 --- a/mediapipe/framework/graph_validation_test.cc +++ b/mediapipe/framework/graph_validation_test.cc @@ -325,10 +325,8 @@ class OptionalSideInputTestCalculator : public CalculatorBase { public: static ::mediapipe::Status GetContract(CalculatorContract* cc) { cc->InputSidePackets().Tag("SIDEINPUT").Set().Optional(); - if (!cc->Outputs().HasTag("OUTPUT")) { - return ::mediapipe::InvalidArgumentError( - "Expected std::string as output."); - } + cc->Inputs().Tag("SELECT").Set().Optional(); + cc->Inputs().Tag("ENABLE").Set().Optional(); cc->Outputs().Tag("OUTPUT").Set(); return ::mediapipe::OkStatus(); } @@ -394,5 +392,64 @@ TEST(GraphValidationTest, OptionalInputNotProvidedForSubgraphCalculator) { MP_EXPECT_OK(graph_1.WaitUntilDone()); } +TEST(GraphValidationTest, MultipleOptionalInputsForSubgraph) { + // A subgraph defining one optional side-packet and two optional inputs. + auto config_1 = ParseTextProtoOrDie(R"( + type: "PassThroughGraph" + input_side_packet: "INPUT:input_0" + input_stream: "SELECT:select" + input_stream: "ENABLE:enable" + output_stream: "OUTPUT:output_0" + node { + calculator: "OptionalSideInputTestCalculator" + input_side_packet: "SIDEINPUT:input_0" # std::string + input_stream: "SELECT:select" + input_stream: "ENABLE:enable" + output_stream: "OUTPUT:output_0" # std::string + } + )"); + + // An enclosing graph that specifies just one optional input. + auto config_2 = ParseTextProtoOrDie(R"( + input_side_packet: "INPUT:foo_in" + input_stream: "SELECT:foo_select" + output_stream: "OUTPUT:foo_out" + node { + calculator: "PassThroughGraph" + input_stream: "SELECT:foo_select" + output_stream: "OUTPUT:foo_out" # std::string + } + )"); + + GraphValidation validation_1; + MP_ASSERT_OK(validation_1.Validate({config_1, config_2}, {})); + CalculatorGraph graph_1; + MP_ASSERT_OK(graph_1.Initialize({config_1, config_2}, {})); + EXPECT_THAT( + graph_1.Config(), + + // The expanded graph includes only the specified input, "SELECT". + // Without the fix to RemoveIgnoredStreams(), the expanded graph + // includes the wrong input. + EqualsProto(::mediapipe::ParseTextProtoOrDie(R"( + input_side_packet: "INPUT:foo_in" + input_stream: "SELECT:foo_select" + output_stream: "OUTPUT:foo_out" + node { + calculator: "OptionalSideInputTestCalculator" + name: "passthroughgraph__OptionalSideInputTestCalculator" + input_stream: "SELECT:foo_select" + output_stream: "OUTPUT:foo_out" + } + executor {} + )"))); + + std::map side_packets; + side_packets.insert({"foo_in", mediapipe::Adopt(new std::string("input"))}); + MP_EXPECT_OK(graph_1.StartRun(side_packets)); + MP_EXPECT_OK(graph_1.CloseAllPacketSources()); + MP_EXPECT_OK(graph_1.WaitUntilDone()); +} + } // namespace } // namespace mediapipe diff --git a/mediapipe/framework/packet.cc b/mediapipe/framework/packet.cc index f19f8ca6f..1b23e521e 100644 --- a/mediapipe/framework/packet.cc +++ b/mediapipe/framework/packet.cc @@ -40,6 +40,13 @@ Packet Create(HolderBase* holder, Timestamp timestamp) { return result; } +Packet Create(std::shared_ptr holder, Timestamp timestamp) { + Packet result; + result.holder_ = std::move(holder); + result.timestamp_ = timestamp; + return result; +} + const HolderBase* GetHolder(const Packet& packet) { return packet.holder_.get(); } diff --git a/mediapipe/framework/packet.h b/mediapipe/framework/packet.h index 9e1946eaa..e08955555 100644 --- a/mediapipe/framework/packet.h +++ b/mediapipe/framework/packet.h @@ -48,7 +48,9 @@ class HolderBase; Packet Create(HolderBase* holder); Packet Create(HolderBase* holder, Timestamp timestamp); +Packet Create(std::shared_ptr holder, Timestamp timestamp); const HolderBase* GetHolder(const Packet& packet); +const std::shared_ptr& GetHolderShared(const Packet& packet); } // namespace packet_internal // A generic container class which can hold data of any type. The type of @@ -201,8 +203,14 @@ class Packet { friend Packet packet_internal::Create(packet_internal::HolderBase* holder); friend Packet packet_internal::Create(packet_internal::HolderBase* holder, class Timestamp timestamp); + friend Packet packet_internal::Create( + std::shared_ptr holder, + class Timestamp timestamp); friend const packet_internal::HolderBase* packet_internal::GetHolder( const Packet& packet); + friend const std::shared_ptr& + packet_internal::GetHolderShared(const Packet& packet); + std::shared_ptr holder_; class Timestamp timestamp_; }; @@ -713,6 +721,15 @@ inline bool operator!=(const Packet& p1, const Packet& p2) { return !(p1 == p2); } +namespace packet_internal { + +inline const std::shared_ptr& GetHolderShared( + const Packet& packet) { + return packet.holder_; +} + +} // namespace packet_internal + } // namespace mediapipe #endif // MEDIAPIPE_FRAMEWORK_PACKET_H_ diff --git a/mediapipe/framework/port.h b/mediapipe/framework/port.h index a1aeae893..fee918a23 100644 --- a/mediapipe/framework/port.h +++ b/mediapipe/framework/port.h @@ -50,4 +50,34 @@ #define MEDIAPIPE_DISABLE_GL_COMPUTE #endif +// Compile time target platform definitions. +// Example: #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 +#define MEDIAPIPE_OPENGL_ES_UNSUPPORTED 0 +#define MEDIAPIPE_OPENGL_ES_20 200 +#define MEDIAPIPE_OPENGL_ES_31 310 + +#if defined(MEDIAPIPE_DISABLE_GPU) +#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED +#define MEDIAPIPE_METAL_ENABLED 0 +#else +#if defined(MEDIAPIPE_ANDROID) +#if defined(MEDIAPIPE_DISABLE_GL_COMPUTE) +#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_20 +#else +#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_31 +#endif +#define MEDIAPIPE_METAL_ENABLED 0 +#elif defined(MEDIAPIPE_IOS) +#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_20 +#define MEDIAPIPE_METAL_ENABLED 1 +#elif defined(MEDIAPIPE_OSX) +#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED +#define MEDIAPIPE_METAL_ENABLED 1 +#else +// GPU is not supported on Linux yet. +#define MEDIAPIPE_OPENGL_ES_VERSION MEDIAPIPE_OPENGL_ES_UNSUPPORTED +#define MEDIAPIPE_METAL_ENABLED 0 +#endif +#endif + #endif // MEDIAPIPE_FRAMEWORK_PORT_H_ diff --git a/mediapipe/framework/port/BUILD b/mediapipe/framework/port/BUILD index 6bb2222d5..2a7a87e11 100644 --- a/mediapipe/framework/port/BUILD +++ b/mediapipe/framework/port/BUILD @@ -351,6 +351,7 @@ cc_library( ":source_location", "//mediapipe/framework:port", "//mediapipe/framework/deps:status", + "@com_google_absl//absl/status", "@com_google_absl//absl/strings", ], ) diff --git a/mediapipe/framework/profiler/graph_tracer_test.cc b/mediapipe/framework/profiler/graph_tracer_test.cc index d37176da4..e34ebb807 100644 --- a/mediapipe/framework/profiler/graph_tracer_test.cc +++ b/mediapipe/framework/profiler/graph_tracer_test.cc @@ -1267,9 +1267,9 @@ TEST_F(GraphTracerE2ETest, GpuTracing) { output_stream: "annotated_buffer" node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_buffer" + input_stream: "IMAGE:input_buffer" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:annotated_buffer" + output_stream: "IMAGE:annotated_buffer" } profiler_config { trace_enabled: true diff --git a/mediapipe/framework/profiler/profiler_resource_util_apple.cc b/mediapipe/framework/profiler/profiler_resource_util_apple.cc index 10363b058..1ce19ff99 100644 --- a/mediapipe/framework/profiler/profiler_resource_util_apple.cc +++ b/mediapipe/framework/profiler/profiler_resource_util_apple.cc @@ -28,7 +28,7 @@ StatusOr GetDefaultTraceLogDirectory() { // Note: "createDirectoryAtURL:..." method doesn't successfully create // the directory, hence this code uses "createDirectoryAtPath:..". - NSString* ns_documents_directory = [documents_directory_url absoluteString]; + NSString* ns_documents_directory = [documents_directory_url path]; NSError* error; BOOL success = [[NSFileManager defaultManager] createDirectoryAtPath:ns_documents_directory diff --git a/mediapipe/framework/profiler/testdata/BUILD b/mediapipe/framework/profiler/testdata/BUILD new file mode 100644 index 000000000..035b00810 --- /dev/null +++ b/mediapipe/framework/profiler/testdata/BUILD @@ -0,0 +1,24 @@ +load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto") + +package(default_visibility = ["//mediapipe/framework/profiler:__subpackages__"]) + +licenses(["notice"]) + +# Compile any proto data into wire format for use in our tests. +[encode_binary_proto( + name = graph.split("/")[-1].rsplit(".", 1)[0], + input = graph, + message_type = "mediapipe.GraphProfile", + output = graph.rsplit(".", 1)[0] + ".binarypb", + deps = ["//mediapipe/framework:calculator_profile_proto"], +) for graph in glob(["profile_*.pbtxt"])] + +filegroup( + name = "mediapipe_profile_graphs", + srcs = [binarypb.rsplit(".", 1)[0] + ".binarypb" for binarypb in glob(["profile_*.pbtxt"])], +) + +filegroup( + name = "pbtxt_files", + srcs = glob(["*.pbtxt"]), +) diff --git a/mediapipe/framework/profiler/testdata/profile_opencv_0.pbtxt b/mediapipe/framework/profiler/testdata/profile_opencv_0.pbtxt new file mode 100644 index 000000000..9d5e30426 --- /dev/null +++ b/mediapipe/framework/profiler/testdata/profile_opencv_0.pbtxt @@ -0,0 +1,8814 @@ +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + calculator_name : [ "OpenCvVideoDecoderCalculator", "OpenCvWriteTextCalculator", "OpenCvVideoEncoderCalculator" ] + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 0 + input_timestamp: -9223372036854775806 + event_type : OPEN + finish_time : 0 + output_trace: { + packet_timestamp: -9223372036854775806 + stream_id : 1 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 574 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : NOT_READY + start_time : 676 + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 0 + event_type : PROCESS + finish_time : 57426 + output_trace: { + packet_timestamp: 0 + stream_id : 2 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 57642 + thread_id : 1 + } + calculator_trace: { + node_id: 1 + input_timestamp: 0 + event_type : PROCESS + start_time : 57788 + finish_time : 70831 + input_trace: { + start_time : 57426 + finish_time : 57788 + packet_timestamp: 0 + stream_id : 2 + packet_id : 1 + } + output_trace: { + packet_timestamp: 0 + stream_id : 3 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 73210 + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 73229 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: -9223372036854775806 + event_type : PROCESS + start_time : 73331 + input_trace: { + finish_time : 73331 + packet_timestamp: -9223372036854775806 + stream_id : 1 + packet_id : 2 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 33333 + event_type : PROCESS + finish_time : 75380 + output_trace: { + packet_timestamp: 33333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 75510 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 33333 + event_type : PROCESS + start_time : 75586 + finish_time : 85937 + input_trace: { + start_time : 75380 + finish_time : 75586 + packet_timestamp: 33333 + stream_id : 2 + packet_id : 3 + } + output_trace: { + packet_timestamp: 33333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 88016 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 66666 + event_type : PROCESS + finish_time : 102033 + output_trace: { + packet_timestamp: 66666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 102157 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 66666 + event_type : PROCESS + start_time : 102231 + finish_time : 113661 + input_trace: { + start_time : 102033 + finish_time : 102231 + packet_timestamp: 66666 + stream_id : 2 + packet_id : 4 + } + output_trace: { + packet_timestamp: 66666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 115545 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 100000 + event_type : PROCESS + finish_time : 129921 + output_trace: { + packet_timestamp: 100000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 130034 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 100000 + event_type : PROCESS + start_time : 130097 + finish_time : 141415 + input_trace: { + start_time : 129921 + finish_time : 130097 + packet_timestamp: 100000 + stream_id : 2 + packet_id : 5 + } + output_trace: { + packet_timestamp: 100000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 143308 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 133333 + event_type : PROCESS + finish_time : 158102 + output_trace: { + packet_timestamp: 133333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 158220 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 133333 + event_type : PROCESS + start_time : 158296 + finish_time : 170013 + input_trace: { + start_time : 158102 + finish_time : 158296 + packet_timestamp: 133333 + stream_id : 2 + packet_id : 6 + } + output_trace: { + packet_timestamp: 133333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 172059 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 178909 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 0 + event_type : PROCESS + start_time : 178974 + input_trace: { + start_time : 70831 + finish_time : 178974 + packet_timestamp: 0 + stream_id : 3 + packet_id : 7 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 166666 + event_type : PROCESS + finish_time : 186538 + output_trace: { + packet_timestamp: 166666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 186673 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 166666 + event_type : PROCESS + start_time : 186770 + finish_time : 197610 + input_trace: { + start_time : 186538 + finish_time : 186770 + packet_timestamp: 166666 + stream_id : 2 + packet_id : 3 + } + output_trace: { + packet_timestamp: 166666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 199273 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 200000 + event_type : PROCESS + finish_time : 211914 + output_trace: { + packet_timestamp: 200000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 212031 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 200000 + event_type : PROCESS + start_time : 212094 + finish_time : 223449 + input_trace: { + start_time : 211914 + finish_time : 212094 + packet_timestamp: 200000 + stream_id : 2 + packet_id : 8 + } + output_trace: { + packet_timestamp: 200000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 224854 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 33333 + event_type : PROCESS + start_time : 224921 + input_trace: { + start_time : 85937 + finish_time : 224921 + packet_timestamp: 33333 + stream_id : 3 + packet_id : 9 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 225207 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 233333 + event_type : PROCESS + finish_time : 238669 + output_trace: { + packet_timestamp: 233333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 238801 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 233333 + event_type : PROCESS + start_time : 238877 + finish_time : 248219 + input_trace: { + start_time : 238669 + finish_time : 238877 + packet_timestamp: 233333 + stream_id : 2 + packet_id : 3 + } + output_trace: { + packet_timestamp: 233333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 244739 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 66666 + event_type : PROCESS + start_time : 244835 + input_trace: { + start_time : 113661 + finish_time : 244835 + packet_timestamp: 66666 + stream_id : 3 + packet_id : 10 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 250207 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 266666 + event_type : PROCESS + finish_time : 264455 + output_trace: { + packet_timestamp: 266666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 264633 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 266666 + event_type : PROCESS + start_time : 264756 + finish_time : 274437 + input_trace: { + start_time : 264455 + finish_time : 264756 + packet_timestamp: 266666 + stream_id : 2 + packet_id : 9 + } + output_trace: { + packet_timestamp: 266666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 268944 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 100000 + event_type : PROCESS + start_time : 269005 + input_trace: { + start_time : 141415 + finish_time : 269005 + packet_timestamp: 100000 + stream_id : 3 + packet_id : 11 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 276102 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 300000 + event_type : PROCESS + finish_time : 288602 + output_trace: { + packet_timestamp: 300000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 288717 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 300000 + event_type : PROCESS + start_time : 288787 + finish_time : 303580 + input_trace: { + start_time : 288602 + finish_time : 288787 + packet_timestamp: 300000 + stream_id : 2 + packet_id : 10 + } + output_trace: { + packet_timestamp: 300000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 297867 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 133333 + event_type : PROCESS + start_time : 297923 + input_trace: { + start_time : 170013 + finish_time : 297923 + packet_timestamp: 133333 + stream_id : 3 + packet_id : 12 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 305572 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 315434 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 166666 + event_type : PROCESS + start_time : 315500 + input_trace: { + start_time : 197610 + finish_time : 315500 + packet_timestamp: 166666 + stream_id : 3 + packet_id : 13 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 333333 + event_type : PROCESS + finish_time : 318876 + output_trace: { + packet_timestamp: 333333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 318996 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 333333 + event_type : PROCESS + start_time : 319060 + finish_time : 328554 + input_trace: { + start_time : 318876 + finish_time : 319060 + packet_timestamp: 333333 + stream_id : 2 + packet_id : 11 + } + output_trace: { + packet_timestamp: 333333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 330367 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 366666 + event_type : PROCESS + finish_time : 343542 + output_trace: { + packet_timestamp: 366666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 343673 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 366666 + event_type : PROCESS + start_time : 343740 + finish_time : 354013 + input_trace: { + start_time : 343542 + finish_time : 343740 + packet_timestamp: 366666 + stream_id : 2 + packet_id : 14 + } + output_trace: { + packet_timestamp: 366666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 355746 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 358533 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 200000 + event_type : PROCESS + start_time : 358594 + input_trace: { + start_time : 223449 + finish_time : 358594 + packet_timestamp: 200000 + stream_id : 3 + packet_id : 1 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 400000 + event_type : PROCESS + finish_time : 371299 + output_trace: { + packet_timestamp: 400000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 371416 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 400000 + event_type : PROCESS + start_time : 371485 + finish_time : 381193 + input_trace: { + start_time : 371299 + finish_time : 371485 + packet_timestamp: 400000 + stream_id : 2 + packet_id : 15 + } + output_trace: { + packet_timestamp: 400000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 381534 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 233333 + event_type : PROCESS + start_time : 381603 + input_trace: { + start_time : 248219 + finish_time : 381603 + packet_timestamp: 233333 + stream_id : 3 + packet_id : 16 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 383074 + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 0 + input_timestamp: 433333 + event_type : PROCESS + finish_time : 398291 + output_trace: { + packet_timestamp: 433333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 398409 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 433333 + event_type : PROCESS + start_time : 398472 + finish_time : 409805 + input_trace: { + start_time : 398291 + finish_time : 398472 + packet_timestamp: 433333 + stream_id : 2 + packet_id : 17 + } + output_trace: { + packet_timestamp: 433333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 405257 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 266666 + event_type : PROCESS + start_time : 405325 + input_trace: { + finish_time : 405325 + packet_timestamp: 266666 + stream_id : 3 + packet_id : 18 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 411597 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 466666 + event_type : PROCESS + finish_time : 423919 + output_trace: { + packet_timestamp: 466666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 424023 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 466666 + event_type : PROCESS + start_time : 424091 + finish_time : 433400 + input_trace: { + start_time : 423919 + finish_time : 424091 + packet_timestamp: 466666 + stream_id : 2 + packet_id : 16 + } + output_trace: { + packet_timestamp: 466666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 425138 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 300000 + event_type : PROCESS + start_time : 425203 + input_trace: { + finish_time : 425203 + packet_timestamp: 300000 + stream_id : 3 + packet_id : 19 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 434987 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 500000 + event_type : PROCESS + finish_time : 448937 + output_trace: { + packet_timestamp: 500000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 449070 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 500000 + event_type : PROCESS + start_time : 449157 + finish_time : 458465 + input_trace: { + start_time : 448937 + finish_time : 449157 + packet_timestamp: 500000 + stream_id : 2 + packet_id : 18 + } + output_trace: { + packet_timestamp: 500000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 458434 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 333333 + event_type : PROCESS + start_time : 458501 + input_trace: { + finish_time : 458501 + packet_timestamp: 333333 + stream_id : 3 + packet_id : 20 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 461531 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 533333 + event_type : PROCESS + finish_time : 475315 + output_trace: { + packet_timestamp: 533333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 475432 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 533333 + event_type : PROCESS + start_time : 475500 + finish_time : 484858 + input_trace: { + start_time : 475315 + finish_time : 475500 + packet_timestamp: 533333 + stream_id : 2 + packet_id : 21 + } + output_trace: { + packet_timestamp: 533333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 477062 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 366666 + event_type : PROCESS + start_time : 477113 + input_trace: { + finish_time : 477113 + packet_timestamp: 366666 + stream_id : 3 + packet_id : 5 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 486387 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 566666 + event_type : PROCESS + finish_time : 499971 + output_trace: { + packet_timestamp: 566666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 500117 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 566666 + event_type : PROCESS + start_time : 500205 + finish_time : 509522 + input_trace: { + start_time : 499971 + finish_time : 500205 + packet_timestamp: 566666 + stream_id : 2 + packet_id : 20 + } + output_trace: { + packet_timestamp: 566666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 506653 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 400000 + event_type : PROCESS + start_time : 506708 + input_trace: { + finish_time : 506708 + packet_timestamp: 400000 + stream_id : 3 + packet_id : 14 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 511579 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 600000 + event_type : PROCESS + finish_time : 523913 + output_trace: { + packet_timestamp: 600000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 524033 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 600000 + event_type : PROCESS + start_time : 524099 + finish_time : 533116 + input_trace: { + start_time : 523913 + finish_time : 524099 + packet_timestamp: 600000 + stream_id : 2 + packet_id : 5 + } + output_trace: { + packet_timestamp: 600000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 527050 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 433333 + event_type : PROCESS + start_time : 527109 + input_trace: { + start_time : 409805 + finish_time : 527109 + packet_timestamp: 433333 + stream_id : 3 + packet_id : 1 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 534639 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 633333 + event_type : PROCESS + finish_time : 549028 + output_trace: { + packet_timestamp: 633333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 549162 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 633333 + event_type : PROCESS + start_time : 549237 + finish_time : 559878 + input_trace: { + start_time : 549028 + finish_time : 549237 + packet_timestamp: 633333 + stream_id : 2 + packet_id : 14 + } + output_trace: { + packet_timestamp: 633333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 562466 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 563912 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 466666 + event_type : PROCESS + start_time : 563962 + input_trace: { + start_time : 433400 + finish_time : 563962 + packet_timestamp: 466666 + stream_id : 3 + packet_id : 10 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 666666 + event_type : PROCESS + finish_time : 579680 + output_trace: { + packet_timestamp: 666666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 579807 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 666666 + event_type : PROCESS + start_time : 579872 + finish_time : 587953 + input_trace: { + start_time : 579680 + finish_time : 579872 + packet_timestamp: 666666 + stream_id : 2 + packet_id : 5 + } + output_trace: { + packet_timestamp: 666666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 587892 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 500000 + event_type : PROCESS + start_time : 587948 + input_trace: { + start_time : 458465 + finish_time : 587948 + packet_timestamp: 500000 + stream_id : 3 + packet_id : 22 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 590053 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 700000 + event_type : PROCESS + finish_time : 602403 + output_trace: { + packet_timestamp: 700000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 602521 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 700000 + event_type : PROCESS + start_time : 602586 + finish_time : 611407 + input_trace: { + start_time : 602403 + finish_time : 602586 + packet_timestamp: 700000 + stream_id : 2 + packet_id : 23 + } + output_trace: { + packet_timestamp: 700000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 607904 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 533333 + event_type : PROCESS + start_time : 607961 + input_trace: { + start_time : 484858 + finish_time : 607961 + packet_timestamp: 533333 + stream_id : 3 + packet_id : 19 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 613239 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 625387 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 566666 + event_type : PROCESS + start_time : 625447 + input_trace: { + start_time : 509522 + finish_time : 625447 + packet_timestamp: 566666 + stream_id : 3 + packet_id : 17 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 733333 + event_type : PROCESS + finish_time : 626947 + output_trace: { + packet_timestamp: 733333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 627074 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 733333 + event_type : PROCESS + start_time : 627152 + finish_time : 644856 + input_trace: { + start_time : 626947 + finish_time : 627152 + packet_timestamp: 733333 + stream_id : 2 + packet_id : 22 + } + output_trace: { + packet_timestamp: 733333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 643265 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 600000 + event_type : PROCESS + start_time : 643316 + input_trace: { + start_time : 533116 + finish_time : 643316 + packet_timestamp: 600000 + stream_id : 3 + packet_id : 24 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 647121 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 766666 + event_type : PROCESS + finish_time : 658968 + output_trace: { + packet_timestamp: 766666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 659084 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 766666 + event_type : PROCESS + start_time : 659152 + finish_time : 668077 + input_trace: { + start_time : 658968 + finish_time : 659152 + packet_timestamp: 766666 + stream_id : 2 + packet_id : 17 + } + output_trace: { + packet_timestamp: 766666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 663883 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 633333 + event_type : PROCESS + start_time : 663938 + input_trace: { + start_time : 559878 + finish_time : 663938 + packet_timestamp: 633333 + stream_id : 3 + packet_id : 7 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 669691 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 800000 + event_type : PROCESS + finish_time : 682886 + output_trace: { + packet_timestamp: 800000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 682993 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 800000 + event_type : PROCESS + start_time : 683058 + finish_time : 691908 + input_trace: { + start_time : 682886 + finish_time : 683058 + packet_timestamp: 800000 + stream_id : 2 + packet_id : 24 + } + output_trace: { + packet_timestamp: 800000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 683180 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 666666 + event_type : PROCESS + start_time : 683231 + input_trace: { + start_time : 587953 + finish_time : 683231 + packet_timestamp: 666666 + stream_id : 3 + packet_id : 25 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 693599 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 833333 + event_type : PROCESS + finish_time : 706694 + output_trace: { + packet_timestamp: 833333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 706832 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 833333 + event_type : PROCESS + start_time : 706942 + finish_time : 715608 + input_trace: { + start_time : 706694 + finish_time : 706942 + packet_timestamp: 833333 + stream_id : 2 + packet_id : 7 + } + output_trace: { + packet_timestamp: 833333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 715469 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 700000 + event_type : PROCESS + start_time : 715525 + input_trace: { + start_time : 611407 + finish_time : 715525 + packet_timestamp: 700000 + stream_id : 3 + packet_id : 26 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 720423 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 866666 + event_type : PROCESS + finish_time : 732291 + output_trace: { + packet_timestamp: 866666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 732472 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 866666 + event_type : PROCESS + start_time : 732560 + finish_time : 741233 + input_trace: { + start_time : 732291 + finish_time : 732560 + packet_timestamp: 866666 + stream_id : 2 + packet_id : 24 + } + output_trace: { + packet_timestamp: 866666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 733607 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 733333 + event_type : PROCESS + start_time : 733658 + input_trace: { + start_time : 644856 + finish_time : 733658 + packet_timestamp: 733333 + stream_id : 3 + packet_id : 27 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 742856 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 900000 + event_type : PROCESS + finish_time : 757121 + output_trace: { + packet_timestamp: 900000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 757257 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 900000 + event_type : PROCESS + start_time : 757339 + finish_time : 766415 + input_trace: { + start_time : 757121 + finish_time : 757339 + packet_timestamp: 900000 + stream_id : 2 + packet_id : 26 + } + output_trace: { + packet_timestamp: 900000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 762947 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 766666 + event_type : PROCESS + start_time : 763007 + input_trace: { + start_time : 668077 + finish_time : 763007 + packet_timestamp: 766666 + stream_id : 3 + packet_id : 23 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 768246 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 933333 + event_type : PROCESS + finish_time : 780219 + output_trace: { + packet_timestamp: 933333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 780323 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 933333 + event_type : PROCESS + start_time : 780390 + finish_time : 788789 + input_trace: { + start_time : 780219 + finish_time : 780390 + packet_timestamp: 933333 + stream_id : 2 + packet_id : 27 + } + output_trace: { + packet_timestamp: 933333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 782267 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 800000 + event_type : PROCESS + start_time : 782325 + input_trace: { + start_time : 691908 + finish_time : 782325 + packet_timestamp: 800000 + stream_id : 3 + packet_id : 17 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 790357 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 966666 + event_type : PROCESS + finish_time : 802403 + output_trace: { + packet_timestamp: 966666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 802542 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 966666 + event_type : PROCESS + start_time : 802623 + finish_time : 815742 + input_trace: { + start_time : 802403 + finish_time : 802623 + packet_timestamp: 966666 + stream_id : 2 + packet_id : 23 + } + output_trace: { + packet_timestamp: 966666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 811216 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 833333 + event_type : PROCESS + start_time : 811275 + input_trace: { + start_time : 715608 + finish_time : 811275 + packet_timestamp: 833333 + stream_id : 3 + packet_id : 10 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 817565 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1000000 + event_type : PROCESS + finish_time : 829276 + output_trace: { + packet_timestamp: 1000000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 829380 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1000000 + event_type : PROCESS + start_time : 829445 + finish_time : 838035 + input_trace: { + start_time : 829276 + finish_time : 829445 + packet_timestamp: 1000000 + stream_id : 2 + packet_id : 17 + } + output_trace: { + packet_timestamp: 1000000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 829749 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 866666 + event_type : PROCESS + start_time : 829793 + input_trace: { + start_time : 741233 + finish_time : 829793 + packet_timestamp: 866666 + stream_id : 3 + packet_id : 28 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 839741 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1033333 + event_type : PROCESS + finish_time : 852781 + output_trace: { + packet_timestamp: 1033333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 852914 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1033333 + event_type : PROCESS + start_time : 852994 + finish_time : 862155 + input_trace: { + start_time : 852781 + finish_time : 852994 + packet_timestamp: 1033333 + stream_id : 2 + packet_id : 10 + } + output_trace: { + packet_timestamp: 1033333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 859587 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 900000 + event_type : PROCESS + start_time : 859663 + input_trace: { + start_time : 766415 + finish_time : 859663 + packet_timestamp: 900000 + stream_id : 3 + packet_id : 29 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 864007 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1066666 + event_type : PROCESS + finish_time : 879541 + output_trace: { + packet_timestamp: 1066666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 879713 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1066666 + event_type : PROCESS + start_time : 879819 + input_trace: { + start_time : 879541 + finish_time : 879819 + packet_timestamp: 1066666 + stream_id : 2 + packet_id : 28 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 883349 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 933333 + event_type : PROCESS + start_time : 883409 + input_trace: { + start_time : 788789 + finish_time : 883409 + packet_timestamp: 933333 + stream_id : 3 + packet_id : 30 + } + thread_id : 1 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 1 + input_timestamp: 1066666 + event_type : PROCESS + finish_time : 889271 + output_trace: { + packet_timestamp: 1066666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 891691 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1100000 + event_type : PROCESS + finish_time : 905188 + output_trace: { + packet_timestamp: 1100000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 905327 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1100000 + event_type : PROCESS + start_time : 905400 + finish_time : 915506 + input_trace: { + start_time : 905188 + finish_time : 905400 + packet_timestamp: 1100000 + stream_id : 2 + packet_id : 29 + } + output_trace: { + packet_timestamp: 1100000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 917892 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 920722 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 966666 + event_type : PROCESS + start_time : 920776 + input_trace: { + finish_time : 920776 + packet_timestamp: 966666 + stream_id : 3 + packet_id : 31 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1133333 + event_type : PROCESS + finish_time : 934091 + output_trace: { + packet_timestamp: 1133333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 934204 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1133333 + event_type : PROCESS + start_time : 934268 + finish_time : 942987 + input_trace: { + start_time : 934091 + finish_time : 934268 + packet_timestamp: 1133333 + stream_id : 2 + packet_id : 32 + } + output_trace: { + packet_timestamp: 1133333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 939541 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1000000 + event_type : PROCESS + start_time : 939597 + input_trace: { + finish_time : 939597 + packet_timestamp: 1000000 + stream_id : 3 + packet_id : 23 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 944766 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1166666 + event_type : PROCESS + finish_time : 957241 + output_trace: { + packet_timestamp: 1166666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 957341 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1166666 + event_type : PROCESS + start_time : 957408 + finish_time : 965553 + input_trace: { + start_time : 957241 + finish_time : 957408 + packet_timestamp: 1166666 + stream_id : 2 + packet_id : 31 + } + output_trace: { + packet_timestamp: 1166666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 957844 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1033333 + event_type : PROCESS + start_time : 957881 + input_trace: { + finish_time : 957881 + packet_timestamp: 1033333 + stream_id : 3 + packet_id : 33 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 967144 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1200000 + event_type : PROCESS + finish_time : 979706 + output_trace: { + packet_timestamp: 1200000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 979859 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1200000 + event_type : PROCESS + start_time : 979949 + finish_time : 992559 + input_trace: { + start_time : 979706 + finish_time : 979949 + packet_timestamp: 1200000 + stream_id : 2 + packet_id : 23 + } + output_trace: { + packet_timestamp: 1200000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 988384 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1066666 + event_type : PROCESS + start_time : 988502 + input_trace: { + start_time : 889271 + finish_time : 988502 + packet_timestamp: 1066666 + stream_id : 3 + packet_id : 34 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 994479 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1007898 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1100000 + event_type : PROCESS + start_time : 1007954 + input_trace: { + start_time : 915506 + finish_time : 1007954 + packet_timestamp: 1100000 + stream_id : 3 + packet_id : 35 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1233333 + event_type : PROCESS + finish_time : 1009754 + output_trace: { + packet_timestamp: 1233333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1009885 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1233333 + event_type : PROCESS + start_time : 1009962 + finish_time : 1033528 + input_trace: { + start_time : 1009754 + finish_time : 1009962 + packet_timestamp: 1233333 + stream_id : 2 + packet_id : 33 + } + output_trace: { + packet_timestamp: 1233333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1025985 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1133333 + event_type : PROCESS + start_time : 1026043 + input_trace: { + start_time : 942987 + finish_time : 1026043 + packet_timestamp: 1133333 + stream_id : 3 + packet_id : 29 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1035699 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1043425 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1166666 + event_type : PROCESS + start_time : 1043492 + input_trace: { + start_time : 965553 + finish_time : 1043492 + packet_timestamp: 1166666 + stream_id : 3 + packet_id : 32 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1266666 + event_type : PROCESS + finish_time : 1049071 + output_trace: { + packet_timestamp: 1266666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1049180 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1266666 + event_type : PROCESS + start_time : 1049243 + finish_time : 1059218 + input_trace: { + start_time : 1049071 + finish_time : 1049243 + packet_timestamp: 1266666 + stream_id : 2 + packet_id : 35 + } + output_trace: { + packet_timestamp: 1266666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1060910 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1073212 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1200000 + event_type : PROCESS + start_time : 1073269 + input_trace: { + start_time : 992559 + finish_time : 1073269 + packet_timestamp: 1200000 + stream_id : 3 + packet_id : 28 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1300000 + event_type : PROCESS + finish_time : 1073655 + output_trace: { + packet_timestamp: 1300000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1073768 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1300000 + event_type : PROCESS + start_time : 1073830 + finish_time : 1089469 + input_trace: { + start_time : 1073655 + finish_time : 1073830 + packet_timestamp: 1300000 + stream_id : 2 + packet_id : 36 + } + output_trace: { + packet_timestamp: 1300000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1092204 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1095678 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1233333 + event_type : PROCESS + start_time : 1095743 + input_trace: { + start_time : 1033528 + finish_time : 1095743 + packet_timestamp: 1233333 + stream_id : 3 + packet_id : 37 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1333333 + event_type : PROCESS + finish_time : 1109697 + output_trace: { + packet_timestamp: 1333333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1109813 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1333333 + event_type : PROCESS + start_time : 1109879 + finish_time : 1119901 + input_trace: { + start_time : 1109697 + finish_time : 1109879 + packet_timestamp: 1333333 + stream_id : 2 + packet_id : 38 + } + output_trace: { + packet_timestamp: 1333333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1114464 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1266666 + event_type : PROCESS + start_time : 1114533 + input_trace: { + start_time : 1059218 + finish_time : 1114533 + packet_timestamp: 1266666 + stream_id : 3 + packet_id : 39 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1121563 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1366666 + event_type : PROCESS + finish_time : 1134077 + output_trace: { + packet_timestamp: 1366666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1134184 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1366666 + event_type : PROCESS + start_time : 1134249 + finish_time : 1145268 + input_trace: { + start_time : 1134077 + finish_time : 1134249 + packet_timestamp: 1366666 + stream_id : 2 + packet_id : 37 + } + output_trace: { + packet_timestamp: 1366666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1134583 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1300000 + event_type : PROCESS + start_time : 1134631 + input_trace: { + start_time : 1089469 + finish_time : 1134631 + packet_timestamp: 1300000 + stream_id : 3 + packet_id : 40 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1147388 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1400000 + event_type : PROCESS + finish_time : 1161680 + output_trace: { + packet_timestamp: 1400000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1161803 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1400000 + event_type : PROCESS + start_time : 1161869 + finish_time : 1171471 + input_trace: { + start_time : 1161680 + finish_time : 1161869 + packet_timestamp: 1400000 + stream_id : 2 + packet_id : 39 + } + output_trace: { + packet_timestamp: 1400000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1170753 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1333333 + event_type : PROCESS + start_time : 1170809 + input_trace: { + start_time : 1119901 + finish_time : 1170809 + packet_timestamp: 1333333 + stream_id : 3 + packet_id : 19 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1174129 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1433333 + event_type : PROCESS + finish_time : 1186942 + output_trace: { + packet_timestamp: 1433333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1187059 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1433333 + event_type : PROCESS + start_time : 1187129 + finish_time : 1197198 + input_trace: { + start_time : 1186942 + finish_time : 1187129 + packet_timestamp: 1433333 + stream_id : 2 + packet_id : 37 + } + output_trace: { + packet_timestamp: 1433333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1190065 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1366666 + event_type : PROCESS + start_time : 1190126 + input_trace: { + start_time : 1145268 + finish_time : 1190126 + packet_timestamp: 1366666 + stream_id : 3 + packet_id : 32 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1198664 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1466666 + event_type : PROCESS + finish_time : 1212660 + output_trace: { + packet_timestamp: 1466666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1212789 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1466666 + event_type : PROCESS + start_time : 1212856 + finish_time : 1226055 + input_trace: { + start_time : 1212660 + finish_time : 1212856 + packet_timestamp: 1466666 + stream_id : 2 + packet_id : 19 + } + output_trace: { + packet_timestamp: 1466666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1217761 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1400000 + event_type : PROCESS + start_time : 1217828 + input_trace: { + start_time : 1171471 + finish_time : 1217828 + packet_timestamp: 1400000 + stream_id : 3 + packet_id : 33 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1228080 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1235837 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1433333 + event_type : PROCESS + start_time : 1235898 + input_trace: { + start_time : 1197198 + finish_time : 1235898 + packet_timestamp: 1433333 + stream_id : 3 + packet_id : 41 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1500000 + event_type : PROCESS + finish_time : 1246332 + output_trace: { + packet_timestamp: 1500000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1246476 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1500000 + event_type : PROCESS + start_time : 1246556 + finish_time : 1258668 + input_trace: { + start_time : 1246332 + finish_time : 1246556 + packet_timestamp: 1500000 + stream_id : 2 + packet_id : 32 + } + output_trace: { + packet_timestamp: 1500000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1261328 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1262104 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1466666 + event_type : PROCESS + start_time : 1262154 + input_trace: { + start_time : 1226055 + finish_time : 1262154 + packet_timestamp: 1466666 + stream_id : 3 + packet_id : 42 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1533333 + event_type : PROCESS + finish_time : 1277248 + output_trace: { + packet_timestamp: 1533333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1277365 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1533333 + event_type : PROCESS + start_time : 1277430 + finish_time : 1287214 + input_trace: { + start_time : 1277248 + finish_time : 1277430 + packet_timestamp: 1533333 + stream_id : 2 + packet_id : 33 + } + output_trace: { + packet_timestamp: 1533333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 1283470 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1500000 + event_type : PROCESS + start_time : 1283528 + input_trace: { + start_time : 1258668 + finish_time : 1283528 + packet_timestamp: 1500000 + stream_id : 3 + packet_id : 19 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1289024 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1566666 + event_type : PROCESS + finish_time : 1303227 + output_trace: { + packet_timestamp: 1566666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1303364 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1566666 + event_type : PROCESS + start_time : 1303444 + finish_time : 1313898 + input_trace: { + start_time : 1303227 + finish_time : 1303444 + packet_timestamp: 1566666 + stream_id : 2 + packet_id : 42 + } + output_trace: { + packet_timestamp: 1566666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1315692 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1600000 + event_type : PROCESS + finish_time : 1329624 + output_trace: { + packet_timestamp: 1600000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1329742 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1600000 + event_type : PROCESS + start_time : 1329808 + finish_time : 1340153 + input_trace: { + start_time : 1329624 + finish_time : 1329808 + packet_timestamp: 1600000 + stream_id : 2 + packet_id : 24 + } + output_trace: { + packet_timestamp: 1600000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1341773 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1633333 + event_type : PROCESS + finish_time : 1354752 + output_trace: { + packet_timestamp: 1633333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1354895 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1633333 + event_type : PROCESS + start_time : 1354971 + finish_time : 1365588 + input_trace: { + start_time : 1354752 + finish_time : 1354971 + packet_timestamp: 1633333 + stream_id : 2 + packet_id : 36 + } + output_trace: { + packet_timestamp: 1633333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1367172 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1666666 + event_type : PROCESS + finish_time : 1380133 + output_trace: { + packet_timestamp: 1666666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1380254 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1666666 + event_type : PROCESS + start_time : 1380340 + input_trace: { + start_time : 1380133 + finish_time : 1380340 + packet_timestamp: 1666666 + stream_id : 2 + packet_id : 43 + } + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 1 + input_timestamp: 1666666 + event_type : PROCESS + finish_time : 1391811 + output_trace: { + packet_timestamp: 1666666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1393899 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1700000 + event_type : PROCESS + finish_time : 1407541 + output_trace: { + packet_timestamp: 1700000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1407662 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1700000 + event_type : PROCESS + start_time : 1407734 + finish_time : 1417631 + input_trace: { + start_time : 1407541 + finish_time : 1407734 + packet_timestamp: 1700000 + stream_id : 2 + packet_id : 25 + } + output_trace: { + packet_timestamp: 1700000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1419190 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1733333 + event_type : PROCESS + finish_time : 1433287 + output_trace: { + packet_timestamp: 1733333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1433404 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1733333 + event_type : PROCESS + start_time : 1433471 + finish_time : 1443656 + input_trace: { + start_time : 1433287 + finish_time : 1433471 + packet_timestamp: 1733333 + stream_id : 2 + packet_id : 40 + } + output_trace: { + packet_timestamp: 1733333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1445295 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1766666 + event_type : PROCESS + finish_time : 1458037 + output_trace: { + packet_timestamp: 1766666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1458150 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1766666 + event_type : PROCESS + start_time : 1458219 + finish_time : 1469082 + input_trace: { + start_time : 1458037 + finish_time : 1458219 + packet_timestamp: 1766666 + stream_id : 2 + packet_id : 38 + } + output_trace: { + packet_timestamp: 1766666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1470827 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1800000 + event_type : PROCESS + finish_time : 1483901 + output_trace: { + packet_timestamp: 1800000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1484023 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1800000 + event_type : PROCESS + start_time : 1484096 + finish_time : 1494954 + input_trace: { + start_time : 1483901 + finish_time : 1484096 + packet_timestamp: 1800000 + stream_id : 2 + packet_id : 13 + } + output_trace: { + packet_timestamp: 1800000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1496474 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1833333 + event_type : PROCESS + finish_time : 1511347 + output_trace: { + packet_timestamp: 1833333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1511468 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1833333 + event_type : PROCESS + start_time : 1511535 + finish_time : 1522050 + input_trace: { + start_time : 1511347 + finish_time : 1511535 + packet_timestamp: 1833333 + stream_id : 2 + packet_id : 44 + } + output_trace: { + packet_timestamp: 1833333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1523667 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1866666 + event_type : PROCESS + finish_time : 1535944 + output_trace: { + packet_timestamp: 1866666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1536059 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1866666 + event_type : PROCESS + start_time : 1536125 + finish_time : 1545913 + input_trace: { + start_time : 1535944 + finish_time : 1536125 + packet_timestamp: 1866666 + stream_id : 2 + packet_id : 24 + } + output_trace: { + packet_timestamp: 1866666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1547345 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1900000 + event_type : PROCESS + finish_time : 1560035 + output_trace: { + packet_timestamp: 1900000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1560160 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1900000 + event_type : PROCESS + start_time : 1560223 + finish_time : 1570127 + input_trace: { + start_time : 1560035 + finish_time : 1560223 + packet_timestamp: 1900000 + stream_id : 2 + packet_id : 10 + } + output_trace: { + packet_timestamp: 1900000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1571584 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1933333 + event_type : PROCESS + finish_time : 1584206 + output_trace: { + packet_timestamp: 1933333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1584324 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1933333 + event_type : PROCESS + start_time : 1584387 + finish_time : 1594497 + input_trace: { + start_time : 1584206 + finish_time : 1584387 + packet_timestamp: 1933333 + stream_id : 2 + packet_id : 13 + } + output_trace: { + packet_timestamp: 1933333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1596230 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 1966666 + event_type : PROCESS + finish_time : 1608877 + output_trace: { + packet_timestamp: 1966666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1609001 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 1966666 + event_type : PROCESS + start_time : 1609066 + finish_time : 1619008 + input_trace: { + start_time : 1608877 + finish_time : 1609066 + packet_timestamp: 1966666 + stream_id : 2 + packet_id : 45 + } + output_trace: { + packet_timestamp: 1966666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1620706 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2000000 + event_type : PROCESS + finish_time : 1633050 + output_trace: { + packet_timestamp: 2000000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1633168 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2000000 + event_type : PROCESS + start_time : 1633232 + finish_time : 1644724 + input_trace: { + start_time : 1633050 + finish_time : 1633232 + packet_timestamp: 2000000 + stream_id : 2 + packet_id : 44 + } + output_trace: { + packet_timestamp: 2000000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1646570 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2033333 + event_type : PROCESS + finish_time : 1661568 + output_trace: { + packet_timestamp: 2033333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1661688 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2033333 + event_type : PROCESS + start_time : 1661755 + finish_time : 1671720 + input_trace: { + start_time : 1661568 + finish_time : 1661755 + packet_timestamp: 2033333 + stream_id : 2 + packet_id : 10 + } + output_trace: { + packet_timestamp: 2033333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1673578 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2066666 + event_type : PROCESS + finish_time : 1687617 + output_trace: { + packet_timestamp: 2066666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1687737 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2066666 + event_type : PROCESS + start_time : 1687803 + finish_time : 1697696 + input_trace: { + start_time : 1687617 + finish_time : 1687803 + packet_timestamp: 2066666 + stream_id : 2 + packet_id : 13 + } + output_trace: { + packet_timestamp: 2066666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1699327 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2100000 + event_type : PROCESS + finish_time : 1712834 + output_trace: { + packet_timestamp: 2100000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1712962 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2100000 + event_type : PROCESS + start_time : 1713027 + finish_time : 1723348 + input_trace: { + start_time : 1712834 + finish_time : 1713027 + packet_timestamp: 2100000 + stream_id : 2 + packet_id : 39 + } + output_trace: { + packet_timestamp: 2100000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1725069 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2133333 + event_type : PROCESS + finish_time : 1737886 + output_trace: { + packet_timestamp: 2133333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1738027 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2133333 + event_type : PROCESS + start_time : 1738094 + finish_time : 1748740 + input_trace: { + start_time : 1737886 + finish_time : 1738094 + packet_timestamp: 2133333 + stream_id : 2 + packet_id : 14 + } + output_trace: { + packet_timestamp: 2133333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1750391 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2166666 + event_type : PROCESS + finish_time : 1763800 + output_trace: { + packet_timestamp: 2166666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1763915 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2166666 + event_type : PROCESS + start_time : 1764014 + finish_time : 1774471 + input_trace: { + start_time : 1763800 + finish_time : 1764014 + packet_timestamp: 2166666 + stream_id : 2 + packet_id : 46 + } + output_trace: { + packet_timestamp: 2166666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1775990 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2200000 + event_type : PROCESS + finish_time : 1789575 + output_trace: { + packet_timestamp: 2200000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1789698 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2200000 + event_type : PROCESS + start_time : 1789761 + finish_time : 1800894 + input_trace: { + start_time : 1789575 + finish_time : 1789761 + packet_timestamp: 2200000 + stream_id : 2 + packet_id : 38 + } + output_trace: { + packet_timestamp: 2200000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1802762 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2233333 + event_type : PROCESS + finish_time : 1816720 + output_trace: { + packet_timestamp: 2233333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1816838 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2233333 + event_type : PROCESS + start_time : 1816903 + finish_time : 1827695 + input_trace: { + start_time : 1816720 + finish_time : 1816903 + packet_timestamp: 2233333 + stream_id : 2 + packet_id : 10 + } + output_trace: { + packet_timestamp: 2233333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1829256 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2266666 + event_type : PROCESS + finish_time : 1842541 + output_trace: { + packet_timestamp: 2266666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1842660 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2266666 + event_type : PROCESS + start_time : 1842723 + finish_time : 1853348 + input_trace: { + start_time : 1842541 + finish_time : 1842723 + packet_timestamp: 2266666 + stream_id : 2 + packet_id : 13 + } + output_trace: { + packet_timestamp: 2266666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1855169 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2300000 + event_type : PROCESS + finish_time : 1869080 + output_trace: { + packet_timestamp: 2300000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1869217 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2300000 + event_type : PROCESS + start_time : 1869293 + finish_time : 1880092 + input_trace: { + start_time : 1869080 + finish_time : 1869293 + packet_timestamp: 2300000 + stream_id : 2 + packet_id : 47 + } + output_trace: { + packet_timestamp: 2300000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1881952 + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 0 + input_timestamp: 2333333 + event_type : PROCESS + finish_time : 1896087 + output_trace: { + packet_timestamp: 2333333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1896203 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2333333 + event_type : PROCESS + start_time : 1896268 + finish_time : 1906304 + input_trace: { + start_time : 1896087 + finish_time : 1896268 + packet_timestamp: 2333333 + stream_id : 2 + packet_id : 34 + } + output_trace: { + packet_timestamp: 2333333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1907849 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2366666 + event_type : PROCESS + finish_time : 1920301 + output_trace: { + packet_timestamp: 2366666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1920470 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2366666 + event_type : PROCESS + start_time : 1920552 + finish_time : 1930493 + input_trace: { + start_time : 1920301 + finish_time : 1920552 + packet_timestamp: 2366666 + stream_id : 2 + packet_id : 29 + } + output_trace: { + packet_timestamp: 2366666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1932063 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2400000 + event_type : PROCESS + finish_time : 1944097 + output_trace: { + packet_timestamp: 2400000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1944218 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2400000 + event_type : PROCESS + start_time : 1944287 + finish_time : 1955061 + input_trace: { + start_time : 1944097 + finish_time : 1944287 + packet_timestamp: 2400000 + stream_id : 2 + packet_id : 38 + } + output_trace: { + packet_timestamp: 2400000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1956788 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2433333 + event_type : PROCESS + finish_time : 1969398 + output_trace: { + packet_timestamp: 2433333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1969518 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2433333 + event_type : PROCESS + start_time : 1969585 + finish_time : 1979569 + input_trace: { + start_time : 1969398 + finish_time : 1969585 + packet_timestamp: 2433333 + stream_id : 2 + packet_id : 47 + } + output_trace: { + packet_timestamp: 2433333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 1981268 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2466666 + event_type : PROCESS + finish_time : 1994991 + output_trace: { + packet_timestamp: 2466666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 1995126 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2466666 + event_type : PROCESS + start_time : 1995205 + finish_time : 2005384 + input_trace: { + start_time : 1994991 + finish_time : 1995205 + packet_timestamp: 2466666 + stream_id : 2 + packet_id : 34 + } + output_trace: { + packet_timestamp: 2466666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2006906 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2500000 + event_type : PROCESS + finish_time : 2020067 + output_trace: { + packet_timestamp: 2500000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2020182 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2500000 + event_type : PROCESS + start_time : 2020248 + finish_time : 2030209 + input_trace: { + start_time : 2020067 + finish_time : 2020248 + packet_timestamp: 2500000 + stream_id : 2 + packet_id : 27 + } + output_trace: { + packet_timestamp: 2500000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2031823 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2533333 + event_type : PROCESS + finish_time : 2045903 + output_trace: { + packet_timestamp: 2533333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2046042 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2533333 + event_type : PROCESS + start_time : 2046125 + finish_time : 2056470 + input_trace: { + start_time : 2045903 + finish_time : 2046125 + packet_timestamp: 2533333 + stream_id : 2 + packet_id : 38 + } + output_trace: { + packet_timestamp: 2533333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2058238 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2062720 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1533333 + event_type : PROCESS + start_time : 2062786 + input_trace: { + finish_time : 2062786 + packet_timestamp: 1533333 + stream_id : 3 + packet_id : 48 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2085951 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1566666 + event_type : PROCESS + start_time : 2086023 + input_trace: { + finish_time : 2086023 + packet_timestamp: 1566666 + stream_id : 3 + packet_id : 37 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2566666 + event_type : PROCESS + finish_time : 2087805 + output_trace: { + packet_timestamp: 2566666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2087933 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2566666 + event_type : PROCESS + start_time : 2088006 + finish_time : 2113613 + input_trace: { + start_time : 2087805 + finish_time : 2088006 + packet_timestamp: 2566666 + stream_id : 2 + packet_id : 29 + } + output_trace: { + packet_timestamp: 2566666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2106235 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1600000 + event_type : PROCESS + start_time : 2106315 + input_trace: { + finish_time : 2106315 + packet_timestamp: 1600000 + stream_id : 3 + packet_id : 49 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2115889 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2600000 + event_type : PROCESS + finish_time : 2131790 + output_trace: { + packet_timestamp: 2600000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2131927 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2600000 + event_type : PROCESS + start_time : 2132008 + finish_time : 2142133 + input_trace: { + start_time : 2131790 + finish_time : 2132008 + packet_timestamp: 2600000 + stream_id : 2 + packet_id : 37 + } + output_trace: { + packet_timestamp: 2600000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2143803 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2159636 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1633333 + event_type : PROCESS + start_time : 2159704 + input_trace: { + finish_time : 2159704 + packet_timestamp: 1633333 + stream_id : 3 + packet_id : 42 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2633333 + event_type : PROCESS + finish_time : 2159838 + output_trace: { + packet_timestamp: 2633333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2159957 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2633333 + event_type : PROCESS + start_time : 2160024 + finish_time : 2193934 + input_trace: { + start_time : 2159838 + finish_time : 2160024 + packet_timestamp: 2633333 + stream_id : 2 + packet_id : 27 + } + output_trace: { + packet_timestamp: 2633333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2186021 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1666666 + event_type : PROCESS + start_time : 2186107 + input_trace: { + finish_time : 2186107 + packet_timestamp: 1666666 + stream_id : 3 + packet_id : 23 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2196126 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2223554 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1700000 + event_type : PROCESS + start_time : 2223621 + input_trace: { + finish_time : 2223621 + packet_timestamp: 1700000 + stream_id : 3 + packet_id : 41 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2666666 + event_type : PROCESS + finish_time : 2226170 + output_trace: { + packet_timestamp: 2666666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2226312 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2666666 + event_type : PROCESS + start_time : 2226387 + finish_time : 2252692 + input_trace: { + start_time : 2226170 + finish_time : 2226387 + packet_timestamp: 2666666 + stream_id : 2 + packet_id : 42 + } + output_trace: { + packet_timestamp: 2666666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2253938 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1733333 + event_type : PROCESS + start_time : 2254021 + input_trace: { + finish_time : 2254021 + packet_timestamp: 1733333 + stream_id : 3 + packet_id : 50 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2254718 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2285495 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1766666 + event_type : PROCESS + start_time : 2285563 + input_trace: { + finish_time : 2285563 + packet_timestamp: 1766666 + stream_id : 3 + packet_id : 43 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2700000 + event_type : PROCESS + finish_time : 2287724 + output_trace: { + packet_timestamp: 2700000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2287850 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2700000 + event_type : PROCESS + start_time : 2287921 + finish_time : 2314418 + input_trace: { + start_time : 2287724 + finish_time : 2287921 + packet_timestamp: 2700000 + stream_id : 2 + packet_id : 7 + } + output_trace: { + packet_timestamp: 2700000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2312436 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1800000 + event_type : PROCESS + start_time : 2312506 + input_trace: { + finish_time : 2312506 + packet_timestamp: 1800000 + stream_id : 3 + packet_id : 25 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2316540 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2349724 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1833333 + event_type : PROCESS + start_time : 2349846 + input_trace: { + finish_time : 2349846 + packet_timestamp: 1833333 + stream_id : 3 + packet_id : 51 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2733333 + event_type : PROCESS + finish_time : 2352259 + output_trace: { + packet_timestamp: 2733333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2352388 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2733333 + event_type : PROCESS + start_time : 2352490 + finish_time : 2378919 + input_trace: { + start_time : 2352259 + finish_time : 2352490 + packet_timestamp: 2733333 + stream_id : 2 + packet_id : 43 + } + output_trace: { + packet_timestamp: 2733333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2381057 + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2386127 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1866666 + event_type : PROCESS + start_time : 2386192 + input_trace: { + finish_time : 2386192 + packet_timestamp: 1866666 + stream_id : 3 + packet_id : 35 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2766666 + event_type : PROCESS + finish_time : 2422595 + output_trace: { + packet_timestamp: 2766666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2422744 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2766666 + event_type : PROCESS + start_time : 2422842 + finish_time : 2442671 + input_trace: { + start_time : 2422595 + finish_time : 2422842 + packet_timestamp: 2766666 + stream_id : 2 + packet_id : 26 + } + output_trace: { + packet_timestamp: 2766666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2431218 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1900000 + event_type : PROCESS + start_time : 2431293 + input_trace: { + finish_time : 2431293 + packet_timestamp: 1900000 + stream_id : 3 + packet_id : 17 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2445016 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2472780 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1933333 + event_type : PROCESS + start_time : 2472853 + input_trace: { + finish_time : 2472853 + packet_timestamp: 1933333 + stream_id : 3 + packet_id : 33 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2800000 + event_type : PROCESS + finish_time : 2476597 + output_trace: { + packet_timestamp: 2800000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2476732 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2800000 + event_type : PROCESS + start_time : 2476809 + finish_time : 2515444 + input_trace: { + start_time : 2476597 + finish_time : 2476809 + packet_timestamp: 2800000 + stream_id : 2 + packet_id : 35 + } + output_trace: { + packet_timestamp: 2800000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2507540 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 1966666 + event_type : PROCESS + start_time : 2507616 + input_trace: { + finish_time : 2507616 + packet_timestamp: 1966666 + stream_id : 3 + packet_id : 36 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2517869 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2833333 + event_type : PROCESS + finish_time : 2535023 + output_trace: { + packet_timestamp: 2833333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2535166 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2833333 + event_type : PROCESS + start_time : 2535248 + finish_time : 2549025 + input_trace: { + start_time : 2535023 + finish_time : 2535248 + packet_timestamp: 2833333 + stream_id : 2 + packet_id : 33 + } + output_trace: { + packet_timestamp: 2833333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2551039 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2866666 + event_type : PROCESS + finish_time : 2568128 + output_trace: { + packet_timestamp: 2866666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2568267 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2866666 + event_type : PROCESS + start_time : 2568350 + finish_time : 2582594 + input_trace: { + start_time : 2568128 + finish_time : 2568350 + packet_timestamp: 2866666 + stream_id : 2 + packet_id : 23 + } + output_trace: { + packet_timestamp: 2866666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2584548 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2605571 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2000000 + event_type : PROCESS + start_time : 2605646 + input_trace: { + finish_time : 2605646 + packet_timestamp: 2000000 + stream_id : 3 + packet_id : 28 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2900000 + event_type : PROCESS + finish_time : 2608543 + output_trace: { + packet_timestamp: 2900000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2608668 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2900000 + event_type : PROCESS + start_time : 2608741 + finish_time : 2638120 + input_trace: { + start_time : 2608543 + finish_time : 2608741 + packet_timestamp: 2900000 + stream_id : 2 + packet_id : 52 + } + output_trace: { + packet_timestamp: 2900000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2640254 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2933333 + event_type : PROCESS + finish_time : 2658656 + output_trace: { + packet_timestamp: 2933333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2658802 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2933333 + event_type : PROCESS + start_time : 2658885 + finish_time : 2670216 + input_trace: { + start_time : 2658656 + finish_time : 2658885 + packet_timestamp: 2933333 + stream_id : 2 + packet_id : 25 + } + output_trace: { + packet_timestamp: 2933333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2672240 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 2966666 + event_type : PROCESS + finish_time : 2687942 + output_trace: { + packet_timestamp: 2966666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2688080 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 2966666 + event_type : PROCESS + start_time : 2688158 + finish_time : 2701742 + input_trace: { + start_time : 2687942 + finish_time : 2688158 + packet_timestamp: 2966666 + stream_id : 2 + packet_id : 33 + } + output_trace: { + packet_timestamp: 2966666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2703859 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3000000 + event_type : PROCESS + finish_time : 2721044 + output_trace: { + packet_timestamp: 3000000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2721199 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3000000 + event_type : PROCESS + start_time : 2721281 + finish_time : 2735141 + input_trace: { + start_time : 2721044 + finish_time : 2721281 + packet_timestamp: 3000000 + stream_id : 2 + packet_id : 36 + } + output_trace: { + packet_timestamp: 3000000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2737296 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3033333 + event_type : PROCESS + finish_time : 2754207 + output_trace: { + packet_timestamp: 3033333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2754347 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3033333 + event_type : PROCESS + start_time : 2754442 + finish_time : 2769407 + input_trace: { + start_time : 2754207 + finish_time : 2754442 + packet_timestamp: 3033333 + stream_id : 2 + packet_id : 52 + } + output_trace: { + packet_timestamp: 3033333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2771471 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3066666 + event_type : PROCESS + finish_time : 2786891 + output_trace: { + packet_timestamp: 3066666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2787031 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3066666 + event_type : PROCESS + start_time : 2787126 + finish_time : 2807316 + input_trace: { + start_time : 2786891 + finish_time : 2787126 + packet_timestamp: 3066666 + stream_id : 2 + packet_id : 25 + } + output_trace: { + packet_timestamp: 3066666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2809511 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3100000 + event_type : PROCESS + finish_time : 2830904 + output_trace: { + packet_timestamp: 3100000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2831041 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3100000 + event_type : PROCESS + start_time : 2831121 + finish_time : 2847428 + input_trace: { + start_time : 2830904 + finish_time : 2831121 + packet_timestamp: 3100000 + stream_id : 2 + packet_id : 33 + } + output_trace: { + packet_timestamp: 3100000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2849562 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3133333 + event_type : PROCESS + finish_time : 2869937 + output_trace: { + packet_timestamp: 3133333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2870071 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3133333 + event_type : PROCESS + start_time : 2870165 + input_trace: { + start_time : 2869937 + finish_time : 2870165 + packet_timestamp: 3133333 + stream_id : 2 + packet_id : 29 + } + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 1 + input_timestamp: 3133333 + event_type : PROCESS + finish_time : 2889922 + output_trace: { + packet_timestamp: 3133333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2892442 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2898412 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2033333 + event_type : PROCESS + start_time : 2898478 + input_trace: { + finish_time : 2898478 + packet_timestamp: 2033333 + stream_id : 3 + packet_id : 45 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3166666 + event_type : PROCESS + finish_time : 2925668 + output_trace: { + packet_timestamp: 3166666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2925877 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3166666 + event_type : PROCESS + start_time : 2925970 + finish_time : 2937370 + input_trace: { + start_time : 2925668 + finish_time : 2925970 + packet_timestamp: 3166666 + stream_id : 2 + packet_id : 52 + } + output_trace: { + packet_timestamp: 3166666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2939416 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3200000 + event_type : PROCESS + finish_time : 2957531 + output_trace: { + packet_timestamp: 3200000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2957670 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3200000 + event_type : PROCESS + start_time : 2957751 + finish_time : 2974512 + input_trace: { + start_time : 2957531 + finish_time : 2957751 + packet_timestamp: 3200000 + stream_id : 2 + packet_id : 25 + } + output_trace: { + packet_timestamp: 3200000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 2976593 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 2984237 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2066666 + event_type : PROCESS + start_time : 2984302 + input_trace: { + finish_time : 2984302 + packet_timestamp: 2066666 + stream_id : 3 + packet_id : 53 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3233333 + event_type : PROCESS + finish_time : 2992497 + output_trace: { + packet_timestamp: 3233333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 2992638 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3233333 + event_type : PROCESS + start_time : 2992719 + finish_time : 3004141 + input_trace: { + start_time : 2992497 + finish_time : 2992719 + packet_timestamp: 3233333 + stream_id : 2 + packet_id : 33 + } + output_trace: { + packet_timestamp: 3233333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3006193 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3266666 + event_type : PROCESS + finish_time : 3021778 + output_trace: { + packet_timestamp: 3266666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3021968 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3266666 + event_type : PROCESS + start_time : 3022115 + finish_time : 3034746 + input_trace: { + start_time : 3021778 + finish_time : 3022115 + packet_timestamp: 3266666 + stream_id : 2 + packet_id : 54 + } + output_trace: { + packet_timestamp: 3266666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3036054 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2100000 + event_type : PROCESS + start_time : 3036136 + input_trace: { + finish_time : 3036136 + packet_timestamp: 2100000 + stream_id : 3 + packet_id : 44 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3037005 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3300000 + event_type : PROCESS + finish_time : 3071477 + output_trace: { + packet_timestamp: 3300000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3071617 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3300000 + event_type : PROCESS + start_time : 3071699 + finish_time : 3085281 + input_trace: { + start_time : 3071477 + finish_time : 3071699 + packet_timestamp: 3300000 + stream_id : 2 + packet_id : 55 + } + output_trace: { + packet_timestamp: 3300000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3087250 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3333333 + event_type : PROCESS + finish_time : 3111343 + output_trace: { + packet_timestamp: 3333333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3111493 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3333333 + event_type : PROCESS + start_time : 3111576 + finish_time : 3128811 + input_trace: { + start_time : 3111343 + finish_time : 3111576 + packet_timestamp: 3333333 + stream_id : 2 + packet_id : 45 + } + output_trace: { + packet_timestamp: 3333333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3130787 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3366666 + event_type : PROCESS + finish_time : 3154874 + output_trace: { + packet_timestamp: 3366666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3155053 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3366666 + event_type : PROCESS + start_time : 3155136 + finish_time : 3169986 + input_trace: { + start_time : 3154874 + finish_time : 3155136 + packet_timestamp: 3366666 + stream_id : 2 + packet_id : 56 + } + output_trace: { + packet_timestamp: 3366666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3172037 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3400000 + event_type : PROCESS + finish_time : 3197477 + output_trace: { + packet_timestamp: 3400000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3197621 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3400000 + event_type : PROCESS + start_time : 3197702 + finish_time : 3226024 + input_trace: { + start_time : 3197477 + finish_time : 3197702 + packet_timestamp: 3400000 + stream_id : 2 + packet_id : 54 + } + output_trace: { + packet_timestamp: 3400000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3208240 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2133333 + event_type : PROCESS + start_time : 3208310 + input_trace: { + finish_time : 3208310 + packet_timestamp: 2133333 + stream_id : 3 + packet_id : 57 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3228095 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3433333 + event_type : PROCESS + finish_time : 3252346 + output_trace: { + packet_timestamp: 3433333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3252514 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3433333 + event_type : PROCESS + start_time : 3252605 + finish_time : 3265535 + input_trace: { + start_time : 3252346 + finish_time : 3252605 + packet_timestamp: 3433333 + stream_id : 2 + packet_id : 44 + } + output_trace: { + packet_timestamp: 3433333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3255272 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2166666 + event_type : PROCESS + start_time : 3255332 + input_trace: { + finish_time : 3255332 + packet_timestamp: 2166666 + stream_id : 3 + packet_id : 58 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3267531 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3466666 + event_type : PROCESS + finish_time : 3285945 + output_trace: { + packet_timestamp: 3466666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3286084 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3466666 + event_type : PROCESS + start_time : 3286163 + finish_time : 3309060 + input_trace: { + start_time : 3285945 + finish_time : 3286163 + packet_timestamp: 3466666 + stream_id : 2 + packet_id : 57 + } + output_trace: { + packet_timestamp: 3466666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3311006 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3500000 + event_type : PROCESS + finish_time : 3326739 + output_trace: { + packet_timestamp: 3500000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3326877 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3500000 + event_type : PROCESS + start_time : 3326956 + finish_time : 3339765 + input_trace: { + start_time : 3326739 + finish_time : 3326956 + packet_timestamp: 3500000 + stream_id : 2 + packet_id : 59 + } + output_trace: { + packet_timestamp: 3500000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3341693 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3345623 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2200000 + event_type : PROCESS + start_time : 3345688 + input_trace: { + finish_time : 3345688 + packet_timestamp: 2200000 + stream_id : 3 + packet_id : 60 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3364754 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2233333 + event_type : PROCESS + start_time : 3364829 + input_trace: { + finish_time : 3364829 + packet_timestamp: 2233333 + stream_id : 3 + packet_id : 39 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3533333 + event_type : PROCESS + finish_time : 3369484 + output_trace: { + packet_timestamp: 3533333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3369622 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3533333 + event_type : PROCESS + start_time : 3369698 + finish_time : 3384559 + input_trace: { + start_time : 3369484 + finish_time : 3369698 + packet_timestamp: 3533333 + stream_id : 2 + packet_id : 36 + } + output_trace: { + packet_timestamp: 3533333 + stream_id : 3 + } + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3386474 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3566666 + event_type : PROCESS + finish_time : 3421420 + output_trace: { + packet_timestamp: 3566666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3421624 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3566666 + event_type : PROCESS + start_time : 3421713 + finish_time : 3435722 + input_trace: { + start_time : 3421420 + finish_time : 3421713 + packet_timestamp: 3566666 + stream_id : 2 + packet_id : 44 + } + output_trace: { + packet_timestamp: 3566666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3437734 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3600000 + event_type : PROCESS + finish_time : 3458335 + output_trace: { + packet_timestamp: 3600000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3458472 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3600000 + event_type : PROCESS + start_time : 3458554 + finish_time : 3473537 + input_trace: { + start_time : 3458335 + finish_time : 3458554 + packet_timestamp: 3600000 + stream_id : 2 + packet_id : 57 + } + output_trace: { + packet_timestamp: 3600000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3461014 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2266666 + event_type : PROCESS + start_time : 3461073 + input_trace: { + finish_time : 3461073 + packet_timestamp: 2266666 + stream_id : 3 + packet_id : 61 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3475534 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3633333 + event_type : PROCESS + finish_time : 3494372 + output_trace: { + packet_timestamp: 3633333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3494511 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3633333 + event_type : PROCESS + start_time : 3494591 + finish_time : 3508944 + input_trace: { + start_time : 3494372 + finish_time : 3494591 + packet_timestamp: 3633333 + stream_id : 2 + packet_id : 39 + } + output_trace: { + packet_timestamp: 3633333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3510868 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3515621 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2300000 + event_type : PROCESS + start_time : 3515680 + input_trace: { + finish_time : 3515680 + packet_timestamp: 2300000 + stream_id : 3 + packet_id : 62 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3666666 + event_type : PROCESS + finish_time : 3541016 + output_trace: { + packet_timestamp: 3666666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3541155 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3666666 + event_type : PROCESS + start_time : 3541236 + finish_time : 3558176 + input_trace: { + start_time : 3541016 + finish_time : 3541236 + packet_timestamp: 3666666 + stream_id : 2 + packet_id : 63 + } + output_trace: { + packet_timestamp: 3666666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3558564 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2333333 + event_type : PROCESS + start_time : 3558632 + input_trace: { + finish_time : 3558632 + packet_timestamp: 2333333 + stream_id : 3 + packet_id : 10 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3560289 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3700000 + event_type : PROCESS + finish_time : 3578421 + output_trace: { + packet_timestamp: 3700000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3578568 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3700000 + event_type : PROCESS + start_time : 3578659 + finish_time : 3591177 + input_trace: { + start_time : 3578421 + finish_time : 3578659 + packet_timestamp: 3700000 + stream_id : 2 + packet_id : 32 + } + output_trace: { + packet_timestamp: 3700000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3581965 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2366666 + event_type : PROCESS + start_time : 3582031 + input_trace: { + finish_time : 3582031 + packet_timestamp: 2366666 + stream_id : 3 + packet_id : 46 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3593111 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3733333 + event_type : PROCESS + finish_time : 3612869 + output_trace: { + packet_timestamp: 3733333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3613012 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3733333 + event_type : PROCESS + start_time : 3613093 + finish_time : 3625734 + input_trace: { + start_time : 3612869 + finish_time : 3613093 + packet_timestamp: 3733333 + stream_id : 2 + packet_id : 10 + } + output_trace: { + packet_timestamp: 3733333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3627669 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3766666 + event_type : PROCESS + finish_time : 3650282 + output_trace: { + packet_timestamp: 3766666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3650422 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3766666 + event_type : PROCESS + start_time : 3650500 + finish_time : 3663018 + input_trace: { + start_time : 3650282 + finish_time : 3650500 + packet_timestamp: 3766666 + stream_id : 2 + packet_id : 50 + } + output_trace: { + packet_timestamp: 3766666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3664943 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3682981 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2400000 + event_type : PROCESS + start_time : 3683049 + input_trace: { + finish_time : 3683049 + packet_timestamp: 2400000 + stream_id : 3 + packet_id : 9 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3800000 + event_type : PROCESS + finish_time : 3693501 + output_trace: { + packet_timestamp: 3800000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3693633 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3800000 + event_type : PROCESS + start_time : 3693713 + finish_time : 3714232 + input_trace: { + start_time : 3693501 + finish_time : 3693713 + packet_timestamp: 3800000 + stream_id : 2 + packet_id : 36 + } + output_trace: { + packet_timestamp: 3800000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3716330 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3833333 + event_type : PROCESS + finish_time : 3731155 + output_trace: { + packet_timestamp: 3833333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3731293 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3833333 + event_type : PROCESS + start_time : 3731391 + finish_time : 3746269 + input_trace: { + start_time : 3731155 + finish_time : 3731391 + packet_timestamp: 3833333 + stream_id : 2 + packet_id : 57 + } + output_trace: { + packet_timestamp: 3833333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3748133 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3758555 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2433333 + event_type : PROCESS + start_time : 3758624 + input_trace: { + finish_time : 3758624 + packet_timestamp: 2433333 + stream_id : 3 + packet_id : 14 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3866666 + event_type : PROCESS + finish_time : 3763462 + output_trace: { + packet_timestamp: 3866666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3763568 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3866666 + event_type : PROCESS + start_time : 3763625 + finish_time : 3775873 + input_trace: { + start_time : 3763462 + finish_time : 3763625 + packet_timestamp: 3866666 + stream_id : 2 + packet_id : 64 + } + output_trace: { + packet_timestamp: 3866666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3777809 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 3794099 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2466666 + event_type : PROCESS + start_time : 3794171 + input_trace: { + finish_time : 3794171 + packet_timestamp: 2466666 + stream_id : 3 + packet_id : 40 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3900000 + event_type : PROCESS + finish_time : 3811931 + output_trace: { + packet_timestamp: 3900000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3812081 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3900000 + event_type : PROCESS + start_time : 3812190 + finish_time : 3826040 + input_trace: { + start_time : 3811931 + finish_time : 3812190 + packet_timestamp: 3900000 + stream_id : 2 + packet_id : 17 + } + output_trace: { + packet_timestamp: 3900000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3827970 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3933333 + event_type : PROCESS + finish_time : 3848504 + output_trace: { + packet_timestamp: 3933333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3848648 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3933333 + event_type : PROCESS + start_time : 3848730 + finish_time : 3863899 + input_trace: { + start_time : 3848504 + finish_time : 3848730 + packet_timestamp: 3933333 + stream_id : 2 + packet_id : 36 + } + output_trace: { + packet_timestamp: 3933333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3865980 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 3966666 + event_type : PROCESS + finish_time : 3881105 + output_trace: { + packet_timestamp: 3966666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3881258 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 3966666 + event_type : PROCESS + start_time : 3881340 + input_trace: { + start_time : 3881105 + finish_time : 3881340 + packet_timestamp: 3966666 + stream_id : 2 + packet_id : 65 + } + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 1 + input_timestamp: 3966666 + event_type : PROCESS + finish_time : 3896440 + output_trace: { + packet_timestamp: 3966666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3898694 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4000000 + event_type : PROCESS + finish_time : 3915256 + output_trace: { + packet_timestamp: 4000000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3915423 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4000000 + event_type : PROCESS + start_time : 3915512 + finish_time : 3931005 + input_trace: { + start_time : 3915256 + finish_time : 3915512 + packet_timestamp: 4000000 + stream_id : 2 + packet_id : 14 + } + output_trace: { + packet_timestamp: 4000000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3932975 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4033333 + event_type : PROCESS + finish_time : 3949632 + output_trace: { + packet_timestamp: 4033333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 3949777 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4033333 + event_type : PROCESS + start_time : 3949860 + finish_time : 3965806 + input_trace: { + start_time : 3949632 + finish_time : 3949860 + packet_timestamp: 4033333 + stream_id : 2 + packet_id : 9 + } + output_trace: { + packet_timestamp: 4033333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 3967867 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4066666 + event_type : PROCESS + finish_time : 4001767 + output_trace: { + packet_timestamp: 4066666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4001927 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4066666 + event_type : PROCESS + start_time : 4002016 + finish_time : 4022822 + input_trace: { + start_time : 4001767 + finish_time : 4002016 + packet_timestamp: 4066666 + stream_id : 2 + packet_id : 36 + } + output_trace: { + packet_timestamp: 4066666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4024844 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4100000 + event_type : PROCESS + finish_time : 4045648 + output_trace: { + packet_timestamp: 4100000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4045788 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4100000 + event_type : PROCESS + start_time : 4045868 + finish_time : 4067076 + input_trace: { + start_time : 4045648 + finish_time : 4045868 + packet_timestamp: 4100000 + stream_id : 2 + packet_id : 65 + } + output_trace: { + packet_timestamp: 4100000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4069297 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4069458 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2500000 + event_type : PROCESS + start_time : 4069505 + input_trace: { + finish_time : 4069505 + packet_timestamp: 2500000 + stream_id : 3 + packet_id : 66 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4133333 + event_type : PROCESS + finish_time : 4094959 + output_trace: { + packet_timestamp: 4133333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4095088 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4133333 + event_type : PROCESS + start_time : 4095171 + finish_time : 4107210 + input_trace: { + start_time : 4094959 + finish_time : 4095171 + packet_timestamp: 4133333 + stream_id : 2 + packet_id : 36 + } + output_trace: { + packet_timestamp: 4133333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4109066 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4166666 + event_type : PROCESS + finish_time : 4132068 + output_trace: { + packet_timestamp: 4166666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4132216 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4166666 + event_type : PROCESS + start_time : 4132294 + finish_time : 4156751 + input_trace: { + start_time : 4132068 + finish_time : 4132294 + packet_timestamp: 4166666 + stream_id : 2 + packet_id : 9 + } + output_trace: { + packet_timestamp: 4166666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4158934 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4183722 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2533333 + event_type : PROCESS + start_time : 4183791 + input_trace: { + finish_time : 4183791 + packet_timestamp: 2533333 + stream_id : 3 + packet_id : 24 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4200000 + event_type : PROCESS + finish_time : 4186996 + output_trace: { + packet_timestamp: 4200000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4187121 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4200000 + event_type : PROCESS + start_time : 4187201 + finish_time : 4213245 + input_trace: { + start_time : 4186996 + finish_time : 4187201 + packet_timestamp: 4200000 + stream_id : 2 + packet_id : 67 + } + output_trace: { + packet_timestamp: 4200000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4215300 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4233333 + event_type : PROCESS + finish_time : 4233182 + output_trace: { + packet_timestamp: 4233333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4233325 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4233333 + event_type : PROCESS + start_time : 4233409 + finish_time : 4248502 + input_trace: { + start_time : 4233182 + finish_time : 4233409 + packet_timestamp: 4233333 + stream_id : 2 + packet_id : 14 + } + output_trace: { + packet_timestamp: 4233333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4250509 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4256911 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2566666 + event_type : PROCESS + start_time : 4256974 + input_trace: { + finish_time : 4256974 + packet_timestamp: 2566666 + stream_id : 3 + packet_id : 5 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4266666 + event_type : PROCESS + finish_time : 4283147 + output_trace: { + packet_timestamp: 4266666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4283275 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4266666 + event_type : PROCESS + start_time : 4283358 + finish_time : 4294250 + input_trace: { + start_time : 4283147 + finish_time : 4283358 + packet_timestamp: 4266666 + stream_id : 2 + packet_id : 35 + } + output_trace: { + packet_timestamp: 4266666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4283657 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2600000 + event_type : PROCESS + start_time : 4283698 + input_trace: { + finish_time : 4283698 + packet_timestamp: 2600000 + stream_id : 3 + packet_id : 68 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4296257 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4300000 + event_type : PROCESS + finish_time : 4314233 + output_trace: { + packet_timestamp: 4300000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4314374 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4300000 + event_type : PROCESS + start_time : 4314457 + finish_time : 4330735 + input_trace: { + start_time : 4314233 + finish_time : 4314457 + packet_timestamp: 4300000 + stream_id : 2 + packet_id : 5 + } + output_trace: { + packet_timestamp: 4300000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4332814 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4333333 + event_type : PROCESS + finish_time : 4347424 + output_trace: { + packet_timestamp: 4333333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4347562 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4333333 + event_type : PROCESS + start_time : 4347642 + finish_time : 4363756 + input_trace: { + start_time : 4347424 + finish_time : 4347642 + packet_timestamp: 4333333 + stream_id : 2 + packet_id : 14 + } + output_trace: { + packet_timestamp: 4333333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4365759 + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 0 + input_timestamp: 4366666 + event_type : PROCESS + finish_time : 4407314 + output_trace: { + packet_timestamp: 4366666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4407458 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4366666 + event_type : PROCESS + start_time : 4407541 + finish_time : 4432235 + input_trace: { + start_time : 4407314 + finish_time : 4407541 + packet_timestamp: 4366666 + stream_id : 2 + packet_id : 60 + } + output_trace: { + packet_timestamp: 4366666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4434655 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4400000 + event_type : PROCESS + finish_time : 4454937 + output_trace: { + packet_timestamp: 4400000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4455086 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4400000 + event_type : PROCESS + start_time : 4455197 + finish_time : 4467122 + input_trace: { + start_time : 4454937 + finish_time : 4455197 + packet_timestamp: 4400000 + stream_id : 2 + packet_id : 35 + } + output_trace: { + packet_timestamp: 4400000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4458236 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2633333 + event_type : PROCESS + start_time : 4458305 + input_trace: { + finish_time : 4458305 + packet_timestamp: 2633333 + stream_id : 3 + packet_id : 38 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4469096 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4433333 + event_type : PROCESS + finish_time : 4487235 + output_trace: { + packet_timestamp: 4433333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4487378 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4433333 + event_type : PROCESS + start_time : 4487469 + finish_time : 4503685 + input_trace: { + start_time : 4487235 + finish_time : 4487469 + packet_timestamp: 4433333 + stream_id : 2 + packet_id : 68 + } + output_trace: { + packet_timestamp: 4433333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4505669 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4466666 + event_type : PROCESS + finish_time : 4522654 + output_trace: { + packet_timestamp: 4466666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4522795 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4466666 + event_type : PROCESS + start_time : 4522877 + finish_time : 4537395 + input_trace: { + start_time : 4522654 + finish_time : 4522877 + packet_timestamp: 4466666 + stream_id : 2 + packet_id : 9 + } + output_trace: { + packet_timestamp: 4466666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4525966 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2666666 + event_type : PROCESS + start_time : 4526023 + input_trace: { + finish_time : 4526023 + packet_timestamp: 2666666 + stream_id : 3 + packet_id : 20 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4539305 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4500000 + event_type : PROCESS + finish_time : 4554352 + output_trace: { + packet_timestamp: 4500000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4554496 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4500000 + event_type : PROCESS + start_time : 4554578 + finish_time : 4568006 + input_trace: { + start_time : 4554352 + finish_time : 4554578 + packet_timestamp: 4500000 + stream_id : 2 + packet_id : 38 + } + output_trace: { + packet_timestamp: 4500000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4562810 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2700000 + event_type : PROCESS + start_time : 4562877 + input_trace: { + finish_time : 4562877 + packet_timestamp: 2700000 + stream_id : 3 + packet_id : 69 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4570082 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4533333 + event_type : PROCESS + finish_time : 4590379 + output_trace: { + packet_timestamp: 4533333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4590515 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4533333 + event_type : PROCESS + start_time : 4590596 + finish_time : 4604996 + input_trace: { + start_time : 4590379 + finish_time : 4590596 + packet_timestamp: 4533333 + stream_id : 2 + packet_id : 20 + } + output_trace: { + packet_timestamp: 4533333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4596837 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2733333 + event_type : PROCESS + start_time : 4596904 + input_trace: { + finish_time : 4596904 + packet_timestamp: 2733333 + stream_id : 3 + packet_id : 70 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4606876 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4566666 + event_type : PROCESS + finish_time : 4628785 + output_trace: { + packet_timestamp: 4566666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4628934 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4566666 + event_type : PROCESS + start_time : 4629015 + finish_time : 4647862 + input_trace: { + start_time : 4628785 + finish_time : 4629015 + packet_timestamp: 4566666 + stream_id : 2 + packet_id : 69 + } + output_trace: { + packet_timestamp: 4566666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4649945 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4600000 + event_type : PROCESS + finish_time : 4670250 + output_trace: { + packet_timestamp: 4600000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4670388 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4600000 + event_type : PROCESS + start_time : 4670468 + finish_time : 4686289 + input_trace: { + start_time : 4670250 + finish_time : 4670468 + packet_timestamp: 4600000 + stream_id : 2 + packet_id : 36 + } + output_trace: { + packet_timestamp: 4600000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4688213 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4633333 + event_type : PROCESS + finish_time : 4706851 + output_trace: { + packet_timestamp: 4633333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4712482 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4633333 + event_type : PROCESS + start_time : 4712581 + finish_time : 4738056 + input_trace: { + start_time : 4706851 + finish_time : 4712581 + packet_timestamp: 4633333 + stream_id : 2 + packet_id : 66 + } + output_trace: { + packet_timestamp: 4633333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4740187 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4666666 + event_type : PROCESS + finish_time : 4758399 + output_trace: { + packet_timestamp: 4666666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4758537 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4666666 + event_type : PROCESS + start_time : 4758617 + finish_time : 4780073 + input_trace: { + start_time : 4758399 + finish_time : 4758617 + packet_timestamp: 4666666 + stream_id : 2 + packet_id : 20 + } + output_trace: { + packet_timestamp: 4666666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4770683 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2766666 + event_type : PROCESS + start_time : 4770758 + input_trace: { + finish_time : 4770758 + packet_timestamp: 2766666 + stream_id : 3 + packet_id : 37 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4782098 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4700000 + event_type : PROCESS + finish_time : 4799531 + output_trace: { + packet_timestamp: 4700000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4799654 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4700000 + event_type : PROCESS + start_time : 4799727 + finish_time : 4819017 + input_trace: { + start_time : 4799531 + finish_time : 4799727 + packet_timestamp: 4700000 + stream_id : 2 + packet_id : 70 + } + output_trace: { + packet_timestamp: 4700000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4810850 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2800000 + event_type : PROCESS + start_time : 4810914 + input_trace: { + finish_time : 4810914 + packet_timestamp: 2800000 + stream_id : 3 + packet_id : 42 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4821118 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4733333 + event_type : PROCESS + finish_time : 4837050 + output_trace: { + packet_timestamp: 4733333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4837196 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4733333 + event_type : PROCESS + start_time : 4837280 + finish_time : 4853478 + input_trace: { + start_time : 4837050 + finish_time : 4837280 + packet_timestamp: 4733333 + stream_id : 2 + packet_id : 37 + } + output_trace: { + packet_timestamp: 4733333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4852917 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2833333 + event_type : PROCESS + start_time : 4852985 + input_trace: { + finish_time : 4852985 + packet_timestamp: 2833333 + stream_id : 3 + packet_id : 41 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4855450 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4766666 + event_type : PROCESS + finish_time : 4877523 + output_trace: { + packet_timestamp: 4766666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4877667 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4766666 + event_type : PROCESS + start_time : 4877749 + input_trace: { + start_time : 4877523 + finish_time : 4877749 + packet_timestamp: 4766666 + stream_id : 2 + packet_id : 71 + } + thread_id : 0 + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 291814 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + process_input_latency: { + total: 1301834 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + process_output_latency: { + total: 1593648 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 886808 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 73327 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 1 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 226098 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_output_latency: { + total: 226098 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 143393 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_input_latency: { + total: 228864 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_output_latency: { + total: 372257 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 2766 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 458057 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 21 ] + } + process_input_latency: { + total: 2844657 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 21 ] + } + process_output_latency: { + total: 3302714 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 21 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 2346045 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 21 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 266057 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 20 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 20 ] + } + process_output_latency: { + total: 266057 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 20 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 186409 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 19 ] + } + process_input_latency: { + total: 254363 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 19 ] + } + process_output_latency: { + total: 440772 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 19 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 3770 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 19 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 363573 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 17 ] + } + process_input_latency: { + total: 1634920 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 17 ] + } + process_output_latency: { + total: 1998493 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 17 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 1209879 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 17 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 254170 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 18 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 18 ] + } + process_output_latency: { + total: 254170 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 18 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 205513 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 18 ] + } + process_input_latency: { + total: 260255 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 18 ] + } + process_output_latency: { + total: 465768 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 18 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 3515 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 18 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_output_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 253091 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 19 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 19 ] + } + process_output_latency: { + total: 253091 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 19 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 209741 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 20 ] + } + process_input_latency: { + total: 269724 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 20 ] + } + process_output_latency: { + total: 479465 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 20 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 3739 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 20 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 1077640 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_input_latency: { + total: 8280430 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_output_latency: { + total: 9358070 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 8005261 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 251509 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_output_latency: { + total: 251509 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 220144 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_input_latency: { + total: 254131 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_output_latency: { + total: 474275 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 2622 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 205872 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + process_input_latency: { + total: 3558818 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + process_output_latency: { + total: 3764690 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 3468393 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 255734 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_output_latency: { + total: 255734 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 206154 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_input_latency: { + total: 237866 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_output_latency: { + total: 444020 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 2427 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 742159 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + process_input_latency: { + total: 9441688 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + process_output_latency: { + total: 10183847 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 9272268 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 276142 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_output_latency: { + total: 276142 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 209514 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_input_latency: { + total: 299531 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_output_latency: { + total: 509045 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 3094 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 413213 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + process_input_latency: { + total: 11757558 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + process_output_latency: { + total: 12170771 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 11591328 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 292413 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_output_latency: { + total: 292413 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 174677 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_input_latency: { + total: 280050 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_output_latency: { + total: 454727 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 2675 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 480571 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + process_input_latency: { + total: 8225896 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + process_output_latency: { + total: 8706467 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 8098750 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 246881 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_output_latency: { + total: 246881 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 209522 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_input_latency: { + total: 264576 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_output_latency: { + total: 474098 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 2657 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 583531 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + process_input_latency: { + total: 18490755 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + process_output_latency: { + total: 19074286 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 18073192 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 265761 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_output_latency: { + total: 265761 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 212315 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_input_latency: { + total: 251898 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_output_latency: { + total: 464213 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 8149 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + } +} +config: { + node: { + name: "OpenCvVideoDecoderCalculator" + calculator : "OpenCvVideoDecoderCalculator" + output_stream: [ "VIDEO:input_video", "VIDEO_PRESTREAM:input_video_header" ] + input_side_packet : [ "INPUT_FILE_PATH:input_video_path" ] + } + node: { + name: "OpenCvWriteTextCalculator" + calculator : "OpenCvWriteTextCalculator" + input_stream : [ "input_video" ] + output_stream: [ "output_video" ] + } + node: { + name: "OpenCvVideoEncoderCalculator" + calculator : "OpenCvVideoEncoderCalculator" + input_stream : [ "VIDEO:output_video", "VIDEO_PRESTREAM:input_video_header" ] + input_side_packet : [ "OUTPUT_FILE_PATH:output_video_path" ] + } + executor: { + } + profiler_config: { + enable_profiler : true + enable_stream_latency : true + trace_log_count : 10 + trace_enabled : true + } +} diff --git a/mediapipe/framework/profiler/testdata/profile_opencv_1.pbtxt b/mediapipe/framework/profiler/testdata/profile_opencv_1.pbtxt new file mode 100644 index 000000000..e174ff88d --- /dev/null +++ b/mediapipe/framework/profiler/testdata/profile_opencv_1.pbtxt @@ -0,0 +1,4957 @@ +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + calculator_name : [ "OpenCvVideoDecoderCalculator", "OpenCvWriteTextCalculator", "OpenCvVideoEncoderCalculator" ] + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 1 + input_timestamp: 4766666 + event_type : PROCESS + finish_time : 4889762 + output_trace: { + packet_timestamp: 4766666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4890168 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2866666 + event_type : PROCESS + start_time : 4890249 + input_trace: { + finish_time : 4890249 + packet_timestamp: 2866666 + stream_id : 3 + packet_id : 49 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4896141 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4800000 + event_type : PROCESS + finish_time : 4912634 + output_trace: { + packet_timestamp: 4800000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4912786 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4800000 + event_type : PROCESS + start_time : 4912870 + finish_time : 4929875 + input_trace: { + start_time : 4912634 + finish_time : 4912870 + packet_timestamp: 4800000 + stream_id : 2 + packet_id : 42 + } + output_trace: { + packet_timestamp: 4800000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4931841 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4936015 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2900000 + event_type : PROCESS + start_time : 4936076 + input_trace: { + finish_time : 4936076 + packet_timestamp: 2900000 + stream_id : 3 + packet_id : 27 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4833333 + event_type : PROCESS + finish_time : 4963336 + output_trace: { + packet_timestamp: 4833333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 4963483 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4833333 + event_type : PROCESS + start_time : 4963567 + finish_time : 4979538 + input_trace: { + start_time : 4963336 + finish_time : 4963567 + packet_timestamp: 4833333 + stream_id : 2 + packet_id : 37 + } + output_trace: { + packet_timestamp: 4833333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 4978554 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2933333 + event_type : PROCESS + start_time : 4978624 + input_trace: { + finish_time : 4978624 + packet_timestamp: 2933333 + stream_id : 3 + packet_id : 51 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 4985234 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4866666 + event_type : PROCESS + finish_time : 5006393 + output_trace: { + packet_timestamp: 4866666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5006535 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4866666 + event_type : PROCESS + start_time : 5006617 + finish_time : 5017985 + input_trace: { + start_time : 5006393 + finish_time : 5006617 + packet_timestamp: 4866666 + stream_id : 2 + packet_id : 72 + } + output_trace: { + packet_timestamp: 4866666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5020038 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4900000 + event_type : PROCESS + finish_time : 5052772 + output_trace: { + packet_timestamp: 4900000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5052918 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4900000 + event_type : PROCESS + start_time : 5053000 + finish_time : 5067721 + input_trace: { + start_time : 5052772 + finish_time : 5053000 + packet_timestamp: 4900000 + stream_id : 2 + packet_id : 73 + } + output_trace: { + packet_timestamp: 4900000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5069673 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5074241 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 2966666 + event_type : PROCESS + start_time : 5074302 + input_trace: { + finish_time : 5074302 + packet_timestamp: 2966666 + stream_id : 3 + packet_id : 7 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4933333 + event_type : PROCESS + finish_time : 5098914 + output_trace: { + packet_timestamp: 4933333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5099055 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4933333 + event_type : PROCESS + start_time : 5099137 + finish_time : 5111333 + input_trace: { + start_time : 5098914 + finish_time : 5099137 + packet_timestamp: 4933333 + stream_id : 2 + packet_id : 71 + } + output_trace: { + packet_timestamp: 4933333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5108911 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3000000 + event_type : PROCESS + start_time : 5108979 + input_trace: { + finish_time : 5108979 + packet_timestamp: 3000000 + stream_id : 3 + packet_id : 31 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5113208 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 4966666 + event_type : PROCESS + finish_time : 5133740 + output_trace: { + packet_timestamp: 4966666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5133878 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 4966666 + event_type : PROCESS + start_time : 5133956 + finish_time : 5144368 + input_trace: { + start_time : 5133740 + finish_time : 5133956 + packet_timestamp: 4966666 + stream_id : 2 + packet_id : 7 + } + output_trace: { + packet_timestamp: 4966666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5146193 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5000000 + event_type : PROCESS + finish_time : 5166311 + output_trace: { + packet_timestamp: 5000000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5166455 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5000000 + event_type : PROCESS + start_time : 5166537 + finish_time : 5185735 + input_trace: { + start_time : 5166311 + finish_time : 5166537 + packet_timestamp: 5000000 + stream_id : 2 + packet_id : 72 + } + output_trace: { + packet_timestamp: 5000000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5187913 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5212552 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3033333 + event_type : PROCESS + start_time : 5212613 + input_trace: { + finish_time : 5212613 + packet_timestamp: 3033333 + stream_id : 3 + packet_id : 74 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5033333 + event_type : PROCESS + finish_time : 5214042 + output_trace: { + packet_timestamp: 5033333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5214152 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5033333 + event_type : PROCESS + start_time : 5214212 + finish_time : 5239389 + input_trace: { + start_time : 5214042 + finish_time : 5214212 + packet_timestamp: 5033333 + stream_id : 2 + packet_id : 75 + } + output_trace: { + packet_timestamp: 5033333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5241639 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5248708 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3066666 + event_type : PROCESS + start_time : 5248772 + input_trace: { + finish_time : 5248772 + packet_timestamp: 3066666 + stream_id : 3 + packet_id : 16 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5066666 + event_type : PROCESS + finish_time : 5272317 + output_trace: { + packet_timestamp: 5066666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5272485 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5066666 + event_type : PROCESS + start_time : 5272571 + finish_time : 5284182 + input_trace: { + start_time : 5272317 + finish_time : 5272571 + packet_timestamp: 5066666 + stream_id : 2 + packet_id : 31 + } + output_trace: { + packet_timestamp: 5066666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5286098 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5307473 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3100000 + event_type : PROCESS + start_time : 5307543 + input_trace: { + finish_time : 5307543 + packet_timestamp: 3100000 + stream_id : 3 + packet_id : 76 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5100000 + event_type : PROCESS + finish_time : 5335811 + output_trace: { + packet_timestamp: 5100000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5335951 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5100000 + event_type : PROCESS + start_time : 5336031 + finish_time : 5346448 + input_trace: { + start_time : 5335811 + finish_time : 5336031 + packet_timestamp: 5100000 + stream_id : 2 + packet_id : 77 + } + output_trace: { + packet_timestamp: 5100000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5348571 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5133333 + event_type : PROCESS + finish_time : 5363298 + output_trace: { + packet_timestamp: 5133333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5363450 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5133333 + event_type : PROCESS + start_time : 5363534 + finish_time : 5378071 + input_trace: { + start_time : 5363298 + finish_time : 5363534 + packet_timestamp: 5133333 + stream_id : 2 + packet_id : 71 + } + output_trace: { + packet_timestamp: 5133333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5380160 + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 0 + input_timestamp: 5166666 + event_type : PROCESS + finish_time : 5394510 + output_trace: { + packet_timestamp: 5166666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5394650 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5166666 + event_type : PROCESS + start_time : 5394729 + finish_time : 5410050 + input_trace: { + start_time : 5394510 + finish_time : 5394729 + packet_timestamp: 5166666 + stream_id : 2 + packet_id : 74 + } + output_trace: { + packet_timestamp: 5166666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5412086 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5200000 + event_type : PROCESS + finish_time : 5426829 + output_trace: { + packet_timestamp: 5200000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5427044 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5200000 + event_type : PROCESS + start_time : 5427127 + finish_time : 5445247 + input_trace: { + start_time : 5426829 + finish_time : 5427127 + packet_timestamp: 5200000 + stream_id : 2 + packet_id : 73 + } + output_trace: { + packet_timestamp: 5200000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5447359 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5233333 + event_type : PROCESS + finish_time : 5485839 + output_trace: { + packet_timestamp: 5233333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5485981 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5233333 + event_type : PROCESS + start_time : 5486066 + finish_time : 5501336 + input_trace: { + start_time : 5485839 + finish_time : 5486066 + packet_timestamp: 5233333 + stream_id : 2 + packet_id : 77 + } + output_trace: { + packet_timestamp: 5233333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5503476 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5266666 + event_type : PROCESS + finish_time : 5518362 + output_trace: { + packet_timestamp: 5266666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5518496 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5266666 + event_type : PROCESS + start_time : 5518571 + finish_time : 5537065 + input_trace: { + start_time : 5518362 + finish_time : 5518571 + packet_timestamp: 5266666 + stream_id : 2 + packet_id : 78 + } + output_trace: { + packet_timestamp: 5266666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5539166 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5300000 + event_type : PROCESS + finish_time : 5554015 + output_trace: { + packet_timestamp: 5300000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5554145 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5300000 + event_type : PROCESS + start_time : 5554239 + finish_time : 5564919 + input_trace: { + start_time : 5554015 + finish_time : 5554239 + packet_timestamp: 5300000 + stream_id : 2 + packet_id : 79 + } + output_trace: { + packet_timestamp: 5300000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5555693 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3133333 + event_type : PROCESS + start_time : 5555750 + input_trace: { + finish_time : 5555750 + packet_timestamp: 3133333 + stream_id : 3 + packet_id : 80 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5566669 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5333333 + event_type : PROCESS + finish_time : 5581987 + output_trace: { + packet_timestamp: 5333333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5582123 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5333333 + event_type : PROCESS + start_time : 5582215 + finish_time : 5616142 + input_trace: { + start_time : 5581987 + finish_time : 5582215 + packet_timestamp: 5333333 + stream_id : 2 + packet_id : 76 + } + output_trace: { + packet_timestamp: 5333333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5602589 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3166666 + event_type : PROCESS + start_time : 5602676 + input_trace: { + finish_time : 5602676 + packet_timestamp: 3166666 + stream_id : 3 + packet_id : 3 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5622903 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5636828 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3200000 + event_type : PROCESS + start_time : 5636894 + input_trace: { + finish_time : 5636894 + packet_timestamp: 3200000 + stream_id : 3 + packet_id : 81 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5366666 + event_type : PROCESS + finish_time : 5664825 + output_trace: { + packet_timestamp: 5366666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5664969 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5366666 + event_type : PROCESS + start_time : 5665051 + finish_time : 5681987 + input_trace: { + start_time : 5664825 + finish_time : 5665051 + packet_timestamp: 5366666 + stream_id : 2 + packet_id : 80 + } + output_trace: { + packet_timestamp: 5366666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5681990 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3233333 + event_type : PROCESS + start_time : 5682062 + input_trace: { + finish_time : 5682062 + packet_timestamp: 3233333 + stream_id : 3 + packet_id : 82 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5688334 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5400000 + event_type : PROCESS + finish_time : 5726182 + output_trace: { + packet_timestamp: 5400000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5726323 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5400000 + event_type : PROCESS + start_time : 5726399 + finish_time : 5740157 + input_trace: { + start_time : 5726182 + finish_time : 5726399 + packet_timestamp: 5400000 + stream_id : 2 + packet_id : 83 + } + output_trace: { + packet_timestamp: 5400000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5742352 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5768220 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3266666 + event_type : PROCESS + start_time : 5768290 + input_trace: { + finish_time : 5768290 + packet_timestamp: 3266666 + stream_id : 3 + packet_id : 23 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5433333 + event_type : PROCESS + finish_time : 5774440 + output_trace: { + packet_timestamp: 5433333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5774627 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5433333 + event_type : PROCESS + start_time : 5774718 + finish_time : 5786052 + input_trace: { + start_time : 5774440 + finish_time : 5774718 + packet_timestamp: 5433333 + stream_id : 2 + packet_id : 11 + } + output_trace: { + packet_timestamp: 5433333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5787894 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5466666 + event_type : PROCESS + finish_time : 5820752 + output_trace: { + packet_timestamp: 5466666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5820890 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5466666 + event_type : PROCESS + start_time : 5820969 + finish_time : 5835199 + input_trace: { + start_time : 5820752 + finish_time : 5820969 + packet_timestamp: 5466666 + stream_id : 2 + packet_id : 73 + } + output_trace: { + packet_timestamp: 5466666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5823568 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3300000 + event_type : PROCESS + start_time : 5823620 + input_trace: { + finish_time : 5823620 + packet_timestamp: 3300000 + stream_id : 3 + packet_id : 84 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5837227 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5500000 + event_type : PROCESS + finish_time : 5856805 + output_trace: { + packet_timestamp: 5500000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5856921 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5500000 + event_type : PROCESS + start_time : 5856982 + finish_time : 5870007 + input_trace: { + start_time : 5856805 + finish_time : 5856982 + packet_timestamp: 5500000 + stream_id : 2 + packet_id : 23 + } + output_trace: { + packet_timestamp: 5500000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5871828 + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5887553 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3333333 + event_type : PROCESS + start_time : 5887619 + input_trace: { + finish_time : 5887619 + packet_timestamp: 3333333 + stream_id : 3 + packet_id : 52 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5533333 + event_type : PROCESS + finish_time : 5893412 + output_trace: { + packet_timestamp: 5533333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5893539 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5533333 + event_type : PROCESS + start_time : 5893613 + finish_time : 5909485 + input_trace: { + start_time : 5893412 + finish_time : 5893613 + packet_timestamp: 5533333 + stream_id : 2 + packet_id : 80 + } + output_trace: { + packet_timestamp: 5533333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5911163 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5566666 + event_type : PROCESS + finish_time : 5930404 + output_trace: { + packet_timestamp: 5566666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5930506 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 5930514 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3366666 + event_type : PROCESS + start_time : 5930578 + input_trace: { + finish_time : 5930578 + packet_timestamp: 3366666 + stream_id : 3 + packet_id : 53 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5566666 + event_type : PROCESS + start_time : 5930579 + finish_time : 5959436 + input_trace: { + start_time : 5930404 + finish_time : 5930579 + packet_timestamp: 5566666 + stream_id : 2 + packet_id : 82 + } + output_trace: { + packet_timestamp: 5566666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 5961376 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5600000 + event_type : PROCESS + finish_time : 5990473 + output_trace: { + packet_timestamp: 5600000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 5990616 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5600000 + event_type : PROCESS + start_time : 5990784 + finish_time : 6009052 + input_trace: { + start_time : 5990473 + finish_time : 5990784 + packet_timestamp: 5600000 + stream_id : 2 + packet_id : 73 + } + output_trace: { + packet_timestamp: 5600000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6010933 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 6015855 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3400000 + event_type : PROCESS + start_time : 6015925 + input_trace: { + finish_time : 6015925 + packet_timestamp: 3400000 + stream_id : 3 + packet_id : 45 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5633333 + event_type : PROCESS + finish_time : 6033886 + output_trace: { + packet_timestamp: 5633333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6034040 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5633333 + event_type : PROCESS + start_time : 6034123 + finish_time : 6047293 + input_trace: { + start_time : 6033886 + finish_time : 6034123 + packet_timestamp: 5633333 + stream_id : 2 + packet_id : 82 + } + output_trace: { + packet_timestamp: 5633333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 6041035 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3433333 + event_type : PROCESS + start_time : 6041106 + input_trace: { + finish_time : 6041106 + packet_timestamp: 3433333 + stream_id : 3 + packet_id : 85 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6049188 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5666666 + event_type : PROCESS + finish_time : 6065092 + output_trace: { + packet_timestamp: 5666666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6065244 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5666666 + event_type : PROCESS + start_time : 6065336 + finish_time : 6078180 + input_trace: { + start_time : 6065092 + finish_time : 6065336 + packet_timestamp: 5666666 + stream_id : 2 + packet_id : 45 + } + output_trace: { + packet_timestamp: 5666666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6084264 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5700000 + event_type : PROCESS + finish_time : 6102338 + output_trace: { + packet_timestamp: 5700000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6102493 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5700000 + event_type : PROCESS + start_time : 6102574 + finish_time : 6125557 + input_trace: { + start_time : 6102338 + finish_time : 6102574 + packet_timestamp: 5700000 + stream_id : 2 + packet_id : 86 + } + output_trace: { + packet_timestamp: 5700000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6127697 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 6128794 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3466666 + event_type : PROCESS + start_time : 6128837 + input_trace: { + finish_time : 6128837 + packet_timestamp: 3466666 + stream_id : 3 + packet_id : 33 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5733333 + event_type : PROCESS + finish_time : 6164077 + output_trace: { + packet_timestamp: 5733333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6164213 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5733333 + event_type : PROCESS + start_time : 6164292 + finish_time : 6180437 + input_trace: { + start_time : 6164077 + finish_time : 6164292 + packet_timestamp: 5733333 + stream_id : 2 + packet_id : 45 + } + output_trace: { + packet_timestamp: 5733333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 6171763 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3500000 + event_type : PROCESS + start_time : 6171826 + input_trace: { + finish_time : 6171826 + packet_timestamp: 3500000 + stream_id : 3 + packet_id : 87 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6182358 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5766666 + event_type : PROCESS + finish_time : 6213674 + output_trace: { + packet_timestamp: 5766666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6213814 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5766666 + event_type : PROCESS + start_time : 6213894 + finish_time : 6230267 + input_trace: { + start_time : 6213674 + finish_time : 6213894 + packet_timestamp: 5766666 + stream_id : 2 + packet_id : 33 + } + output_trace: { + packet_timestamp: 5766666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6232413 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5800000 + event_type : PROCESS + finish_time : 6258550 + output_trace: { + packet_timestamp: 5800000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6258688 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5800000 + event_type : PROCESS + start_time : 6258767 + finish_time : 6279703 + input_trace: { + start_time : 6258550 + finish_time : 6258767 + packet_timestamp: 5800000 + stream_id : 2 + packet_id : 88 + } + output_trace: { + packet_timestamp: 5800000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6281581 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 6298404 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3533333 + event_type : PROCESS + start_time : 6298470 + input_trace: { + finish_time : 6298470 + packet_timestamp: 3533333 + stream_id : 3 + packet_id : 89 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5833333 + event_type : PROCESS + finish_time : 6301917 + output_trace: { + packet_timestamp: 5833333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6302051 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5833333 + event_type : PROCESS + start_time : 6302123 + finish_time : 6314804 + input_trace: { + start_time : 6301917 + finish_time : 6302123 + packet_timestamp: 5833333 + stream_id : 2 + packet_id : 52 + } + output_trace: { + packet_timestamp: 5833333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6316569 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5866666 + event_type : PROCESS + finish_time : 6336503 + output_trace: { + packet_timestamp: 5866666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6336637 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5866666 + event_type : PROCESS + start_time : 6336724 + finish_time : 6353016 + input_trace: { + start_time : 6336503 + finish_time : 6336724 + packet_timestamp: 5866666 + stream_id : 2 + packet_id : 45 + } + output_trace: { + packet_timestamp: 5866666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6355006 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5900000 + event_type : PROCESS + finish_time : 6378384 + output_trace: { + packet_timestamp: 5900000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6378528 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5900000 + event_type : PROCESS + start_time : 6378613 + input_trace: { + start_time : 6378384 + finish_time : 6378613 + packet_timestamp: 5900000 + stream_id : 2 + packet_id : 42 + } + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 1 + input_timestamp: 5900000 + event_type : PROCESS + finish_time : 6397079 + output_trace: { + packet_timestamp: 5900000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6399531 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 6407372 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3566666 + event_type : PROCESS + start_time : 6407444 + input_trace: { + finish_time : 6407444 + packet_timestamp: 3566666 + stream_id : 3 + packet_id : 43 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5933333 + event_type : PROCESS + finish_time : 6418367 + output_trace: { + packet_timestamp: 5933333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6418509 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5933333 + event_type : PROCESS + start_time : 6418588 + finish_time : 6429268 + input_trace: { + start_time : 6418367 + finish_time : 6418588 + packet_timestamp: 5933333 + stream_id : 2 + packet_id : 87 + } + output_trace: { + packet_timestamp: 5933333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6431296 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 5966666 + event_type : PROCESS + finish_time : 6447597 + output_trace: { + packet_timestamp: 5966666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6447737 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 5966666 + event_type : PROCESS + start_time : 6447815 + finish_time : 6463279 + input_trace: { + start_time : 6447597 + finish_time : 6447815 + packet_timestamp: 5966666 + stream_id : 2 + packet_id : 52 + } + output_trace: { + packet_timestamp: 5966666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6465321 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6000000 + event_type : PROCESS + finish_time : 6482319 + output_trace: { + packet_timestamp: 6000000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6482465 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6000000 + event_type : PROCESS + start_time : 6482548 + finish_time : 6498075 + input_trace: { + start_time : 6482319 + finish_time : 6482548 + packet_timestamp: 6000000 + stream_id : 2 + packet_id : 45 + } + output_trace: { + packet_timestamp: 6000000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6500207 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 6502302 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3600000 + event_type : PROCESS + start_time : 6502362 + input_trace: { + finish_time : 6502362 + packet_timestamp: 3600000 + stream_id : 3 + packet_id : 8 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6033333 + event_type : PROCESS + finish_time : 6518818 + output_trace: { + packet_timestamp: 6033333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6518958 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6033333 + event_type : PROCESS + start_time : 6519040 + finish_time : 6529094 + input_trace: { + start_time : 6518818 + finish_time : 6519040 + packet_timestamp: 6033333 + stream_id : 2 + packet_id : 52 + } + output_trace: { + packet_timestamp: 6033333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6530885 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 6537040 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3633333 + event_type : PROCESS + start_time : 6537106 + input_trace: { + finish_time : 6537106 + packet_timestamp: 3633333 + stream_id : 3 + packet_id : 47 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6066666 + event_type : PROCESS + finish_time : 6577309 + output_trace: { + packet_timestamp: 6066666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6577454 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6066666 + event_type : PROCESS + start_time : 6577536 + finish_time : 6594210 + input_trace: { + start_time : 6577309 + finish_time : 6577536 + packet_timestamp: 6066666 + stream_id : 2 + packet_id : 90 + } + output_trace: { + packet_timestamp: 6066666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6596142 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6100000 + event_type : PROCESS + finish_time : 6611945 + output_trace: { + packet_timestamp: 6100000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6612087 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6100000 + event_type : PROCESS + start_time : 6612170 + finish_time : 6630155 + input_trace: { + start_time : 6611945 + finish_time : 6612170 + packet_timestamp: 6100000 + stream_id : 2 + packet_id : 91 + } + output_trace: { + packet_timestamp: 6100000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6632255 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6133333 + event_type : PROCESS + finish_time : 6659283 + output_trace: { + packet_timestamp: 6133333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6659426 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6133333 + event_type : PROCESS + start_time : 6659509 + finish_time : 6674942 + input_trace: { + start_time : 6659283 + finish_time : 6659509 + packet_timestamp: 6133333 + stream_id : 2 + packet_id : 36 + } + output_trace: { + packet_timestamp: 6133333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6677039 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6166666 + event_type : PROCESS + finish_time : 6696290 + output_trace: { + packet_timestamp: 6166666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6696483 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6166666 + event_type : PROCESS + start_time : 6696591 + finish_time : 6712666 + input_trace: { + start_time : 6696290 + finish_time : 6696591 + packet_timestamp: 6166666 + stream_id : 2 + packet_id : 92 + } + output_trace: { + packet_timestamp: 6166666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6714703 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6200000 + event_type : PROCESS + finish_time : 6733096 + output_trace: { + packet_timestamp: 6200000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6733246 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6200000 + event_type : PROCESS + start_time : 6733330 + finish_time : 6750533 + input_trace: { + start_time : 6733096 + finish_time : 6733330 + packet_timestamp: 6200000 + stream_id : 2 + packet_id : 90 + } + output_trace: { + packet_timestamp: 6200000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6752612 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6233333 + event_type : PROCESS + finish_time : 6769276 + output_trace: { + packet_timestamp: 6233333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6769415 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6233333 + event_type : PROCESS + start_time : 6769493 + finish_time : 6786916 + input_trace: { + start_time : 6769276 + finish_time : 6769493 + packet_timestamp: 6233333 + stream_id : 2 + packet_id : 91 + } + output_trace: { + packet_timestamp: 6233333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6789065 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6266666 + event_type : PROCESS + finish_time : 6808297 + output_trace: { + packet_timestamp: 6266666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6808479 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6266666 + event_type : PROCESS + start_time : 6808565 + finish_time : 6825600 + input_trace: { + start_time : 6808297 + finish_time : 6808565 + packet_timestamp: 6266666 + stream_id : 2 + packet_id : 83 + } + output_trace: { + packet_timestamp: 6266666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6827685 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6300000 + event_type : PROCESS + finish_time : 6844905 + output_trace: { + packet_timestamp: 6300000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6845046 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6300000 + event_type : PROCESS + start_time : 6845130 + finish_time : 6863499 + input_trace: { + start_time : 6844905 + finish_time : 6845130 + packet_timestamp: 6300000 + stream_id : 2 + packet_id : 92 + } + output_trace: { + packet_timestamp: 6300000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6865498 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6333333 + event_type : PROCESS + finish_time : 6885798 + output_trace: { + packet_timestamp: 6333333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6885946 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6333333 + event_type : PROCESS + start_time : 6886034 + input_trace: { + start_time : 6885798 + finish_time : 6886034 + packet_timestamp: 6333333 + stream_id : 2 + packet_id : 74 + } + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 1 + input_timestamp: 6333333 + event_type : PROCESS + finish_time : 6902421 + output_trace: { + packet_timestamp: 6333333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6904619 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6366666 + event_type : PROCESS + finish_time : 6929047 + output_trace: { + packet_timestamp: 6366666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6929191 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6366666 + event_type : PROCESS + start_time : 6929273 + finish_time : 6944349 + input_trace: { + start_time : 6929047 + finish_time : 6929273 + packet_timestamp: 6366666 + stream_id : 2 + packet_id : 91 + } + output_trace: { + packet_timestamp: 6366666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6946524 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 6952186 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3666666 + event_type : PROCESS + start_time : 6952248 + input_trace: { + finish_time : 6952248 + packet_timestamp: 3666666 + stream_id : 3 + packet_id : 93 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6400000 + event_type : PROCESS + finish_time : 6977289 + output_trace: { + packet_timestamp: 6400000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 6977439 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6400000 + event_type : PROCESS + start_time : 6977536 + finish_time : 6988487 + input_trace: { + start_time : 6977289 + finish_time : 6977536 + packet_timestamp: 6400000 + stream_id : 2 + packet_id : 83 + } + output_trace: { + packet_timestamp: 6400000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 6990442 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6433333 + event_type : PROCESS + finish_time : 7008609 + output_trace: { + packet_timestamp: 6433333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7008755 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6433333 + event_type : PROCESS + start_time : 7008835 + finish_time : 7024250 + input_trace: { + start_time : 7008609 + finish_time : 7008835 + packet_timestamp: 6433333 + stream_id : 2 + packet_id : 94 + } + output_trace: { + packet_timestamp: 6433333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7026232 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7039626 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3700000 + event_type : PROCESS + start_time : 7039702 + input_trace: { + finish_time : 7039702 + packet_timestamp: 3700000 + stream_id : 3 + packet_id : 62 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6466666 + event_type : PROCESS + finish_time : 7041768 + output_trace: { + packet_timestamp: 6466666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7041897 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6466666 + event_type : PROCESS + start_time : 7041966 + finish_time : 7068671 + input_trace: { + start_time : 7041768 + finish_time : 7041966 + packet_timestamp: 6466666 + stream_id : 2 + packet_id : 74 + } + output_trace: { + packet_timestamp: 6466666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7060247 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3733333 + event_type : PROCESS + start_time : 7060314 + input_trace: { + finish_time : 7060314 + packet_timestamp: 3733333 + stream_id : 3 + packet_id : 25 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7070844 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7084944 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3766666 + event_type : PROCESS + start_time : 7085011 + input_trace: { + finish_time : 7085011 + packet_timestamp: 3766666 + stream_id : 3 + packet_id : 32 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6500000 + event_type : PROCESS + finish_time : 7091552 + output_trace: { + packet_timestamp: 6500000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7091693 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6500000 + event_type : PROCESS + start_time : 7091780 + finish_time : 7108984 + input_trace: { + start_time : 7091552 + finish_time : 7091780 + packet_timestamp: 6500000 + stream_id : 2 + packet_id : 62 + } + output_trace: { + packet_timestamp: 6500000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7111040 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7127641 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3800000 + event_type : PROCESS + start_time : 7127711 + input_trace: { + finish_time : 7127711 + packet_timestamp: 3800000 + stream_id : 3 + packet_id : 95 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6533333 + event_type : PROCESS + finish_time : 7127839 + output_trace: { + packet_timestamp: 6533333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7127955 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6533333 + event_type : PROCESS + start_time : 7128033 + finish_time : 7150698 + input_trace: { + start_time : 7127839 + finish_time : 7128033 + packet_timestamp: 6533333 + stream_id : 2 + packet_id : 83 + } + output_trace: { + packet_timestamp: 6533333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7149631 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3833333 + event_type : PROCESS + start_time : 7149694 + input_trace: { + finish_time : 7149694 + packet_timestamp: 3833333 + stream_id : 3 + packet_id : 46 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7152727 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6566666 + event_type : PROCESS + finish_time : 7167848 + output_trace: { + packet_timestamp: 6566666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7167989 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6566666 + event_type : PROCESS + start_time : 7168073 + finish_time : 7179990 + input_trace: { + start_time : 7167848 + finish_time : 7168073 + packet_timestamp: 6566666 + stream_id : 2 + packet_id : 96 + } + output_trace: { + packet_timestamp: 6566666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7179843 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3866666 + event_type : PROCESS + start_time : 7179915 + input_trace: { + finish_time : 7179915 + packet_timestamp: 3866666 + stream_id : 3 + packet_id : 97 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7181851 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7216838 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3900000 + event_type : PROCESS + start_time : 7216907 + input_trace: { + finish_time : 7216907 + packet_timestamp: 3900000 + stream_id : 3 + packet_id : 64 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6600000 + event_type : PROCESS + finish_time : 7217088 + output_trace: { + packet_timestamp: 6600000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7217224 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6600000 + event_type : PROCESS + start_time : 7217293 + finish_time : 7243871 + input_trace: { + start_time : 7217088 + finish_time : 7217293 + packet_timestamp: 6600000 + stream_id : 2 + packet_id : 98 + } + output_trace: { + packet_timestamp: 6600000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7245976 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7259158 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3933333 + event_type : PROCESS + start_time : 7259223 + input_trace: { + finish_time : 7259223 + packet_timestamp: 3933333 + stream_id : 3 + packet_id : 57 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6633333 + event_type : PROCESS + finish_time : 7266357 + output_trace: { + packet_timestamp: 6633333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7266502 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6633333 + event_type : PROCESS + start_time : 7266595 + finish_time : 7277706 + input_trace: { + start_time : 7266357 + finish_time : 7266595 + packet_timestamp: 6633333 + stream_id : 2 + packet_id : 99 + } + output_trace: { + packet_timestamp: 6633333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7279699 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6666666 + event_type : PROCESS + finish_time : 7298290 + output_trace: { + packet_timestamp: 6666666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7298432 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6666666 + event_type : PROCESS + start_time : 7298513 + finish_time : 7309881 + input_trace: { + start_time : 7298290 + finish_time : 7298513 + packet_timestamp: 6666666 + stream_id : 2 + packet_id : 100 + } + output_trace: { + packet_timestamp: 6666666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7311588 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7318524 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 3966666 + event_type : PROCESS + start_time : 7318592 + input_trace: { + finish_time : 7318592 + packet_timestamp: 3966666 + stream_id : 3 + packet_id : 101 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6700000 + event_type : PROCESS + finish_time : 7328068 + output_trace: { + packet_timestamp: 6700000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7328218 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6700000 + event_type : PROCESS + start_time : 7328296 + finish_time : 7341968 + input_trace: { + start_time : 7328068 + finish_time : 7328296 + packet_timestamp: 6700000 + stream_id : 2 + packet_id : 97 + } + output_trace: { + packet_timestamp: 6700000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7343996 + thread_id : 0 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6733333 + event_type : PROCESS + finish_time : 7361146 + output_trace: { + packet_timestamp: 6733333 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7361304 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6733333 + event_type : PROCESS + start_time : 7361385 + finish_time : 7382269 + input_trace: { + start_time : 7361146 + finish_time : 7361385 + packet_timestamp: 6733333 + stream_id : 2 + packet_id : 102 + } + output_trace: { + packet_timestamp: 6733333 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7367619 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4000000 + event_type : PROCESS + start_time : 7367687 + input_trace: { + finish_time : 7367687 + packet_timestamp: 4000000 + stream_id : 3 + packet_id : 10 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7384048 + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 0 + input_timestamp: 6766666 + event_type : PROCESS + finish_time : 7409057 + output_trace: { + packet_timestamp: 6766666 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7409195 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6766666 + event_type : PROCESS + start_time : 7409273 + finish_time : 7437418 + input_trace: { + start_time : 7409057 + finish_time : 7409273 + packet_timestamp: 6766666 + stream_id : 2 + packet_id : 101 + } + output_trace: { + packet_timestamp: 6766666 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7414294 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4033333 + event_type : PROCESS + start_time : 7414367 + input_trace: { + finish_time : 7414367 + packet_timestamp: 4033333 + stream_id : 3 + packet_id : 29 + } + thread_id : 1 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7439439 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7449830 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4066666 + event_type : PROCESS + start_time : 7449901 + input_trace: { + finish_time : 7449901 + packet_timestamp: 4066666 + stream_id : 3 + packet_id : 30 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7483697 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4100000 + event_type : PROCESS + start_time : 7483769 + input_trace: { + finish_time : 7483769 + packet_timestamp: 4100000 + stream_id : 3 + packet_id : 1 + } + thread_id : 1 + } + calculator_trace: { + node_id: 0 + input_timestamp: 6800000 + event_type : PROCESS + finish_time : 7491516 + output_trace: { + packet_timestamp: 6800000 + stream_id : 2 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_PROCESS + start_time : 7491656 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + input_timestamp: 6800000 + event_type : PROCESS + start_time : 7491738 + finish_time : 7522729 + input_trace: { + start_time : 7491516 + finish_time : 7491738 + packet_timestamp: 6800000 + stream_id : 2 + packet_id : 10 + } + output_trace: { + packet_timestamp: 6800000 + stream_id : 3 + } + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : NOT_READY + start_time : 7524884 + thread_id : 0 + } + calculator_trace: { + node_id: 1 + event_type : READY_FOR_CLOSE + start_time : 7568535 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7713076 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4133333 + event_type : PROCESS + start_time : 7713174 + input_trace: { + finish_time : 7713174 + packet_timestamp: 4133333 + stream_id : 3 + packet_id : 22 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7810586 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4166666 + event_type : PROCESS + start_time : 7810691 + input_trace: { + finish_time : 7810691 + packet_timestamp: 4166666 + stream_id : 3 + packet_id : 103 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7831792 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4200000 + event_type : PROCESS + start_time : 7831880 + input_trace: { + finish_time : 7831880 + packet_timestamp: 4200000 + stream_id : 3 + packet_id : 34 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7857177 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4233333 + event_type : PROCESS + start_time : 7857271 + input_trace: { + finish_time : 7857271 + packet_timestamp: 4233333 + stream_id : 3 + packet_id : 104 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7878501 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4266666 + event_type : PROCESS + start_time : 7878601 + input_trace: { + finish_time : 7878601 + packet_timestamp: 4266666 + stream_id : 3 + packet_id : 24 + } + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7901711 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4300000 + event_type : PROCESS + start_time : 7901813 + input_trace: { + finish_time : 7901813 + packet_timestamp: 4300000 + stream_id : 3 + packet_id : 67 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 7967805 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4333333 + event_type : PROCESS + start_time : 7967897 + input_trace: { + finish_time : 7967897 + packet_timestamp: 4333333 + stream_id : 3 + packet_id : 105 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8014397 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4366666 + event_type : PROCESS + start_time : 8014466 + input_trace: { + finish_time : 8014466 + packet_timestamp: 4366666 + stream_id : 3 + packet_id : 106 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8171611 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4400000 + event_type : PROCESS + start_time : 8171718 + input_trace: { + finish_time : 8171718 + packet_timestamp: 4400000 + stream_id : 3 + packet_id : 60 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8208950 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4433333 + event_type : PROCESS + start_time : 8209047 + input_trace: { + finish_time : 8209047 + packet_timestamp: 4433333 + stream_id : 3 + packet_id : 40 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8251999 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4466666 + event_type : PROCESS + start_time : 8252072 + input_trace: { + finish_time : 8252072 + packet_timestamp: 4466666 + stream_id : 3 + packet_id : 48 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8316927 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4500000 + event_type : PROCESS + start_time : 8317029 + input_trace: { + finish_time : 8317029 + packet_timestamp: 4500000 + stream_id : 3 + packet_id : 19 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8355292 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4533333 + event_type : PROCESS + start_time : 8355408 + input_trace: { + finish_time : 8355408 + packet_timestamp: 4533333 + stream_id : 3 + packet_id : 38 + } + thread_id : 0 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8396763 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4566666 + event_type : PROCESS + start_time : 8396832 + input_trace: { + finish_time : 8396832 + packet_timestamp: 4566666 + stream_id : 3 + packet_id : 17 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8432759 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4600000 + event_type : PROCESS + start_time : 8432860 + input_trace: { + finish_time : 8432860 + packet_timestamp: 4600000 + stream_id : 3 + packet_id : 50 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8743760 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4633333 + event_type : PROCESS + start_time : 8743853 + input_trace: { + finish_time : 8743853 + packet_timestamp: 4633333 + stream_id : 3 + packet_id : 63 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4666666 + event_type : PROCESS + start_time : 8787092 + input_trace: { + finish_time : 8787092 + packet_timestamp: 4666666 + stream_id : 3 + packet_id : 107 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8793733 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4700000 + event_type : PROCESS + start_time : 8868426 + input_trace: { + finish_time : 8868426 + packet_timestamp: 4700000 + stream_id : 3 + packet_id : 108 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8873712 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4733333 + event_type : PROCESS + start_time : 8890592 + input_trace: { + finish_time : 8890592 + packet_timestamp: 4733333 + stream_id : 3 + packet_id : 109 + } + thread_id : 1 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8896714 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4766666 + event_type : PROCESS + start_time : 8926773 + input_trace: { + finish_time : 8926773 + packet_timestamp: 4766666 + stream_id : 3 + packet_id : 110 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8928666 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8948844 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4800000 + event_type : PROCESS + start_time : 8948914 + input_trace: { + finish_time : 8948914 + packet_timestamp: 4800000 + stream_id : 3 + packet_id : 41 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 8968868 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4833333 + event_type : PROCESS + start_time : 8968938 + input_trace: { + finish_time : 8968938 + packet_timestamp: 4833333 + stream_id : 3 + packet_id : 111 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 9004937 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4866666 + event_type : PROCESS + start_time : 9005013 + input_trace: { + finish_time : 9005013 + packet_timestamp: 4866666 + stream_id : 3 + packet_id : 68 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 9048677 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4900000 + event_type : PROCESS + start_time : 9048751 + input_trace: { + finish_time : 9048751 + packet_timestamp: 4900000 + stream_id : 3 + packet_id : 112 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4933333 + event_type : PROCESS + start_time : 9081720 + input_trace: { + finish_time : 9081720 + packet_timestamp: 4933333 + stream_id : 3 + packet_id : 113 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 9087002 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 9183903 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 4966666 + event_type : PROCESS + start_time : 9183973 + input_trace: { + finish_time : 9183973 + packet_timestamp: 4966666 + stream_id : 3 + packet_id : 114 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 9222146 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 5000000 + event_type : PROCESS + start_time : 9222274 + input_trace: { + finish_time : 9222274 + packet_timestamp: 5000000 + stream_id : 3 + packet_id : 65 + } + thread_id : 1 + } +} +graph_trace: { + base_time : 1580845770612736 + base_timestamp : 0 + stream_name : [ "", "input_video_header", "input_video", "output_video" ] + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 9556204 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 5033333 + event_type : PROCESS + start_time : 9556273 + input_trace: { + finish_time : 9556273 + packet_timestamp: 5033333 + stream_id : 3 + packet_id : 115 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 9650880 + thread_id : 1 + } + calculator_trace: { + node_id: 2 + input_timestamp: 5066666 + event_type : PROCESS + start_time : 9650983 + input_trace: { + finish_time : 9650983 + packet_timestamp: 5066666 + stream_id : 3 + packet_id : 75 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 9720998 + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 5100000 + event_type : PROCESS + start_time : 9721061 + input_trace: { + finish_time : 9721061 + packet_timestamp: 5100000 + stream_id : 3 + packet_id : 116 + } + thread_id : 0 + } + calculator_trace: { + node_id: 2 + input_timestamp: 5133333 + event_type : PROCESS + start_time : 9762838 + input_trace: { + finish_time : 9762838 + packet_timestamp: 5133333 + stream_id : 3 + packet_id : 117 + } + thread_id : 1 + } + calculator_trace: { + node_id: 2 + event_type : READY_FOR_PROCESS + start_time : 9780444 + thread_id : 0 + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 401379 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + process_input_latency: { + total: 16788293 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + process_output_latency: { + total: 17189672 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 16545046 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 292166 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_output_latency: { + total: 292166 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 174676 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_input_latency: { + total: 316816 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_output_latency: { + total: 491492 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 2638 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 564837 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + process_input_latency: { + total: 18865536 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + process_output_latency: { + total: 19430373 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 18603405 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 7 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 275929 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_output_latency: { + total: 275929 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 181147 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_input_latency: { + total: 278404 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_output_latency: { + total: 459551 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 2475 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 387320 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 6 ] + } + process_input_latency: { + total: 16978649 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 6 ] + } + process_output_latency: { + total: 17365969 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 6 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 16736681 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 6 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 283407 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_output_latency: { + total: 283407 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 194470 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_input_latency: { + total: 262541 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_output_latency: { + total: 457011 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 2439 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 216677 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 3 ] + } + process_input_latency: { + total: 9042125 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 3 ] + } + process_output_latency: { + total: 9258802 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 3 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 8914445 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 3 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 269999 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_output_latency: { + total: 269999 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 206450 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_input_latency: { + total: 276065 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_output_latency: { + total: 482515 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 2982 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 805324 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_input_latency: { + total: 37556775 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + process_output_latency: { + total: 38362099 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 37156576 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 11 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + process_runtime: { + total: 248443 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } + process_output_latency: { + total: 248443 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 12 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + process_runtime: { + total: 219987 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_input_latency: { + total: 271525 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + process_output_latency: { + total: 491512 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 2861 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 13 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 492012 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + process_input_latency: { + total: 28551089 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + process_output_latency: { + total: 29043101 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 28216287 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + close_runtime: 38295 + process_runtime: { + total: 81520 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 3 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 3 ] + } + process_output_latency: { + total: 81520 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 3 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + close_runtime: 7 + process_runtime: { + total: 59152 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 2 ] + } + process_input_latency: { + total: 77349 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 2 ] + } + process_output_latency: { + total: 136501 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 2 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 429 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 2 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 451450 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + process_input_latency: { + total: 29517887 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + process_output_latency: { + total: 29969337 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 29215428 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + close_runtime: 38295 + process_runtime: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_output_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + close_runtime: 7 + process_runtime: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_output_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 520320 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 6 ] + } + process_input_latency: { + total: 23546551 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 6 ] + } + process_output_latency: { + total: 24066871 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 6 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 23308153 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 6 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + close_runtime: 38295 + process_runtime: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_output_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + close_runtime: 7 + process_runtime: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_output_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 313080 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + process_input_latency: { + total: 32361341 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + process_output_latency: { + total: 32674421 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 32060586 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 8 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + close_runtime: 38295 + process_runtime: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_output_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + close_runtime: 7 + process_runtime: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_output_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoEncoderCalculator" + open_runtime : 29 + process_runtime: { + total: 530317 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + process_input_latency: { + total: 17288413 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + process_output_latency: { + total: 17818730 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + input_stream_profiles: { + name : "output_video" + back_edge: false + latency: { + total: 17094817 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 4 ] + } + } + input_stream_profiles: { + name : "input_video_header" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +calculator_profiles: { + name: "OpenCvVideoDecoderCalculator" + open_runtime : 115829 + close_runtime: 38295 + process_runtime: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_output_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } +} +calculator_profiles: { + name: "OpenCvWriteTextCalculator" + open_runtime : 17 + close_runtime: 7 + process_runtime: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_input_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + process_output_latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + input_stream_profiles: { + name : "input_video" + back_edge: false + latency: { + total: 0 + interval_size_usec: 1000000 + num_intervals : 1 + count: [ 0 ] + } + } +} +config: { + node: { + name: "OpenCvVideoDecoderCalculator" + calculator : "OpenCvVideoDecoderCalculator" + output_stream: [ "VIDEO:input_video", "VIDEO_PRESTREAM:input_video_header" ] + input_side_packet : [ "INPUT_FILE_PATH:input_video_path" ] + } + node: { + name: "OpenCvWriteTextCalculator" + calculator : "OpenCvWriteTextCalculator" + input_stream : [ "input_video" ] + output_stream: [ "output_video" ] + } + node: { + name: "OpenCvVideoEncoderCalculator" + calculator : "OpenCvVideoEncoderCalculator" + input_stream : [ "VIDEO:output_video", "VIDEO_PRESTREAM:input_video_header" ] + input_side_packet : [ "OUTPUT_FILE_PATH:output_video_path" ] + } + executor: { + } + profiler_config: { + enable_profiler : true + enable_stream_latency : true + trace_log_count : 10 + trace_enabled : true + } +} diff --git a/mediapipe/framework/tool/BUILD b/mediapipe/framework/tool/BUILD index bcf741ac7..e169ec37c 100644 --- a/mediapipe/framework/tool/BUILD +++ b/mediapipe/framework/tool/BUILD @@ -177,7 +177,7 @@ cc_library( deps = [ "//mediapipe/framework:packet", "//mediapipe/framework/port:statusor", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) diff --git a/mediapipe/framework/tool/status_util_test.cc b/mediapipe/framework/tool/status_util_test.cc index 2719545e5..8ab9d1a35 100644 --- a/mediapipe/framework/tool/status_util_test.cc +++ b/mediapipe/framework/tool/status_util_test.cc @@ -52,8 +52,10 @@ TEST(StatusTest, CombinedStatus) { errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument, "error_with_that_string"); status = tool::CombinedStatus(prefix_error_message, errors); - EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[0].error_message())); - EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[1].error_message())); + EXPECT_THAT(status.ToString(), + testing::HasSubstr(std::string(errors[0].message()))); + EXPECT_THAT(status.ToString(), + testing::HasSubstr(std::string(errors[1].message()))); EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message)); EXPECT_EQ(::mediapipe::StatusCode::kInvalidArgument, status.code()); @@ -63,8 +65,10 @@ TEST(StatusTest, CombinedStatus) { errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument, "error_with_that_string"); status = tool::CombinedStatus(prefix_error_message, errors); - EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[0].error_message())); - EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[1].error_message())); + EXPECT_THAT(status.ToString(), + testing::HasSubstr(std::string(errors[0].message()))); + EXPECT_THAT(status.ToString(), + testing::HasSubstr(std::string(errors[1].message()))); EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message)); EXPECT_EQ(::mediapipe::StatusCode::kUnknown, status.code()); errors.clear(); @@ -72,7 +76,8 @@ TEST(StatusTest, CombinedStatus) { errors.emplace_back(::mediapipe::StatusCode::kInvalidArgument, "error_with_that_string"); status = tool::CombinedStatus(prefix_error_message, errors); - EXPECT_THAT(status.ToString(), testing::HasSubstr(errors[1].error_message())); + EXPECT_THAT(status.ToString(), + testing::HasSubstr(std::string(errors[1].message()))); EXPECT_THAT(status.ToString(), testing::HasSubstr(prefix_error_message)); EXPECT_EQ(::mediapipe::StatusCode::kInvalidArgument, status.code()); diff --git a/mediapipe/framework/tool/subgraph_expansion.cc b/mediapipe/framework/tool/subgraph_expansion.cc index a597879ca..6d2ce40e9 100644 --- a/mediapipe/framework/tool/subgraph_expansion.cc +++ b/mediapipe/framework/tool/subgraph_expansion.cc @@ -76,10 +76,11 @@ namespace tool { ::mediapipe::Status RemoveIgnoredStreams( proto_ns::RepeatedPtrField* streams, const std::set& missing_streams) { - ASSIGN_OR_RETURN(auto src_map, tool::TagMap::Create(*streams)); - std::vector src_names = src_map->Names(); for (int i = streams->size() - 1; i >= 0; --i) { - if (missing_streams.count(src_names[i]) > 0) { + std::string tag, name; + int index; + MP_RETURN_IF_ERROR(ParseTagIndexName(streams->Get(i), &tag, &index, &name)); + if (missing_streams.count(name) > 0) { streams->DeleteSubrange(i, 1); } } diff --git a/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt b/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt index 2d22b7a14..61b6f935d 100644 --- a/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt +++ b/mediapipe/graphs/face_detection/face_detection_desktop_live.pbtxt @@ -177,8 +177,8 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:throttled_input_video" + input_stream: "IMAGE:throttled_input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } diff --git a/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt b/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt index 1b6ecbf47..4f866286e 100644 --- a/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt +++ b/mediapipe/graphs/face_detection/face_detection_mobile_cpu.pbtxt @@ -188,9 +188,9 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_video_cpu" + input_stream: "IMAGE:input_video_cpu" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video_cpu" + output_stream: "IMAGE:output_video_cpu" } # Transfers the annotated image from CPU back to GPU memory, to be sent out of diff --git a/mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt b/mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt index 2fb85bc00..e25d42f27 100644 --- a/mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt +++ b/mediapipe/graphs/face_detection/face_detection_mobile_gpu.pbtxt @@ -178,7 +178,7 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:throttled_input_video" + input_stream: "IMAGE_GPU:throttled_input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME_GPU:output_video" + output_stream: "IMAGE_GPU:output_video" } diff --git a/mediapipe/graphs/hand_tracking/BUILD b/mediapipe/graphs/hand_tracking/BUILD index da6776b8d..3c419d8be 100644 --- a/mediapipe/graphs/hand_tracking/BUILD +++ b/mediapipe/graphs/hand_tracking/BUILD @@ -21,6 +21,10 @@ licenses(["notice"]) # Apache 2.0 package(default_visibility = ["//visibility:public"]) +exports_files(glob([ + "*.pbtxt", +])) + cc_library( name = "desktop_offline_calculators", deps = [ diff --git a/mediapipe/graphs/hand_tracking/hand_detection_desktop.pbtxt b/mediapipe/graphs/hand_tracking/hand_detection_desktop.pbtxt index ac8e7a401..813f555bf 100644 --- a/mediapipe/graphs/hand_tracking/hand_detection_desktop.pbtxt +++ b/mediapipe/graphs/hand_tracking/hand_detection_desktop.pbtxt @@ -41,9 +41,9 @@ node { # the graph. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_video" + input_stream: "IMAGE:input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } # Encodes the annotated images into a video file, adopting properties specified diff --git a/mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt b/mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt index 363bb5182..26f8d1b46 100644 --- a/mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt +++ b/mediapipe/graphs/hand_tracking/hand_detection_desktop_live.pbtxt @@ -32,7 +32,7 @@ node { # the graph. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_video" + input_stream: "IMAGE:input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } diff --git a/mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt b/mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt index 7495c62fc..df8ca6dbf 100644 --- a/mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt +++ b/mediapipe/graphs/hand_tracking/hand_detection_mobile.pbtxt @@ -66,8 +66,8 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:throttled_input_video" + input_stream: "IMAGE_GPU:throttled_input_video" input_stream: "detection_render_data" input_stream: "rect_render_data" - output_stream: "OUTPUT_FRAME_GPU:output_video" + output_stream: "IMAGE_GPU:output_video" } diff --git a/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_cpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_cpu.pbtxt index cecffca9f..c0b8e542b 100644 --- a/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_cpu.pbtxt +++ b/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_cpu.pbtxt @@ -14,6 +14,11 @@ node { input_stream: "IMAGE:input_video" input_stream: "NORM_RECT:hand_rect" output_stream: "IMAGE:hand_image" + node_options: { + [type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] { + border_mode: BORDER_REPLICATE + } + } } # Transforms the input image on CPU to a 256x256 image. To scale the input @@ -33,13 +38,9 @@ node: { } } -# Converts the transformed input image on GPU into an image tensor stored in -# tflite::gpu::GlBuffer. The zero_center option is set to true to normalize the -# pixel values to [-1.f, 1.f] as opposed to [0.f, 1.f]. The flip_vertically -# option is set to true to account for the descrepancy between the -# representation of the input image (origin at the bottom-left corner, the -# OpenGL convention) and what the model used in this graph is expecting (origin -# at the top-left corner). +# Converts the transformed input image on CPU into an image tensor stored in +# TfliteTensor. The zero_center option is set to false to normalize the +# pixel values to [0.f, 1.f]. node { calculator: "TfLiteConverterCalculator" input_stream: "IMAGE:transformed_input_video" diff --git a/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt index 229463454..a45e0e1e1 100644 --- a/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt +++ b/mediapipe/graphs/hand_tracking/subgraphs/hand_landmark_gpu.pbtxt @@ -14,6 +14,11 @@ node { input_stream: "IMAGE_GPU:input_video" input_stream: "NORM_RECT:hand_rect" output_stream: "IMAGE_GPU:hand_image" + node_options: { + [type.googleapis.com/mediapipe.ImageCroppingCalculatorOptions] { + border_mode: BORDER_REPLICATE + } + } } # Transforms the input image on GPU to a 256x256 image. To scale the input diff --git a/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_cpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_cpu.pbtxt index 8406712e9..5bc99c033 100644 --- a/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_cpu.pbtxt +++ b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_cpu.pbtxt @@ -135,10 +135,10 @@ node { # a vector of RenderData objects and draws each of them on the input frame. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_image" + input_stream: "IMAGE:input_image" input_stream: "detection_render_data" input_stream: "multi_hand_rects_render_data" input_stream: "multi_palm_rects_render_data" input_stream: "VECTOR:0:multi_hand_landmarks_render_data" - output_stream: "OUTPUT_FRAME:output_image" + output_stream: "IMAGE:output_image" } diff --git a/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt index d7e300c02..82bd4ce90 100644 --- a/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt +++ b/mediapipe/graphs/hand_tracking/subgraphs/multi_hand_renderer_gpu.pbtxt @@ -135,10 +135,10 @@ node { # a vector of RenderData objects and draws each of them on the input frame. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:input_image" + input_stream: "IMAGE_GPU:input_image" input_stream: "detection_render_data" input_stream: "multi_hand_rects_render_data" input_stream: "multi_palm_rects_render_data" input_stream: "VECTOR:0:multi_hand_landmarks_render_data" - output_stream: "OUTPUT_FRAME_GPU:output_image" + output_stream: "IMAGE_GPU:output_image" } diff --git a/mediapipe/graphs/hand_tracking/subgraphs/renderer_cpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/renderer_cpu.pbtxt index c3033155d..f8aa07a7b 100644 --- a/mediapipe/graphs/hand_tracking/subgraphs/renderer_cpu.pbtxt +++ b/mediapipe/graphs/hand_tracking/subgraphs/renderer_cpu.pbtxt @@ -94,9 +94,9 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_image" + input_stream: "IMAGE:input_image" input_stream: "detection_render_data" input_stream: "landmark_render_data" input_stream: "rect_render_data" - output_stream: "OUTPUT_FRAME:output_image" + output_stream: "IMAGE:output_image" } diff --git a/mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt b/mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt index 6635958a1..5acffec69 100644 --- a/mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt +++ b/mediapipe/graphs/hand_tracking/subgraphs/renderer_gpu.pbtxt @@ -94,9 +94,9 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:input_image" + input_stream: "IMAGE_GPU:input_image" input_stream: "detection_render_data" input_stream: "landmark_render_data" input_stream: "rect_render_data" - output_stream: "OUTPUT_FRAME_GPU:output_image" + output_stream: "IMAGE_GPU:output_image" } diff --git a/mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt b/mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt index 7cf19b2ed..98b9fabd8 100644 --- a/mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt +++ b/mediapipe/graphs/object_detection/object_detection_desktop_live.pbtxt @@ -168,7 +168,7 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:throttled_input_video" + input_stream: "IMAGE:throttled_input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } diff --git a/mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt b/mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt index b55290c10..f12eeb607 100644 --- a/mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt +++ b/mediapipe/graphs/object_detection/object_detection_desktop_tensorflow_graph.pbtxt @@ -109,9 +109,9 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_video" + input_stream: "IMAGE:input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } # Encodes the annotated images into a video file, adopting properties specified diff --git a/mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt b/mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt index fd63fd97a..15aa2cdd2 100644 --- a/mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt +++ b/mediapipe/graphs/object_detection/object_detection_desktop_tflite_graph.pbtxt @@ -159,9 +159,9 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_video" + input_stream: "IMAGE:input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } # Encodes the annotated images into a video file, adopting properties specified diff --git a/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt b/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt index 4de82c07f..82561798a 100644 --- a/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt +++ b/mediapipe/graphs/object_detection/object_detection_mobile_cpu.pbtxt @@ -179,9 +179,9 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:throttled_input_video_cpu" + input_stream: "IMAGE:throttled_input_video_cpu" input_stream: "render_data" - output_stream: "OUTPUT_FRAME:output_video_cpu" + output_stream: "IMAGE:output_video_cpu" } # Transfers the annotated image from CPU back to GPU memory, to be sent out of diff --git a/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt b/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt index f3dc1d9e9..1ed66e8ba 100644 --- a/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt +++ b/mediapipe/graphs/object_detection/object_detection_mobile_gpu.pbtxt @@ -169,7 +169,7 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:throttled_input_video" + input_stream: "IMAGE_GPU:throttled_input_video" input_stream: "render_data" - output_stream: "OUTPUT_FRAME_GPU:output_video" + output_stream: "IMAGE_GPU:output_video" } diff --git a/mediapipe/graphs/tracking/subgraphs/renderer_cpu.pbtxt b/mediapipe/graphs/tracking/subgraphs/renderer_cpu.pbtxt index 41e05d1f0..665126a6c 100644 --- a/mediapipe/graphs/tracking/subgraphs/renderer_cpu.pbtxt +++ b/mediapipe/graphs/tracking/subgraphs/renderer_cpu.pbtxt @@ -23,7 +23,7 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_image" + input_stream: "IMAGE:input_image" input_stream: "detections_render_data" - output_stream: "OUTPUT_FRAME:output_image" + output_stream: "IMAGE:output_image" } diff --git a/mediapipe/graphs/tracking/subgraphs/renderer_gpu.pbtxt b/mediapipe/graphs/tracking/subgraphs/renderer_gpu.pbtxt index cf8d6f6b4..e94fb6d04 100644 --- a/mediapipe/graphs/tracking/subgraphs/renderer_gpu.pbtxt +++ b/mediapipe/graphs/tracking/subgraphs/renderer_gpu.pbtxt @@ -23,7 +23,7 @@ node { # Draws annotations and overlays them on top of the input images. node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME_GPU:input_image" + input_stream: "IMAGE_GPU:input_image" input_stream: "detections_render_data" - output_stream: "OUTPUT_FRAME_GPU:output_image" + output_stream: "IMAGE_GPU:output_image" } diff --git a/mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt b/mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt index 3b598a534..12ed2cb6c 100644 --- a/mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt +++ b/mediapipe/graphs/youtube8m/local_video_model_inference.pbtxt @@ -158,9 +158,9 @@ node { node { calculator: "AnnotationOverlayCalculator" - input_stream: "INPUT_FRAME:input_video" + input_stream: "IMAGE:input_video" input_stream: "synchronized_render_data" - output_stream: "OUTPUT_FRAME:output_video" + output_stream: "IMAGE:output_video" } node { diff --git a/mediapipe/java/com/google/mediapipe/components/FrameProcessor.java b/mediapipe/java/com/google/mediapipe/components/FrameProcessor.java index 8c901606e..c8063b1c2 100644 --- a/mediapipe/java/com/google/mediapipe/components/FrameProcessor.java +++ b/mediapipe/java/com/google/mediapipe/components/FrameProcessor.java @@ -16,6 +16,7 @@ package com.google.mediapipe.components; import android.content.Context; import android.graphics.Bitmap; +import android.media.AudioFormat; import android.util.Log; import com.google.common.base.Preconditions; import com.google.mediapipe.framework.AndroidAssetUtil; @@ -29,6 +30,7 @@ import com.google.mediapipe.framework.PacketGetter; import com.google.mediapipe.framework.SurfaceOutput; import com.google.mediapipe.framework.TextureFrame; import java.io.File; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -38,12 +40,16 @@ import javax.annotation.Nullable; /** * A {@link com.google.mediapipe.components.TextureFrameProcessor} that sends video frames through a - * MediaPipe graph. + * MediaPipe graph and a {@link com.google.mediapipe.components.AudioDataProcessor} that sends audio + * data samples through a MediaPipe graph. */ -public class FrameProcessor implements TextureFrameProcessor { +public class FrameProcessor implements TextureFrameProcessor, AudioDataProcessor { private static final String TAG = "FrameProcessor"; + private static final int BYTES_PER_MONO_SAMPLE = 2; // 16 bit PCM encoding. + private static final int AUDIO_ENCODING = AudioFormat.ENCODING_PCM_16BIT; - private List consumers = new ArrayList<>(); + private List videoConsumers = new ArrayList<>(); + private List audioConsumers = new ArrayList<>(); private Graph mediapipeGraph; private AndroidPacketCreator packetCreator; private OnWillAddFrameListener addFrameListener; @@ -52,7 +58,15 @@ public class FrameProcessor implements TextureFrameProcessor { private String videoOutputStream; private SurfaceOutput videoSurfaceOutput; private final AtomicBoolean started = new AtomicBoolean(false); - private boolean hybridPath = false; + // Input stream of audio data. Can be null. + private String audioInputStream; + // Output stream of audio data. Can be null. + private String audioOutputStream; + // Number of channels of audio data read in the input stream. This can be only 1 or 2, as + // AudioRecord supports only AudioFormat.CHANNEL_IN_MONO and AudioFormat.CHANNEL_IN_STEREO. + private int numAudioChannels = 1; + // Sample rate of audio data sent to the MediaPipe graph. + private double audioSampleRate; /** * Constructor. @@ -91,7 +105,7 @@ public class FrameProcessor implements TextureFrameProcessor { public void process(Packet packet) { List currentConsumers; synchronized (this) { - currentConsumers = consumers; + currentConsumers = videoConsumers; } for (TextureFrameConsumer consumer : currentConsumers) { TextureFrame frame = PacketGetter.getTextureFrame(packet); @@ -115,6 +129,56 @@ public class FrameProcessor implements TextureFrameProcessor { videoSurfaceOutput = mediapipeGraph.addSurfaceOutput(videoOutputStream); } + /** + * Adds input streams to process audio data and output streams that output processed audio data. + * + * @param inputStream the graph input stream that will receive input audio samples. + * @param outputStream the output stream from which output audio samples will be produced. + * @param numChannels the number of audio channels in the input audio stream. + * @param audioSampleRateInHz the sample rate for audio samples in hertz (Hz). + */ + public void addAudioStreams( + @Nullable String inputStream, + @Nullable String outputStream, + int numChannels, + double audioSampleRateInHz) { + audioInputStream = inputStream; + audioOutputStream = outputStream; + numAudioChannels = numChannels; + audioSampleRate = audioSampleRateInHz; + + if (audioInputStream != null) { + Packet audioHeader = + packetCreator.createTimeSeriesHeader(numAudioChannels, audioSampleRateInHz); + mediapipeGraph.setStreamHeader(audioInputStream, audioHeader); + } + + if (audioOutputStream != null) { + AudioFormat audioFormat = + new AudioFormat.Builder() + .setEncoding(AUDIO_ENCODING) + .setSampleRate((int) audioSampleRateInHz) + .setChannelMask(numAudioChannels) + .build(); + mediapipeGraph.addPacketCallback( + audioOutputStream, + new PacketCallback() { + @Override + public void process(Packet packet) { + List currentAudioConsumers; + synchronized (this) { + currentAudioConsumers = audioConsumers; + } + for (AudioDataConsumer consumer : currentAudioConsumers) { + byte[] buffer = PacketGetter.getAudioByteData(packet); + ByteBuffer audioData = ByteBuffer.wrap(buffer); + consumer.onNewAudioData(audioData, packet.getTimestamp(), audioFormat); + } + } + }); + } + } + /** * Interface to be used so that this class can receive a callback when onNewFrame has determined * it will process an input frame. Can be used to feed packets to accessory streams. @@ -134,9 +198,16 @@ public class FrameProcessor implements TextureFrameProcessor { } @Override - public void setConsumer(TextureFrameConsumer listener) { + public void setConsumer(TextureFrameConsumer consumer) { synchronized (this) { - consumers = Arrays.asList(listener); + videoConsumers = Arrays.asList(consumer); + } + } + + @Override + public void setAudioConsumer(AudioDataConsumer consumer) { + synchronized (this) { + audioConsumers = Arrays.asList(consumer); } } @@ -144,29 +215,25 @@ public class FrameProcessor implements TextureFrameProcessor { videoInputStreamCpu = inputStream; } - public void setHybridPath() { - hybridPath = true; - } - /** Adds a callback to the graph to process packets from the specified output stream. */ public void addPacketCallback(String outputStream, PacketCallback callback) { mediapipeGraph.addPacketCallback(outputStream, callback); } - public void addConsumer(TextureFrameConsumer listener) { + public void addConsumer(TextureFrameConsumer consumer) { synchronized (this) { - List newConsumers = new ArrayList<>(consumers); - newConsumers.add(listener); - consumers = newConsumers; + List newConsumers = new ArrayList<>(videoConsumers); + newConsumers.add(consumer); + videoConsumers = newConsumers; } } public boolean removeConsumer(TextureFrameConsumer listener) { boolean existed; synchronized (this) { - List newConsumers = new ArrayList<>(consumers); + List newConsumers = new ArrayList<>(videoConsumers); existed = newConsumers.remove(listener); - consumers = newConsumers; + videoConsumers = newConsumers; } return existed; } @@ -273,8 +340,7 @@ public class FrameProcessor implements TextureFrameProcessor { if (!maybeAcceptNewFrame()) { return; } - - if (!hybridPath && addFrameListener != null) { + if (addFrameListener != null) { addFrameListener.onWillAddFrame(timestamp); } @@ -305,4 +371,49 @@ public class FrameProcessor implements TextureFrameProcessor { private void startGraph() { mediapipeGraph.startRunningGraph(); } + + @Override + public void onNewAudioData(ByteBuffer audioData, long timestampMicros, AudioFormat audioFormat) { + if (!started.getAndSet(true)) { + startGraph(); + } + + if (audioFormat.getChannelCount() != numAudioChannels + || audioFormat.getSampleRate() != audioSampleRate + || audioFormat.getEncoding() != AUDIO_ENCODING) { + Log.e(TAG, "Producer's AudioFormat doesn't match FrameProcessor's AudioFormat"); + return; + } + Preconditions.checkNotNull(audioInputStream); + + int numSamples = audioData.limit() / BYTES_PER_MONO_SAMPLE / numAudioChannels; + Packet audioPacket = packetCreator.createAudioPacket(audioData, numAudioChannels, numSamples); + try { + // addConsumablePacketToInputStream allows the graph to take exclusive ownership of the + // packet, which may allow for more memory optimizations. + mediapipeGraph.addConsumablePacketToInputStream( + audioInputStream, audioPacket, timestampMicros); + } catch (MediaPipeException e) { + Log.e(TAG, "Mediapipe error: ", e); + } + audioPacket.release(); + } + + public void addAudioConsumer(AudioDataConsumer consumer) { + synchronized (this) { + List newConsumers = new ArrayList<>(audioConsumers); + newConsumers.add(consumer); + audioConsumers = newConsumers; + } + } + + public boolean removeAudioConsumer(AudioDataConsumer consumer) { + boolean existed; + synchronized (this) { + List newConsumers = new ArrayList<>(audioConsumers); + existed = newConsumers.remove(consumer); + audioConsumers = newConsumers; + } + return existed; + } } diff --git a/mediapipe/java/com/google/mediapipe/components/MicrophoneHelper.java b/mediapipe/java/com/google/mediapipe/components/MicrophoneHelper.java index 0c7da3627..4775bd7ee 100644 --- a/mediapipe/java/com/google/mediapipe/components/MicrophoneHelper.java +++ b/mediapipe/java/com/google/mediapipe/components/MicrophoneHelper.java @@ -22,6 +22,7 @@ import android.os.Build.VERSION; import android.os.Build.VERSION_CODES; import android.util.Log; import com.google.common.base.Preconditions; +import java.io.IOException; import java.nio.ByteBuffer; /** Provides access to audio data from a microphone. */ @@ -31,25 +32,24 @@ public class MicrophoneHelper implements AudioDataProducer { private static final int AUDIO_ENCODING = AudioFormat.ENCODING_PCM_16BIT; private static final int AUDIO_SOURCE = AudioSource.MIC; - // A small constant valued multiplier for setting bufferSize. This is useful + // A small constant valued multiplier for setting audioRecordBufferSize. This is useful // to reduce buffer overflows when a lot of data needs to be read at a high // sample rate from the audio stream. Note that it is desirable to keep this // multiplier small, because very large buffer sizes can slow down blocking // calls to AudioRecord.read(...) when the sample rate is low for instance. private static final int BUFFER_SIZE_MULTIPLIER = 2; - // A small constant value to decide the number of seconds of audio data that - // will be read in a single AudioRecord.read(...) call when - // AudioRecord.minBufferSize(...) is unavailable. Smaller values for this - // constant favor faster blocking calls to AudioRecord.read(...). - private static final int MAX_READ_INTERVAL_SEC = 1; + // Number of microseconds of data to be read before sending audio data to a client. Smaller values + // for this constant favor faster blocking calls to readAudioPacket(...). + private static final long DEFAULT_READ_INTERVAL_MICROS = 10_000; // This class uses AudioFormat.ENCODING_PCM_16BIT, i.e. 16 bits per sample. private static final int BYTES_PER_SAMPLE = 2; private static final long UNINITIALIZED_TIMESTAMP = Long.MIN_VALUE; - private static final long NANOS_PER_MICROS = 1000; - private static final long MICROS_PER_SECOND = 1000000; + private static final long NANOS_PER_MICROS = 1_000; + private static final long MICROS_PER_SECOND = 1_000_000; + private static final long NANOS_PER_SECOND = 1_000_000_000; // Number of audio samples recorded per second. private final int sampleRateInHz; @@ -59,13 +59,23 @@ public class MicrophoneHelper implements AudioDataProducer { // Bytes per audio frame. A frame is defined as a multi-channel audio sample. Possible values are // 2 bytes for 1 channel, or 4 bytes for 2 channel audio. private final int bytesPerFrame; - // Data storage allocated to record audio samples in a single function call to AudioRecord.read(). - private final int bufferSize; + // The minimum buffer size required by AudioRecord. + private final int minBufferSize; + + // Number of microseconds of data to be read before sending audio data to a client. This is + // initialized to DEFAULT_READ_INTERVAL_MICROS but can be changed by the client before calling + // startMicrophone(...). + private long readIntervalMicros = DEFAULT_READ_INTERVAL_MICROS; + // Data storage allocated to internal buffer used by AudioRecord for reading audio data. + private int audioRecordBufferSize; + // Size of audio packet sent to an AudioConsumer with every call to consumer.onNewAudioData(...). + private int audioPacketBufferSize; // Initial timestamp base. Can be set by the client so that all timestamps calculated using the - // number of samples read per AudioRecord.read() function call start from this timestamp. If it - // is not set by the client, then every startMicrophone(...) call marks a value for it. - private long initialTimestampMicros = UNINITIALIZED_TIMESTAMP; + // number of samples read per AudioRecord.read() function call start from this timestamp. + private long initialTimestampNanos = UNINITIALIZED_TIMESTAMP; + // The timestamp marked when startMicrophone(...) call starts recording. + private long startRecordingTimestampNanos = UNINITIALIZED_TIMESTAMP; // AudioRecord is used to setup a way to record data from the audio source. See // https://developer.android.com/reference/android/media/AudioRecord.htm for details. @@ -83,6 +93,8 @@ public class MicrophoneHelper implements AudioDataProducer { // the data read after recording stopped. private AudioDataConsumer consumer; + // TODO: Add a constructor that takes an AudioFormat. + /** * MicrophoneHelper class constructor. Arugments: * @@ -100,26 +112,40 @@ public class MicrophoneHelper implements AudioDataProducer { bytesPerFrame = BYTES_PER_SAMPLE * numChannels; // The minimum buffer size required by AudioRecord. - final int minBufferSize = + minBufferSize = AudioRecord.getMinBufferSize( sampleRateInHz, channelConfig, /*audioFormat=*/ AUDIO_ENCODING); - // Set bufferSize. If the minimum buffer size permitted by the hardware is - // unavailable, use the the sampleRateInHz value as the number of bytes. - // This is arguably better than another arbitrary constant because a higher - // value of sampleRateInHz implies the need for reading large chunks of data - // from the audio stream in each AudioRecord.read(...) call. - if (minBufferSize == AudioRecord.ERROR || minBufferSize == AudioRecord.ERROR_BAD_VALUE) { - Log.e(TAG, "AudioRecord minBufferSize unavailable."); - bufferSize = sampleRateInHz * MAX_READ_INTERVAL_SEC * bytesPerFrame * BUFFER_SIZE_MULTIPLIER; - } else { - bufferSize = minBufferSize * BUFFER_SIZE_MULTIPLIER; - } + updateBufferSizes(readIntervalMicros); + } + + /** + * Sets readIntervalMicros. This should be set before calling {@link #startMicrophone()}. + * + * @param micros the number of microseconds of data MicrophoneHelper should read before calling + * consumer.onNewAudioData(...). + */ + public void setReadIntervalMicros(long micros) { + readIntervalMicros = micros; + updateBufferSizes(readIntervalMicros); + } + + /** + * Updates audioPacketBufferSize and audioRecordBufferSize. + * + * @param micros The interval size in microseconds of the amount of audio data to be read. + */ + private void updateBufferSizes(long micros) { + audioPacketBufferSize = + (int) Math.ceil(1.0 * bytesPerFrame * sampleRateInHz * micros / MICROS_PER_SECOND); + // The size of the internal buffer should be greater than the size of the audio packet read + // and sent to the AudioDataConsumer so that AudioRecord. + audioRecordBufferSize = Math.max(audioPacketBufferSize, minBufferSize) * BUFFER_SIZE_MULTIPLIER; } private void setupAudioRecord() { - Log.d(TAG, "AudioRecord(" + sampleRateInHz + ", " + bufferSize + ")"); + Log.d(TAG, "AudioRecord(" + sampleRateInHz + ", " + audioRecordBufferSize + ")"); audioFormat = new AudioFormat.Builder() .setEncoding(AUDIO_ENCODING) @@ -130,7 +156,7 @@ public class MicrophoneHelper implements AudioDataProducer { new AudioRecord.Builder() .setAudioSource(AUDIO_SOURCE) .setAudioFormat(audioFormat) - .setBufferSizeInBytes(bufferSize) + .setBufferSizeInBytes(audioRecordBufferSize) .build(); if (audioRecord.getState() != AudioRecord.STATE_INITIALIZED) { audioRecord.release(); @@ -143,6 +169,8 @@ public class MicrophoneHelper implements AudioDataProducer { () -> { android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO); + startRecordingTimestampNanos = System.nanoTime(); + long timestampOffsetNanos = 0; // The total number of frames read from multiple calls to AudioRecord.read() in this // recording thread. int totalNumFramesRead = 0; @@ -150,67 +178,106 @@ public class MicrophoneHelper implements AudioDataProducer { if (audioRecord == null) { break; } + // TODO: Fix audio data cloning. - ByteBuffer audioData = ByteBuffer.allocateDirect(bufferSize); - final int numBytesRead = audioRecord.read(audioData, /*sizeInBytes=*/ bufferSize); - // Get the timestamp of the first audio frame in the latest read call. - long timestampMicros = getTimestampMicros(totalNumFramesRead); - if (numBytesRead <= 0) { - if (numBytesRead == AudioRecord.ERROR_INVALID_OPERATION) { - Log.e(TAG, "ERROR_INVALID_OPERATION"); - } else if (numBytesRead == AudioRecord.ERROR_BAD_VALUE) { - Log.e(TAG, "ERROR_BAD_VALUE"); - } + ByteBuffer audioData = ByteBuffer.allocateDirect(audioPacketBufferSize); + try { + readAudioPacket(audioData); + } catch (IOException ioException) { + // Reading audio data failed in this loop iteration, continue to next iteration if + // recording is still enabled. + Log.e(TAG, ioException.getMessage()); continue; } + // Get the timestamp of the first audio frame which marks the beginning of reading + // the audio data for the entire audioPacketBufferSize bytes to be read before + // sending them to the AudioDataConsumer. We deliberately do this _after_ the read + // call, even though we are getting the timestamp corresponding to the beginning of + // the read chunk. This is because experimentation has shown that calling + // AudioRecord.getTimestamp(...) before the first AudioRecord.read(...) call fails. + long timestampNanos = getTimestampNanos(totalNumFramesRead); + if (totalNumFramesRead == 0 && initialTimestampNanos != UNINITIALIZED_TIMESTAMP) { + timestampOffsetNanos = timestampNanos - initialTimestampNanos; + } + long timestampMicros = (timestampNanos - timestampOffsetNanos) / NANOS_PER_MICROS; + + // It is expected that audioRecord.read() will read full samples and therefore + // number of bytes read is expected to be a multiple of bytesPerFrame. + int numFramesRead = audioData.limit() / bytesPerFrame; + totalNumFramesRead += numFramesRead; + // Confirm that the consumer is still interested in receiving audio data and // stopMicrophone() wasn't called. If the consumer called stopMicrophone(), discard // the data read in the latest AudioRecord.read(...) function call. if (recording && consumer != null) { consumer.onNewAudioData(audioData, timestampMicros, audioFormat); } - - // It is expected that audioRecord.read() will read full samples and therefore - // numBytesRead is expected to be a multiple of bytesPerFrame. - int numFramesRead = numBytesRead / bytesPerFrame; - totalNumFramesRead += numFramesRead; } }, "microphoneHelperRecordingThread"); } - // If AudioRecord.getTimestamp() is available and returns without error, this function returns the - // timestamp using AudioRecord.getTimestamp(). If the function is unavailable, it returns a - // fallback timestamp calculated using number of samples read so far. - // Use numFramesRead to be the frame count before the latest AudioRecord.read(...) call to get - // the timestamp of the first audio frame in the latest AudioRecord.read(...) call. - private long getTimestampMicros(long numFramesRead) { - AudioTimestamp audioTimestamp = getAudioRecordTimestamp(); - if (audioTimestamp == null) { - if (numFramesRead == 0) { - initialTimestampMicros = markInitialTimestamp(); + /** + * Reads audio data into a packet. + * + * @param audioPacket the ByteBuffer in which audio data is read. + * @throws java.io.IOException when AudioRecord.read(...) fails. + */ + private void readAudioPacket(ByteBuffer audioPacket) throws IOException { + int totalNumBytesRead = 0; + while (totalNumBytesRead < audioPacket.capacity()) { + int bytesRemaining = audioPacket.capacity() - totalNumBytesRead; + int numBytesRead = 0; + // Blocking reads are available in only API Level 23 and above. + // https://developer.android.com/reference/android/media/AudioRecord.html#read(java.nio.ByteBuffer,%20int,%20int). + if (VERSION.SDK_INT >= VERSION_CODES.M) { + numBytesRead = + audioRecord.read( + audioPacket, /*sizeInBytes=*/ bytesRemaining, AudioRecord.READ_BLOCKING); + } else { + numBytesRead = audioRecord.read(audioPacket, /*sizeInBytes=*/ bytesRemaining); + } + if (numBytesRead <= 0) { + String error = "ERROR"; + if (numBytesRead == AudioRecord.ERROR_INVALID_OPERATION) { + error = "ERROR_INVALID_OPERATION"; + } else if (numBytesRead == AudioRecord.ERROR_BAD_VALUE) { + error = "ERROR_BAD_VALUE"; + } else if (numBytesRead == AudioRecord.ERROR_DEAD_OBJECT) { + error = "ERROR_DEAD_OBJECT"; + } + throw new IOException("AudioRecord.read(...) failed due to " + error); } - // If AudioRecord.getTimestamp() is unavailable, calculate the timestamp using the - // number of frames read in the call to AudioRecord.read(). - return initialTimestampMicros + numFramesRead * getMicrosPerSample(); + // Advance the position of the ByteBuffer for the next read. + totalNumBytesRead += numBytesRead; + audioPacket.position(totalNumBytesRead); } - // If audioTimestamp.framePosition is ahead of numFramesRead so far, then the offset is - // negative. - long frameOffset = numFramesRead - audioTimestamp.framePosition; - long audioTsMicros = audioTimestamp.nanoTime / NANOS_PER_MICROS; - return audioTsMicros + frameOffset * getMicrosPerSample(); + // Reset the position of the ByteBuffer for consumption. + audioPacket.position(0); } - private long markInitialTimestamp() { - return initialTimestampMicros != UNINITIALIZED_TIMESTAMP - ? initialTimestampMicros - : System.nanoTime() / NANOS_PER_MICROS; - } - - private long getMicrosPerSample() { - return MICROS_PER_SECOND / sampleRateInHz; + /** + * If AudioRecord.getTimestamp() is available and returns without error, this function returns the + * timestamp using AudioRecord.getTimestamp(). If the function is unavailable, it returns a + * fallback timestamp calculated using number of samples read so far and the initial + * System.nanoTime(). Use framePosition to be the frame count before the latest + * AudioRecord.read(...) call to get the timestamp of the first audio frame in the latest + * AudioRecord.read(...) call. + */ + private long getTimestampNanos(long framePosition) { + long referenceFrame = 0; + long referenceTimestamp = startRecordingTimestampNanos; + AudioTimestamp audioTimestamp = getAudioRecordTimestamp(); + if (audioTimestamp != null) { + referenceFrame = audioTimestamp.framePosition; + referenceTimestamp = audioTimestamp.nanoTime; + } + // Assuming the first frame is read at 0 ns, this timestamp can be at most + // (2**63-1) / 48000 nanoseconds for a sampleRateInHz = 48kHz. + return referenceTimestamp + + (framePosition - referenceFrame) * NANOS_PER_SECOND / sampleRateInHz; } private AudioTimestamp getAudioRecordTimestamp() { @@ -229,21 +296,36 @@ public class MicrophoneHelper implements AudioDataProducer { return null; } - // Returns the buffer size read by this class per AudioRecord.read() call. - public int getBufferSize() { - return bufferSize; + /* + * Returns the buffer size of the internal buffer used by AudioRecord. + */ + public int getAudioRecordBufferSize() { + return audioRecordBufferSize; + } + + /* + * Returns the packet size of the audio packet that clients will receive. + */ + public int getAudioPacketBufferSize() { + return audioPacketBufferSize; } /** - * Overrides the use of system time as the source of timestamps for audio packets. Not - * recommended. Provided to maintain compatibility with existing usage by CameraRecorder. + * Sets initialTimestampNanos. Overrides the use of system time as the first timestamp for audio + * packets. Not recommended. Provided to maintain compatibility with existing usage by + * CameraRecorder. + * + * @param initialTimestampNanos The timestamp to be used by the first audio packet read by this + * class when {@link #startMicrophone()} is called. */ - public void setInitialTimestampMicros(long initialTimestampMicros) { - this.initialTimestampMicros = initialTimestampMicros; + public void setInitialTimestampNanos(long initialTimestampNanos) { + this.initialTimestampNanos = initialTimestampNanos; } - // This method sets up a new AudioRecord object for reading audio data from the microphone. It - // can be called multiple times to restart the recording if necessary. + /** + * This method sets up a new AudioRecord object for reading audio data from the microphone. It can + * be called multiple times to restart the recording if necessary. + */ public void startMicrophone() { if (recording) { return; @@ -263,14 +345,18 @@ public class MicrophoneHelper implements AudioDataProducer { Log.d(TAG, "AudioRecord is recording audio."); } - // Stops the AudioRecord object from reading data from the microphone and releases it. + /* + * Stops the AudioRecord object from reading data from the microphone and releases it. + */ public void stopMicrophone() { stopMicrophoneWithoutCleanup(); cleanup(); Log.d(TAG, "AudioRecord stopped recording audio."); } - // Stops the AudioRecord object from reading data from the microphone. + /* + * Stops the AudioRecord object from reading data from the microphone. + */ public void stopMicrophoneWithoutCleanup() { Preconditions.checkNotNull(audioRecord); if (!recording) { @@ -292,7 +378,9 @@ public class MicrophoneHelper implements AudioDataProducer { } } - // Releases the AudioRecord object when there is no ongoing recording. + /* + * Releases the AudioRecord object when there is no ongoing recording. + */ public void cleanup() { Preconditions.checkNotNull(audioRecord); if (recording) { diff --git a/mediapipe/java/com/google/mediapipe/framework/BUILD b/mediapipe/java/com/google/mediapipe/framework/BUILD index 5e582ebff..69520ec97 100644 --- a/mediapipe/java/com/google/mediapipe/framework/BUILD +++ b/mediapipe/java/com/google/mediapipe/framework/BUILD @@ -16,13 +16,14 @@ licenses(["notice"]) # Apache 2.0 # MediaPipe Android framework. -exports_files(["proguard.pgcfg"]) +exports_files([ + "proguard.pgcfg", + "proguard_allowobfuscation.pgcfg", +]) android_library( name = "android_framework", - proguard_specs = [ - ":proguard.pgcfg", - ], + proguard_specs = [":proguard.pgcfg"], visibility = ["//visibility:public"], exports = [ ":android_core", @@ -33,6 +34,14 @@ android_library( # TODO: Rename android_framework_no_mff. android_library( name = "android_framework_no_mff", + proguard_specs = [":proguard.pgcfg"], + exports = [ + ":android_framework_no_proguard", + ], +) + +android_library( + name = "android_framework_no_proguard", srcs = glob( ["Android*.java"], ) + [ @@ -40,8 +49,8 @@ android_library( "AssetCacheDbHelper.java", "MediaPipeRunner.java", ], - proguard_specs = [ - ":proguard.pgcfg", + visibility = [ + "//mediapipe/java/com/google/mediapipe:__subpackages__", ], exports = [ ":android_core", diff --git a/mediapipe/java/com/google/mediapipe/framework/PacketCreator.java b/mediapipe/java/com/google/mediapipe/framework/PacketCreator.java index f078150b3..d1e8089ce 100644 --- a/mediapipe/java/com/google/mediapipe/framework/PacketCreator.java +++ b/mediapipe/java/com/google/mediapipe/framework/PacketCreator.java @@ -16,6 +16,7 @@ package com.google.mediapipe.framework; import com.google.protobuf.MessageLite; import java.nio.ByteBuffer; +import java.nio.FloatBuffer; // TODO: use Preconditions in this file. /** @@ -152,6 +153,19 @@ public class PacketCreator { nativeCreateRgbaImageFrame(mediapipeGraph.getNativeHandle(), buffer, width, height)); } + /** + * Creates a 1 channel float ImageFrame packet. + * + *

Use {@link ByteBuffer#allocateDirect} when allocating the buffer. + */ + public Packet createFloatImageFrame(FloatBuffer buffer, int width, int height) { + if (buffer.capacity() != width * height * 4) { + throw new RuntimeException("buffer doesn't have the correct size."); + } + return Packet.create( + nativeCreateFloatImageFrame(mediapipeGraph.getNativeHandle(), buffer, width, height)); + } + public Packet createInt16(short value) { return Packet.create(nativeCreateInt16(mediapipeGraph.getNativeHandle(), value)); } @@ -327,6 +341,8 @@ public class PacketCreator { private native long nativeCreateRgbaImageFrame( long context, ByteBuffer buffer, int width, int height); + private native long nativeCreateFloatImageFrame( + long context, FloatBuffer buffer, int width, int height); private native long nativeCreateInt16(long context, short value); private native long nativeCreateInt32(long context, int value); private native long nativeCreateInt64(long context, long value); diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/BUILD b/mediapipe/java/com/google/mediapipe/framework/jni/BUILD index 0e6e71815..3c5793253 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/BUILD +++ b/mediapipe/java/com/google/mediapipe/framework/jni/BUILD @@ -82,6 +82,7 @@ cc_library( }), visibility = ["//visibility:public"], deps = [ + ":class_registry", ":jni_util", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_profile_cc_proto", @@ -90,6 +91,7 @@ cc_library( "//mediapipe/framework/formats:matrix_data_cc_proto", "//mediapipe/framework/formats:time_series_header_cc_proto", "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/synchronization", "@eigen_archive//:eigen", "//mediapipe/framework:camera_intrinsics", @@ -132,6 +134,7 @@ cc_library( srcs = (["jni_util.cc"]), hdrs = (["jni_util.h"]), deps = [ + ":class_registry", "@com_google_absl//absl/synchronization", "//mediapipe/framework/port:logging", "//mediapipe/framework/port:status", @@ -142,3 +145,37 @@ cc_library( ], }), ) + +cc_library( + name = "class_registry", + srcs = (["class_registry.cc"]), + hdrs = (["class_registry.h"]), + deps = [ + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/container:node_hash_map", + ] + select({ + "//conditions:default": [ + ], + "//mediapipe:android": [ + ], + }), +) + +cc_library( + name = "register_natives", + srcs = (["register_natives.cc"]), + hdrs = (["register_natives.h"]), + deps = [ + ":class_registry", + ":mediapipe_framework_jni", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/container:node_hash_map", + ] + select({ + "//conditions:default": [ + ], + "//mediapipe:android": [ + ], + }), +) diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/class_registry.cc b/mediapipe/java/com/google/mediapipe/framework/jni/class_registry.cc new file mode 100644 index 000000000..d917b8af2 --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/framework/jni/class_registry.cc @@ -0,0 +1,52 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h" + +#include "absl/strings/str_format.h" + +namespace mediapipe { +namespace android { + +ClassRegistry::ClassRegistry() {} + +ClassRegistry& ClassRegistry::GetInstance() { + static ClassRegistry* instance_ = new ClassRegistry(); + return *instance_; +} + +void ClassRegistry::InstallRenamingMap( + absl::node_hash_map renaming_map) { + renaming_map_ = renaming_map; +} + +std::string ClassRegistry::GetClassName(std::string cls) { + auto match = renaming_map_.find(cls); + if (match != renaming_map_.end()) { + return match->second; + } + return cls; +} + +std::string ClassRegistry::GetMethodName(std::string cls, std::string method) { + std::string key = absl::StrFormat("%s#%s", cls, method); + auto match = renaming_map_.find(key); + if (match != renaming_map_.end()) { + return match->second; + } + return method; +} + +} // namespace android +} // namespace mediapipe diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h b/mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h new file mode 100644 index 000000000..998c070ee --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h @@ -0,0 +1,67 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef JAVA_COM_GOOGLE_MEDIAPIPE_FRAMEWORK_JNI_CLASS_REGISTRY_H_ +#define JAVA_COM_GOOGLE_MEDIAPIPE_FRAMEWORK_JNI_CLASS_REGISTRY_H_ + +#include + +#include + +#include "absl/container/node_hash_map.h" + +namespace mediapipe { +namespace android { + +// ClassRegistry maintains the correct names of Java classes and methods and +// should be queried before any calls to FindClass() or GetMethodID(). +class ClassRegistry { + public: + static ClassRegistry& GetInstance(); + void InstallRenamingMap( + absl::node_hash_map renaming_map); + std::string GetClassName(std::string cls); + std::string GetMethodName(std::string cls, std::string method); + + // TODO: Just have the prefix instead of all these constants. + static constexpr char const* kAndroidAssetUtilClassName = + "com/google/mediapipe/framework/AndroidAssetUtil"; + static constexpr char const* kAndroidPacketCreatorClassName = + "com/google/mediapipe/framework/AndroidPacketCreator"; + static constexpr char const* kCompatClassName = + "com/google/mediapipe/framework/Compat"; + static constexpr char const* kGraphClassName = + "com/google/mediapipe/framework/Graph"; + static constexpr char const* kPacketClassName = + "com/google/mediapipe/framework/Packet"; + static constexpr char const* kMediaPipeExceptionClassName = + "com/google/mediapipe/framework/MediaPipeException"; + static constexpr char const* kPacketCallbackClassName = + "com/google/mediapipe/framework/PacketCallback"; + static constexpr char const* kPacketCreatorClassName = + "com/google/mediapipe/framework/PacketCreator"; + static constexpr char const* kPacketGetterClassName = + "com/google/mediapipe/framework/PacketGetter"; + static constexpr char const* kPacketWithHeaderCallbackClassName = + "com/google/mediapipe/framework/PacketWithHeaderCallback"; + + private: + ClassRegistry(); + absl::node_hash_map renaming_map_; +}; + +} // namespace android +} // namespace mediapipe + +#endif // JAVA_COM_GOOGLE_MEDIAPIPE_FRAMEWORK_JNI_CLASS_REGISTRY_H_ diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc b/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc index 18e8c7c5c..a65de2992 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/graph.cc @@ -31,6 +31,7 @@ #include "mediapipe/framework/tool/name_util.h" #include "mediapipe/gpu/gpu_shared_data_internal.h" #include "mediapipe/gpu/graph_support.h" +#include "mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/jni_util.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/packet_context_jni.h" #ifdef __ANDROID__ @@ -274,10 +275,16 @@ CalculatorGraphConfig Graph::GetCalculatorGraphConfig() { void Graph::CallbackToJava(JNIEnv* env, jobject java_callback_obj, const Packet& packet) { jclass callback_cls = env->GetObjectClass(java_callback_obj); - jmethodID processMethod = env->GetMethodID( - callback_cls, "process", - absl::StrFormat("(L%s;)V", std::string(Graph::kJavaPacketClassName)) - .c_str()); + + auto& class_registry = mediapipe::android::ClassRegistry::GetInstance(); + std::string packet_class_name = class_registry.GetClassName( + mediapipe::android::ClassRegistry::kPacketClassName); + std::string process_method_name = class_registry.GetMethodName( + mediapipe::android::ClassRegistry::kPacketCallbackClassName, "process"); + + jmethodID processMethod = + env->GetMethodID(callback_cls, process_method_name.c_str(), + absl::StrFormat("(L%s;)V", packet_class_name).c_str()); int64_t packet_handle = WrapPacketIntoContext(packet); // Creates a Java Packet. @@ -296,10 +303,17 @@ void Graph::CallbackToJava(JNIEnv* env, jobject java_callback_obj, void Graph::CallbackToJava(JNIEnv* env, jobject java_callback_obj, const Packet& packet, const Packet& header_packet) { jclass callback_cls = env->GetObjectClass(java_callback_obj); + + auto& class_registry = mediapipe::android::ClassRegistry::GetInstance(); + std::string packet_class_name = class_registry.GetClassName( + mediapipe::android::ClassRegistry::kPacketClassName); + std::string process_method_name = class_registry.GetMethodName( + mediapipe::android::ClassRegistry::kPacketWithHeaderCallbackClassName, + "process"); + jmethodID processMethod = env->GetMethodID( - callback_cls, "process", - absl::StrFormat("(L%s;L%s;)V", std::string(Graph::kJavaPacketClassName), - std::string(Graph::kJavaPacketClassName)) + callback_cls, process_method_name.c_str(), + absl::StrFormat("(L%s;L%s;)V", packet_class_name, packet_class_name) .c_str()); int64_t packet_handle = WrapPacketIntoContext(packet); @@ -321,8 +335,10 @@ void Graph::CallbackToJava(JNIEnv* env, jobject java_callback_obj, void Graph::SetPacketJavaClass(JNIEnv* env) { if (global_java_packet_cls_ == nullptr) { - jclass packet_cls = - env->FindClass(mediapipe::android::Graph::kJavaPacketClassName); + auto& class_registry = ClassRegistry::GetInstance(); + std::string packet_class_name = class_registry.GetClassName( + mediapipe::android::ClassRegistry::kPacketClassName); + jclass packet_cls = env->FindClass(packet_class_name.c_str()); global_java_packet_cls_ = reinterpret_cast(env->NewGlobalRef(packet_cls)); } diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/graph.h b/mediapipe/java/com/google/mediapipe/framework/jni/graph.h index 02be3709e..6bb310b5f 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/graph.h +++ b/mediapipe/java/com/google/mediapipe/framework/jni/graph.h @@ -43,10 +43,6 @@ class PacketWithContext; // so that we can clean up or query later. class Graph { public: - // The Packet java class name. - static constexpr char const* kJavaPacketClassName = - "com/google/mediapipe/framework/Packet"; - Graph(); Graph(const Graph&) = delete; Graph& operator=(const Graph&) = delete; diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/graph_jni.cc b/mediapipe/java/com/google/mediapipe/framework/jni/graph_jni.cc index 6df03f2a8..6060e4ea1 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/graph_jni.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/graph_jni.cc @@ -19,6 +19,7 @@ #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/logging.h" +#include "mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/graph.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/jni_util.h" diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/jni_util.cc b/mediapipe/java/com/google/mediapipe/framework/jni/jni_util.cc index 57635da40..079767512 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/jni_util.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/jni_util.cc @@ -18,6 +18,7 @@ #include "absl/synchronization/mutex.h" #include "mediapipe/framework/port/logging.h" +#include "mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h" namespace { @@ -111,9 +112,16 @@ std::string JStringToStdString(JNIEnv* env, jstring jstr) { } jthrowable CreateMediaPipeException(JNIEnv* env, mediapipe::Status status) { - jclass status_cls = - env->FindClass("com/google/mediapipe/framework/MediaPipeException"); - jmethodID status_ctr = env->GetMethodID(status_cls, "", "(I[B)V"); + auto& class_registry = mediapipe::android::ClassRegistry::GetInstance(); + std::string mpe_class_name = class_registry.GetClassName( + mediapipe::android::ClassRegistry::kMediaPipeExceptionClassName); + std::string mpe_constructor_name = class_registry.GetMethodName( + mediapipe::android::ClassRegistry::kMediaPipeExceptionClassName, + ""); + + jclass status_cls = env->FindClass(mpe_class_name.c_str()); + jmethodID status_ctr = + env->GetMethodID(status_cls, mpe_constructor_name.c_str(), "(I[B)V"); int length = status.message().length(); jbyteArray message_bytes = env->NewByteArray(length); env->SetByteArrayRegion(message_bytes, 0, length, diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/packet_context_jni.cc b/mediapipe/java/com/google/mediapipe/framework/jni/packet_context_jni.cc index 60ca862d2..12a2a92c1 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/packet_context_jni.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/packet_context_jni.cc @@ -15,6 +15,7 @@ #include "mediapipe/java/com/google/mediapipe/framework/jni/packet_context_jni.h" #include "absl/strings/str_format.h" +#include "mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h" #include "mediapipe/java/com/google/mediapipe/framework/jni/graph.h" // Releases a native mediapipe packet. @@ -44,11 +45,15 @@ JNIEXPORT jlong JNICALL PACKET_METHOD(nativeCopyPacket)(JNIEnv* env, } jobject CreateJavaPacket(JNIEnv* env, jclass packet_cls, jlong packet) { + auto& class_registry = mediapipe::android::ClassRegistry::GetInstance(); + + std::string packet_class_name = class_registry.GetClassName( + mediapipe::android::ClassRegistry::kPacketClassName); + std::string create_method_name = class_registry.GetMethodName( + mediapipe::android::ClassRegistry::kPacketClassName, "create"); + + std::string signature = absl::StrFormat("(J)L%s;", packet_class_name); jmethodID createMethod = env->GetStaticMethodID( - packet_cls, "create", - absl::StrFormat( - "(J)L%s;", - std::string(mediapipe::android::Graph::kJavaPacketClassName)) - .c_str()); + packet_cls, create_method_name.c_str(), signature.c_str()); return env->CallStaticObjectMethod(packet_cls, createMethod, packet); } diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.cc b/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.cc index 047803e13..45a4955b3 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.cc @@ -136,6 +136,27 @@ JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateGrayscaleImage)( return CreatePacketWithContext(context, packet); } +JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateFloatImageFrame)( + JNIEnv* env, jobject thiz, jlong context, jobject byte_buffer, jint width, + jint height) { + const void* data = env->GetDirectBufferAddress(byte_buffer); + auto image_frame = absl::make_unique<::mediapipe::ImageFrame>( + mediapipe::ImageFormat::VEC32F1, width, height, + ::mediapipe::ImageFrame::kGlDefaultAlignmentBoundary); + int64_t buffer_size = env->GetDirectBufferCapacity(byte_buffer); + if (buffer_size != image_frame->PixelDataSize()) { + LOG(ERROR) << "Please check the input buffer size."; + LOG(ERROR) << "Buffer size: " << buffer_size + << ", Buffer size needed: " << image_frame->PixelDataSize() + << ", Image width: " << width; + return 0L; + } + std::memcpy(image_frame->MutablePixelData(), data, + image_frame->PixelDataSize()); + mediapipe::Packet packet = mediapipe::Adopt(image_frame.release()); + return CreatePacketWithContext(context, packet); +} + JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateRgbaImageFrame)( JNIEnv* env, jobject thiz, jlong context, jobject byte_buffer, jint width, jint height) { diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.h b/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.h index d7dfbd42b..9d0e73165 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.h +++ b/mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.h @@ -31,6 +31,10 @@ JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateRgbImage)( JNIEnv* env, jobject thiz, jlong context, jobject byte_buffer, jint width, jint height); +JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateFloatImageFrame)( + JNIEnv* env, jobject thiz, jlong context, jobject byte_buffer, jint width, + jint height); + JNIEXPORT jlong JNICALL PACKET_CREATOR_METHOD(nativeCreateRgbaImageFrame)( JNIEnv* env, jobject thiz, jlong context, jobject byte_buffer, jint width, jint height); diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/packet_getter_jni.cc b/mediapipe/java/com/google/mediapipe/framework/jni/packet_getter_jni.cc index 1cab1aca7..e9de00e2b 100644 --- a/mediapipe/java/com/google/mediapipe/framework/jni/packet_getter_jni.cc +++ b/mediapipe/java/com/google/mediapipe/framework/jni/packet_getter_jni.cc @@ -242,7 +242,6 @@ JNIEXPORT jboolean JNICALL PACKET_GETTER_METHOD(nativeGetImageData)( JNIEnv* env, jobject thiz, jlong packet, jobject byte_buffer) { const ::mediapipe::ImageFrame& image = GetFromNativeHandle<::mediapipe::ImageFrame>(packet); - uint8* data = static_cast(env->GetDirectBufferAddress(byte_buffer)); int64_t buffer_size = env->GetDirectBufferCapacity(byte_buffer); @@ -257,7 +256,29 @@ JNIEXPORT jboolean JNICALL PACKET_GETTER_METHOD(nativeGetImageData)( return false; } - image.CopyToBuffer(data, expected_buffer_size); + switch (image.ByteDepth()) { + case 1: { + uint8* data = + static_cast(env->GetDirectBufferAddress(byte_buffer)); + image.CopyToBuffer(data, expected_buffer_size); + break; + } + case 2: { + uint16* data = + static_cast(env->GetDirectBufferAddress(byte_buffer)); + image.CopyToBuffer(data, expected_buffer_size); + break; + } + case 4: { + float* data = + static_cast(env->GetDirectBufferAddress(byte_buffer)); + image.CopyToBuffer(data, expected_buffer_size); + break; + } + default: { + return false; + } + } return true; } diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/register_natives.cc b/mediapipe/java/com/google/mediapipe/framework/jni/register_natives.cc new file mode 100644 index 000000000..f584ade1a --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/framework/jni/register_natives.cc @@ -0,0 +1,233 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/java/com/google/mediapipe/framework/jni/register_natives.h" + +#include "absl/strings/str_format.h" +#include "mediapipe/java/com/google/mediapipe/framework/jni/class_registry.h" + +#if defined(__ANDROID__) +#include "mediapipe/java/com/google/mediapipe/framework/jni/android_asset_util_jni.h" +#include "mediapipe/java/com/google/mediapipe/framework/jni/android_packet_creator_jni.h" +#endif +#include "mediapipe/java/com/google/mediapipe/framework/jni/compat_jni.h" +#include "mediapipe/java/com/google/mediapipe/framework/jni/graph_jni.h" +#include "mediapipe/java/com/google/mediapipe/framework/jni/packet_context_jni.h" +#include "mediapipe/java/com/google/mediapipe/framework/jni/packet_creator_jni.h" +#include "mediapipe/java/com/google/mediapipe/framework/jni/packet_getter_jni.h" + +namespace mediapipe { +namespace android { +namespace registration { +namespace { + +// TODO: Ideally all these methods would live in their own JNI files. +// We should have a JniOnLoadRegistry which collects a series of function ptrs +// to call when JNI_OnLoad is called. Each module would add its own hook with a +// static initializer. + +struct JNINativeMethodStrings { + std::string name; + std::string signature; + void *fnPtr; +}; + +void AddJNINativeMethod(std::vector *methods, + std::string cls, std::string method, + std::string signature, void *fn) { + auto &class_registry = mediapipe::android::ClassRegistry::GetInstance(); + std::string method_name = class_registry.GetMethodName(cls, method); + if (method_name != method) { + JNINativeMethodStrings jniNativeMethod{method_name, signature, fn}; + methods->push_back(jniNativeMethod); + } +} + +void RegisterNativesVector(JNIEnv *env, jclass cls, + const std::vector &methods) { + JNINativeMethod *methods_array = new JNINativeMethod[methods.size()]; + for (int i = 0; i < methods.size(); i++) { + JNINativeMethod jniNativeMethod{ + const_cast(methods[i].name.c_str()), + const_cast(methods[i].signature.c_str()), methods[i].fnPtr}; + methods_array[i] = jniNativeMethod; + } + env->RegisterNatives(cls, methods_array, methods.size()); + delete[] methods_array; +} + +void RegisterGraphNatives(JNIEnv *env) { + auto &class_registry = mediapipe::android::ClassRegistry::GetInstance(); + std::string graph(mediapipe::android::ClassRegistry::kGraphClassName); + std::string graph_name = class_registry.GetClassName(graph); + jclass graph_class = env->FindClass(graph_name.c_str()); + + std::vector graph_methods; + AddJNINativeMethod(&graph_methods, graph, "nativeCreateGraph", "()J", + (void *)&GRAPH_METHOD(nativeCreateGraph)); + AddJNINativeMethod(&graph_methods, graph, "nativeLoadBinaryGraph", + "(JLjava/lang/String;)V", + (void *)&GRAPH_METHOD(nativeLoadBinaryGraph)); + AddJNINativeMethod(&graph_methods, graph, "nativeLoadBinaryGraphBytes", + "(J[B)V", + (void *)&GRAPH_METHOD(nativeLoadBinaryGraphBytes)); + std::string packet_callback_name = class_registry.GetClassName( + mediapipe::android::ClassRegistry::kPacketCallbackClassName); + std::string native_add_packet_callback_signature = + absl::StrFormat("(JLjava/lang/String;L%s;)V", packet_callback_name); + AddJNINativeMethod(&graph_methods, graph, "nativeAddPacketCallback", + native_add_packet_callback_signature.c_str(), + (void *)&GRAPH_METHOD(nativeAddPacketCallback)); + AddJNINativeMethod(&graph_methods, graph, "nativeMovePacketToInputStream", + "(JLjava/lang/String;JJ)V", + (void *)&GRAPH_METHOD(nativeMovePacketToInputStream)); + AddJNINativeMethod(&graph_methods, graph, "nativeStartRunningGraph", + "(J[Ljava/lang/String;[J[Ljava/lang/String;[J)V", + (void *)&GRAPH_METHOD(nativeStartRunningGraph)); + AddJNINativeMethod(&graph_methods, graph, "nativeSetParentGlContext", "(JJ)V", + (void *)&GRAPH_METHOD(nativeSetParentGlContext)); + RegisterNativesVector(env, graph_class, graph_methods); +} + +void RegisterAndroidAssetUtilNatives(JNIEnv *env) { +#if defined(__ANDROID__) + auto &class_registry = mediapipe::android::ClassRegistry::GetInstance(); + std::string android_asset_util( + mediapipe::android::ClassRegistry::kAndroidAssetUtilClassName); + std::string android_asset_util_name = + class_registry.GetClassName(android_asset_util); + jclass android_asset_util_class = + env->FindClass(android_asset_util_name.c_str()); + + std::vector android_asset_util_methods; + AddJNINativeMethod( + &android_asset_util_methods, android_asset_util, + "nativeInitializeAssetManager", + "(Landroid/content/Context;Ljava/lang/String;)Z", + (void *)&ANDROID_ASSET_UTIL_METHOD(nativeInitializeAssetManager)); + RegisterNativesVector(env, android_asset_util_class, + android_asset_util_methods); +#endif +} + +void RegisterAndroidPacketCreatorNatives(JNIEnv *env) { +#if defined(__ANDROID__) + auto &class_registry = mediapipe::android::ClassRegistry::GetInstance(); + std::string android_packet_creator( + mediapipe::android::ClassRegistry::kAndroidPacketCreatorClassName); + std::string android_packet_creator_name = + class_registry.GetClassName(android_packet_creator); + jclass android_packet_creator_class = + env->FindClass(android_packet_creator_name.c_str()); + + std::vector android_packet_creator_methods; + AddJNINativeMethod( + &android_packet_creator_methods, android_packet_creator, + "nativeCreateRgbImageFrame", "(JLandroid/graphics/Bitmap;)J", + (void *)&ANDROID_PACKET_CREATOR_METHOD(nativeCreateRgbImageFrame)); + RegisterNativesVector(env, android_packet_creator_class, + android_packet_creator_methods); +#endif +} + +void RegisterPacketCreatorNatives(JNIEnv *env) { + auto &class_registry = mediapipe::android::ClassRegistry::GetInstance(); + std::string packet_creator( + mediapipe::android::ClassRegistry::kPacketCreatorClassName); + std::string packet_creator_name = class_registry.GetClassName(packet_creator); + jclass packet_creator_class = env->FindClass(packet_creator_name.c_str()); + + std::vector packet_creator_methods; + AddJNINativeMethod(&packet_creator_methods, packet_creator, + "nativeCreateRgbImage", "(JLjava/nio/ByteBuffer;II)J", + (void *)&PACKET_CREATOR_METHOD(nativeCreateRgbImage)); + AddJNINativeMethod( + &packet_creator_methods, packet_creator, "nativeCreateRgbaImageFrame", + "(JLjava/nio/ByteBuffer;II)J", + (void *)&PACKET_CREATOR_METHOD(nativeCreateRgbaImageFrame)); + AddJNINativeMethod( + &packet_creator_methods, packet_creator, "nativeCreateFloatImageFrame", + "(JLjava/nio/ByteBuffer;II)J", + (void *)&PACKET_CREATOR_METHOD(nativeCreateFloatImageFrame)); + RegisterNativesVector(env, packet_creator_class, packet_creator_methods); +} + +void RegisterPacketGetterNatives(JNIEnv *env) { + auto &class_registry = mediapipe::android::ClassRegistry::GetInstance(); + std::string packet_getter( + mediapipe::android::ClassRegistry::kPacketGetterClassName); + std::string packet_getter_name = class_registry.GetClassName(packet_getter); + jclass packet_getter_class = env->FindClass(packet_getter_name.c_str()); + + std::vector packet_getter_methods; + AddJNINativeMethod(&packet_getter_methods, packet_getter, "nativeGetBytes", + "(J)[B", (void *)&PACKET_GETTER_METHOD(nativeGetBytes)); + AddJNINativeMethod(&packet_getter_methods, packet_getter, + "nativeGetProtoBytes", "(J)[B", + (void *)&PACKET_GETTER_METHOD(nativeGetProtoBytes)); + AddJNINativeMethod(&packet_getter_methods, packet_getter, + "nativeGetImageData", "(JLjava/nio/ByteBuffer;)Z", + (void *)&PACKET_GETTER_METHOD(nativeGetImageData)); + AddJNINativeMethod(&packet_getter_methods, packet_getter, + "nativeGetFloat32Vector", "(J)[F", + (void *)&PACKET_GETTER_METHOD(nativeGetFloat32Vector)); + RegisterNativesVector(env, packet_getter_class, packet_getter_methods); +} + +void RegisterPacketNatives(JNIEnv *env) { + auto &class_registry = mediapipe::android::ClassRegistry::GetInstance(); + std::string packet(mediapipe::android::ClassRegistry::kPacketClassName); + std::string packet_name = class_registry.GetClassName(packet); + jclass packet_class = env->FindClass(packet_name.c_str()); + + std::vector packet_methods; + AddJNINativeMethod(&packet_methods, packet, "nativeReleasePacket", "(J)V", + (void *)&PACKET_METHOD(nativeReleasePacket)); + AddJNINativeMethod(&packet_methods, packet, "nativeCopyPacket", "(J)J", + (void *)&PACKET_METHOD(nativeCopyPacket)); + AddJNINativeMethod(&packet_methods, packet, "nativeGetTimestamp", "(J)J", + (void *)&PACKET_METHOD(nativeGetTimestamp)); + RegisterNativesVector(env, packet_class, packet_methods); +} + +void RegisterCompatNatives(JNIEnv *env) { + auto &class_registry = mediapipe::android::ClassRegistry::GetInstance(); + std::string compat(mediapipe::android::ClassRegistry::kCompatClassName); + std::string compat_name = class_registry.GetClassName(compat); + jclass compat_class = env->FindClass(compat_name.c_str()); + + std::vector compat_methods; + AddJNINativeMethod(&compat_methods, compat, "getCurrentNativeEGLContext", + "()J", (void *)&COMPAT_METHOD(getCurrentNativeEGLContext)); + AddJNINativeMethod(&compat_methods, compat, "getCurrentNativeEGLSurface", + "(I)J", + (void *)&COMPAT_METHOD(getCurrentNativeEGLSurface)); + RegisterNativesVector(env, compat_class, compat_methods); +} + +} // namespace + +void RegisterAllNatives(JNIEnv *env) { + RegisterGraphNatives(env); + RegisterAndroidAssetUtilNatives(env); + RegisterAndroidPacketCreatorNatives(env); + RegisterPacketCreatorNatives(env); + RegisterPacketGetterNatives(env); + RegisterPacketNatives(env); + RegisterCompatNatives(env); +} + +} // namespace registration +} // namespace android +} // namespace mediapipe diff --git a/mediapipe/java/com/google/mediapipe/framework/jni/register_natives.h b/mediapipe/java/com/google/mediapipe/framework/jni/register_natives.h new file mode 100644 index 000000000..23d2ca327 --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/framework/jni/register_natives.h @@ -0,0 +1,30 @@ +// Copyright 2019 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef JAVA_COM_GOOGLE_MEDIAPIPE_FRAMEWORK_JNI_REGISTER_NATIVES_H_ +#define JAVA_COM_GOOGLE_MEDIAPIPE_FRAMEWORK_JNI_REGISTER_NATIVES_H_ + +#include + +namespace mediapipe { +namespace android { +namespace registration { + +void RegisterAllNatives(JNIEnv* env); + +} // namespace registration +} // namespace android +} // namespace mediapipe + +#endif // JAVA_COM_GOOGLE_MEDIAPIPE_FRAMEWORK_JNI_REGISTER_NATIVES_H_ diff --git a/mediapipe/java/com/google/mediapipe/framework/proguard_allowobfuscation.pgcfg b/mediapipe/java/com/google/mediapipe/framework/proguard_allowobfuscation.pgcfg new file mode 100644 index 000000000..1de918205 --- /dev/null +++ b/mediapipe/java/com/google/mediapipe/framework/proguard_allowobfuscation.pgcfg @@ -0,0 +1,29 @@ +# Additional flags to pass to Proguard when processing a binary that uses +# MediaPipe wherein classes and methods invoked from native are kept but +# they are allowed to be obfuscated. In such cases, you must use the +# go/proguard-jni system to feed the proguard obfuscation map using the +# generate_obfuscation_mapping genrule. + +# Keep public members of our public interfaces. This also prevents the +# obfuscation of the corresponding methods in classes implementing them, +# such as implementations of PacketCallback#process. +-keep,allowobfuscation public interface com.google.mediapipe.framework.* { + public *; +} + +# This method is invoked by native code. +-keep,allowobfuscation public class com.google.mediapipe.framework.Packet { + public static *** create(***); + public long getNativeHandle(); + public void release(); +} + +# This method is invoked by native code. +-keep,allowobfuscation public class com.google.mediapipe.framework.PacketCreator { + *** releaseWithSyncToken(...); +} + +# This method is invoked by native code. +-keep,allowobfuscation public class com.google.mediapipe.framework.MediaPipeException { + (int, byte[]); +} diff --git a/mediapipe/java/com/google/mediapipe/glutil/BUILD b/mediapipe/java/com/google/mediapipe/glutil/BUILD index 4ad0d16d9..97ebdb187 100644 --- a/mediapipe/java/com/google/mediapipe/glutil/BUILD +++ b/mediapipe/java/com/google/mediapipe/glutil/BUILD @@ -23,7 +23,7 @@ android_library( srcs = glob(["**/*.java"]), visibility = ["//visibility:public"], deps = [ - "//mediapipe/java/com/google/mediapipe/framework:android_framework", + "//mediapipe/java/com/google/mediapipe/framework:android_framework_no_proguard", "@com_google_code_findbugs//jar", "@com_google_common_flogger//jar", "@com_google_common_flogger_system_backend//jar", diff --git a/mediapipe/java/com/google/mediapipe/glutil/TextureRenderer.java b/mediapipe/java/com/google/mediapipe/glutil/TextureRenderer.java index da785fd8a..c5d4b2320 100644 --- a/mediapipe/java/com/google/mediapipe/glutil/TextureRenderer.java +++ b/mediapipe/java/com/google/mediapipe/glutil/TextureRenderer.java @@ -30,6 +30,14 @@ public class TextureRenderer { 1.0f, 1.0f // top right ); + private static final FloatBuffer FLIPPED_TEXTURE_VERTICES = + ShaderUtil.floatBuffer( + 0.0f, 1.0f, // top left + 1.0f, 1.0f, // top right + 0.0f, 0.0f, // bottom left + 1.0f, 0.0f // bottom right + ); + private static final String TAG = "TextureRenderer"; private static final int ATTRIB_POSITION = 1; private static final int ATTRIB_TEXTURE_COORDINATE = 2; @@ -38,6 +46,7 @@ public class TextureRenderer { private int frameUniform; private int textureTransformUniform; private float[] textureTransformMatrix = new float[16]; + private boolean flipY; /** Call this to setup the shader program before rendering. */ public void setup() { @@ -53,6 +62,14 @@ public class TextureRenderer { Matrix.setIdentityM(textureTransformMatrix, 0 /* offset */); } + /** + * Flips rendering output vertically, useful for conversion between coordinate systems with + * top-left v.s. bottom-left origins. Effective in subsequent {@link #render(int)} calls. + */ + public void setFlipY(boolean flip) { + flipY = flip; + } + /** * Renders a texture to the framebuffer. * @@ -83,7 +100,12 @@ public class TextureRenderer { GLES20.glEnableVertexAttribArray(ATTRIB_TEXTURE_COORDINATE); GLES20.glVertexAttribPointer( - ATTRIB_TEXTURE_COORDINATE, 2, GLES20.GL_FLOAT, false, 0, TEXTURE_VERTICES); + ATTRIB_TEXTURE_COORDINATE, + 2, + GLES20.GL_FLOAT, + false, + 0, + flipY ? FLIPPED_TEXTURE_VERTICES : TEXTURE_VERTICES); ShaderUtil.checkGlError("program setup"); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); diff --git a/mediapipe/objc/MPPGraphTests.mm b/mediapipe/objc/MPPGraphTests.mm index f37273093..e4562da38 100644 --- a/mediapipe/objc/MPPGraphTests.mm +++ b/mediapipe/objc/MPPGraphTests.mm @@ -245,9 +245,9 @@ REGISTER_CALCULATOR(ErrorCalculator); [self waitForExpectationsWithTimeout:3.0 handler:NULL]; XCTAssertNotNil(error); status = error.gus_status; - XCTAssertNotEqual(status.error_message().find(kExpectedError), std::string::npos, + XCTAssertNotEqual(status.message().find(kExpectedError), std::string::npos, @"Missing expected std::string '%s' from error messge '%s'", kExpectedError, - status.error_message().c_str()); + status.message().data()); } - (void)testSetStreamHeader { diff --git a/mediapipe/objc/NSError+util_status.mm b/mediapipe/objc/NSError+util_status.mm index 715be2674..f8ba4802d 100644 --- a/mediapipe/objc/NSError+util_status.mm +++ b/mediapipe/objc/NSError+util_status.mm @@ -30,7 +30,7 @@ - (NSString *)description { return [NSString stringWithFormat:@"<%@: %p; status = %s>", - [self class], self, _status.error_message().c_str()]; + [self class], self, _status.message().data()]; } @end @@ -42,7 +42,7 @@ NSString *const kGUSGoogleUtilStatusErrorKey = @"GUSGoogleUtilStatusErrorKey"; + (NSError *)gus_errorWithStatus:(const ::mediapipe::Status &)status { NSDictionary *userInfo = @{ - NSLocalizedDescriptionKey : @(status.error_message().c_str()), + NSLocalizedDescriptionKey : @(status.message().data()), kGUSGoogleUtilStatusErrorKey : [GUSUtilStatusWrapper wrapStatus:status], }; NSError *error = [NSError errorWithDomain:kGUSGoogleUtilStatusErrorDomain diff --git a/mediapipe/util/android/BUILD b/mediapipe/util/android/BUILD index 4d9e742a0..9b848374d 100644 --- a/mediapipe/util/android/BUILD +++ b/mediapipe/util/android/BUILD @@ -22,20 +22,47 @@ cc_library( "asset_manager_util.cc", ], }), - hdrs = [ - "asset_manager_util.h", - ], + hdrs = ["asset_manager_util.h"], linkopts = select({ "//conditions:default": [], "//mediapipe:android": ["-landroid"], }), visibility = ["//visibility:public"], deps = [ + "@com_google_absl//absl/strings", "//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:singleton", "//mediapipe/framework/port:status", "//mediapipe/framework/port:statusor", "//mediapipe/util/android/file/base", - "@com_google_absl//absl/strings", + ] + select({ + "//conditions:default": [], + "//mediapipe:android": [":jni_helper"], + }), +) + +cc_library( + name = "jni_helper", + srcs = select({ + "//conditions:default": [], + "//mediapipe:android": ["jni_helper.cc"], + }), + hdrs = select({ + "//conditions:default": [], + "//mediapipe:android": ["jni_helper.h"], + }), + linkopts = ["-landroid"], + deps = [":logging"], +) + +cc_library( + name = "logging", + hdrs = select({ + "//conditions:default": [], + "//mediapipe:android": ["logging.h"], + }), + linkopts = [ + "-landroid", + "-llog", ], ) diff --git a/mediapipe/util/android/asset_manager_util.cc b/mediapipe/util/android/asset_manager_util.cc index dffff9b73..81e2b2be7 100644 --- a/mediapipe/util/android/asset_manager_util.cc +++ b/mediapipe/util/android/asset_manager_util.cc @@ -20,6 +20,7 @@ #include "mediapipe/framework/port/ret_check.h" #include "mediapipe/util/android/file/base/file.h" #include "mediapipe/util/android/file/base/filesystem.h" +#include "mediapipe/util/android/jni_helper.h" namespace mediapipe { @@ -46,22 +47,43 @@ bool AssetManager::InitializeFromAssetManager( return false; } -bool AssetManager::InitializeFromActivity(JNIEnv* env, jobject activity, - const std::string& cache_dir_path) { +bool AssetManager::InitializeFromContext(JNIEnv* env, jobject context, + const std::string& cache_dir_path) { + jni_common::JniHelper jni_helper(env, __LINE__, true); + + int status = env->GetJavaVM(&jvm_); + if (status != 0) { + return false; + } + + if (context_ != nullptr) { + env->DeleteGlobalRef(context_); + } + context_ = env->NewGlobalRef(context); + // Get the class of the Java activity that calls this JNI method. - jclass activity_class = env->GetObjectClass(activity); - + jclass context_class = env->GetObjectClass(context_); // Get the id of the getAssets method for the activity. - jmethodID activity_class_getAssets = env->GetMethodID( - activity_class, "getAssets", "()Landroid/content/res/AssetManager;"); - + jmethodID context_class_get_assets = env->GetMethodID( + context_class, "getAssets", "()Landroid/content/res/AssetManager;"); // Call activity.getAssets(); jobject local_asset_manager = - env->CallObjectMethod(activity, activity_class_getAssets); + env->CallObjectMethod(context_, context_class_get_assets); + + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + return false; + } return InitializeFromAssetManager(env, local_asset_manager, cache_dir_path); } +bool AssetManager::InitializeFromActivity(JNIEnv* env, jobject activity, + const std::string& cache_dir_path) { + return InitializeFromContext(env, activity, cache_dir_path); +} + bool AssetManager::FileExists(const std::string& filename) { if (!asset_manager_) { LOG(ERROR) << "Asset manager was not initialized from JNI"; @@ -140,4 +162,54 @@ bool AssetManager::ReadFile(const std::string& filename, return file_path; } +::mediapipe::StatusOr AssetManager::OpenContentUri( + const std::string& content_uri) { + jni_common::JniHelper jni_helper(jvm_, JNI_VERSION_1_6, __LINE__); + JNIEnv* env = jni_helper.GetEnv(); + if (env == nullptr) { + return ::mediapipe::UnavailableError("Couldn't get JNI env."); + } + + // ContentResolver contentResolver = context.getContentResolver(); + jclass context_class = env->FindClass("android/content/Context"); + jmethodID context_get_content_resolver = + env->GetMethodID(context_class, "getContentResolver", + "()Landroid/content/ContentResolver;"); + jclass content_resolver_class = + env->FindClass("android/content/ContentResolver"); + jobject content_resolver = + env->CallObjectMethod(context_, context_get_content_resolver); + + // Uri uri = Uri.parse(content_uri) + jclass uri_class = env->FindClass("android/net/Uri"); + jmethodID uri_parse = env->GetStaticMethodID( + uri_class, "parse", "(Ljava/lang/String;)Landroid/net/Uri;"); + jobject uri = env->CallStaticObjectMethod( + uri_class, uri_parse, env->NewStringUTF(content_uri.c_str())); + + // ParcelFileDescriptor descriptor = + // contentResolver.openAssetFileDescriptor(uri, "r"); + jmethodID content_resolver_open_file_descriptor = env->GetMethodID( + content_resolver_class, "openFileDescriptor", + "(Landroid/net/Uri;Ljava/lang/String;)Landroid/os/ParcelFileDescriptor;"); + jobject parcel_file_descriptor = env->CallObjectMethod( + content_resolver, content_resolver_open_file_descriptor, uri, + env->NewStringUTF("r")); + + // int fd = parcelDescriptor.detachFd(); + jclass parcel_descriptor_class = + env->FindClass("android/os/ParcelFileDescriptor"); + jmethodID parcel_class_detach_fd = + env->GetMethodID(parcel_descriptor_class, "detachFd", "()I"); + jint fd = env->CallIntMethod(parcel_file_descriptor, parcel_class_detach_fd); + + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + return ::mediapipe::NotFoundError("Content URI not found"); + } + + return static_cast(fd); +} + } // namespace mediapipe diff --git a/mediapipe/util/android/asset_manager_util.h b/mediapipe/util/android/asset_manager_util.h index c5e1e41b4..4644c4825 100644 --- a/mediapipe/util/android/asset_manager_util.h +++ b/mediapipe/util/android/asset_manager_util.h @@ -20,6 +20,7 @@ #include #include +#include "mediapipe/framework/port/canonical_errors.h" #include "mediapipe/framework/port/singleton.h" #include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/statusor.h" @@ -54,12 +55,23 @@ class AssetManager { ABSL_DEPRECATED("Use InitializeFromActivity instead.") bool InitializeFromAssetManager(JNIEnv* env, jobject local_asset_manager); + // Returns true if AAssetManager was successfully initialized. + // cache_dir_path should be set to context.getCacheDir().getAbsolutePath(). + // We could get it from the context, but we have the Java layer pass it + // directly for convenience. + bool InitializeFromContext(JNIEnv* env, jobject context, + const std::string& cache_dir_path); + // Checks if a file exists. Returns true on success, false otherwise. bool FileExists(const std::string& filename); // Reads a file into raw_bytes. Returns true on success, false otherwise. bool ReadFile(const std::string& filename, std::vector* raw_bytes); + // Returns the open file descriptor from an Android content URI, the caller + // is responsible to close the file descriptor. + ::mediapipe::StatusOr OpenContentUri(const std::string& content_uri); + // Returns the path to the Android cache directory. Will be empty if // InitializeFromActivity has not been called. const std::string& GetCacheDirPath(); @@ -77,6 +89,12 @@ class AssetManager { // Pointer to asset manager from JNI. AAssetManager* asset_manager_ = nullptr; + // The context from which assets should be loaded. + jobject context_; + + // Pointer to the JVM, used to get the JNIEnv on background threads. + JavaVM* jvm_; + // Path to the Android cache directory for our context. std::string cache_dir_path_; diff --git a/mediapipe/util/android/file/base/helpers.cc b/mediapipe/util/android/file/base/helpers.cc index 930f916fa..34411b272 100644 --- a/mediapipe/util/android/file/base/helpers.cc +++ b/mediapipe/util/android/file/base/helpers.cc @@ -42,17 +42,7 @@ class FdCloser { } // namespace // Read contents of a file to a std::string. -::mediapipe::Status GetContents(absl::string_view file_name, - std::string* output, - const file::Options& /*options*/) { - int fd = open(std::string(file_name).c_str(), O_RDONLY); - if (fd < 0) { - return ::mediapipe::Status( - mediapipe::StatusCode::kUnknown, - "Failed to open file: " + std::string(file_name)); - } - FdCloser closer(fd); - +::mediapipe::Status GetContents(int fd, std::string* output) { // Determine the length of the file. struct stat buf; if (fstat(fd, &buf) != 0) { @@ -80,6 +70,21 @@ class FdCloser { return ::mediapipe::OkStatus(); } +// Read contents of a file to a std::string. +::mediapipe::Status GetContents(absl::string_view file_name, + std::string* output, + const file::Options& /*options*/) { + int fd = open(std::string(file_name).c_str(), O_RDONLY); + if (fd < 0) { + return ::mediapipe::Status( + mediapipe::StatusCode::kUnknown, + "Failed to open file: " + std::string(file_name)); + } + + FdCloser closer(fd); + return GetContents(fd, output); +} + ::mediapipe::Status GetContents(absl::string_view file_name, std::string* output) { return GetContents(file_name, output, file::Defaults()); @@ -112,5 +117,10 @@ class FdCloser { } } +::mediapipe::Status SetContents(absl::string_view file_name, + absl::string_view content) { + return SetContents(file_name, content, file::Defaults()); +} + } // namespace file } // namespace mediapipe diff --git a/mediapipe/util/android/file/base/helpers.h b/mediapipe/util/android/file/base/helpers.h index ff8f9f2c6..4dc10d6e9 100644 --- a/mediapipe/util/android/file/base/helpers.h +++ b/mediapipe/util/android/file/base/helpers.h @@ -33,11 +33,18 @@ namespace file { ::mediapipe::Status GetContents(absl::string_view file_name, std::string* output); +// Read contents of a file to a std::string from an open file descriptor. +::mediapipe::Status GetContents(int fd, std::string* output); + // Write std::string to file. ::mediapipe::Status SetContents(absl::string_view file_name, absl::string_view content, const file::Options& options); +// Write std::string to file with default file options. +::mediapipe::Status SetContents(absl::string_view file_name, + absl::string_view content); + } // namespace file } // namespace mediapipe diff --git a/mediapipe/util/android/jni_helper.cc b/mediapipe/util/android/jni_helper.cc new file mode 100644 index 000000000..fb171ff59 --- /dev/null +++ b/mediapipe/util/android/jni_helper.cc @@ -0,0 +1,115 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/util/android/jni_helper.h" + +#include "mediapipe/util/android/logging.h" + +namespace mediapipe { +namespace jni_common { + +JniHelper::JniHelper(JavaVM* vm, jint version, int caller_line, + bool enable_logging) + : vm_(vm), + env_(nullptr), + need_to_detach_(false), + caller_line_(caller_line), + enable_logging_(enable_logging) { + JNI_COMMON_CHECK(vm_); + const int code = vm_->GetEnv(reinterpret_cast(&env_), version); + if (code == JNI_OK) { + if (0 != env_->PushLocalFrame(0)) { // Create new stack frame. + ExceptionPrintClear(env_); + if (enable_logging_) { + JNI_COMMON_LOG(VERBOSE, "JniHelper: failed to push local frame."); + } + env_ = nullptr; + } + } else if (code == JNI_EDETACHED) { + if (vm_->AttachCurrentThread(&env_, nullptr) == JNI_OK) { + if (enable_logging_) { + JNI_COMMON_LOG(VERBOSE, + "JniHelper: attached thread (Called from line %d).", + caller_line_); + } + need_to_detach_ = true; + } else { + if (enable_logging_) { + JNI_COMMON_LOG( + ERROR, + "JniHelper: couldn't attach current thread (Called from line %d).", + caller_line_); + } + env_ = nullptr; + } + } else { + if (enable_logging_) { + JNI_COMMON_LOG(ERROR, + "JniHelper: couldn't get env (Called from line %d).", + caller_line_); + } + env_ = nullptr; + } +} + +JniHelper::JniHelper(JNIEnv* env, int caller_line, bool enable_logging) + : vm_(nullptr), + env_(env), + need_to_detach_(false), + caller_line_(caller_line), + enable_logging_(enable_logging) { + JNI_COMMON_CHECK(env_); + if (0 != env_->PushLocalFrame(0)) { // Create new stack frame. + ExceptionPrintClear(env_); + if (enable_logging_) { + JNI_COMMON_LOG( + VERBOSE, + "JniHelper: failed to push local frame (Called from line %d).", + caller_line_); + } + env_ = nullptr; + } +} + +JniHelper::~JniHelper() { + if (need_to_detach_) { + if (enable_logging_) { + JNI_COMMON_LOG( + VERBOSE, "~JniHelper: about to detach thread (Called from line %d).", + caller_line_); + } + if (vm_->DetachCurrentThread() == JNI_OK) { + if (enable_logging_) { + JNI_COMMON_LOG(VERBOSE, + "~JniHelper: detached thread (Called from line %d).", + caller_line_); + } + } else { + if (enable_logging_) { + JNI_COMMON_LOG( + ERROR, "~JniHelper: couldn't detach thread (Called from line %d).", + caller_line_); + } + } + } else { + if (env_ != nullptr) { + env_->PopLocalFrame(nullptr); // Clean up local references. + } + } +} + +JNIEnv* JniHelper::GetEnv() const { return env_; } + +} // namespace jni_common +} // namespace mediapipe diff --git a/mediapipe/util/android/jni_helper.h b/mediapipe/util/android/jni_helper.h new file mode 100644 index 000000000..0e603e931 --- /dev/null +++ b/mediapipe/util/android/jni_helper.h @@ -0,0 +1,69 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_UTIL_ANDROID_JNI_HELPER_H_ +#define MEDIAPIPE_UTIL_ANDROID_JNI_HELPER_H_ + +#include + +namespace mediapipe { +namespace jni_common { + +inline bool ExceptionPrintClear(JNIEnv* env) { + if (env->ExceptionCheck()) { + env->ExceptionDescribe(); + env->ExceptionClear(); + return true; + } + return false; +} + +class JniHelper { + public: + // This constructor should be used when a JavaVM pointer is available, and the + // JNIEnv needs to be obtained using AttachCurrentThread. This will also push + // a local stack frame, and pop it when this object is destroyed. If + // enable_logging is true, it will log verbosely in the constructor and + // destructor. + JniHelper(JavaVM* vm, jint version, int caller_line, + bool enable_logging = true); + + // This constructor should be used then the JNIEnv pointer itself is + // available, and the only thing that needs to be taken care of is pushing and + // popping the stack frames. If enable_logging is true, it will log verbosely + // in the constructor and destructor. + JniHelper(JNIEnv* env, int caller_line, bool enable_logging = true); + + // Detaches the current thread, if necessary, and pops the local stack frame + // that was pushed during construction. + ~JniHelper(); + + // Copy and assignment are disallowed because it could cause double-detaching. + JniHelper(const JniHelper& other) = delete; + JniHelper& operator=(const JniHelper& other) = delete; + + JNIEnv* GetEnv() const; + + private: + JavaVM* vm_; + JNIEnv* env_; + bool need_to_detach_; + const int caller_line_; + const bool enable_logging_; +}; + +} // namespace jni_common +} // namespace mediapipe + +#endif // MEDIAPIPE_UTIL_ANDROID_JNI_HELPER_H_ diff --git a/mediapipe/util/android/logging.h b/mediapipe/util/android/logging.h new file mode 100644 index 000000000..113428166 --- /dev/null +++ b/mediapipe/util/android/logging.h @@ -0,0 +1,39 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_UTIL_ANDROID_JNI_COMMON_LOGGING_H_ +#define MEDIAPIPE_UTIL_ANDROID_JNI_COMMON_LOGGING_H_ + +#include +#include + +#define JNI_COMMON_LOG(priority, ...) \ + __android_log_print(ANDROID_LOG_##priority, __FILE__, __VA_ARGS__) + +#define JNI_COMMON_CHECK(condition) \ + if (!(condition)) { \ + JNI_COMMON_LOG(ERROR, "CHECK FAILED at %s:%d: %s", __FILE__, __LINE__, \ + #condition); \ + abort(); \ + } + +#define JNI_COMMON_CHECK_WITH_LOG(condition, message, ...) \ + if (!(condition)) { \ + __android_log_print(ANDROID_LOG_ERROR, __FILE__, \ + "CHECK FAILED at %s:%d: %s " message, __FILE__, \ + __LINE__, #condition, ##__VA_ARGS__); \ + abort(); \ + } + +#endif // MEDIAPIPE_UTIL_ANDROID_JNI_COMMON_LOGGING_H_ diff --git a/mediapipe/util/resource_util_android.cc b/mediapipe/util/resource_util_android.cc index 643739566..449fbf376 100644 --- a/mediapipe/util/resource_util_android.cc +++ b/mediapipe/util/resource_util_android.cc @@ -61,6 +61,19 @@ namespace { return file::GetContents(path, output, file::Defaults()); } + if (absl::StartsWith(path, "content://")) { + auto fd_status = Singleton::get()->OpenContentUri(path); + if (!fd_status.ok()) { + return ::mediapipe::Status(mediapipe::StatusCode::kUnknown, + "Failed to open file: " + std::string(path)); + } + int fd = fd_status.ValueOrDie(); + auto status = file::GetContents(fd, output); + + close(fd); + return status; + } + std::vector data; RET_CHECK(Singleton::get()->ReadFile(path, &data)) << "could not read asset: " << path; diff --git a/mediapipe/util/sequence/BUILD b/mediapipe/util/sequence/BUILD index 1e07d0ca4..82f9e8c98 100644 --- a/mediapipe/util/sequence/BUILD +++ b/mediapipe/util/sequence/BUILD @@ -28,7 +28,7 @@ cc_library( "//mediapipe/framework/port:core_proto", "//mediapipe/framework/port:integral_types", "//mediapipe/framework/port:logging", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -50,7 +50,7 @@ cc_library( "//mediapipe/framework/port:status", "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -61,7 +61,7 @@ cc_test( ":media_sequence_util", "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:parse_text_proto", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -74,6 +74,6 @@ cc_test( "//mediapipe/framework/port:gtest_main", "//mediapipe/framework/port:opencv_imgcodecs", "//mediapipe/framework/port:status", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) diff --git a/third_party/org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff b/third_party/org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff new file mode 100644 index 000000000..6b28b0548 --- /dev/null +++ b/third_party/org_tensorflow_528e22eae8bf3206189a066032c66e9e5c9b4a61.diff @@ -0,0 +1,65 @@ +diff --git a/tensorflow/c/tf_tensor.cc b/tensorflow/c/tf_tensor.cc +index 6bb2cafbbc..60250d6f92 100644 +--- a/tensorflow/c/tf_tensor.cc ++++ b/tensorflow/c/tf_tensor.cc +@@ -114,7 +114,7 @@ TF_Tensor* TF_NewTensor(TF_DataType dtype, const int64_t* dims, int num_dims, + if (elem_size > 0 && len < (elem_size * ret.NumElements())) { + return nullptr; + } +- return new TF_Tensor{std::make_unique(ret)}; ++ return new TF_Tensor{absl::make_unique(ret)}; + } + + TF_Tensor* TF_TensorMaybeMove(TF_Tensor* t) { +@@ -317,7 +317,7 @@ TF_Tensor* TF_TensorFromTensor(const tensorflow::Tensor& src, Status* status) { + if (!tensor.CopyFrom(src, src.shape())) { + return nullptr; + } +- return new TF_Tensor{std::make_unique(tensor)}; ++ return new TF_Tensor{absl::make_unique(tensor)}; + } + // DT_STRING tensors require a copying since TF_Tensor.buffer expects a flatly + // encoded sequence of strings. +diff --git a/tensorflow/core/lib/monitoring/percentile_sampler.cc b/tensorflow/core/lib/monitoring/percentile_sampler.cc +index 988e50ded5..a8a412dc25 100644 +--- a/tensorflow/core/lib/monitoring/percentile_sampler.cc ++++ b/tensorflow/core/lib/monitoring/percentile_sampler.cc +@@ -29,7 +29,8 @@ namespace monitoring { + void PercentileSamplerCell::Add(double sample) { + uint64 nstime = EnvTime::NowNanos(); + mutex_lock l(mu_); +- samples_[next_position_] = {nstime, sample}; ++ samples_[next_position_].nstime = nstime; ++ samples_[next_position_].value = sample; + ++next_position_; + if (TF_PREDICT_FALSE(next_position_ >= samples_.size())) { + next_position_ = 0; +@@ -72,7 +73,9 @@ Percentiles PercentileSamplerCell::value() const { + size_t index = std::min( + static_cast(percentile * pct_samples.num_samples / 100.0), + pct_samples.num_samples - 1); +- PercentilePoint pct = {percentile, samples[index].value}; ++ PercentilePoint pct; ++ pct.percentile = percentile; ++ pct.value = samples[index].value; + pct_samples.points.push_back(pct); + } + } +diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl +index 8947038da8..75a1259b88 100755 +--- a/tensorflow/workspace.bzl ++++ b/tensorflow/workspace.bzl +@@ -927,9 +927,10 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): + # https://github.com/bazelbuild/rules_apple/releases + tf_http_archive( + name = "build_bazel_rules_apple", +- sha256 = "a045a436b642c70fb0c10ca84ff0fd2dcbd59cc89100d597a61e8374afafb366", ++ sha256 = "bdc8e66e70b8a75da23b79f1f8c6207356df07d041d96d2189add7ee0780cf4e", ++ strip_prefix = "rules_apple-b869b0d3868d78a1d4ffd866ccb304fb68aa12c3", + urls = [ +- "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/rules_apple/releases/download/0.18.0/rules_apple.0.18.0.tar.gz", +- "https://github.com/bazelbuild/rules_apple/releases/download/0.18.0/rules_apple.0.18.0.tar.gz", ++ "https://github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz", ++ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/rules_apple/archive/b869b0d3868d78a1d4ffd866ccb304fb68aa12c3.tar.gz", + ], + ) diff --git a/third_party/org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff b/third_party/org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff deleted file mode 100644 index 4f8a10f77..000000000 --- a/third_party/org_tensorflow_e3a7bdbebb99352351a19e2e403136166aa52934.diff +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl -index a0546b5f83..c7a9f39ded 100755 ---- a/tensorflow/workspace.bzl -+++ b/tensorflow/workspace.bzl -@@ -172,9 +172,9 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): - name = "eigen_archive", - build_file = clean_dep("//third_party:eigen.BUILD"), - patch_file = clean_dep("//third_party/eigen3:gpu_packet_math.patch"), -- sha256 = "6d8ed482addd14892d7b0bd98fec2c02f18fdab97775bda68c3f2a99ffb190fb", -- strip_prefix = "eigen-eigen-66be6c76fc01", -+ sha256 = "add24720f99ab4f3222f4c8a887f2609554cf9187d4f7d24a777a151a0ee2548", -+ strip_prefix = "eigen-git-mirror-4898dcdb06f1b1b0441b8e15119764793f8997e2", - urls = [ -- "https://storage.googleapis.com/mirror.tensorflow.org/bitbucket.org/eigen/eigen/get/66be6c76fc01.tar.gz", -- "https://bitbucket.org/eigen/eigen/get/66be6c76fc01.tar.gz", -+ "https://storage.googleapis.com/mirror.tensorflow.org/github.com/eigenteam/eigen-git-mirror/archive/4898dcdb06f1b1b0441b8e15119764793f8997e2.tar.gz", -+ "https://github.com/eigenteam/eigen-git-mirror/archive/4898dcdb06f1b1b0441b8e15119764793f8997e2.tar.gz", - ], - ) \ No newline at end of file