diff --git a/mediapipe/calculators/core/gate_calculator.cc b/mediapipe/calculators/core/gate_calculator.cc index 9c142c70a..8b8e1424d 100644 --- a/mediapipe/calculators/core/gate_calculator.cc +++ b/mediapipe/calculators/core/gate_calculator.cc @@ -134,13 +134,7 @@ class GateCalculator : public CalculatorBase { } ::mediapipe::Status Open(CalculatorContext* cc) final { - const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>(); - use_calculator_option_for_allow_disallow_ = - options.has_allowance_override(); - if (use_calculator_option_for_allow_disallow_) { - allow_by_calculator_option_ = options.allowance_override(); - } - + use_side_packet_for_allow_disallow_ = false; if (cc->InputSidePackets().HasTag("ALLOW")) { use_side_packet_for_allow_disallow_ = true; allow_by_side_packet_decision_ = @@ -156,27 +150,24 @@ class GateCalculator : public CalculatorBase { last_gate_state_ = GATE_UNINITIALIZED; RET_CHECK_OK(CopyInputHeadersToOutputs(cc->Inputs(), &cc->Outputs())); + const auto& options = cc->Options<::mediapipe::GateCalculatorOptions>(); empty_packets_as_allow_ = options.empty_packets_as_allow(); + return ::mediapipe::OkStatus(); } ::mediapipe::Status Process(CalculatorContext* cc) final { - // The allow/disallow signal in the calculator option has the highest - // priority. If it's not set, use the stream/side packet signal. - bool allow = allow_by_calculator_option_; - if (!use_calculator_option_for_allow_disallow_) { - allow = empty_packets_as_allow_; - if (use_side_packet_for_allow_disallow_) { - allow = allow_by_side_packet_decision_; - } else { - if (cc->Inputs().HasTag("ALLOW") && - !cc->Inputs().Tag("ALLOW").IsEmpty()) { - allow = cc->Inputs().Tag("ALLOW").Get(); - } - if (cc->Inputs().HasTag("DISALLOW") && - !cc->Inputs().Tag("DISALLOW").IsEmpty()) { - allow = !cc->Inputs().Tag("DISALLOW").Get(); - } + bool allow = empty_packets_as_allow_; + if (use_side_packet_for_allow_disallow_) { + allow = allow_by_side_packet_decision_; + } else { + if (cc->Inputs().HasTag("ALLOW") && + !cc->Inputs().Tag("ALLOW").IsEmpty()) { + allow = cc->Inputs().Tag("ALLOW").Get(); + } + if (cc->Inputs().HasTag("DISALLOW") && + !cc->Inputs().Tag("DISALLOW").IsEmpty()) { + allow = !cc->Inputs().Tag("DISALLOW").Get(); } } const GateState new_gate_state = allow ? GATE_ALLOW : GATE_DISALLOW; @@ -196,6 +187,14 @@ class GateCalculator : public CalculatorBase { last_gate_state_ = new_gate_state; if (!allow) { + // Close the output streams if the gate will be permanently closed. + // Prevents buffering in calculators whose parents do no use SetOffset. + for (int i = 0; i < num_data_streams_; ++i) { + if (!cc->Outputs().Get("", i).IsClosed() && + use_side_packet_for_allow_disallow_) { + cc->Outputs().Get("", i).Close(); + } + } return ::mediapipe::OkStatus(); } @@ -212,11 +211,9 @@ class GateCalculator : public CalculatorBase { private: GateState last_gate_state_ = GATE_UNINITIALIZED; int num_data_streams_; - bool empty_packets_as_allow_ = false; - bool use_side_packet_for_allow_disallow_ = false; - bool allow_by_side_packet_decision_ = false; - bool use_calculator_option_for_allow_disallow_ = false; - bool allow_by_calculator_option_ = false; + bool empty_packets_as_allow_; + bool use_side_packet_for_allow_disallow_; + bool allow_by_side_packet_decision_; }; REGISTER_CALCULATOR(GateCalculator); diff --git a/mediapipe/calculators/core/gate_calculator.proto b/mediapipe/calculators/core/gate_calculator.proto index 63850774a..d9a1b69d4 100644 --- a/mediapipe/calculators/core/gate_calculator.proto +++ b/mediapipe/calculators/core/gate_calculator.proto @@ -27,9 +27,4 @@ message GateCalculatorOptions { // disallowing the corresponding packets in the data input streams. Setting // this option to true inverts that, allowing the data packets to go through. optional bool empty_packets_as_allow = 1; - - // If set, the calculator will always allow (if set to yes) or disallow (if - // set to no) the input streams to pass through, and ignore the ALLOW or - // DISALLOW input stream or side input packets. - optional bool allowance_override = 2; } diff --git a/mediapipe/calculators/core/gate_calculator_test.cc b/mediapipe/calculators/core/gate_calculator_test.cc index d6d2c5530..ec838f6b6 100644 --- a/mediapipe/calculators/core/gate_calculator_test.cc +++ b/mediapipe/calculators/core/gate_calculator_test.cc @@ -330,52 +330,5 @@ TEST_F(GateCalculatorTest, AllowInitialNoStateTransition) { ASSERT_EQ(0, output.size()); } -TEST_F(GateCalculatorTest, - TestCalculatorOptionDecisionOverrideOverStreamSingal) { - SetRunner(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_stream: "ALLOW:gating_stream" - output_stream: "test_output" - options: { - [mediapipe.GateCalculatorOptions.ext] { - allowance_override: false - } - } - )"); - - constexpr int64 kTimestampValue0 = 42; - // The CalculatorOptions says disallow and the stream says allow. Should - // follow the CalculatorOptions' decision to disallow outputting anything. - RunTimeStep(kTimestampValue0, "ALLOW", true); - - const std::vector& output = runner()->Outputs().Get("", 0).packets; - ASSERT_EQ(0, output.size()); -} - -TEST_F(GateCalculatorTest, - TestCalculatorOptionDecisionOverrideOverSidePacketSingal) { - SetRunner(R"( - calculator: "GateCalculator" - input_stream: "test_input" - input_side_packet: "ALLOW:gating_packet" - output_stream: "test_output" - options: { - [mediapipe.GateCalculatorOptions.ext] { - allowance_override: true - } - } - )"); - - constexpr int64 kTimestampValue0 = 42; - // The CalculatorOptions says allow and the side packet says disallow. Should - // follow the CalculatorOptions' decision to allow outputting a packet. - runner()->MutableSidePackets()->Tag("ALLOW") = Adopt(new bool(false)); - RunTimeStep(kTimestampValue0, true); - - const std::vector& output = runner()->Outputs().Get("", 0).packets; - ASSERT_EQ(1, output.size()); -} - } // namespace } // namespace mediapipe diff --git a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc index a12af47a5..cee5b40fb 100644 --- a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc +++ b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.cc @@ -2,11 +2,24 @@ namespace mediapipe { namespace autoflip { +namespace { +int Median(const std::deque>& positions_raw) { + std::deque positions; + for (const auto& position : positions_raw) { + positions.push_back(position.second); + } + size_t n = positions.size() / 2; + nth_element(positions.begin(), positions.begin() + n, positions.end()); + return positions[n]; +} +} // namespace ::mediapipe::Status KinematicPathSolver::AddObservation(int position, const uint64 time_us) { if (!initialized_) { current_position_px_ = position; + raw_positions_at_time_.push_front( + std::pair(time_us, position)); current_time_ = time_us; initialized_ = true; current_velocity_deg_per_s_ = 0; @@ -16,13 +29,26 @@ namespace autoflip { << "update_rate_seconds must be greater than 0."; RET_CHECK_GE(options_.min_motion_to_reframe(), options_.reframe_window()) << "Reframe window cannot exceed min_motion_to_reframe."; + RET_CHECK_GE(options_.filtering_time_window_us(), 0) + << "update_rate_seconds must be greater than 0."; return ::mediapipe::OkStatus(); } RET_CHECK(current_time_ < time_us) << "Observation added before a prior observations."; - double delta_degs = (position - current_position_px_) / pixels_per_degree_; + raw_positions_at_time_.push_front(std::pair(time_us, position)); + while (raw_positions_at_time_.size() > 1) { + if (static_cast(raw_positions_at_time_.back().first) < + static_cast(time_us) - options_.filtering_time_window_us()) { + raw_positions_at_time_.pop_back(); + } else { + break; + } + } + + double delta_degs = (Median(raw_positions_at_time_) - current_position_px_) / + pixels_per_degree_; // If the motion is smaller than the min, don't use the update. if (abs(delta_degs) < options_.min_motion_to_reframe()) { diff --git a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h index f2d26f0b0..9634b6ee9 100644 --- a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h +++ b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.h @@ -15,6 +15,8 @@ #ifndef MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UNIFORM_ACCELERATION_PATH_SOLVER_H_ #define MEDIAPIPE_EXAMPLES_DESKTOP_AUTOFLIP_QUALITY_UNIFORM_ACCELERATION_PATH_SOLVER_H_ +#include + #include "mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.pb.h" #include "mediapipe/framework/port/integral_types.h" #include "mediapipe/framework/port/ret_check.h" @@ -61,6 +63,8 @@ class KinematicPathSolver { double current_position_px_; double current_velocity_deg_per_s_; uint64 current_time_; + // History of observations (second) and their time (first). + std::deque> raw_positions_at_time_; }; } // namespace autoflip diff --git a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto index ac2595328..525bcdb62 100644 --- a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto +++ b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver.proto @@ -20,4 +20,6 @@ message KinematicOptions { // where delta_time_s is the time since the last frame. optional double update_rate_seconds = 5 [default = 0.20]; optional double max_update_rate = 6 [default = 0.8]; + // History time window of observations to be median filtered. + optional int64 filtering_time_window_us = 7 [default = 0]; } diff --git a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc index 38e3e4f0b..c9ad2690c 100644 --- a/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc +++ b/mediapipe/examples/desktop/autoflip/quality/kinematic_path_solver_test.cc @@ -81,6 +81,46 @@ TEST(KinematicPathSolverTest, PassNotEnoughMotionSmallImg) { EXPECT_EQ(state, 400); } +TEST(KinematicPathSolverTest, PassEnoughMotionFiltered) { + KinematicOptions options; + // Set min motion to 2deg + options.set_min_motion_to_reframe(1.0); + options.set_update_rate(1); + options.set_max_velocity(1000); + options.set_filtering_time_window_us(3000000); + // Set degrees / pixel to 16.6 + KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); + int state; + MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); + // Move target by 20px / 16.6 = 1.2deg + MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 1)); + MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 2)); + MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 3)); + MP_ASSERT_OK(solver.GetState(&state)); + // Expect cam to not move. + EXPECT_EQ(state, 500); +} + +TEST(KinematicPathSolverTest, PassEnoughMotionNotFiltered) { + KinematicOptions options; + // Set min motion to 2deg + options.set_min_motion_to_reframe(1.0); + options.set_update_rate(1); + options.set_max_velocity(1000); + options.set_filtering_time_window_us(0); + // Set degrees / pixel to 16.6 + KinematicPathSolver solver(options, 0, 1000, 1000.0 / kWidthFieldOfView); + int state; + MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 0)); + // Move target by 20px / 16.6 = 1.2deg + MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 1)); + MP_ASSERT_OK(solver.AddObservation(520, kMicroSecInSec * 2)); + MP_ASSERT_OK(solver.AddObservation(500, kMicroSecInSec * 3)); + MP_ASSERT_OK(solver.GetState(&state)); + // Expect cam to not move. + EXPECT_EQ(state, 519); +} + TEST(KinematicPathSolverTest, PassEnoughMotionLargeImg) { KinematicOptions options; // Set min motion to 1deg diff --git a/mediapipe/framework/profiler/graph_profiler_test.cc b/mediapipe/framework/profiler/graph_profiler_test.cc index 0f1497d76..b5eb953ff 100644 --- a/mediapipe/framework/profiler/graph_profiler_test.cc +++ b/mediapipe/framework/profiler/graph_profiler_test.cc @@ -14,6 +14,7 @@ #include "mediapipe/framework/profiler/graph_profiler.h" +#include "absl/status/statusor.h" #include "absl/synchronization/mutex.h" #include "absl/time/time.h" #include "mediapipe/framework/calculator_framework.h"