From 36be94f861e57dd21e4e7a64e5a1650e73313753 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 11 Jan 2023 14:21:02 -0800 Subject: [PATCH] Internal change PiperOrigin-RevId: 501378130 --- .../desktop/autoflip/autoflip_messages.proto | 4 +++ .../calculators/scene_cropping_calculator.cc | 21 +++++++++-- .../scene_cropping_calculator_test.cc | 35 +++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/mediapipe/examples/desktop/autoflip/autoflip_messages.proto b/mediapipe/examples/desktop/autoflip/autoflip_messages.proto index 8507c9ad7..c89a6aea6 100644 --- a/mediapipe/examples/desktop/autoflip/autoflip_messages.proto +++ b/mediapipe/examples/desktop/autoflip/autoflip_messages.proto @@ -185,6 +185,10 @@ message ExternalRenderFrame { // original dimensions of the input video. The first step to render this // frame is to crop this rect from the input frame. optional Rect crop_from_location = 1; + // Rect that must be cropped out of the input frame. It is defined in the + // ratio of the frame of the input video. The first step to render this frame + // is to crop this rect from the input frame. + optional Rect normalized_crop_from_location = 7; // The placement location where the above rect is placed on the output frame. // This will always have the same aspect ratio as the above rect but scaling // may be required. diff --git a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc b/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc index 89170dc6a..7e286b743 100644 --- a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc +++ b/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator.cc @@ -201,13 +201,26 @@ absl::Status ParseAspectRatioString(const std::string& aspect_ratio_string, void ConstructExternalRenderMessage( const cv::Rect& crop_from_location, const cv::Rect& render_to_location, const cv::Scalar& padding_color, const uint64 timestamp_us, - ExternalRenderFrame* external_render_message) { + ExternalRenderFrame* external_render_message, int frame_width, + int frame_height) { auto crop_from_message = external_render_message->mutable_crop_from_location(); crop_from_message->set_x(crop_from_location.x); crop_from_message->set_y(crop_from_location.y); crop_from_message->set_width(crop_from_location.width); crop_from_message->set_height(crop_from_location.height); + + auto normalized_crop_from_message = + external_render_message->mutable_normalized_crop_from_location(); + normalized_crop_from_message->set_x(crop_from_location.x / + static_cast(frame_width)); + normalized_crop_from_message->set_y(crop_from_location.y / + static_cast(frame_height)); + normalized_crop_from_message->set_width(crop_from_location.width / + static_cast(frame_width)); + normalized_crop_from_message->set_height(crop_from_location.height / + static_cast(frame_height)); + auto render_to_message = external_render_message->mutable_render_to_location(); render_to_message->set_x(render_to_location.x); @@ -627,7 +640,8 @@ absl::Status SceneCroppingCalculator::ProcessScene(const bool is_end_of_scene, auto external_render_message = absl::make_unique(); ConstructExternalRenderMessage( crop_from_locations[i], render_to_locations[i], padding_colors[i], - scene_frame_timestamps_[i], external_render_message.get()); + scene_frame_timestamps_[i], external_render_message.get(), + frame_width_, frame_height_); cc->Outputs() .Tag(kExternalRenderingPerFrame) .Add(external_render_message.release(), @@ -640,7 +654,8 @@ absl::Status SceneCroppingCalculator::ProcessScene(const bool is_end_of_scene, ExternalRenderFrame render_frame; ConstructExternalRenderMessage(crop_from_locations[i], render_to_locations[i], padding_colors[i], - scene_frame_timestamps_[i], &render_frame); + scene_frame_timestamps_[i], &render_frame, + frame_width_, frame_height_); external_render_list_->push_back(render_frame); } } diff --git a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc b/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc index 88728860a..c3285ea58 100644 --- a/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc +++ b/mediapipe/examples/desktop/autoflip/calculators/scene_cropping_calculator_test.cc @@ -920,6 +920,41 @@ TEST(SceneCroppingCalculatorTest, OutputsCropMessageKinematicPathNoVideo) { EXPECT_EQ(ext_render_message.render_to_location().height(), 1124); } } + +// Checks external render message with default poly path solver using +// normalized crops. +TEST(SceneCroppingCalculatorTest, OutputsCropMessagePolyPathNormalized) { + const CalculatorGraphConfig::Node config = + ParseTextProtoOrDie( + absl::Substitute(kExternalRenderConfig, kTargetWidth, kTargetHeight)); + auto runner = absl::make_unique(config); + const int num_frames = kSceneSize; + AddScene(0, num_frames, kInputFrameWidth, kInputFrameHeight, kKeyFrameWidth, + kKeyFrameHeight, 1, runner->MutableInputs()); + + MP_EXPECT_OK(runner->Run()); + const auto& outputs = runner->Outputs(); + const auto& ext_render_per_frame = + outputs.Tag(kExternalRenderingPerFrameTag).packets; + EXPECT_EQ(ext_render_per_frame.size(), num_frames); + + for (int i = 0; i < num_frames - 1; ++i) { + const auto& ext_render_message = + ext_render_per_frame[i].Get(); + EXPECT_EQ(ext_render_message.timestamp_us(), i * 20000); + EXPECT_EQ(ext_render_message.normalized_crop_from_location().x(), + 725 / static_cast(kInputFrameWidth)); + EXPECT_EQ(ext_render_message.normalized_crop_from_location().y(), 0); + EXPECT_EQ(ext_render_message.normalized_crop_from_location().width(), + 461 / static_cast(kInputFrameWidth)); + EXPECT_EQ(ext_render_message.normalized_crop_from_location().height(), + 720 / static_cast(kInputFrameHeight)); + EXPECT_EQ(ext_render_message.render_to_location().x(), 0); + EXPECT_EQ(ext_render_message.render_to_location().y(), 0); + EXPECT_EQ(ext_render_message.render_to_location().width(), 720); + EXPECT_EQ(ext_render_message.render_to_location().height(), 1124); + } +} } // namespace } // namespace autoflip } // namespace mediapipe