From 056881f4a96f6c59106655b8c5d737f8f6ddd6a3 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Tue, 30 May 2023 11:14:09 -0700 Subject: [PATCH] Only apply face landmarks smoothing for stream mode (VIDEO and LIVE_STREAM). PiperOrigin-RevId: 536455842 --- .../face_landmarker/face_landmarker_graph.cc | 2 +- .../face_landmarker_graph_test.cc | 96 +++++++++++++++++++ 2 files changed, 97 insertions(+), 1 deletion(-) diff --git a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc index 0f9caec0d..643f40624 100644 --- a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc +++ b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc @@ -454,7 +454,7 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph { if (face_detector_options.num_faces() == 1) { face_landmarks_detector_graph .GetOptions() - .set_smooth_landmarks(true); + .set_smooth_landmarks(tasks_options.base_options().use_stream_mode()); } else if (face_detector_options.num_faces() > 1 && face_landmarks_detector_graph .GetOptions() diff --git a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph_test.cc b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph_test.cc index a6fade602..a8ab0b9e9 100644 --- a/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph_test.cc +++ b/mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph_test.cc @@ -14,11 +14,13 @@ limitations under the License. ==============================================================================*/ #include +#include #include "absl/flags/flag.h" #include "absl/status/statusor.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" #include "mediapipe/framework/api2/builder.h" #include "mediapipe/framework/api2/port.h" #include "mediapipe/framework/calculator_framework.h" @@ -31,6 +33,7 @@ limitations under the License. #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/gmock.h" #include "mediapipe/framework/port/gtest.h" +#include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" #include "mediapipe/tasks/cc/core/proto/base_options.pb.h" #include "mediapipe/tasks/cc/core/proto/external_file.pb.h" @@ -95,6 +98,8 @@ constexpr float kLandmarksDiffMargin = 0.03; constexpr float kBlendshapesDiffMargin = 0.1; constexpr float kFaceGeometryDiffMargin = 0.02; +constexpr char kLandmarksSmoothingCalculator[] = "LandmarksSmoothingCalculator"; + template ProtoT GetExpectedProto(absl::string_view filename) { ProtoT expected_proto; @@ -103,6 +108,13 @@ ProtoT GetExpectedProto(absl::string_view filename) { return expected_proto; } +struct VerifyExpandedConfigTestParams { + std::string test_name; + bool use_stream_mode; + int num_faces; + bool has_smoothing_calculator; +}; + // Struct holding the parameters for parameterized FaceLandmarkerGraphTest // class. struct FaceLandmarkerGraphTestParams { @@ -165,6 +177,25 @@ absl::StatusOr> CreateFaceLandmarkerGraphTaskRunner( absl::make_unique()); } +absl::StatusOr ExpandConfig( + const std::string& config_str) { + auto config = + mediapipe::ParseTextProtoOrDie(config_str); + CalculatorGraph graph; + MP_RETURN_IF_ERROR(graph.Initialize(config)); + return graph.Config(); +} + +bool HasCalculatorInConfig(const std::string& calculator_name, + const CalculatorGraphConfig& config) { + for (const auto& node : config.node()) { + if (node.calculator() == calculator_name) { + return true; + } + } + return false; +} + // Helper function to construct NormalizeRect proto. NormalizedRect MakeNormRect(float x_center, float y_center, float width, float height, float rotation) { @@ -177,6 +208,71 @@ NormalizedRect MakeNormRect(float x_center, float y_center, float width, return face_rect; } +constexpr char kGraphConfigString[] = R"pb( + node { + calculator: "mediapipe.tasks.vision.face_landmarker.FaceLandmarkerGraph" + input_stream: "IMAGE:image_in" + output_stream: "NORM_LANDMARKS:face_landmarks" + options { + [mediapipe.tasks.vision.face_landmarker.proto.FaceLandmarkerGraphOptions + .ext] { + base_options { + model_asset { + file_name: "mediapipe/tasks/testdata/vision/face_landmarker_v2_with_blendshapes.task" + } + use_stream_mode: $0 + } + face_detector_graph_options { num_faces: $1 } + } + } + } + input_stream: "IMAGE:image_in" +)pb"; + +class VerifyExpandedConfig + : public testing::TestWithParam {}; + +TEST_P(VerifyExpandedConfig, Succeeds) { + MP_ASSERT_OK_AND_ASSIGN( + auto actual_graph, + ExpandConfig(absl::Substitute( + kGraphConfigString, GetParam().use_stream_mode ? "true" : "false", + std::to_string(GetParam().num_faces)))); + if (GetParam().has_smoothing_calculator) { + EXPECT_TRUE( + HasCalculatorInConfig(kLandmarksSmoothingCalculator, actual_graph)); + } else { + EXPECT_FALSE( + HasCalculatorInConfig(kLandmarksSmoothingCalculator, actual_graph)); + } +} + +INSTANTIATE_TEST_SUITE_P( + VerifyExpandedConfig, VerifyExpandedConfig, + Values(VerifyExpandedConfigTestParams{ + /*test_name=*/"NonStreamOneFaceHasNoSmoothing", + /*use_stream_mode=*/false, + /*num_faces=*/1, + /*has_smoothing_calculator=*/false}, + VerifyExpandedConfigTestParams{ + /*test_name=*/"NonStreamTwoFaceHasNoSmoothing", + /*use_stream_mode=*/false, + /*num_faces=*/2, + /*has_smoothing_calculator=*/false}, + VerifyExpandedConfigTestParams{ + /*test_name=*/"StreamOneFaceHasSmoothing", + /*use_stream_mode=*/true, + /*num_faces=*/1, + /*has_smoothing_calculator=*/true}, + VerifyExpandedConfigTestParams{ + /*test_name=*/"StreamTwoFaceHasNoSmoothing", + /*use_stream_mode=*/true, + /*num_faces=*/2, + /*has_smoothing_calculator=*/false}), + [](const TestParamInfo& info) { + return info.param.test_name; + }); + class FaceLandmarkerGraphTest : public testing::TestWithParam {};