cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Blendshapes graph take smoothed face landmarks as input.

Browse Source 
PiperOrigin-RevId: 527640341
This commit is contained in:
MediaPipe Team 2023-04-27 11:43:58 -07:00 committed by Copybara-Service
parent 82b8e4d7bf
commit 3ca2427cc8
5 changed files with 178 additions and 187 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
mediapipe/tasks/cc/vision/face_landmarker/BUILD
View File

@ -73,6 +73,8 @@ cc_library(
":tensors_to_face_landmarks_graph", ":tensors_to_face_landmarks_graph",
"//mediapipe/calculators/core:begin_loop_calculator", "//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:end_loop_calculator", "//mediapipe/calculators/core:end_loop_calculator",
"//mediapipe/calculators/core:get_vector_item_calculator",
"//mediapipe/calculators/core:get_vector_item_calculator_cc_proto",
"//mediapipe/calculators/core:split_vector_calculator", "//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/core:split_vector_calculator_cc_proto", "//mediapipe/calculators/core:split_vector_calculator_cc_proto",
"//mediapipe/calculators/image:image_properties_calculator", "//mediapipe/calculators/image:image_properties_calculator",
@ -86,6 +88,8 @@ cc_library(
"//mediapipe/calculators/util:detections_to_rects_calculator_cc_proto", "//mediapipe/calculators/util:detections_to_rects_calculator_cc_proto",
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator", "//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
"//mediapipe/calculators/util:landmark_projection_calculator", "//mediapipe/calculators/util:landmark_projection_calculator",
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
"//mediapipe/calculators/util:landmarks_smoothing_calculator_cc_proto",
"//mediapipe/calculators/util:landmarks_to_detection_calculator", "//mediapipe/calculators/util:landmarks_to_detection_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator", "//mediapipe/calculators/util:rect_transformation_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator_cc_proto", "//mediapipe/calculators/util:rect_transformation_calculator_cc_proto",
@ -194,8 +198,6 @@ cc_library(
"//mediapipe/calculators/util:association_norm_rect_calculator", "//mediapipe/calculators/util:association_norm_rect_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator", "//mediapipe/calculators/util:collection_has_min_size_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator_cc_proto", "//mediapipe/calculators/util:collection_has_min_size_calculator_cc_proto",
"//mediapipe/calculators/util:landmarks_smoothing_calculator",
"//mediapipe/calculators/util:landmarks_smoothing_calculator_cc_proto",
"//mediapipe/framework/api2:builder", "//mediapipe/framework/api2:builder",
"//mediapipe/framework/api2:port", "//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:classification_cc_proto", "//mediapipe/framework/formats:classification_cc_proto",

59
mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc
View File

@ -26,7 +26,6 @@ limitations under the License.
#include "mediapipe/calculators/core/get_vector_item_calculator.pb.h" #include "mediapipe/calculators/core/get_vector_item_calculator.pb.h"
#include "mediapipe/calculators/util/association_calculator.pb.h" #include "mediapipe/calculators/util/association_calculator.pb.h"
#include "mediapipe/calculators/util/collection_has_min_size_calculator.pb.h" #include "mediapipe/calculators/util/collection_has_min_size_calculator.pb.h"
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
#include "mediapipe/framework/api2/builder.h" #include "mediapipe/framework/api2/builder.h"
#include "mediapipe/framework/api2/port.h" #include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/formats/classification.pb.h" #include "mediapipe/framework/formats/classification.pb.h"
@ -172,19 +171,6 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
return absl::OkStatus(); return absl::OkStatus();
} }
void ConfigureLandmarksSmoothingCalculator(
mediapipe::LandmarksSmoothingCalculatorOptions& options) {
// Min cutoff 0.05 results into ~0.01 alpha in landmark EMA filter when
// landmark is static.
options.mutable_one_euro_filter()->set_min_cutoff(0.05f);
// Beta 80.0 in combintation with min_cutoff 0.05 results into ~0.94
// alpha in landmark EMA filter when landmark is moving fast.
options.mutable_one_euro_filter()->set_beta(80.0f);
// Derivative cutoff 1.0 results into ~0.17 alpha in landmark velocity
// EMA filter.
options.mutable_one_euro_filter()->set_derivate_cutoff(1.0f);
}
} // namespace } // namespace
// A "mediapipe.tasks.vision.face_landmarker.FaceLandmarkerGraph" performs face // A "mediapipe.tasks.vision.face_landmarker.FaceLandmarkerGraph" performs face
@ -464,32 +450,17 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
auto image_size = image_properties.Out(kSizeTag); auto image_size = image_properties.Out(kSizeTag);
// Apply smoothing filter only on the single face landmarks, because // Apply smoothing filter only on the single face landmarks, because
// landmakrs smoothing calculator doesn't support multiple landmarks yet. // landmarks smoothing calculator doesn't support multiple landmarks yet.
if (face_detector_options.num_faces() == 1) { if (face_detector_options.num_faces() == 1) {
// Get the single face landmarks face_landmarks_detector_graph
auto& get_vector_item = .GetOptions<FaceLandmarksDetectorGraphOptions>()
graph.AddNode("GetNormalizedLandmarkListVectorItemCalculator"); .set_smooth_landmarks(true);
get_vector_item.GetOptions<mediapipe::GetVectorItemCalculatorOptions>() } else if (face_detector_options.num_faces() > 1 &&
.set_item_index(0); face_landmarks_detector_graph
face_landmarks >> get_vector_item.In(kVectorTag); .GetOptions<FaceLandmarksDetectorGraphOptions>()
auto single_face_landmarks = get_vector_item.Out(kItemTag); .smooth_landmarks()) {
return absl::InvalidArgumentError(
// Apply smoothing filter on face landmarks. "Currently face landmarks smoothing only support a single face.");
auto& landmarks_smoothing = graph.AddNode("LandmarksSmoothingCalculator");
ConfigureLandmarksSmoothingCalculator(
landmarks_smoothing
.GetOptions<mediapipe::LandmarksSmoothingCalculatorOptions>());
single_face_landmarks >> landmarks_smoothing.In(kNormLandmarksTag);
image_size >> landmarks_smoothing.In(kImageSizeTag);
auto smoothed_single_face_landmarks =
landmarks_smoothing.Out(kNormFilteredLandmarksTag);
// Wrap the single face landmarks into a vector of landmarks.
auto& concatenate_vector =
graph.AddNode("ConcatenateNormalizedLandmarkListVectorCalculator");
smoothed_single_face_landmarks >> concatenate_vector.In("");
face_landmarks = concatenate_vector.Out("")
.Cast<std::vector<NormalizedLandmarkList>>();
} }
if (tasks_options.base_options().use_stream_mode()) { if (tasks_options.base_options().use_stream_mode()) {
@ -533,9 +504,10 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
// Back edge. // Back edge.
face_rects_for_next_frame >> previous_loopback.In(kLoopTag); face_rects_for_next_frame >> previous_loopback.In(kLoopTag);
} else { } else {
// While not in stream mode, the input images are not guaranteed to be in // While not in stream mode, the input images are not guaranteed to be
// series, and we don't want to enable the tracking and rect associations // in series, and we don't want to enable the tracking and rect
// between input images. Always use the face detector graph. // associations between input images. Always use the face detector
// graph.
image_in >> face_detector.In(kImageTag); image_in >> face_detector.In(kImageTag);
if (norm_rect_in) { if (norm_rect_in) {
*norm_rect_in >> face_detector.In(kNormRectTag); *norm_rect_in >> face_detector.In(kNormRectTag);
@ -571,7 +543,8 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
} }
// TODO: Replace PassThroughCalculator with a calculator that // TODO: Replace PassThroughCalculator with a calculator that
// converts the pixel data to be stored on the target storage (CPU vs GPU). // converts the pixel data to be stored on the target storage (CPU vs
// GPU).
auto& pass_through = graph.AddNode("PassThroughCalculator"); auto& pass_through = graph.AddNode("PassThroughCalculator");
image_in >> pass_through.In(""); image_in >> pass_through.In("");

219
mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_detector_graph.cc
View File

@ -19,10 +19,13 @@ limitations under the License.
#include <utility> #include <utility>
#include <vector> #include <vector>
#include "mediapipe/calculators/core/get_vector_item_calculator.h"
#include "mediapipe/calculators/core/get_vector_item_calculator.pb.h"
#include "mediapipe/calculators/core/split_vector_calculator.pb.h" #include "mediapipe/calculators/core/split_vector_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_floats_calculator.pb.h" #include "mediapipe/calculators/tensor/tensors_to_floats_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_landmarks_calculator.pb.h" #include "mediapipe/calculators/tensor/tensors_to_landmarks_calculator.pb.h"
#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h" #include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h"
#include "mediapipe/calculators/util/landmarks_smoothing_calculator.pb.h"
#include "mediapipe/calculators/util/rect_transformation_calculator.pb.h" #include "mediapipe/calculators/util/rect_transformation_calculator.pb.h"
#include "mediapipe/calculators/util/thresholding_calculator.pb.h" #include "mediapipe/calculators/util/thresholding_calculator.pb.h"
#include "mediapipe/framework/api2/builder.h" #include "mediapipe/framework/api2/builder.h"
@ -79,6 +82,9 @@ constexpr char kBatchEndTag[] = "BATCH_END";
constexpr char kItemTag[] = "ITEM"; constexpr char kItemTag[] = "ITEM";
constexpr char kDetectionTag[] = "DETECTION"; constexpr char kDetectionTag[] = "DETECTION";
constexpr char kBlendshapesTag[] = "BLENDSHAPES"; constexpr char kBlendshapesTag[] = "BLENDSHAPES";
constexpr char kNormFilteredLandmarksTag[] = "NORM_FILTERED_LANDMARKS";
constexpr char kSizeTag[] = "SIZE";
constexpr char kVectorTag[] = "VECTOR";
// a landmarks tensor and a scores tensor // a landmarks tensor and a scores tensor
constexpr int kFaceLandmarksOutputTensorsNum = 2; constexpr int kFaceLandmarksOutputTensorsNum = 2;
@ -88,7 +94,6 @@ struct SingleFaceLandmarksOutputs {
Stream<NormalizedRect> rect_next_frame; Stream<NormalizedRect> rect_next_frame;
Stream<bool> presence; Stream<bool> presence;
Stream<float> presence_score; Stream<float> presence_score;
std::optional<Stream<ClassificationList>> face_blendshapes;
}; };
struct MultiFaceLandmarksOutputs { struct MultiFaceLandmarksOutputs {
@ -148,6 +153,19 @@ void ConfigureFaceRectTransformationCalculator(
options->set_square_long(true); options->set_square_long(true);
} }
void ConfigureLandmarksSmoothingCalculator(
mediapipe::LandmarksSmoothingCalculatorOptions& options) {
// Min cutoff 0.05 results into ~0.01 alpha in landmark EMA filter when
// landmark is static.
options.mutable_one_euro_filter()->set_min_cutoff(0.05f);
// Beta 80.0 in combintation with min_cutoff 0.05 results into ~0.94
// alpha in landmark EMA filter when landmark is moving fast.
options.mutable_one_euro_filter()->set_beta(80.0f);
// Derivative cutoff 1.0 results into ~0.17 alpha in landmark velocity
// EMA filter.
options.mutable_one_euro_filter()->set_derivate_cutoff(1.0f);
}
} // namespace } // namespace
// A "mediapipe.tasks.vision.face_landmarker.SingleFaceLandmarksDetectorGraph" // A "mediapipe.tasks.vision.face_landmarker.SingleFaceLandmarksDetectorGraph"
@ -171,62 +189,6 @@ void ConfigureFaceRectTransformationCalculator(
// Boolean value indicates whether the face is present. // Boolean value indicates whether the face is present.
// PRESENCE_SCORE - float // PRESENCE_SCORE - float
// Float value indicates the probability that the face is present. // Float value indicates the probability that the face is present.
// BLENDSHAPES - ClassificationList @optional
// Blendshape classification, available when face_blendshapes_graph_options
// is set.
// All 52 blendshape coefficients:
// 0 - _neutral (ignore it)
// 1 - browDownLeft
// 2 - browDownRight
// 3 - browInnerUp
// 4 - browOuterUpLeft
// 5 - browOuterUpRight
// 6 - cheekPuff
// 7 - cheekSquintLeft
// 8 - cheekSquintRight
// 9 - eyeBlinkLeft
// 10 - eyeBlinkRight
// 11 - eyeLookDownLeft
// 12 - eyeLookDownRight
// 13 - eyeLookInLeft
// 14 - eyeLookInRight
// 15 - eyeLookOutLeft
// 16 - eyeLookOutRight
// 17 - eyeLookUpLeft
// 18 - eyeLookUpRight
// 19 - eyeSquintLeft
// 20 - eyeSquintRight
// 21 - eyeWideLeft
// 22 - eyeWideRight
// 23 - jawForward
// 24 - jawLeft
// 25 - jawOpen
// 26 - jawRight
// 27 - mouthClose
// 28 - mouthDimpleLeft
// 29 - mouthDimpleRight
// 30 - mouthFrownLeft
// 31 - mouthFrownRight
// 32 - mouthFunnel
// 33 - mouthLeft
// 34 - mouthLowerDownLeft
// 35 - mouthLowerDownRight
// 36 - mouthPressLeft
// 37 - mouthPressRight
// 38 - mouthPucker
// 39 - mouthRight
// 40 - mouthRollLower
// 41 - mouthRollUpper
// 42 - mouthShrugLower
// 43 - mouthShrugUpper
// 44 - mouthSmileLeft
// 45 - mouthSmileRight
// 46 - mouthStretchLeft
// 47 - mouthStretchRight
// 48 - mouthUpperUpLeft
// 49 - mouthUpperUpRight
// 50 - noseSneerLeft
// 51 - noseSneerRight
// //
// Example: // Example:
// node { // node {
@ -238,7 +200,6 @@ void ConfigureFaceRectTransformationCalculator(
// output_stream: "FACE_RECT_NEXT_FRAME:face_rect_next_frame" // output_stream: "FACE_RECT_NEXT_FRAME:face_rect_next_frame"
// output_stream: "PRESENCE:presence" // output_stream: "PRESENCE:presence"
// output_stream: "PRESENCE_SCORE:presence_score" // output_stream: "PRESENCE_SCORE:presence_score"
// output_stream: "BLENDSHAPES:blendshapes"
// options { // options {
// [mediapipe.tasks.vision.face_landmarker.proto.FaceLandmarksDetectorGraphOptions.ext] // [mediapipe.tasks.vision.face_landmarker.proto.FaceLandmarksDetectorGraphOptions.ext]
// { // {
@ -278,10 +239,6 @@ class SingleFaceLandmarksDetectorGraph : public core::ModelTaskGraph {
graph.Out(kFaceRectNextFrameTag).Cast<NormalizedRect>(); graph.Out(kFaceRectNextFrameTag).Cast<NormalizedRect>();
outs.presence >> graph.Out(kPresenceTag).Cast<bool>(); outs.presence >> graph.Out(kPresenceTag).Cast<bool>();
outs.presence_score >> graph.Out(kPresenceScoreTag).Cast<float>(); outs.presence_score >> graph.Out(kPresenceScoreTag).Cast<float>();
if (outs.face_blendshapes) {
outs.face_blendshapes.value() >>
graph.Out(kBlendshapesTag).Cast<ClassificationList>();
}
return graph.GetConfig(); return graph.GetConfig();
} }
@ -378,7 +335,7 @@ class SingleFaceLandmarksDetectorGraph : public core::ModelTaskGraph {
auto& landmark_projection = graph.AddNode("LandmarkProjectionCalculator"); auto& landmark_projection = graph.AddNode("LandmarkProjectionCalculator");
landmarks_letterbox_removed >> landmark_projection.In(kNormLandmarksTag); landmarks_letterbox_removed >> landmark_projection.In(kNormLandmarksTag);
face_rect >> landmark_projection.In(kNormRectTag); face_rect >> landmark_projection.In(kNormRectTag);
auto projected_landmarks = AllowIf( Stream<NormalizedLandmarkList> projected_landmarks = AllowIf(
landmark_projection[Output<NormalizedLandmarkList>(kNormLandmarksTag)], landmark_projection[Output<NormalizedLandmarkList>(kNormLandmarksTag)],
presence, graph); presence, graph);
@ -409,25 +366,11 @@ class SingleFaceLandmarksDetectorGraph : public core::ModelTaskGraph {
AllowIf(face_rect_transformation.Out("").Cast<NormalizedRect>(), AllowIf(face_rect_transformation.Out("").Cast<NormalizedRect>(),
presence, graph); presence, graph);
std::optional<Stream<ClassificationList>> face_blendshapes;
if (subgraph_options.has_face_blendshapes_graph_options()) {
auto& face_blendshapes_graph = graph.AddNode(
"mediapipe.tasks.vision.face_landmarker.FaceBlendshapesGraph");
face_blendshapes_graph.GetOptions<proto::FaceBlendshapesGraphOptions>()
.Swap(subgraph_options.mutable_face_blendshapes_graph_options());
projected_landmarks >> face_blendshapes_graph.In(kLandmarksTag);
image_size >> face_blendshapes_graph.In(kImageSizeTag);
face_blendshapes =
std::make_optional(face_blendshapes_graph.Out(kBlendshapesTag)
.Cast<ClassificationList>());
}
return {{ return {{
/* landmarks= */ projected_landmarks, /* landmarks= */ projected_landmarks,
/* rect_next_frame= */ face_rect_next_frame, /* rect_next_frame= */ face_rect_next_frame,
/* presence= */ presence, /* presence= */ presence,
/* presence_score= */ presence_score, /* presence_score= */ presence_score,
/* face_blendshapes= */ face_blendshapes,
}}; }};
} }
}; };
@ -465,6 +408,59 @@ REGISTER_MEDIAPIPE_GRAPH(
// BLENDSHAPES - std::vector<ClassificationList> @optional // BLENDSHAPES - std::vector<ClassificationList> @optional
// Vector of face blendshape classification, available when // Vector of face blendshape classification, available when
// face_blendshapes_graph_options is set. // face_blendshapes_graph_options is set.
// All 52 blendshape coefficients:
// 0 - _neutral (ignore it)
// 1 - browDownLeft
// 2 - browDownRight
// 3 - browInnerUp
// 4 - browOuterUpLeft
// 5 - browOuterUpRight
// 6 - cheekPuff
// 7 - cheekSquintLeft
// 8 - cheekSquintRight
// 9 - eyeBlinkLeft
// 10 - eyeBlinkRight
// 11 - eyeLookDownLeft
// 12 - eyeLookDownRight
// 13 - eyeLookInLeft
// 14 - eyeLookInRight
// 15 - eyeLookOutLeft
// 16 - eyeLookOutRight
// 17 - eyeLookUpLeft
// 18 - eyeLookUpRight
// 19 - eyeSquintLeft
// 20 - eyeSquintRight
// 21 - eyeWideLeft
// 22 - eyeWideRight
// 23 - jawForward
// 24 - jawLeft
// 25 - jawOpen
// 26 - jawRight
// 27 - mouthClose
// 28 - mouthDimpleLeft
// 29 - mouthDimpleRight
// 30 - mouthFrownLeft
// 31 - mouthFrownRight
// 32 - mouthFunnel
// 33 - mouthLeft
// 34 - mouthLowerDownLeft
// 35 - mouthLowerDownRight
// 36 - mouthPressLeft
// 37 - mouthPressRight
// 38 - mouthPucker
// 39 - mouthRight
// 40 - mouthRollLower
// 41 - mouthRollUpper
// 42 - mouthShrugLower
// 43 - mouthShrugUpper
// 44 - mouthSmileLeft
// 45 - mouthSmileRight
// 46 - mouthStretchLeft
// 47 - mouthStretchRight
// 48 - mouthUpperUpLeft
// 49 - mouthUpperUpRight
// 50 - noseSneerLeft
// 51 - noseSneerRight
// //
// Example: // Example:
// node { // node {
@ -566,7 +562,8 @@ class MultiFaceLandmarksDetectorGraph : public core::ModelTaskGraph {
graph.AddNode("EndLoopNormalizedLandmarkListVectorCalculator"); graph.AddNode("EndLoopNormalizedLandmarkListVectorCalculator");
batch_end >> end_loop_landmarks.In(kBatchEndTag); batch_end >> end_loop_landmarks.In(kBatchEndTag);
landmarks >> end_loop_landmarks.In(kItemTag); landmarks >> end_loop_landmarks.In(kItemTag);
auto landmark_lists = end_loop_landmarks.Out(kIterableTag) Stream<std::vector<NormalizedLandmarkList>> landmark_lists =
end_loop_landmarks.Out(kIterableTag)
.Cast<std::vector<NormalizedLandmarkList>>(); .Cast<std::vector<NormalizedLandmarkList>>();
auto& end_loop_rects_next_frame = auto& end_loop_rects_next_frame =
@ -576,16 +573,78 @@ class MultiFaceLandmarksDetectorGraph : public core::ModelTaskGraph {
auto face_rects_next_frame = end_loop_rects_next_frame.Out(kIterableTag) auto face_rects_next_frame = end_loop_rects_next_frame.Out(kIterableTag)
.Cast<std::vector<NormalizedRect>>(); .Cast<std::vector<NormalizedRect>>();
// Apply smoothing filter only on the single face landmarks, because
// landmarks smoothing calculator doesn't support multiple landmarks yet.
// Notice the landmarks smoothing calculator cannot be put inside the for
// loop calculator, because the smoothing calculator utilize the timestamp
// to smoote landmarks across frames but the for loop calculator makes fake
// timestamps for the streams.
if (face_landmark_subgraph
.GetOptions<proto::FaceLandmarksDetectorGraphOptions>()
.smooth_landmarks()) {
// Get the single face landmarks
auto& get_vector_item =
graph.AddNode("GetNormalizedLandmarkListVectorItemCalculator");
get_vector_item.GetOptions<mediapipe::GetVectorItemCalculatorOptions>()
.set_item_index(0);
landmark_lists >> get_vector_item.In(kVectorTag);
Stream<NormalizedLandmarkList> single_landmarks =
get_vector_item.Out(kItemTag).Cast<NormalizedLandmarkList>();
auto& image_properties = graph.AddNode("ImagePropertiesCalculator");
image_in >> image_properties.In(kImageTag);
auto image_size = image_properties.Out(kSizeTag);
// Apply smoothing filter on face landmarks.
auto& landmarks_smoothing = graph.AddNode("LandmarksSmoothingCalculator");
ConfigureLandmarksSmoothingCalculator(
landmarks_smoothing
.GetOptions<mediapipe::LandmarksSmoothingCalculatorOptions>());
single_landmarks >> landmarks_smoothing.In(kNormLandmarksTag);
image_size >> landmarks_smoothing.In(kImageSizeTag);
single_landmarks = landmarks_smoothing.Out(kNormFilteredLandmarksTag)
.Cast<NormalizedLandmarkList>();
// Wrap the single face landmarks into a vector of landmarks.
auto& concatenate_vector =
graph.AddNode("ConcatenateNormalizedLandmarkListVectorCalculator");
single_landmarks >> concatenate_vector.In("");
landmark_lists = concatenate_vector.Out("")
.Cast<std::vector<NormalizedLandmarkList>>();
}
std::optional<Stream<std::vector<ClassificationList>>> std::optional<Stream<std::vector<ClassificationList>>>
face_blendshapes_vector; face_blendshapes_vector;
if (face_landmark_subgraph if (face_landmark_subgraph
.GetOptions<proto::FaceLandmarksDetectorGraphOptions>() .GetOptions<proto::FaceLandmarksDetectorGraphOptions>()
.has_face_blendshapes_graph_options()) { .has_face_blendshapes_graph_options()) {
auto blendshapes = face_landmark_subgraph.Out(kBlendshapesTag); auto& begin_loop_multi_face_landmarks =
graph.AddNode("BeginLoopNormalizedLandmarkListVectorCalculator");
landmark_lists >> begin_loop_multi_face_landmarks.In(kIterableTag);
image_in >> begin_loop_multi_face_landmarks.In(kCloneTag);
auto image = begin_loop_multi_face_landmarks.Out(kCloneTag);
auto batch_end = begin_loop_multi_face_landmarks.Out(kBatchEndTag);
auto landmarks = begin_loop_multi_face_landmarks.Out(kItemTag);
auto& image_properties = graph.AddNode("ImagePropertiesCalculator");
image >> image_properties.In(kImageTag);
auto image_size = image_properties.Out(kSizeTag);
auto& face_blendshapes_graph = graph.AddNode(
"mediapipe.tasks.vision.face_landmarker.FaceBlendshapesGraph");
face_blendshapes_graph.GetOptions<proto::FaceBlendshapesGraphOptions>()
.Swap(face_landmark_subgraph
.GetOptions<proto::FaceLandmarksDetectorGraphOptions>()
.mutable_face_blendshapes_graph_options());
landmarks >> face_blendshapes_graph.In(kLandmarksTag);
image_size >> face_blendshapes_graph.In(kImageSizeTag);
auto face_blendshapes = face_blendshapes_graph.Out(kBlendshapesTag)
.Cast<ClassificationList>();
auto& end_loop_blendshapes = auto& end_loop_blendshapes =
graph.AddNode("EndLoopClassificationListCalculator"); graph.AddNode("EndLoopClassificationListCalculator");
batch_end >> end_loop_blendshapes.In(kBatchEndTag); batch_end >> end_loop_blendshapes.In(kBatchEndTag);
blendshapes >> end_loop_blendshapes.In(kItemTag); face_blendshapes >> end_loop_blendshapes.In(kItemTag);
face_blendshapes_vector = face_blendshapes_vector =
std::make_optional(end_loop_blendshapes.Out(kIterableTag) std::make_optional(end_loop_blendshapes.Out(kIterableTag)
.Cast<std::vector<ClassificationList>>()); .Cast<std::vector<ClassificationList>>());

56
mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_detector_graph_test.cc
View File

@ -99,8 +99,7 @@ constexpr float kBlendshapesDiffMargin = 0.1;
// Helper function to create a Single Face Landmark TaskRunner. // Helper function to create a Single Face Landmark TaskRunner.
absl::StatusOr<std::unique_ptr<TaskRunner>> CreateSingleFaceLandmarksTaskRunner( absl::StatusOr<std::unique_ptr<TaskRunner>> CreateSingleFaceLandmarksTaskRunner(
absl::string_view landmarks_model_name, absl::string_view landmarks_model_name) {
std::optional<absl::string_view> blendshapes_model_name) {
Graph graph; Graph graph;
auto& face_landmark_detection = graph.AddNode( auto& face_landmark_detection = graph.AddNode(
@ -112,14 +111,6 @@ absl::StatusOr<std::unique_ptr<TaskRunner>> CreateSingleFaceLandmarksTaskRunner(
JoinPath("./", kTestDataDirectory, landmarks_model_name)); JoinPath("./", kTestDataDirectory, landmarks_model_name));
options->set_min_detection_confidence(0.5); options->set_min_detection_confidence(0.5);
if (blendshapes_model_name.has_value()) {
options->mutable_face_blendshapes_graph_options()
->mutable_base_options()
->mutable_model_asset()
->set_file_name(
JoinPath("./", kTestDataDirectory, *blendshapes_model_name));
}
face_landmark_detection.GetOptions<proto::FaceLandmarksDetectorGraphOptions>() face_landmark_detection.GetOptions<proto::FaceLandmarksDetectorGraphOptions>()
.Swap(options.get()); .Swap(options.get());
@ -137,11 +128,6 @@ absl::StatusOr<std::unique_ptr<TaskRunner>> CreateSingleFaceLandmarksTaskRunner(
face_landmark_detection.Out(kFaceRectNextFrameTag) face_landmark_detection.Out(kFaceRectNextFrameTag)
.SetName(kFaceRectNextFrameName) >> .SetName(kFaceRectNextFrameName) >>
graph[Output<NormalizedRect>(kFaceRectNextFrameTag)]; graph[Output<NormalizedRect>(kFaceRectNextFrameTag)];
if (blendshapes_model_name.has_value()) {
face_landmark_detection.Out(kBlendshapesTag).SetName(kBlendshapesName) >>
graph[Output<ClassificationList>(kBlendshapesTag)];
}
return TaskRunner::Create( return TaskRunner::Create(
graph.GetConfig(), absl::make_unique<core::MediaPipeBuiltinOpResolver>()); graph.GetConfig(), absl::make_unique<core::MediaPipeBuiltinOpResolver>());
} }
@ -227,8 +213,6 @@ struct SingeFaceTestParams {
std::string test_name; std::string test_name;
// The filename of landmarks model name. // The filename of landmarks model name.
std::string landmarks_model_name; std::string landmarks_model_name;
// The filename of blendshape model name.
std::optional<std::string> blendshape_model_name;
// The filename of the test image. // The filename of the test image.
std::string test_image_name; std::string test_image_name;
// RoI on image to detect faces. // RoI on image to detect faces.
@ -237,13 +221,8 @@ struct SingeFaceTestParams {
bool expected_presence; bool expected_presence;
// The expected output landmarks positions. // The expected output landmarks positions.
NormalizedLandmarkList expected_landmarks; NormalizedLandmarkList expected_landmarks;
// The expected output blendshape classification;
std::optional<ClassificationList> expected_blendshapes;
// The max value difference between expected_positions and detected positions. // The max value difference between expected_positions and detected positions.
float landmarks_diff_threshold; float landmarks_diff_threshold;
// The max value difference between expected blendshapes and actual
// blendshapes.
float blendshapes_diff_threshold;
}; };
struct MultiFaceTestParams { struct MultiFaceTestParams {
@ -279,8 +258,7 @@ TEST_P(SingleFaceLandmarksDetectionTest, Succeeds) {
GetParam().test_image_name))); GetParam().test_image_name)));
MP_ASSERT_OK_AND_ASSIGN( MP_ASSERT_OK_AND_ASSIGN(
auto task_runner, auto task_runner,
CreateSingleFaceLandmarksTaskRunner(GetParam().landmarks_model_name, CreateSingleFaceLandmarksTaskRunner(GetParam().landmarks_model_name));
GetParam().blendshape_model_name));
auto output_packets = task_runner->Process( auto output_packets = task_runner->Process(
{{kImageName, MakePacket<Image>(std::move(image))}, {{kImageName, MakePacket<Image>(std::move(image))},
@ -301,15 +279,6 @@ TEST_P(SingleFaceLandmarksDetectionTest, Succeeds) {
Approximately(Partially(EqualsProto(expected_landmarks)), Approximately(Partially(EqualsProto(expected_landmarks)),
/*margin=*/kAbsMargin, /*margin=*/kAbsMargin,
/*fraction=*/GetParam().landmarks_diff_threshold)); /*fraction=*/GetParam().landmarks_diff_threshold));
if (GetParam().expected_blendshapes) {
const ClassificationList& actual_blendshapes =
(*output_packets)[kBlendshapesName].Get<ClassificationList>();
const ClassificationList& expected_blendshapes =
*GetParam().expected_blendshapes;
EXPECT_THAT(actual_blendshapes,
Approximately(EqualsProto(expected_blendshapes),
GetParam().blendshapes_diff_threshold));
}
} }
} }
@ -363,31 +332,12 @@ INSTANTIATE_TEST_SUITE_P(
/* test_name= */ "PortraitV2", /* test_name= */ "PortraitV2",
/* landmarks_model_name= */ /* landmarks_model_name= */
kFaceLandmarksV2Model, kFaceLandmarksV2Model,
/* blendshape_model_name= */ std::nullopt,
/* test_image_name= */ kPortraitImageName, /* test_image_name= */ kPortraitImageName,
/* norm_rect= */ MakeNormRect(0.4987, 0.2211, 0.2877, 0.2303, 0), /* norm_rect= */ MakeNormRect(0.4987, 0.2211, 0.2877, 0.2303, 0),
/* expected_presence= */ true, /* expected_presence= */ true,
/* expected_landmarks= */ /* expected_landmarks= */
GetExpectedLandmarkList(kPortraitExpectedFaceLandmarksName), GetExpectedLandmarkList(kPortraitExpectedFaceLandmarksName),
/* expected_blendshapes= */ std::nullopt, /* landmarks_diff_threshold= */ kFractionDiff}),
/* landmarks_diff_threshold= */ kFractionDiff,
/* blendshapes_diff_threshold= */ kBlendshapesDiffMargin},
SingeFaceTestParams{
/* test_name= */ "PortraitV2WithBlendshapes",
/* landmarks_model_name= */
kFaceLandmarksV2Model,
/* blendshape_model_name= */ kFaceBlendshapesModel,
/* test_image_name= */ kPortraitImageName,
/* norm_rect= */
MakeNormRect(0.48906386, 0.22731927, 0.42905223, 0.34357703,
0.008304443),
/* expected_presence= */ true,
/* expected_landmarks= */
GetExpectedLandmarkList(kPortraitExpectedFaceLandmarksName),
/* expected_blendshapes= */
GetBlendshapes(kPortraitExpectedBlendshapesName),
/* landmarks_diff_threshold= */ kFractionDiff,
/* blendshapes_diff_threshold= */ kBlendshapesDiffMargin}),
[](const TestParamInfo<SingleFaceLandmarksDetectionTest::ParamType>& info) { [](const TestParamInfo<SingleFaceLandmarksDetectionTest::ParamType>& info) {
return info.param.test_name; return info.param.test_name;

7
mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.proto
View File

@ -37,6 +37,13 @@ message FaceLandmarksDetectorGraphOptions {
// successfully detecting a face in the image. // successfully detecting a face in the image.
optional float min_detection_confidence = 2 [default = 0.5]; optional float min_detection_confidence = 2 [default = 0.5];
// Whether to smooth the detected landmarks over timestamps. Note that
// landmarks smoothing is only applicable for a single face. If multiple faces
// landmarks are given, and smooth_landmarks is true, only the first face
// landmarks would be smoothed, and the remaining landmarks are discarded in
// the returned landmarks list.
optional bool smooth_landmarks = 4;
// Optional options for FaceBlendshapeGraph. If this options is set, the // Optional options for FaceBlendshapeGraph. If this options is set, the
// FaceLandmarksDetectorGraph would output the face blendshapes. // FaceLandmarksDetectorGraph would output the face blendshapes.
optional FaceBlendshapesGraphOptions face_blendshapes_graph_options = 3; optional FaceBlendshapesGraphOptions face_blendshapes_graph_options = 3;