Internal change
PiperOrigin-RevId: 515187906
This commit is contained in:
parent
8d9f627fd9
commit
c12eae229f
|
@ -322,10 +322,8 @@ class FaceDetectorGraph : public core::ModelTaskGraph {
|
||||||
// detection bounding boxes.
|
// detection bounding boxes.
|
||||||
auto& detection_transformation =
|
auto& detection_transformation =
|
||||||
graph.AddNode("DetectionTransformationCalculator");
|
graph.AddNode("DetectionTransformationCalculator");
|
||||||
detection_projection.Out(kDetectionsTag) >>
|
face_detections >> detection_transformation.In(kDetectionsTag);
|
||||||
detection_transformation.In(kDetectionsTag);
|
image_size >> detection_transformation.In(kImageSizeTag);
|
||||||
preprocessing.Out(kImageSizeTag) >>
|
|
||||||
detection_transformation.In(kImageSizeTag);
|
|
||||||
auto face_pixel_detections =
|
auto face_pixel_detections =
|
||||||
detection_transformation.Out(kPixelDetectionsTag)
|
detection_transformation.Out(kPixelDetectionsTag)
|
||||||
.Cast<std::vector<Detection>>();
|
.Cast<std::vector<Detection>>();
|
||||||
|
|
|
@ -97,7 +97,8 @@ void ConfigureFaceGeometryEnvGeneratorCalculator(
|
||||||
// SideInputs:
|
// SideInputs:
|
||||||
// ENVIRONMENT - ENVIRONMENT
|
// ENVIRONMENT - ENVIRONMENT
|
||||||
// Environment that describes the current virtual scene. If not provided, a
|
// Environment that describes the current virtual scene. If not provided, a
|
||||||
// default environment will be used which can be applied to common webcam.
|
// default environment will be used which is good enough for most general
|
||||||
|
// use cases.
|
||||||
//
|
//
|
||||||
//
|
//
|
||||||
// Outputs:
|
// Outputs:
|
||||||
|
@ -142,7 +143,7 @@ class FaceGeometryFromLandmarksGraph : public Subgraph {
|
||||||
std::optional<SidePacket<Environment>> environment, Graph& graph) {
|
std::optional<SidePacket<Environment>> environment, Graph& graph) {
|
||||||
if (!environment.has_value()) {
|
if (!environment.has_value()) {
|
||||||
// If there is no provided Environment, use a a default environment which
|
// If there is no provided Environment, use a a default environment which
|
||||||
// is good enough for most general use case.
|
// is good enough for most general use cases.
|
||||||
auto& env_generator = graph.AddNode(
|
auto& env_generator = graph.AddNode(
|
||||||
"mediapipe.tasks.vision.face_geometry."
|
"mediapipe.tasks.vision.face_geometry."
|
||||||
"FaceGeometryEnvGeneratorCalculator");
|
"FaceGeometryEnvGeneratorCalculator");
|
||||||
|
|
|
@ -142,3 +142,50 @@ cc_library(
|
||||||
"//mediapipe/tasks/cc/components/containers:landmark",
|
"//mediapipe/tasks/cc/components/containers:landmark",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
cc_library(
|
||||||
|
name = "face_landmarker_graph",
|
||||||
|
srcs = ["face_landmarker_graph.cc"],
|
||||||
|
deps = [
|
||||||
|
":face_landmarks_detector_graph",
|
||||||
|
"//mediapipe/calculators/core:begin_loop_calculator",
|
||||||
|
"//mediapipe/calculators/core:clip_vector_size_calculator_cc_proto",
|
||||||
|
"//mediapipe/calculators/core:end_loop_calculator",
|
||||||
|
"//mediapipe/calculators/core:gate_calculator",
|
||||||
|
"//mediapipe/calculators/core:gate_calculator_cc_proto",
|
||||||
|
"//mediapipe/calculators/core:pass_through_calculator",
|
||||||
|
"//mediapipe/calculators/core:previous_loopback_calculator",
|
||||||
|
"//mediapipe/calculators/image:image_properties_calculator",
|
||||||
|
"//mediapipe/calculators/util:association_calculator_cc_proto",
|
||||||
|
"//mediapipe/calculators/util:association_norm_rect_calculator",
|
||||||
|
"//mediapipe/calculators/util:collection_has_min_size_calculator",
|
||||||
|
"//mediapipe/calculators/util:collection_has_min_size_calculator_cc_proto",
|
||||||
|
"//mediapipe/framework/api2:builder",
|
||||||
|
"//mediapipe/framework/api2:port",
|
||||||
|
"//mediapipe/framework/formats:classification_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:detection_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:image",
|
||||||
|
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:rect_cc_proto",
|
||||||
|
"//mediapipe/framework/formats:tensor",
|
||||||
|
"//mediapipe/framework/port:status",
|
||||||
|
"//mediapipe/tasks/cc:common",
|
||||||
|
"//mediapipe/tasks/cc/components/utils:gate",
|
||||||
|
"//mediapipe/tasks/cc/core:model_asset_bundle_resources",
|
||||||
|
"//mediapipe/tasks/cc/core:model_resources_cache",
|
||||||
|
"//mediapipe/tasks/cc/core:model_task_graph",
|
||||||
|
"//mediapipe/tasks/cc/core:utils",
|
||||||
|
"//mediapipe/tasks/cc/metadata/utils:zip_utils",
|
||||||
|
"//mediapipe/tasks/cc/vision/face_detector:face_detector_graph",
|
||||||
|
"//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto",
|
||||||
|
"//mediapipe/tasks/cc/vision/face_geometry:face_geometry_from_landmarks_graph",
|
||||||
|
"//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto",
|
||||||
|
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto",
|
||||||
|
"//mediapipe/tasks/cc/vision/face_landmarker/proto:face_blendshapes_graph_options_cc_proto",
|
||||||
|
"//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_cc_proto",
|
||||||
|
"//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_cc_proto",
|
||||||
|
"//mediapipe/util:graph_builder_utils",
|
||||||
|
"@com_google_absl//absl/strings:str_format",
|
||||||
|
],
|
||||||
|
alwayslink = 1,
|
||||||
|
)
|
||||||
|
|
|
@ -0,0 +1,520 @@
|
||||||
|
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
==============================================================================*/
|
||||||
|
|
||||||
|
#include <memory>
|
||||||
|
#include <type_traits>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#include "absl/strings/str_format.h"
|
||||||
|
#include "mediapipe/calculators/core/clip_vector_size_calculator.pb.h"
|
||||||
|
#include "mediapipe/calculators/core/gate_calculator.pb.h"
|
||||||
|
#include "mediapipe/calculators/util/association_calculator.pb.h"
|
||||||
|
#include "mediapipe/calculators/util/collection_has_min_size_calculator.pb.h"
|
||||||
|
#include "mediapipe/framework/api2/builder.h"
|
||||||
|
#include "mediapipe/framework/api2/port.h"
|
||||||
|
#include "mediapipe/framework/formats/classification.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/detection.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/image.h"
|
||||||
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/rect.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/tensor.h"
|
||||||
|
#include "mediapipe/framework/port/status_macros.h"
|
||||||
|
#include "mediapipe/tasks/cc/common.h"
|
||||||
|
#include "mediapipe/tasks/cc/components/utils/gate.h"
|
||||||
|
#include "mediapipe/tasks/cc/core/model_asset_bundle_resources.h"
|
||||||
|
#include "mediapipe/tasks/cc/core/model_resources_cache.h"
|
||||||
|
#include "mediapipe/tasks/cc/core/model_task_graph.h"
|
||||||
|
#include "mediapipe/tasks/cc/core/utils.h"
|
||||||
|
#include "mediapipe/tasks/cc/metadata/utils/zip_utils.h"
|
||||||
|
#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h"
|
||||||
|
#include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
|
||||||
|
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
|
||||||
|
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h"
|
||||||
|
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.pb.h"
|
||||||
|
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h"
|
||||||
|
#include "mediapipe/util/graph_builder_utils.h"
|
||||||
|
|
||||||
|
namespace mediapipe {
|
||||||
|
namespace tasks {
|
||||||
|
namespace vision {
|
||||||
|
namespace face_landmarker {
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
using ::mediapipe::NormalizedRect;
|
||||||
|
using ::mediapipe::api2::Input;
|
||||||
|
using ::mediapipe::api2::Output;
|
||||||
|
using ::mediapipe::api2::builder::Graph;
|
||||||
|
using ::mediapipe::api2::builder::SidePacket;
|
||||||
|
using ::mediapipe::api2::builder::Source;
|
||||||
|
using ::mediapipe::tasks::components::utils::DisallowIf;
|
||||||
|
using ::mediapipe::tasks::core::ModelAssetBundleResources;
|
||||||
|
using ::mediapipe::tasks::metadata::SetExternalFile;
|
||||||
|
using ::mediapipe::tasks::vision::face_detector::proto::
|
||||||
|
FaceDetectorGraphOptions;
|
||||||
|
using ::mediapipe::tasks::vision::face_geometry::proto::Environment;
|
||||||
|
using ::mediapipe::tasks::vision::face_geometry::proto::FaceGeometry;
|
||||||
|
using ::mediapipe::tasks::vision::face_landmarker::proto::
|
||||||
|
FaceLandmarkerGraphOptions;
|
||||||
|
using ::mediapipe::tasks::vision::face_landmarker::proto::
|
||||||
|
FaceLandmarksDetectorGraphOptions;
|
||||||
|
|
||||||
|
constexpr char kImageTag[] = "IMAGE";
|
||||||
|
constexpr char kNormRectTag[] = "NORM_RECT";
|
||||||
|
constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS";
|
||||||
|
constexpr char kFaceRectsTag[] = "FACE_RECTS";
|
||||||
|
constexpr char kFaceRectsNextFrameTag[] = "FACE_RECTS_NEXT_FRAME";
|
||||||
|
constexpr char kExpandedFaceRectsTag[] = "EXPANDED_FACE_RECTS";
|
||||||
|
constexpr char kDetectionsTag[] = "DETECTIONS";
|
||||||
|
constexpr char kLoopTag[] = "LOOP";
|
||||||
|
constexpr char kPrevLoopTag[] = "PREV_LOOP";
|
||||||
|
constexpr char kMainTag[] = "MAIN";
|
||||||
|
constexpr char kIterableTag[] = "ITERABLE";
|
||||||
|
constexpr char kFaceLandmarksTag[] = "FACE_LANDMARKS";
|
||||||
|
constexpr char kFaceGeometryTag[] = "FACE_GEOMETRY";
|
||||||
|
constexpr char kEnvironmentTag[] = "ENVIRONMENT";
|
||||||
|
constexpr char kBlendshapesTag[] = "BLENDSHAPES";
|
||||||
|
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
|
||||||
|
constexpr char kSizeTag[] = "SIZE";
|
||||||
|
constexpr char kFaceDetectorTFLiteName[] = "face_detector.tflite";
|
||||||
|
constexpr char kFaceLandmarksDetectorTFLiteName[] =
|
||||||
|
"face_landmarks_detector.tflite";
|
||||||
|
constexpr char kFaceBlendshapeTFLiteName[] = "face_blendshapes.tflite";
|
||||||
|
|
||||||
|
struct FaceLandmarkerOutputs {
|
||||||
|
Source<std::vector<NormalizedLandmarkList>> landmark_lists;
|
||||||
|
Source<std::vector<NormalizedRect>> face_rects_next_frame;
|
||||||
|
Source<std::vector<NormalizedRect>> face_rects;
|
||||||
|
Source<std::vector<Detection>> detections;
|
||||||
|
std::optional<Source<std::vector<ClassificationList>>> face_blendshapes;
|
||||||
|
std::optional<Source<std::vector<FaceGeometry>>> face_geometry;
|
||||||
|
Source<Image> image;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Sets the base options in the sub tasks.
|
||||||
|
absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
|
||||||
|
FaceLandmarkerGraphOptions* options,
|
||||||
|
bool is_copy) {
|
||||||
|
auto* face_detector_graph_options =
|
||||||
|
options->mutable_face_detector_graph_options();
|
||||||
|
if (!face_detector_graph_options->base_options().has_model_asset()) {
|
||||||
|
ASSIGN_OR_RETURN(const auto face_detector_file,
|
||||||
|
resources.GetModelFile(kFaceDetectorTFLiteName));
|
||||||
|
SetExternalFile(face_detector_file,
|
||||||
|
face_detector_graph_options->mutable_base_options()
|
||||||
|
->mutable_model_asset(),
|
||||||
|
is_copy);
|
||||||
|
}
|
||||||
|
face_detector_graph_options->mutable_base_options()
|
||||||
|
->mutable_acceleration()
|
||||||
|
->CopyFrom(options->base_options().acceleration());
|
||||||
|
face_detector_graph_options->mutable_base_options()->set_use_stream_mode(
|
||||||
|
options->base_options().use_stream_mode());
|
||||||
|
auto* face_landmarks_detector_graph_options =
|
||||||
|
options->mutable_face_landmarks_detector_graph_options();
|
||||||
|
if (!face_landmarks_detector_graph_options->base_options()
|
||||||
|
.has_model_asset()) {
|
||||||
|
ASSIGN_OR_RETURN(const auto face_landmarks_detector_file,
|
||||||
|
resources.GetModelFile(kFaceLandmarksDetectorTFLiteName));
|
||||||
|
SetExternalFile(
|
||||||
|
face_landmarks_detector_file,
|
||||||
|
face_landmarks_detector_graph_options->mutable_base_options()
|
||||||
|
->mutable_model_asset(),
|
||||||
|
is_copy);
|
||||||
|
}
|
||||||
|
face_landmarks_detector_graph_options->mutable_base_options()
|
||||||
|
->mutable_acceleration()
|
||||||
|
->CopyFrom(options->base_options().acceleration());
|
||||||
|
face_landmarks_detector_graph_options->mutable_base_options()
|
||||||
|
->set_use_stream_mode(options->base_options().use_stream_mode());
|
||||||
|
|
||||||
|
absl::StatusOr<absl::string_view> face_blendshape_model =
|
||||||
|
resources.GetModelFile(kFaceBlendshapeTFLiteName);
|
||||||
|
if (face_blendshape_model.ok()) {
|
||||||
|
SetExternalFile(*face_blendshape_model,
|
||||||
|
face_landmarks_detector_graph_options
|
||||||
|
->mutable_face_blendshapes_graph_options()
|
||||||
|
->mutable_base_options()
|
||||||
|
->mutable_model_asset(),
|
||||||
|
is_copy);
|
||||||
|
face_landmarks_detector_graph_options
|
||||||
|
->mutable_face_blendshapes_graph_options()
|
||||||
|
->mutable_base_options()
|
||||||
|
->mutable_acceleration()
|
||||||
|
->mutable_xnnpack();
|
||||||
|
LOG(WARNING) << "Face blendshape model contains CPU only ops. Sets "
|
||||||
|
<< "FaceBlendshapesGraph acceleartion to Xnnpack.";
|
||||||
|
}
|
||||||
|
|
||||||
|
return absl::OkStatus();
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
// A "mediapipe.tasks.vision.face_landmarker.FaceLandmarkerGraph" performs face
|
||||||
|
// landmarks detection. The FaceLandmarkerGraph consists of three subgraphs:
|
||||||
|
// FaceDetectorGraph, MultipleFaceLandmarksDetectorGraph and
|
||||||
|
// FaceGeometryFromLandmarksGraph.
|
||||||
|
//
|
||||||
|
// MultipleFaceLandmarksDetectorGraph detects landmarks from bounding boxes
|
||||||
|
// produced by FaceDetectorGraph. FaceLandmarkerGraph tracks the landmarks over
|
||||||
|
// time, and skips the FaceDetectorGraph. If the tracking is lost or the
|
||||||
|
// detected faces are less than configured max number faces, FaceDetectorGraph
|
||||||
|
// would be triggered to detect faces.
|
||||||
|
//
|
||||||
|
// FaceGeometryFromLandmarksGraph finds the transformation from canonical face
|
||||||
|
// to the detected faces. This transformation is useful for renderring face
|
||||||
|
// effects on the detected faces. This subgraph is added if users request a
|
||||||
|
// FaceGeometry Tag.
|
||||||
|
//
|
||||||
|
//
|
||||||
|
// Inputs:
|
||||||
|
// IMAGE - Image
|
||||||
|
// Image to perform face landmarks detection on.
|
||||||
|
// NORM_RECT - NormalizedRect @Optional
|
||||||
|
// Describes image rotation and region of image to perform landmarks
|
||||||
|
// detection on. If not provided, whole image is used for face landmarks
|
||||||
|
// detection.
|
||||||
|
//
|
||||||
|
// SideInputs:
|
||||||
|
// ENVIRONMENT - ENVIRONMENT @optional
|
||||||
|
// Environment that describes the current virtual scene. If not provided, a
|
||||||
|
// default environment will be used which is good enough for most general
|
||||||
|
// use case
|
||||||
|
//
|
||||||
|
// Outputs:
|
||||||
|
// NORM_LANDMARKS: - std::vector<NormalizedLandmarkList>
|
||||||
|
// Vector of detected face landmarks.
|
||||||
|
// BLENDSHAPES: - std::vector<ClassificationList> @optional
|
||||||
|
// Blendshape classification, available when the given model asset contains
|
||||||
|
// blendshapes model.
|
||||||
|
// All 52 blendshape coefficients:
|
||||||
|
// 0 - _neutral (ignore it)
|
||||||
|
// 1 - browDownLeft
|
||||||
|
// 2 - browDownRight
|
||||||
|
// 3 - browInnerUp
|
||||||
|
// 4 - browOuterUpLeft
|
||||||
|
// 5 - browOuterUpRight
|
||||||
|
// 6 - cheekPuff
|
||||||
|
// 7 - cheekSquintLeft
|
||||||
|
// 8 - cheekSquintRight
|
||||||
|
// 9 - eyeBlinkLeft
|
||||||
|
// 10 - eyeBlinkRight
|
||||||
|
// 11 - eyeLookDownLeft
|
||||||
|
// 12 - eyeLookDownRight
|
||||||
|
// 13 - eyeLookInLeft
|
||||||
|
// 14 - eyeLookInRight
|
||||||
|
// 15 - eyeLookOutLeft
|
||||||
|
// 16 - eyeLookOutRight
|
||||||
|
// 17 - eyeLookUpLeft
|
||||||
|
// 18 - eyeLookUpRight
|
||||||
|
// 19 - eyeSquintLeft
|
||||||
|
// 20 - eyeSquintRight
|
||||||
|
// 21 - eyeWideLeft
|
||||||
|
// 22 - eyeWideRight
|
||||||
|
// 23 - jawForward
|
||||||
|
// 24 - jawLeft
|
||||||
|
// 25 - jawOpen
|
||||||
|
// 26 - jawRight
|
||||||
|
// 27 - mouthClose
|
||||||
|
// 28 - mouthDimpleLeft
|
||||||
|
// 29 - mouthDimpleRight
|
||||||
|
// 30 - mouthFrownLeft
|
||||||
|
// 31 - mouthFrownRight
|
||||||
|
// 32 - mouthFunnel
|
||||||
|
// 33 - mouthLeft
|
||||||
|
// 34 - mouthLowerDownLeft
|
||||||
|
// 35 - mouthLowerDownRight
|
||||||
|
// 36 - mouthPressLeft
|
||||||
|
// 37 - mouthPressRight
|
||||||
|
// 38 - mouthPucker
|
||||||
|
// 39 - mouthRight
|
||||||
|
// 40 - mouthRollLower
|
||||||
|
// 41 - mouthRollUpper
|
||||||
|
// 42 - mouthShrugLower
|
||||||
|
// 43 - mouthShrugUpper
|
||||||
|
// 44 - mouthSmileLeft
|
||||||
|
// 45 - mouthSmileRight
|
||||||
|
// 46 - mouthStretchLeft
|
||||||
|
// 47 - mouthStretchRight
|
||||||
|
// 48 - mouthUpperUpLeft
|
||||||
|
// 49 - mouthUpperUpRight
|
||||||
|
// 50 - noseSneerLeft
|
||||||
|
// 51 - noseSneerRight
|
||||||
|
// FACE_GEOMETRY - std::vector<FaceGeometry> @optional
|
||||||
|
// A vector of 3D transform data for each detected face.
|
||||||
|
// FACE_RECTS_NEXT_FRAME - std::vector<NormalizedRect>
|
||||||
|
// Vector of the expanded rects enclosing the whole face RoI for landmark
|
||||||
|
// detection on the next frame.
|
||||||
|
// FACE_RECTS - std::vector<NormalizedRect>
|
||||||
|
// Detected face bounding boxes in normalized coordinates from face
|
||||||
|
// detection.
|
||||||
|
// DETECTIONS - std::vector<Detection>
|
||||||
|
// Detected faces with maximum `num_faces` specified in options.
|
||||||
|
// IMAGE - Image
|
||||||
|
// The input image that the face landmarker runs on and has the pixel data
|
||||||
|
// stored on the target storage (CPU vs GPU).
|
||||||
|
// All returned coordinates are in the unrotated and uncropped input image
|
||||||
|
// coordinates system.
|
||||||
|
//
|
||||||
|
// Example:
|
||||||
|
// node {
|
||||||
|
// calculator: "mediapipe.tasks.vision.face_landmarker.FaceLandmarkerGraph"
|
||||||
|
// input_stream: "IMAGE:image_in"
|
||||||
|
// input_stream: "NORM_RECT:norm_rect"
|
||||||
|
// output_stream: "NORM_LANDMARKS:face_landmarks"
|
||||||
|
// output_stream: "BLENDSHAPES:face_blendshapes"
|
||||||
|
// output_stream: "FACE_GEOMETRY:face_geometry"
|
||||||
|
// output_stream: "FACE_RECTS_NEXT_FRAME:face_rects_next_frame"
|
||||||
|
// output_stream: "FACE_RECTS:face_rects"
|
||||||
|
// output_stream: "DETECTIONS:detections"
|
||||||
|
// output_stream: "IMAGE:image_out"
|
||||||
|
// options {
|
||||||
|
// [mediapipe.tasks.vision.face_landmarker.proto.FaceLandmarkerGraphOptions.ext]
|
||||||
|
// {
|
||||||
|
// base_options {
|
||||||
|
// model_asset {
|
||||||
|
// file_name: "face_landmarker.task"
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// face_detector_graph_options {
|
||||||
|
// min_detection_confidence: 0.5
|
||||||
|
// num_faces: 2
|
||||||
|
// }
|
||||||
|
// face_landmarks_detector_graph_options {
|
||||||
|
// min_detection_confidence: 0.5
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
class FaceLandmarkerGraph : public core::ModelTaskGraph {
|
||||||
|
public:
|
||||||
|
absl::StatusOr<CalculatorGraphConfig> GetConfig(
|
||||||
|
SubgraphContext* sc) override {
|
||||||
|
Graph graph;
|
||||||
|
if (sc->Options<FaceLandmarkerGraphOptions>()
|
||||||
|
.base_options()
|
||||||
|
.has_model_asset()) {
|
||||||
|
ASSIGN_OR_RETURN(
|
||||||
|
const auto* model_asset_bundle_resources,
|
||||||
|
CreateModelAssetBundleResources<FaceLandmarkerGraphOptions>(sc));
|
||||||
|
// Copies the file content instead of passing the pointer of file in
|
||||||
|
// memory if the subgraph model resource service is not available.
|
||||||
|
MP_RETURN_IF_ERROR(SetSubTaskBaseOptions(
|
||||||
|
*model_asset_bundle_resources,
|
||||||
|
sc->MutableOptions<FaceLandmarkerGraphOptions>(),
|
||||||
|
!sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService)
|
||||||
|
.IsAvailable()));
|
||||||
|
}
|
||||||
|
std::optional<SidePacket<Environment>> environment;
|
||||||
|
if (HasSideInput(sc->OriginalNode(), kEnvironmentTag)) {
|
||||||
|
environment = std::make_optional<>(
|
||||||
|
graph.SideIn(kEnvironmentTag).Cast<Environment>());
|
||||||
|
}
|
||||||
|
bool output_blendshapes = HasOutput(sc->OriginalNode(), kBlendshapesTag);
|
||||||
|
if (output_blendshapes && !sc->Options<FaceLandmarkerGraphOptions>()
|
||||||
|
.face_landmarks_detector_graph_options()
|
||||||
|
.has_face_blendshapes_graph_options()) {
|
||||||
|
return absl::InvalidArgumentError(absl::StrFormat(
|
||||||
|
"BLENDSHAPES Tag and blendshapes model must be both set. Get "
|
||||||
|
"BLENDSHAPES is set: %v, blendshapes "
|
||||||
|
"model "
|
||||||
|
"is set: %v",
|
||||||
|
output_blendshapes,
|
||||||
|
sc->Options<FaceLandmarkerGraphOptions>()
|
||||||
|
.face_landmarks_detector_graph_options()
|
||||||
|
.has_face_blendshapes_graph_options()));
|
||||||
|
}
|
||||||
|
bool output_geometry = HasOutput(sc->OriginalNode(), kFaceGeometryTag);
|
||||||
|
ASSIGN_OR_RETURN(
|
||||||
|
auto outs,
|
||||||
|
BuildFaceLandmarkerGraph(
|
||||||
|
*sc->MutableOptions<FaceLandmarkerGraphOptions>(),
|
||||||
|
graph[Input<Image>(kImageTag)],
|
||||||
|
graph[Input<NormalizedRect>::Optional(kNormRectTag)], environment,
|
||||||
|
output_blendshapes, output_geometry, graph));
|
||||||
|
outs.landmark_lists >>
|
||||||
|
graph[Output<std::vector<NormalizedLandmarkList>>(kNormLandmarksTag)];
|
||||||
|
outs.face_rects_next_frame >>
|
||||||
|
graph[Output<std::vector<NormalizedRect>>(kFaceRectsNextFrameTag)];
|
||||||
|
outs.face_rects >>
|
||||||
|
graph[Output<std::vector<NormalizedRect>>(kFaceRectsTag)];
|
||||||
|
outs.detections >> graph[Output<std::vector<Detection>>(kDetectionsTag)];
|
||||||
|
outs.image >> graph[Output<Image>(kImageTag)];
|
||||||
|
if (outs.face_blendshapes) {
|
||||||
|
*outs.face_blendshapes >>
|
||||||
|
graph[Output<std::vector<ClassificationList>>(kBlendshapesTag)];
|
||||||
|
}
|
||||||
|
if (outs.face_geometry) {
|
||||||
|
*outs.face_geometry >>
|
||||||
|
graph[Output<std::vector<FaceGeometry>>(kFaceGeometryTag)];
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO remove when support is fixed.
|
||||||
|
// As mediapipe GraphBuilder currently doesn't support configuring
|
||||||
|
// InputStreamInfo, modifying the CalculatorGraphConfig proto directly.
|
||||||
|
CalculatorGraphConfig config = graph.GetConfig();
|
||||||
|
for (int i = 0; i < config.node_size(); ++i) {
|
||||||
|
if (config.node(i).calculator() == "PreviousLoopbackCalculator") {
|
||||||
|
auto* info = config.mutable_node(i)->add_input_stream_info();
|
||||||
|
info->set_tag_index(kLoopTag);
|
||||||
|
info->set_back_edge(true);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return config;
|
||||||
|
}
|
||||||
|
|
||||||
|
private:
|
||||||
|
// Adds a mediapipe face landmarker graph into the provided builder::Graph
|
||||||
|
// instance.
|
||||||
|
//
|
||||||
|
// tasks_options: the mediapipe tasks module FaceLandmarkerGraphOptions.
|
||||||
|
// image_in: (mediapipe::Image) stream to run face landmark detection on.
|
||||||
|
// graph: the mediapipe graph instance to be updated.
|
||||||
|
absl::StatusOr<FaceLandmarkerOutputs> BuildFaceLandmarkerGraph(
|
||||||
|
FaceLandmarkerGraphOptions& tasks_options, Source<Image> image_in,
|
||||||
|
Source<NormalizedRect> norm_rect_in,
|
||||||
|
std::optional<SidePacket<Environment>> environment,
|
||||||
|
bool output_blendshapes, bool output_geometry, Graph& graph) {
|
||||||
|
const int max_num_faces =
|
||||||
|
tasks_options.face_detector_graph_options().num_faces();
|
||||||
|
|
||||||
|
auto& face_detector =
|
||||||
|
graph.AddNode("mediapipe.tasks.vision.face_detector.FaceDetectorGraph");
|
||||||
|
face_detector.GetOptions<FaceDetectorGraphOptions>().Swap(
|
||||||
|
tasks_options.mutable_face_detector_graph_options());
|
||||||
|
auto& clip_face_rects =
|
||||||
|
graph.AddNode("ClipNormalizedRectVectorSizeCalculator");
|
||||||
|
clip_face_rects.GetOptions<ClipVectorSizeCalculatorOptions>()
|
||||||
|
.set_max_vec_size(max_num_faces);
|
||||||
|
auto clipped_face_rects = clip_face_rects.Out("");
|
||||||
|
|
||||||
|
auto& face_landmarks_detector_graph = graph.AddNode(
|
||||||
|
"mediapipe.tasks.vision.face_landmarker."
|
||||||
|
"MultiFaceLandmarksDetectorGraph");
|
||||||
|
face_landmarks_detector_graph
|
||||||
|
.GetOptions<FaceLandmarksDetectorGraphOptions>()
|
||||||
|
.Swap(tasks_options.mutable_face_landmarks_detector_graph_options());
|
||||||
|
image_in >> face_landmarks_detector_graph.In(kImageTag);
|
||||||
|
clipped_face_rects >> face_landmarks_detector_graph.In(kNormRectTag);
|
||||||
|
|
||||||
|
// TODO: add landmarks smoothing calculators.
|
||||||
|
auto landmarks = face_landmarks_detector_graph.Out(kNormLandmarksTag)
|
||||||
|
.Cast<std::vector<NormalizedLandmarkList>>();
|
||||||
|
auto face_rects_for_next_frame =
|
||||||
|
face_landmarks_detector_graph.Out(kFaceRectsNextFrameTag)
|
||||||
|
.Cast<std::vector<NormalizedRect>>();
|
||||||
|
|
||||||
|
if (tasks_options.base_options().use_stream_mode()) {
|
||||||
|
auto& previous_loopback = graph.AddNode("PreviousLoopbackCalculator");
|
||||||
|
image_in >> previous_loopback.In(kMainTag);
|
||||||
|
auto prev_face_rects_from_landmarks =
|
||||||
|
previous_loopback[Output<std::vector<NormalizedRect>>(kPrevLoopTag)];
|
||||||
|
|
||||||
|
auto& min_size_node =
|
||||||
|
graph.AddNode("NormalizedRectVectorHasMinSizeCalculator");
|
||||||
|
prev_face_rects_from_landmarks >> min_size_node.In(kIterableTag);
|
||||||
|
min_size_node.GetOptions<CollectionHasMinSizeCalculatorOptions>()
|
||||||
|
.set_min_size(max_num_faces);
|
||||||
|
auto has_enough_faces = min_size_node.Out("").Cast<bool>();
|
||||||
|
|
||||||
|
// While in stream mode, skip face detector graph when we successfully
|
||||||
|
// track the faces from the last frame.
|
||||||
|
auto image_for_face_detector =
|
||||||
|
DisallowIf(image_in, has_enough_faces, graph);
|
||||||
|
auto norm_rect_in_for_face_detector =
|
||||||
|
DisallowIf(norm_rect_in, has_enough_faces, graph);
|
||||||
|
image_for_face_detector >> face_detector.In(kImageTag);
|
||||||
|
norm_rect_in_for_face_detector >> face_detector.In(kNormRectTag);
|
||||||
|
auto expanded_face_rects_from_face_detector =
|
||||||
|
face_detector.Out(kExpandedFaceRectsTag);
|
||||||
|
auto& face_association = graph.AddNode("AssociationNormRectCalculator");
|
||||||
|
face_association.GetOptions<mediapipe::AssociationCalculatorOptions>()
|
||||||
|
.set_min_similarity_threshold(
|
||||||
|
tasks_options.min_tracking_confidence());
|
||||||
|
prev_face_rects_from_landmarks >>
|
||||||
|
face_association[Input<std::vector<NormalizedRect>>::Multiple("")][0];
|
||||||
|
expanded_face_rects_from_face_detector >>
|
||||||
|
face_association[Input<std::vector<NormalizedRect>>::Multiple("")][1];
|
||||||
|
auto face_rects = face_association.Out("");
|
||||||
|
face_rects >> clip_face_rects.In("");
|
||||||
|
// Back edge.
|
||||||
|
face_rects_for_next_frame >> previous_loopback.In(kLoopTag);
|
||||||
|
} else {
|
||||||
|
// While not in stream mode, the input images are not guaranteed to be in
|
||||||
|
// series, and we don't want to enable the tracking and rect associations
|
||||||
|
// between input images. Always use the face detector graph.
|
||||||
|
image_in >> face_detector.In(kImageTag);
|
||||||
|
norm_rect_in >> face_detector.In(kNormRectTag);
|
||||||
|
auto face_rects = face_detector.Out(kExpandedFaceRectsTag);
|
||||||
|
face_rects >> clip_face_rects.In("");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Optional blendshape output.
|
||||||
|
std::optional<Source<std::vector<ClassificationList>>> blendshapes;
|
||||||
|
if (output_blendshapes) {
|
||||||
|
blendshapes = std::make_optional<>(
|
||||||
|
face_landmarks_detector_graph.Out(kBlendshapesTag)
|
||||||
|
.Cast<std::vector<ClassificationList>>());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Optional face geometry output.
|
||||||
|
std::optional<Source<std::vector<FaceGeometry>>> face_geometry;
|
||||||
|
if (output_geometry) {
|
||||||
|
auto& image_properties = graph.AddNode("ImagePropertiesCalculator");
|
||||||
|
image_in >> image_properties.In(kImageTag);
|
||||||
|
auto image_size = image_properties.Out(kSizeTag);
|
||||||
|
auto& face_geometry_from_landmarks = graph.AddNode(
|
||||||
|
"mediapipe.tasks.vision.face_geometry."
|
||||||
|
"FaceGeometryFromLandmarksGraph");
|
||||||
|
if (environment.has_value()) {
|
||||||
|
*environment >> face_geometry_from_landmarks.SideIn(kEnvironmentTag);
|
||||||
|
}
|
||||||
|
landmarks >> face_geometry_from_landmarks.In(kFaceLandmarksTag);
|
||||||
|
image_size >> face_geometry_from_landmarks.In(kImageSizeTag);
|
||||||
|
face_geometry = face_geometry_from_landmarks.Out(kFaceGeometryTag)
|
||||||
|
.Cast<std::vector<FaceGeometry>>();
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Replace PassThroughCalculator with a calculator that
|
||||||
|
// converts the pixel data to be stored on the target storage (CPU vs GPU).
|
||||||
|
auto& pass_through = graph.AddNode("PassThroughCalculator");
|
||||||
|
image_in >> pass_through.In("");
|
||||||
|
|
||||||
|
return {{
|
||||||
|
/* landmark_lists= */ landmarks,
|
||||||
|
/* face_rects_next_frame= */
|
||||||
|
face_rects_for_next_frame,
|
||||||
|
/* face_rects= */
|
||||||
|
face_detector.Out(kFaceRectsTag).Cast<std::vector<NormalizedRect>>(),
|
||||||
|
/* face_detections */
|
||||||
|
face_detector.Out(kDetectionsTag).Cast<std::vector<Detection>>(),
|
||||||
|
/* face_blendshapes= */ blendshapes,
|
||||||
|
/* face_geometry= */ face_geometry,
|
||||||
|
/* image= */
|
||||||
|
pass_through[Output<Image>("")],
|
||||||
|
}};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
REGISTER_MEDIAPIPE_GRAPH(
|
||||||
|
::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerGraph);
|
||||||
|
|
||||||
|
} // namespace face_landmarker
|
||||||
|
} // namespace vision
|
||||||
|
} // namespace tasks
|
||||||
|
} // namespace mediapipe
|
|
@ -0,0 +1,311 @@
|
||||||
|
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
==============================================================================*/
|
||||||
|
|
||||||
|
#include <optional>
|
||||||
|
|
||||||
|
#include "absl/flags/flag.h"
|
||||||
|
#include "absl/status/statusor.h"
|
||||||
|
#include "absl/strings/str_format.h"
|
||||||
|
#include "absl/strings/string_view.h"
|
||||||
|
#include "mediapipe/framework/api2/builder.h"
|
||||||
|
#include "mediapipe/framework/api2/port.h"
|
||||||
|
#include "mediapipe/framework/calculator_framework.h"
|
||||||
|
#include "mediapipe/framework/deps/file_path.h"
|
||||||
|
#include "mediapipe/framework/formats/classification.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/image.h"
|
||||||
|
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||||
|
#include "mediapipe/framework/formats/rect.pb.h"
|
||||||
|
#include "mediapipe/framework/packet.h"
|
||||||
|
#include "mediapipe/framework/port/file_helpers.h"
|
||||||
|
#include "mediapipe/framework/port/gmock.h"
|
||||||
|
#include "mediapipe/framework/port/gtest.h"
|
||||||
|
#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h"
|
||||||
|
#include "mediapipe/tasks/cc/core/proto/base_options.pb.h"
|
||||||
|
#include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
|
||||||
|
#include "mediapipe/tasks/cc/core/task_runner.h"
|
||||||
|
#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h"
|
||||||
|
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
|
||||||
|
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.pb.h"
|
||||||
|
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h"
|
||||||
|
#include "mediapipe/tasks/cc/vision/utils/image_utils.h"
|
||||||
|
|
||||||
|
namespace mediapipe {
|
||||||
|
namespace tasks {
|
||||||
|
namespace vision {
|
||||||
|
namespace face_landmarker {
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
using ::file::Defaults;
|
||||||
|
using ::file::GetTextProto;
|
||||||
|
using ::mediapipe::api2::Input;
|
||||||
|
using ::mediapipe::api2::Output;
|
||||||
|
using ::mediapipe::api2::builder::Graph;
|
||||||
|
using ::mediapipe::api2::builder::Source;
|
||||||
|
using ::mediapipe::file::JoinPath;
|
||||||
|
using ::mediapipe::tasks::core::TaskRunner;
|
||||||
|
using ::mediapipe::tasks::vision::DecodeImageFromFile;
|
||||||
|
using ::mediapipe::tasks::vision::face_geometry::proto::FaceGeometry;
|
||||||
|
using ::mediapipe::tasks::vision::face_landmarker::proto::
|
||||||
|
FaceLandmarkerGraphOptions;
|
||||||
|
using ::testing::EqualsProto;
|
||||||
|
using ::testing::Pointwise;
|
||||||
|
using ::testing::TestParamInfo;
|
||||||
|
using ::testing::TestWithParam;
|
||||||
|
using ::testing::Values;
|
||||||
|
using ::testing::proto::Approximately;
|
||||||
|
using ::testing::proto::Partially;
|
||||||
|
|
||||||
|
constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/";
|
||||||
|
constexpr char kFaceLandmarkerModelBundleName[] = "face_landmarker.task";
|
||||||
|
constexpr char kFaceLandmarkerWithBlendshapesModelBundleName[] =
|
||||||
|
"face_landmarker_with_blendshapes.task";
|
||||||
|
constexpr char kPortraitImageName[] = "portrait.jpg";
|
||||||
|
constexpr char kCatImageName[] = "cat.jpg";
|
||||||
|
constexpr char kPortraitExpectedFaceLandamrksName[] =
|
||||||
|
"portrait_expected_face_landmarks.pbtxt";
|
||||||
|
constexpr char kPortraitExpectedFaceLandamrksWithAttentionName[] =
|
||||||
|
"portrait_expected_face_landmarks_with_attention.pbtxt";
|
||||||
|
constexpr char kPortraitExpectedBlendshapesName[] =
|
||||||
|
"portrait_expected_blendshapes_with_attention.pbtxt";
|
||||||
|
constexpr char kPortraitExpectedFaceGeometryName[] =
|
||||||
|
"portrait_expected_face_geometry_with_attention.pbtxt";
|
||||||
|
|
||||||
|
constexpr char kImageTag[] = "IMAGE";
|
||||||
|
constexpr char kImageName[] = "image";
|
||||||
|
constexpr char kNormRectTag[] = "NORM_RECT";
|
||||||
|
constexpr char kNormRectName[] = "norm_rect";
|
||||||
|
constexpr char kNormLandmarksTag[] = "NORM_LANDMARKS";
|
||||||
|
constexpr char kNormLandmarksName[] = "norm_landmarks";
|
||||||
|
constexpr char kBlendshapesTag[] = "BLENDSHAPES";
|
||||||
|
constexpr char kBlendshapesName[] = "blendshapes";
|
||||||
|
constexpr char kFaceGeometryTag[] = "FACE_GEOMETRY";
|
||||||
|
constexpr char kFaceGeometryName[] = "face_geometry";
|
||||||
|
|
||||||
|
constexpr float kLandmarksDiffMargin = 0.03;
|
||||||
|
constexpr float kBlendshapesDiffMargin = 0.1;
|
||||||
|
constexpr float kFaceGeometryDiffMargin = 0.01;
|
||||||
|
|
||||||
|
template <typename ProtoT>
|
||||||
|
ProtoT GetExpectedProto(absl::string_view filename) {
|
||||||
|
ProtoT expected_proto;
|
||||||
|
MP_EXPECT_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, filename),
|
||||||
|
&expected_proto, Defaults()));
|
||||||
|
return expected_proto;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Struct holding the parameters for parameterized FaceLandmarkerGraphTest
|
||||||
|
// class.
|
||||||
|
struct FaceLandmarkerGraphTestParams {
|
||||||
|
// The name of this test, for convenience when displaying test results.
|
||||||
|
std::string test_name;
|
||||||
|
// The filename of the model to test.
|
||||||
|
std::string input_model_name;
|
||||||
|
// The filename of the test image.
|
||||||
|
std::string test_image_name;
|
||||||
|
// The expected output landmarks positions.
|
||||||
|
std::optional<std::vector<NormalizedLandmarkList>> expected_landmarks_list;
|
||||||
|
// The expected output blendshape classification.
|
||||||
|
std::optional<std::vector<ClassificationList>> expected_blendshapes;
|
||||||
|
// The expected output face geometry.
|
||||||
|
std::optional<std::vector<FaceGeometry>> expected_face_geometry;
|
||||||
|
// The max value difference between expected_positions and detected positions.
|
||||||
|
float landmarks_diff_threshold;
|
||||||
|
// The max value difference between expected blendshapes and actual
|
||||||
|
// blendshapes.
|
||||||
|
float blendshapes_diff_threshold;
|
||||||
|
// The max value difference between expected blendshape and actual face
|
||||||
|
// geometry.
|
||||||
|
float face_geometry_diff_threshold;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Helper function to create a FaceLandmarkerGraph TaskRunner.
|
||||||
|
absl::StatusOr<std::unique_ptr<TaskRunner>> CreateFaceLandmarkerGraphTaskRunner(
|
||||||
|
absl::string_view model_name, bool output_blendshape,
|
||||||
|
bool output_face_geometry) {
|
||||||
|
Graph graph;
|
||||||
|
|
||||||
|
auto& face_landmarker = graph.AddNode(
|
||||||
|
"mediapipe.tasks.vision.face_landmarker."
|
||||||
|
"FaceLandmarkerGraph");
|
||||||
|
|
||||||
|
auto* options = &face_landmarker.GetOptions<FaceLandmarkerGraphOptions>();
|
||||||
|
options->mutable_base_options()->mutable_model_asset()->set_file_name(
|
||||||
|
JoinPath("./", kTestDataDirectory, model_name));
|
||||||
|
options->mutable_face_detector_graph_options()->set_num_faces(1);
|
||||||
|
options->mutable_base_options()->set_use_stream_mode(true);
|
||||||
|
|
||||||
|
graph[Input<Image>(kImageTag)].SetName(kImageName) >>
|
||||||
|
face_landmarker.In(kImageTag);
|
||||||
|
graph[Input<NormalizedRect>(kNormRectTag)].SetName(kNormRectName) >>
|
||||||
|
face_landmarker.In(kNormRectTag);
|
||||||
|
|
||||||
|
face_landmarker.Out(kNormLandmarksTag).SetName(kNormLandmarksName) >>
|
||||||
|
graph[Output<std::vector<NormalizedLandmarkList>>(kNormLandmarksTag)];
|
||||||
|
if (output_blendshape) {
|
||||||
|
face_landmarker.Out(kBlendshapesTag).SetName(kBlendshapesName) >>
|
||||||
|
graph[Output<std::vector<ClassificationList>>(kBlendshapesTag)];
|
||||||
|
}
|
||||||
|
if (output_face_geometry) {
|
||||||
|
face_landmarker.Out(kFaceGeometryTag).SetName(kFaceGeometryName) >>
|
||||||
|
graph[Output<std::vector<FaceGeometry>>(kFaceGeometryTag)];
|
||||||
|
}
|
||||||
|
|
||||||
|
return TaskRunner::Create(
|
||||||
|
graph.GetConfig(),
|
||||||
|
absl::make_unique<tasks::core::MediaPipeBuiltinOpResolver>());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to construct NormalizeRect proto.
|
||||||
|
NormalizedRect MakeNormRect(float x_center, float y_center, float width,
|
||||||
|
float height, float rotation) {
|
||||||
|
NormalizedRect face_rect;
|
||||||
|
face_rect.set_x_center(x_center);
|
||||||
|
face_rect.set_y_center(y_center);
|
||||||
|
face_rect.set_width(width);
|
||||||
|
face_rect.set_height(height);
|
||||||
|
face_rect.set_rotation(rotation);
|
||||||
|
return face_rect;
|
||||||
|
}
|
||||||
|
|
||||||
|
class FaceLandmarkerGraphTest
|
||||||
|
: public testing::TestWithParam<FaceLandmarkerGraphTestParams> {};
|
||||||
|
|
||||||
|
TEST(FaceLandmarkerGraphTest, FailsWithNoBlendshapesModel) {
|
||||||
|
MP_ASSERT_OK_AND_ASSIGN(
|
||||||
|
Image image, DecodeImageFromFile(
|
||||||
|
JoinPath("./", kTestDataDirectory, kPortraitImageName)));
|
||||||
|
auto result =
|
||||||
|
CreateFaceLandmarkerGraphTaskRunner(kFaceLandmarkerModelBundleName,
|
||||||
|
/*output_blendshape=*/true,
|
||||||
|
/*output_face_geometry=*/false);
|
||||||
|
EXPECT_EQ(result.status().code(), absl::StatusCode::kInvalidArgument);
|
||||||
|
EXPECT_THAT(result.status().message(),
|
||||||
|
testing::HasSubstr(
|
||||||
|
"BLENDSHAPES Tag and blendshapes model must be both set."));
|
||||||
|
}
|
||||||
|
|
||||||
|
TEST_P(FaceLandmarkerGraphTest, Succeeds) {
|
||||||
|
MP_ASSERT_OK_AND_ASSIGN(
|
||||||
|
Image image, DecodeImageFromFile(JoinPath("./", kTestDataDirectory,
|
||||||
|
GetParam().test_image_name)));
|
||||||
|
MP_ASSERT_OK_AND_ASSIGN(auto task_runner,
|
||||||
|
CreateFaceLandmarkerGraphTaskRunner(
|
||||||
|
GetParam().input_model_name,
|
||||||
|
GetParam().expected_blendshapes.has_value(),
|
||||||
|
GetParam().expected_face_geometry.has_value()));
|
||||||
|
|
||||||
|
auto output_packets = task_runner->Process(
|
||||||
|
{{kImageName, MakePacket<Image>(std::move(image))},
|
||||||
|
{kNormRectName,
|
||||||
|
MakePacket<NormalizedRect>(MakeNormRect(0.5, 0.5, 1.0, 1.0, 0))}});
|
||||||
|
MP_ASSERT_OK(output_packets);
|
||||||
|
|
||||||
|
if (GetParam().expected_landmarks_list) {
|
||||||
|
const std::vector<NormalizedLandmarkList>& landmarks_lists =
|
||||||
|
(*output_packets)[kNormLandmarksName]
|
||||||
|
.Get<std::vector<NormalizedLandmarkList>>();
|
||||||
|
EXPECT_THAT(landmarks_lists,
|
||||||
|
Pointwise(Approximately(Partially(EqualsProto()),
|
||||||
|
GetParam().landmarks_diff_threshold),
|
||||||
|
*GetParam().expected_landmarks_list));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (GetParam().expected_blendshapes) {
|
||||||
|
const std::vector<ClassificationList>& blendshapes =
|
||||||
|
(*output_packets)[kBlendshapesName]
|
||||||
|
.Get<std::vector<ClassificationList>>();
|
||||||
|
EXPECT_THAT(blendshapes,
|
||||||
|
Pointwise(Approximately(Partially(EqualsProto()),
|
||||||
|
GetParam().blendshapes_diff_threshold),
|
||||||
|
*GetParam().expected_blendshapes));
|
||||||
|
}
|
||||||
|
if (GetParam().expected_face_geometry) {
|
||||||
|
const std::vector<FaceGeometry>& face_geometry =
|
||||||
|
(*output_packets)[kFaceGeometryName].Get<std::vector<FaceGeometry>>();
|
||||||
|
EXPECT_THAT(
|
||||||
|
face_geometry,
|
||||||
|
Pointwise(Approximately(Partially(EqualsProto()),
|
||||||
|
GetParam().face_geometry_diff_threshold),
|
||||||
|
*GetParam().expected_face_geometry));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
INSTANTIATE_TEST_SUITE_P(
|
||||||
|
FaceLandmarkerGraphTests, FaceLandmarkerGraphTest,
|
||||||
|
Values(FaceLandmarkerGraphTestParams{
|
||||||
|
/* test_name= */ "Portrait",
|
||||||
|
/* input_model_name= */ kFaceLandmarkerModelBundleName,
|
||||||
|
/* test_image_name= */ kPortraitImageName,
|
||||||
|
/* expected_landmarks_list= */
|
||||||
|
{{GetExpectedProto<NormalizedLandmarkList>(
|
||||||
|
kPortraitExpectedFaceLandamrksName)}},
|
||||||
|
/* expected_blendshapes= */ std::nullopt,
|
||||||
|
/* expected_face_geometry= */ std::nullopt,
|
||||||
|
/* landmarks_diff_threshold= */ kLandmarksDiffMargin,
|
||||||
|
/* blendshapes_diff_threshold= */ kBlendshapesDiffMargin,
|
||||||
|
/* face_geometry_diff_threshold= */
|
||||||
|
kFaceGeometryDiffMargin},
|
||||||
|
FaceLandmarkerGraphTestParams{
|
||||||
|
/* test_name= */ "NoFace",
|
||||||
|
/* input_model_name= */ kFaceLandmarkerModelBundleName,
|
||||||
|
/* test_image_name= */ kCatImageName,
|
||||||
|
/* expected_landmarks_list= */ std::nullopt,
|
||||||
|
/* expected_blendshapes= */ std::nullopt,
|
||||||
|
/* expected_face_geometry= */ std::nullopt,
|
||||||
|
/* landmarks_diff_threshold= */ kLandmarksDiffMargin,
|
||||||
|
/* blendshapes_diff_threshold= */ kBlendshapesDiffMargin,
|
||||||
|
/* face_geometry_diff_threshold= */
|
||||||
|
kFaceGeometryDiffMargin},
|
||||||
|
FaceLandmarkerGraphTestParams{
|
||||||
|
/* test_name= */ "PortraitWithBlendshape",
|
||||||
|
/* input_model_name= */
|
||||||
|
kFaceLandmarkerWithBlendshapesModelBundleName,
|
||||||
|
/* test_image_name= */ kPortraitImageName,
|
||||||
|
/* expected_landmarks_list= */
|
||||||
|
{{GetExpectedProto<NormalizedLandmarkList>(
|
||||||
|
kPortraitExpectedFaceLandamrksWithAttentionName)}},
|
||||||
|
/* expected_blendshapes= */
|
||||||
|
{{GetExpectedProto<ClassificationList>(
|
||||||
|
kPortraitExpectedBlendshapesName)}},
|
||||||
|
/* expected_face_geometry= */ std::nullopt,
|
||||||
|
/*landmarks_diff_threshold= */ kLandmarksDiffMargin,
|
||||||
|
/*blendshapes_diff_threshold= */ kBlendshapesDiffMargin,
|
||||||
|
/*face_geometry_diff_threshold= */ kFaceGeometryDiffMargin},
|
||||||
|
FaceLandmarkerGraphTestParams{
|
||||||
|
/* test_name= */ "PortraitWithBlendshapeWithFaceGeometry",
|
||||||
|
/* input_model_name= */
|
||||||
|
kFaceLandmarkerWithBlendshapesModelBundleName,
|
||||||
|
/* test_image_name= */ kPortraitImageName,
|
||||||
|
/* expected_landmarks_list= */
|
||||||
|
{{GetExpectedProto<NormalizedLandmarkList>(
|
||||||
|
kPortraitExpectedFaceLandamrksWithAttentionName)}},
|
||||||
|
/* expected_blendshapes= */
|
||||||
|
{{GetExpectedProto<ClassificationList>(
|
||||||
|
kPortraitExpectedBlendshapesName)}},
|
||||||
|
/* expected_face_geometry= */
|
||||||
|
{{GetExpectedProto<FaceGeometry>(
|
||||||
|
kPortraitExpectedFaceGeometryName)}},
|
||||||
|
/*landmarks_diff_threshold= */ kLandmarksDiffMargin,
|
||||||
|
/*blendshapes_diff_threshold= */ kBlendshapesDiffMargin,
|
||||||
|
/*face_geometry_diff_threshold= */ kFaceGeometryDiffMargin}),
|
||||||
|
[](const TestParamInfo<FaceLandmarkerGraphTest::ParamType>& info) {
|
||||||
|
return info.param.test_name;
|
||||||
|
});
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
} // namespace face_landmarker
|
||||||
|
} // namespace vision
|
||||||
|
} // namespace tasks
|
||||||
|
} // namespace mediapipe
|
|
@ -214,13 +214,13 @@ ClassificationList GetBlendshapes(absl::string_view filename) {
|
||||||
// Helper function to construct NormalizeRect proto.
|
// Helper function to construct NormalizeRect proto.
|
||||||
NormalizedRect MakeNormRect(float x_center, float y_center, float width,
|
NormalizedRect MakeNormRect(float x_center, float y_center, float width,
|
||||||
float height, float rotation) {
|
float height, float rotation) {
|
||||||
NormalizedRect hand_rect;
|
NormalizedRect face_rect;
|
||||||
hand_rect.set_x_center(x_center);
|
face_rect.set_x_center(x_center);
|
||||||
hand_rect.set_y_center(y_center);
|
face_rect.set_y_center(y_center);
|
||||||
hand_rect.set_width(width);
|
face_rect.set_width(width);
|
||||||
hand_rect.set_height(height);
|
face_rect.set_height(height);
|
||||||
hand_rect.set_rotation(rotation);
|
face_rect.set_rotation(rotation);
|
||||||
return hand_rect;
|
return face_rect;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Struct holding the parameters for parameterized FaceLandmarksDetectionTest
|
// Struct holding the parameters for parameterized FaceLandmarksDetectionTest
|
||||||
|
@ -234,9 +234,9 @@ struct SingeFaceTestParams {
|
||||||
std::optional<std::string> blendshape_model_name;
|
std::optional<std::string> blendshape_model_name;
|
||||||
// The filename of the test image.
|
// The filename of the test image.
|
||||||
std::string test_image_name;
|
std::string test_image_name;
|
||||||
// RoI on image to detect hands.
|
// RoI on image to detect faces.
|
||||||
NormalizedRect norm_rect;
|
NormalizedRect norm_rect;
|
||||||
// Expected hand presence value.
|
// Expected face presence value.
|
||||||
bool expected_presence;
|
bool expected_presence;
|
||||||
// The expected output landmarks positions.
|
// The expected output landmarks positions.
|
||||||
NormalizedLandmarkList expected_landmarks;
|
NormalizedLandmarkList expected_landmarks;
|
||||||
|
@ -258,9 +258,9 @@ struct MultiFaceTestParams {
|
||||||
std::optional<std::string> blendshape_model_name;
|
std::optional<std::string> blendshape_model_name;
|
||||||
// The filename of the test image.
|
// The filename of the test image.
|
||||||
std::string test_image_name;
|
std::string test_image_name;
|
||||||
// RoI on image to detect hands.
|
// RoI on image to detect faces.
|
||||||
std::vector<NormalizedRect> norm_rects;
|
std::vector<NormalizedRect> norm_rects;
|
||||||
// Expected hand presence value.
|
// Expected face presence value.
|
||||||
std::vector<bool> expected_presence;
|
std::vector<bool> expected_presence;
|
||||||
// The expected output landmarks positions.
|
// The expected output landmarks positions.
|
||||||
std::optional<std::vector<NormalizedLandmarkList>> expected_landmarks_lists;
|
std::optional<std::vector<NormalizedLandmarkList>> expected_landmarks_lists;
|
||||||
|
@ -299,7 +299,6 @@ TEST_P(SingleFaceLandmarksDetectionTest, Succeeds) {
|
||||||
(*output_packets)[kNormLandmarksName].Get<NormalizedLandmarkList>();
|
(*output_packets)[kNormLandmarksName].Get<NormalizedLandmarkList>();
|
||||||
const NormalizedLandmarkList& expected_landmarks =
|
const NormalizedLandmarkList& expected_landmarks =
|
||||||
GetParam().expected_landmarks;
|
GetParam().expected_landmarks;
|
||||||
|
|
||||||
EXPECT_THAT(
|
EXPECT_THAT(
|
||||||
landmarks,
|
landmarks,
|
||||||
Approximately(Partially(EqualsProto(expected_landmarks)),
|
Approximately(Partially(EqualsProto(expected_landmarks)),
|
||||||
|
@ -395,7 +394,9 @@ INSTANTIATE_TEST_SUITE_P(
|
||||||
kFaceLandmarksDetectionWithAttentionModel,
|
kFaceLandmarksDetectionWithAttentionModel,
|
||||||
/* blendshape_model_name= */ kFaceBlendshapesModel,
|
/* blendshape_model_name= */ kFaceBlendshapesModel,
|
||||||
/* test_image_name= */ kPortraitImageName,
|
/* test_image_name= */ kPortraitImageName,
|
||||||
/* norm_rect= */ MakeNormRect(0.4987, 0.2211, 0.2877, 0.2303, 0),
|
/* norm_rect= */
|
||||||
|
MakeNormRect(0.48906386, 0.22731927, 0.42905223, 0.34357703,
|
||||||
|
0.008304443),
|
||||||
/* expected_presence= */ true,
|
/* expected_presence= */ true,
|
||||||
/* expected_landmarks= */
|
/* expected_landmarks= */
|
||||||
GetExpectedLandmarkList(
|
GetExpectedLandmarkList(
|
||||||
|
@ -444,7 +445,9 @@ INSTANTIATE_TEST_SUITE_P(
|
||||||
kFaceLandmarksDetectionWithAttentionModel,
|
kFaceLandmarksDetectionWithAttentionModel,
|
||||||
/* blendshape_model_name= */ kFaceBlendshapesModel,
|
/* blendshape_model_name= */ kFaceBlendshapesModel,
|
||||||
/* test_image_name= */ kPortraitImageName,
|
/* test_image_name= */ kPortraitImageName,
|
||||||
/* norm_rects= */ {MakeNormRect(0.4987, 0.2211, 0.2877, 0.2303, 0)},
|
/* norm_rects= */
|
||||||
|
{MakeNormRect(0.48906386, 0.22731927, 0.42905223, 0.34357703,
|
||||||
|
0.008304443)},
|
||||||
/* expected_presence= */ {true},
|
/* expected_presence= */ {true},
|
||||||
/* expected_landmarks_list= */
|
/* expected_landmarks_list= */
|
||||||
{{GetExpectedLandmarkList(
|
{{GetExpectedLandmarkList(
|
||||||
|
|
|
@ -50,3 +50,15 @@ mediapipe_proto_library(
|
||||||
"//mediapipe/framework:calculator_proto",
|
"//mediapipe/framework:calculator_proto",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
mediapipe_proto_library(
|
||||||
|
name = "face_landmarker_graph_options_proto",
|
||||||
|
srcs = ["face_landmarker_graph_options.proto"],
|
||||||
|
deps = [
|
||||||
|
":face_landmarks_detector_graph_options_proto",
|
||||||
|
"//mediapipe/framework:calculator_options_proto",
|
||||||
|
"//mediapipe/framework:calculator_proto",
|
||||||
|
"//mediapipe/tasks/cc/core/proto:base_options_proto",
|
||||||
|
"//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_proto",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
|
@ -0,0 +1,48 @@
|
||||||
|
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
==============================================================================*/
|
||||||
|
|
||||||
|
syntax = "proto2";
|
||||||
|
|
||||||
|
package mediapipe.tasks.vision.face_landmarker.proto;
|
||||||
|
|
||||||
|
import "mediapipe/framework/calculator.proto";
|
||||||
|
import "mediapipe/framework/calculator_options.proto";
|
||||||
|
import "mediapipe/tasks/cc/core/proto/base_options.proto";
|
||||||
|
import "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.proto";
|
||||||
|
import "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.proto";
|
||||||
|
|
||||||
|
option java_package = "com.google.mediapipe.tasks.vision.facelandmarker.proto";
|
||||||
|
option java_outer_classname = "FaceLandmarkerGraphOptionsProto";
|
||||||
|
|
||||||
|
message FaceLandmarkerGraphOptions {
|
||||||
|
extend mediapipe.CalculatorOptions {
|
||||||
|
optional FaceLandmarkerGraphOptions ext = 508968150;
|
||||||
|
}
|
||||||
|
// Base options for configuring Task library, such as specifying the TfLite
|
||||||
|
// model file with metadata, accelerator options, etc.
|
||||||
|
optional core.proto.BaseOptions base_options = 1;
|
||||||
|
|
||||||
|
// Options for face detector graph.
|
||||||
|
optional face_detector.proto.FaceDetectorGraphOptions
|
||||||
|
face_detector_graph_options = 2;
|
||||||
|
|
||||||
|
// Options for face landmarks detector graph.
|
||||||
|
optional FaceLandmarksDetectorGraphOptions
|
||||||
|
face_landmarks_detector_graph_options = 3;
|
||||||
|
|
||||||
|
// Minimum confidence for face landmarks tracking to be considered
|
||||||
|
// successfully.
|
||||||
|
optional float min_tracking_confidence = 4 [default = 0.5];
|
||||||
|
}
|
5
mediapipe/tasks/testdata/vision/BUILD
vendored
5
mediapipe/tasks/testdata/vision/BUILD
vendored
|
@ -44,6 +44,7 @@ mediapipe_files(srcs = [
|
||||||
"face_detection_short_range.tflite",
|
"face_detection_short_range.tflite",
|
||||||
"face_landmark.tflite",
|
"face_landmark.tflite",
|
||||||
"face_landmark_with_attention.tflite",
|
"face_landmark_with_attention.tflite",
|
||||||
|
"face_landmarker.task",
|
||||||
"fist.jpg",
|
"fist.jpg",
|
||||||
"fist.png",
|
"fist.png",
|
||||||
"hair_segmentation.tflite",
|
"hair_segmentation.tflite",
|
||||||
|
@ -92,6 +93,7 @@ exports_files(
|
||||||
"face_geometry_expected_out.pbtxt",
|
"face_geometry_expected_out.pbtxt",
|
||||||
"gesture_recognizer.task",
|
"gesture_recognizer.task",
|
||||||
"portrait_expected_detection.pbtxt",
|
"portrait_expected_detection.pbtxt",
|
||||||
|
"portrait_expected_face_geometry_with_attention.pbtxt",
|
||||||
"portrait_rotated_expected_detection.pbtxt",
|
"portrait_rotated_expected_detection.pbtxt",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
@ -153,6 +155,7 @@ filegroup(
|
||||||
"face_detection_short_range.tflite",
|
"face_detection_short_range.tflite",
|
||||||
"face_landmark.tflite",
|
"face_landmark.tflite",
|
||||||
"face_landmark_with_attention.tflite",
|
"face_landmark_with_attention.tflite",
|
||||||
|
"face_landmarker.task",
|
||||||
"hair_segmentation.tflite",
|
"hair_segmentation.tflite",
|
||||||
"hand_landmark_full.tflite",
|
"hand_landmark_full.tflite",
|
||||||
"hand_landmark_lite.tflite",
|
"hand_landmark_lite.tflite",
|
||||||
|
@ -166,7 +169,6 @@ filegroup(
|
||||||
"mobilenet_v2_1.0_224.tflite",
|
"mobilenet_v2_1.0_224.tflite",
|
||||||
"mobilenet_v3_small_100_224_embedder.tflite",
|
"mobilenet_v3_small_100_224_embedder.tflite",
|
||||||
"palm_detection_full.tflite",
|
"palm_detection_full.tflite",
|
||||||
"portrait_expected_face_landmarks.pbtxt",
|
|
||||||
"selfie_segm_128_128_3.tflite",
|
"selfie_segm_128_128_3.tflite",
|
||||||
"selfie_segm_144_256_3.tflite",
|
"selfie_segm_144_256_3.tflite",
|
||||||
],
|
],
|
||||||
|
@ -189,6 +191,7 @@ filegroup(
|
||||||
"pointing_up_landmarks.pbtxt",
|
"pointing_up_landmarks.pbtxt",
|
||||||
"pointing_up_rotated_landmarks.pbtxt",
|
"pointing_up_rotated_landmarks.pbtxt",
|
||||||
"portrait_expected_detection.pbtxt",
|
"portrait_expected_detection.pbtxt",
|
||||||
|
"portrait_expected_face_geometry_with_attention.pbtxt",
|
||||||
"portrait_expected_face_landmarks.pbtxt",
|
"portrait_expected_face_landmarks.pbtxt",
|
||||||
"portrait_expected_face_landmarks_with_attention.pbtxt",
|
"portrait_expected_face_landmarks_with_attention.pbtxt",
|
||||||
"portrait_rotated_expected_detection.pbtxt",
|
"portrait_rotated_expected_detection.pbtxt",
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
# proto-file: mediapipe/tasks/cc/vision/face_geometry/proto/environment.proto
|
# proto-file: mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.proto
|
||||||
# proto-message: Environment
|
# proto-message: FaceGeometry
|
||||||
mesh {
|
mesh {
|
||||||
vertex_type: VERTEX_PT
|
vertex_type: VERTEX_PT
|
||||||
primitive_type: TRIANGLE
|
primitive_type: TRIANGLE
|
||||||
|
|
BIN
mediapipe/tasks/testdata/vision/face_landmarker.task
vendored
Normal file
BIN
mediapipe/tasks/testdata/vision/face_landmarker.task
vendored
Normal file
Binary file not shown.
BIN
mediapipe/tasks/testdata/vision/face_landmarker_with_blendshapes.task
vendored
Normal file
BIN
mediapipe/tasks/testdata/vision/face_landmarker_with_blendshapes.task
vendored
Normal file
Binary file not shown.
|
@ -2,261 +2,261 @@
|
||||||
# proto-message: ClassificationList
|
# proto-message: ClassificationList
|
||||||
classification {
|
classification {
|
||||||
index: 0
|
index: 0
|
||||||
score: 4.9559007e-06
|
score: 2.922153e-05
|
||||||
label: "_neutral"
|
label: "_neutral"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 1
|
index: 1
|
||||||
score: 0.22943014
|
score: 0.7923543
|
||||||
label: "browDownLeft"
|
label: "browDownLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 2
|
index: 2
|
||||||
score: 0.22297752
|
score: 0.81483257
|
||||||
label: "browDownRight"
|
label: "browDownRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 3
|
index: 3
|
||||||
score: 0.015948873
|
score: 0.0002845049
|
||||||
label: "browInnerUp"
|
label: "browInnerUp"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 4
|
index: 4
|
||||||
score: 0.006946607
|
score: 0.00049632485
|
||||||
label: "browOuterUpLeft"
|
label: "browOuterUpLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 5
|
index: 5
|
||||||
score: 0.0070318673
|
score: 0.00050830405
|
||||||
label: "browOuterUpRight"
|
label: "browOuterUpRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 6
|
index: 6
|
||||||
score: 0.0013679645
|
score: 0.0014010799
|
||||||
label: "cheekPuff"
|
label: "cheekPuff"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 7
|
index: 7
|
||||||
score: 7.1003383e-06
|
score: 3.1523705e-06
|
||||||
label: "cheekSquintLeft"
|
label: "cheekSquintLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 8
|
index: 8
|
||||||
score: 5.78299e-06
|
score: 4.7487533e-06
|
||||||
label: "cheekSquintRight"
|
label: "cheekSquintRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 9
|
index: 9
|
||||||
score: 0.20132238
|
score: 0.2633514
|
||||||
label: "eyeBlinkLeft"
|
label: "eyeBlinkLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 10
|
index: 10
|
||||||
score: 0.16521452
|
score: 0.29687604
|
||||||
label: "eyeBlinkRight"
|
label: "eyeBlinkRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 11
|
index: 11
|
||||||
score: 0.03764786
|
score: 0.15565884
|
||||||
label: "eyeLookDownLeft"
|
label: "eyeLookDownLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 12
|
index: 12
|
||||||
score: 0.04828824
|
score: 0.18480143
|
||||||
label: "eyeLookDownRight"
|
label: "eyeLookDownRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 13
|
index: 13
|
||||||
score: 0.016539993
|
score: 0.054997284
|
||||||
label: "eyeLookInLeft"
|
label: "eyeLookInLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 14
|
index: 14
|
||||||
score: 0.20026363
|
score: 0.10678173
|
||||||
label: "eyeLookInRight"
|
label: "eyeLookInRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 15
|
index: 15
|
||||||
score: 0.21363346
|
score: 0.099223085
|
||||||
label: "eyeLookOutLeft"
|
label: "eyeLookOutLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 16
|
index: 16
|
||||||
score: 0.024430025
|
score: 0.06485316
|
||||||
label: "eyeLookOutRight"
|
label: "eyeLookOutRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 17
|
index: 17
|
||||||
score: 0.30147508
|
score: 0.09108365
|
||||||
label: "eyeLookUpLeft"
|
label: "eyeLookUpLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 18
|
index: 18
|
||||||
score: 0.28701693
|
score: 0.090983614
|
||||||
label: "eyeLookUpRight"
|
label: "eyeLookUpRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 19
|
index: 19
|
||||||
score: 0.67143106
|
score: 0.7040458
|
||||||
label: "eyeSquintLeft"
|
label: "eyeSquintLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 20
|
index: 20
|
||||||
score: 0.5306328
|
score: 0.5759947
|
||||||
label: "eyeSquintRight"
|
label: "eyeSquintRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 21
|
index: 21
|
||||||
score: 0.0041342233
|
score: 0.002923721
|
||||||
label: "eyeWideLeft"
|
label: "eyeWideLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 22
|
index: 22
|
||||||
score: 0.005231879
|
score: 0.0027232585
|
||||||
label: "eyeWideRight"
|
label: "eyeWideRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 23
|
index: 23
|
||||||
score: 0.009427094
|
score: 0.0028721786
|
||||||
label: "jawForward"
|
label: "jawForward"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 24
|
index: 24
|
||||||
score: 0.0015789346
|
score: 0.0014818686
|
||||||
label: "jawLeft"
|
label: "jawLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 25
|
index: 25
|
||||||
score: 0.073719256
|
score: 0.13099338
|
||||||
label: "jawOpen"
|
label: "jawOpen"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 26
|
index: 26
|
||||||
score: 0.00046979196
|
score: 0.00025238062
|
||||||
label: "jawRight"
|
label: "jawRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 27
|
index: 27
|
||||||
score: 0.0011400756
|
score: 0.013557238
|
||||||
label: "mouthClose"
|
label: "mouthClose"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 28
|
index: 28
|
||||||
score: 0.0060502808
|
score: 0.040049534
|
||||||
label: "mouthDimpleLeft"
|
label: "mouthDimpleLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 29
|
index: 29
|
||||||
score: 0.013351685
|
score: 0.017489946
|
||||||
label: "mouthDimpleRight"
|
label: "mouthDimpleRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 30
|
index: 30
|
||||||
score: 0.09859665
|
score: 0.00069562776
|
||||||
label: "mouthFrownLeft"
|
label: "mouthFrownLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 31
|
index: 31
|
||||||
score: 0.08897466
|
score: 0.00082510535
|
||||||
label: "mouthFrownRight"
|
label: "mouthFrownRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 32
|
index: 32
|
||||||
score: 0.0020718675
|
score: 0.008079494
|
||||||
label: "mouthFunnel"
|
label: "mouthFunnel"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 33
|
index: 33
|
||||||
score: 6.42887e-06
|
score: 0.000491791
|
||||||
label: "mouthLeft"
|
label: "mouthLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 34
|
index: 34
|
||||||
score: 0.68950605
|
score: 0.067228556
|
||||||
label: "mouthLowerDownLeft"
|
label: "mouthLowerDownLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 35
|
index: 35
|
||||||
score: 0.7864029
|
score: 0.17653553
|
||||||
label: "mouthLowerDownRight"
|
label: "mouthLowerDownRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 36
|
index: 36
|
||||||
score: 0.056456964
|
score: 0.14764099
|
||||||
label: "mouthPressLeft"
|
label: "mouthPressLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 37
|
index: 37
|
||||||
score: 0.037348792
|
score: 0.11692603
|
||||||
label: "mouthPressRight"
|
label: "mouthPressRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 38
|
index: 38
|
||||||
score: 0.00067001814
|
score: 0.0016829089
|
||||||
label: "mouthPucker"
|
label: "mouthPucker"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 39
|
index: 39
|
||||||
score: 0.005189785
|
score: 0.0010300461
|
||||||
label: "mouthRight"
|
label: "mouthRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 40
|
index: 40
|
||||||
score: 0.018723497
|
score: 0.01507229
|
||||||
label: "mouthRollLower"
|
label: "mouthRollLower"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 41
|
index: 41
|
||||||
score: 0.052819636
|
score: 0.11626337
|
||||||
label: "mouthRollUpper"
|
label: "mouthRollUpper"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 42
|
index: 42
|
||||||
score: 0.0033772716
|
score: 0.004975724
|
||||||
label: "mouthShrugLower"
|
label: "mouthShrugLower"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 43
|
index: 43
|
||||||
score: 0.0031609535
|
score: 0.004570691
|
||||||
label: "mouthShrugUpper"
|
label: "mouthShrugUpper"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 44
|
index: 44
|
||||||
score: 0.49639142
|
score: 0.9271871
|
||||||
label: "mouthSmileLeft"
|
label: "mouthSmileLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 45
|
index: 45
|
||||||
score: 0.4014515
|
score: 0.9062751
|
||||||
label: "mouthSmileRight"
|
label: "mouthSmileRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 46
|
index: 46
|
||||||
score: 0.5825701
|
score: 0.037454817
|
||||||
label: "mouthStretchLeft"
|
label: "mouthStretchLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 47
|
index: 47
|
||||||
score: 0.73058575
|
score: 0.1777958
|
||||||
label: "mouthStretchRight"
|
label: "mouthStretchRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 48
|
index: 48
|
||||||
score: 0.13561466
|
score: 0.8198947
|
||||||
label: "mouthUpperUpLeft"
|
label: "mouthUpperUpLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 49
|
index: 49
|
||||||
score: 0.20078722
|
score: 0.769086
|
||||||
label: "mouthUpperUpRight"
|
label: "mouthUpperUpRight"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 50
|
index: 50
|
||||||
score: 3.3396598e-06
|
score: 7.096563e-07
|
||||||
label: "noseSneerLeft"
|
label: "noseSneerLeft"
|
||||||
}
|
}
|
||||||
classification {
|
classification {
|
||||||
index: 51
|
index: 51
|
||||||
score: 1.3096546e-05
|
score: 9.818824e-06
|
||||||
label: "noseSneerRight"
|
label: "noseSneerRight"
|
||||||
}
|
}
|
||||||
|
|
5060
mediapipe/tasks/testdata/vision/portrait_expected_face_geometry_with_attention.pbtxt
vendored
Normal file
5060
mediapipe/tasks/testdata/vision/portrait_expected_face_geometry_with_attention.pbtxt
vendored
Normal file
File diff suppressed because it is too large
Load Diff
50
third_party/external_files.bzl
vendored
50
third_party/external_files.bzl
vendored
|
@ -70,6 +70,12 @@ def external_files():
|
||||||
urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD?generation=1661875663693976"],
|
urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD?generation=1661875663693976"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
http_file(
|
||||||
|
name = "com_google_mediapipe_BUILD_orig",
|
||||||
|
sha256 = "64d5343a6a5f9be06db0a5074a2260f9ae63a989fe01702832cd215680dc19c1",
|
||||||
|
urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD.orig?generation=1678323576393653"],
|
||||||
|
)
|
||||||
|
|
||||||
http_file(
|
http_file(
|
||||||
name = "com_google_mediapipe_burger_crop_jpg",
|
name = "com_google_mediapipe_burger_crop_jpg",
|
||||||
sha256 = "8f58de573f0bf59a49c3d86cfabb9ad4061481f574aa049177e8da3963dddc50",
|
sha256 = "8f58de573f0bf59a49c3d86cfabb9ad4061481f574aa049177e8da3963dddc50",
|
||||||
|
@ -300,8 +306,20 @@ def external_files():
|
||||||
|
|
||||||
http_file(
|
http_file(
|
||||||
name = "com_google_mediapipe_face_geometry_expected_out_pbtxt",
|
name = "com_google_mediapipe_face_geometry_expected_out_pbtxt",
|
||||||
sha256 = "611b203bca40e547ae75bf0822fda0695d512d02940e7af08a70068eaa8524f7",
|
sha256 = "4a4ed08055a5bc9281472dd60180d11f0cdc9a15fa1788d87a58af3d06b2c6e4",
|
||||||
urls = ["https://storage.googleapis.com/mediapipe-assets/face_geometry_expected_out.pbtxt?generation=1677787710308910"],
|
urls = ["https://storage.googleapis.com/mediapipe-assets/face_geometry_expected_out.pbtxt?generation=1678323580380646"],
|
||||||
|
)
|
||||||
|
|
||||||
|
http_file(
|
||||||
|
name = "com_google_mediapipe_face_landmarker_task",
|
||||||
|
sha256 = "7cf2bbf1842c429e9defee38e7f1c4238978d8a6faf2da145bb19846f86bd2f4",
|
||||||
|
urls = ["https://storage.googleapis.com/mediapipe-assets/face_landmarker.task?generation=1678323583183024"],
|
||||||
|
)
|
||||||
|
|
||||||
|
http_file(
|
||||||
|
name = "com_google_mediapipe_face_landmarker_with_blendshapes_task",
|
||||||
|
sha256 = "a75c1ba70e4b8568000af2ad0b355ed559ab5d5793db50fa9ad241f8dc4fad5f",
|
||||||
|
urls = ["https://storage.googleapis.com/mediapipe-assets/face_landmarker_with_blendshapes.task?generation=1678323586260800"],
|
||||||
)
|
)
|
||||||
|
|
||||||
http_file(
|
http_file(
|
||||||
|
@ -316,6 +334,12 @@ def external_files():
|
||||||
urls = ["https://storage.googleapis.com/mediapipe-assets/face_landmark_with_attention.tflite?generation=1676415468821650"],
|
urls = ["https://storage.googleapis.com/mediapipe-assets/face_landmark_with_attention.tflite?generation=1676415468821650"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
http_file(
|
||||||
|
name = "com_google_mediapipe_face_stylization_dummy_tflite",
|
||||||
|
sha256 = "f57fd2d5638def25466f6fec142eb3397d8ad99a9bd0a9344b622bad7c3f0376",
|
||||||
|
urls = ["https://storage.googleapis.com/mediapipe-assets/face_stylization_dummy.tflite?generation=1678323589048063"],
|
||||||
|
)
|
||||||
|
|
||||||
http_file(
|
http_file(
|
||||||
name = "com_google_mediapipe_feature_tensor_meta_json",
|
name = "com_google_mediapipe_feature_tensor_meta_json",
|
||||||
sha256 = "b2c30ddfd495956ce81085f8a143422f4310b002cfbf1c594ff2ee0576e29d6f",
|
sha256 = "b2c30ddfd495956ce81085f8a143422f4310b002cfbf1c594ff2ee0576e29d6f",
|
||||||
|
@ -508,6 +532,12 @@ def external_files():
|
||||||
urls = ["https://storage.googleapis.com/mediapipe-assets/labels.txt?generation=1667892497527642"],
|
urls = ["https://storage.googleapis.com/mediapipe-assets/labels.txt?generation=1667892497527642"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
http_file(
|
||||||
|
name = "com_google_mediapipe_language_detector_tflite",
|
||||||
|
sha256 = "5f64d821110dd2a3280546e8cd59dff09547e25d5f5c9711ec3f03416414dbb2",
|
||||||
|
urls = ["https://storage.googleapis.com/mediapipe-assets/language_detector.tflite?generation=1678323592870401"],
|
||||||
|
)
|
||||||
|
|
||||||
http_file(
|
http_file(
|
||||||
name = "com_google_mediapipe_left_hands_jpg",
|
name = "com_google_mediapipe_left_hands_jpg",
|
||||||
sha256 = "4b5134daa4cb60465535239535f9f74c2842aba3aa5fd30bf04ef5678f93d87f",
|
sha256 = "4b5134daa4cb60465535239535f9f74c2842aba3aa5fd30bf04ef5678f93d87f",
|
||||||
|
@ -780,8 +810,8 @@ def external_files():
|
||||||
|
|
||||||
http_file(
|
http_file(
|
||||||
name = "com_google_mediapipe_portrait_expected_blendshapes_with_attention_pbtxt",
|
name = "com_google_mediapipe_portrait_expected_blendshapes_with_attention_pbtxt",
|
||||||
sha256 = "0142d56705093c3d79ea5ee79b8e9454499abee00fc059491e6ca14f5fbab862",
|
sha256 = "3f8f698d8ed81346c6f13d1cc85190fd4a58b021e664d336997d29818b8ffbb6",
|
||||||
urls = ["https://storage.googleapis.com/mediapipe-assets/portrait_expected_blendshapes_with_attention.pbtxt?generation=1678218364703223"],
|
urls = ["https://storage.googleapis.com/mediapipe-assets/portrait_expected_blendshapes_with_attention.pbtxt?generation=1678323598426417"],
|
||||||
)
|
)
|
||||||
|
|
||||||
http_file(
|
http_file(
|
||||||
|
@ -790,6 +820,12 @@ def external_files():
|
||||||
urls = ["https://storage.googleapis.com/mediapipe-assets/portrait_expected_detection.pbtxt?generation=1677044311581104"],
|
urls = ["https://storage.googleapis.com/mediapipe-assets/portrait_expected_detection.pbtxt?generation=1677044311581104"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
http_file(
|
||||||
|
name = "com_google_mediapipe_portrait_expected_face_geometry_with_attention_pbtxt",
|
||||||
|
sha256 = "5cc57b8da3ad0527dce581fe1309f6b36043e5837e3f4f5af5e24005a99dc52a",
|
||||||
|
urls = ["https://storage.googleapis.com/mediapipe-assets/portrait_expected_face_geometry_with_attention.pbtxt?generation=1678323601064393"],
|
||||||
|
)
|
||||||
|
|
||||||
http_file(
|
http_file(
|
||||||
name = "com_google_mediapipe_portrait_expected_face_landmarks_pbtxt",
|
name = "com_google_mediapipe_portrait_expected_face_landmarks_pbtxt",
|
||||||
sha256 = "4ac8587379bd072c36cda0d7345f5e592fae51b30522475e0b49c18aab108ce7",
|
sha256 = "4ac8587379bd072c36cda0d7345f5e592fae51b30522475e0b49c18aab108ce7",
|
||||||
|
@ -850,6 +886,12 @@ def external_files():
|
||||||
urls = ["https://storage.googleapis.com/mediapipe-assets/pose_landmark_lite.tflite?generation=1661875901231143"],
|
urls = ["https://storage.googleapis.com/mediapipe-assets/pose_landmark_lite.tflite?generation=1661875901231143"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
http_file(
|
||||||
|
name = "com_google_mediapipe_ptm_512_hdt_ptm_woid_tflite",
|
||||||
|
sha256 = "2baa1c9783d03dd26f91e3c49efbcab11dd1361ff80e40e7209e81f84f281b6a",
|
||||||
|
urls = ["https://storage.googleapis.com/mediapipe-assets/ptm_512_hdt_ptm_woid.tflite?generation=1678323604771164"],
|
||||||
|
)
|
||||||
|
|
||||||
http_file(
|
http_file(
|
||||||
name = "com_google_mediapipe_README_md",
|
name = "com_google_mediapipe_README_md",
|
||||||
sha256 = "a96d08c9c70cd9717207ed72c926e02e5eada751f00bdc5d3a7e82e3492b72cb",
|
sha256 = "a96d08c9c70cd9717207ed72c926e02e5eada751f00bdc5d3a7e82e3492b72cb",
|
||||||
|
|
Loading…
Reference in New Issue
Block a user