Use ExternalFile to set metadata of GeometryPipelineCalculator.

PiperOrigin-RevId: 516384491
This commit is contained in:
MediaPipe Team 2023-03-13 18:46:54 -07:00 committed by Copybara-Service
parent 57f106e0a7
commit 46ba1d8051
14 changed files with 137 additions and 61 deletions

View File

@ -19,9 +19,6 @@ package(default_visibility = ["//mediapipe/tasks:internal"])
cc_library( cc_library(
name = "face_geometry_from_landmarks_graph", name = "face_geometry_from_landmarks_graph",
srcs = ["face_geometry_from_landmarks_graph.cc"], srcs = ["face_geometry_from_landmarks_graph.cc"],
data = [
"//mediapipe/tasks/cc/vision/face_geometry/data:geometry_pipeline_metadata_landmarks",
],
deps = [ deps = [
"//mediapipe/calculators/core:begin_loop_calculator", "//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:end_loop_calculator", "//mediapipe/calculators/core:end_loop_calculator",
@ -39,6 +36,7 @@ cc_library(
"//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator_cc_proto", "//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto", "//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto", "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_graph_options_cc_proto",
"//mediapipe/util:graph_builder_utils", "//mediapipe/util:graph_builder_utils",
"@com_google_absl//absl/status:statusor", "@com_google_absl//absl/status:statusor",
], ],

View File

@ -45,6 +45,7 @@ mediapipe_proto_library(
srcs = ["geometry_pipeline_calculator.proto"], srcs = ["geometry_pipeline_calculator.proto"],
deps = [ deps = [
"//mediapipe/framework:calculator_options_proto", "//mediapipe/framework:calculator_options_proto",
"//mediapipe/tasks/cc/core/proto:external_file_proto",
], ],
) )
@ -59,6 +60,8 @@ cc_library(
"//mediapipe/framework/port:ret_check", "//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor", "//mediapipe/framework/port:statusor",
"//mediapipe/tasks/cc/core:external_file_handler",
"//mediapipe/tasks/cc/core/proto:external_file_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/libs:geometry_pipeline", "//mediapipe/tasks/cc/vision/face_geometry/libs:geometry_pipeline",
"//mediapipe/tasks/cc/vision/face_geometry/libs:validation_utils", "//mediapipe/tasks/cc/vision/face_geometry/libs:validation_utils",
"//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto", "//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto",

View File

@ -24,6 +24,8 @@
#include "mediapipe/framework/port/status.h" #include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h" #include "mediapipe/framework/port/statusor.h"
#include "mediapipe/tasks/cc/core/external_file_handler.h"
#include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h" #include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/libs/geometry_pipeline.h" #include "mediapipe/tasks/cc/vision/face_geometry/libs/geometry_pipeline.h"
#include "mediapipe/tasks/cc/vision/face_geometry/libs/validation_utils.h" #include "mediapipe/tasks/cc/vision/face_geometry/libs/validation_utils.h"
@ -69,8 +71,8 @@ using ::mediapipe::tasks::vision::face_geometry::proto::
// A vector of face geometry data. // A vector of face geometry data.
// //
// Options: // Options:
// metadata_path (`string`, optional): // metadata_file (`ExternalFile`, optional):
// Defines a path for the geometry pipeline metadata file. // Defines an ExternalFile for the geometry pipeline metadata file.
// //
// The geometry pipeline metadata file format must be the binary // The geometry pipeline metadata file format must be the binary
// `GeometryPipelineMetadata` proto. // `GeometryPipelineMetadata` proto.
@ -95,7 +97,7 @@ class GeometryPipelineCalculator : public CalculatorBase {
ASSIGN_OR_RETURN( ASSIGN_OR_RETURN(
GeometryPipelineMetadata metadata, GeometryPipelineMetadata metadata,
ReadMetadataFromFile(options.metadata_path()), ReadMetadataFromFile(options.metadata_file()),
_ << "Failed to read the geometry pipeline metadata from file!"); _ << "Failed to read the geometry pipeline metadata from file!");
MP_RETURN_IF_ERROR(ValidateGeometryPipelineMetadata(metadata)) MP_RETURN_IF_ERROR(ValidateGeometryPipelineMetadata(metadata))
@ -155,32 +157,19 @@ class GeometryPipelineCalculator : public CalculatorBase {
private: private:
static absl::StatusOr<GeometryPipelineMetadata> ReadMetadataFromFile( static absl::StatusOr<GeometryPipelineMetadata> ReadMetadataFromFile(
const std::string& metadata_path) { const core::proto::ExternalFile& metadata_file) {
ASSIGN_OR_RETURN(std::string metadata_blob, ASSIGN_OR_RETURN(
ReadContentBlobFromFile(metadata_path), const auto file_handler,
_ << "Failed to read a metadata blob from file!"); core::ExternalFileHandler::CreateFromExternalFile(&metadata_file));
GeometryPipelineMetadata metadata; GeometryPipelineMetadata metadata;
RET_CHECK(metadata.ParseFromString(metadata_blob)) RET_CHECK(
metadata.ParseFromString(std::string(file_handler->GetFileContent())))
<< "Failed to parse a metadata proto from a binary blob!"; << "Failed to parse a metadata proto from a binary blob!";
return metadata; return metadata;
} }
static absl::StatusOr<std::string> ReadContentBlobFromFile(
const std::string& unresolved_path) {
ASSIGN_OR_RETURN(std::string resolved_path,
mediapipe::PathToResourceAsFile(unresolved_path),
_ << "Failed to resolve path! Path = " << unresolved_path);
std::string content_blob;
MP_RETURN_IF_ERROR(
mediapipe::GetResourceContents(resolved_path, &content_blob))
<< "Failed to read content blob! Resolved path = " << resolved_path;
return content_blob;
}
std::unique_ptr<GeometryPipeline> geometry_pipeline_; std::unique_ptr<GeometryPipeline> geometry_pipeline_;
}; };

View File

@ -17,11 +17,12 @@ syntax = "proto2";
package mediapipe.tasks.vision.face_geometry; package mediapipe.tasks.vision.face_geometry;
import "mediapipe/framework/calculator_options.proto"; import "mediapipe/framework/calculator_options.proto";
import "mediapipe/tasks/cc/core/proto/external_file.proto";
message FaceGeometryPipelineCalculatorOptions { message FaceGeometryPipelineCalculatorOptions {
extend mediapipe.CalculatorOptions { extend mediapipe.CalculatorOptions {
optional FaceGeometryPipelineCalculatorOptions ext = 512499200; optional FaceGeometryPipelineCalculatorOptions ext = 512499200;
} }
optional string metadata_path = 1; optional core.proto.ExternalFile metadata_file = 1;
} }

View File

@ -28,6 +28,7 @@ limitations under the License.
#include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h" #include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h" #include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h" #include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry_graph_options.pb.h"
#include "mediapipe/util/graph_builder_utils.h" #include "mediapipe/util/graph_builder_utils.h"
namespace mediapipe::tasks::vision::face_geometry { namespace mediapipe::tasks::vision::face_geometry {
@ -49,10 +50,6 @@ constexpr char kIterableTag[] = "ITERABLE";
constexpr char kBatchEndTag[] = "BATCH_END"; constexpr char kBatchEndTag[] = "BATCH_END";
constexpr char kItemTag[] = "ITEM"; constexpr char kItemTag[] = "ITEM";
constexpr char kGeometryPipelineMetadataPath[] =
"mediapipe/tasks/cc/vision/face_geometry/data/"
"geometry_pipeline_metadata_landmarks.binarypb";
struct FaceGeometryOuts { struct FaceGeometryOuts {
Stream<std::vector<FaceGeometry>> multi_face_geometry; Stream<std::vector<FaceGeometry>> multi_face_geometry;
}; };
@ -127,6 +124,7 @@ class FaceGeometryFromLandmarksGraph : public Subgraph {
} }
ASSIGN_OR_RETURN(auto outs, ASSIGN_OR_RETURN(auto outs,
BuildFaceGeometryFromLandmarksGraph( BuildFaceGeometryFromLandmarksGraph(
*sc->MutableOptions<proto::FaceGeometryGraphOptions>(),
graph.In(kFaceLandmarksTag) graph.In(kFaceLandmarksTag)
.Cast<std::vector<NormalizedLandmarkList>>(), .Cast<std::vector<NormalizedLandmarkList>>(),
graph.In(kImageSizeTag).Cast<std::pair<int, int>>(), graph.In(kImageSizeTag).Cast<std::pair<int, int>>(),
@ -138,6 +136,7 @@ class FaceGeometryFromLandmarksGraph : public Subgraph {
private: private:
absl::StatusOr<FaceGeometryOuts> BuildFaceGeometryFromLandmarksGraph( absl::StatusOr<FaceGeometryOuts> BuildFaceGeometryFromLandmarksGraph(
proto::FaceGeometryGraphOptions& graph_options,
Stream<std::vector<NormalizedLandmarkList>> multi_face_landmarks, Stream<std::vector<NormalizedLandmarkList>> multi_face_landmarks,
Stream<std::pair<int, int>> image_size, Stream<std::pair<int, int>> image_size,
std::optional<SidePacket<Environment>> environment, Graph& graph) { std::optional<SidePacket<Environment>> environment, Graph& graph) {
@ -185,7 +184,8 @@ class FaceGeometryFromLandmarksGraph : public Subgraph {
"mediapipe.tasks.vision.face_geometry.FaceGeometryPipelineCalculator"); "mediapipe.tasks.vision.face_geometry.FaceGeometryPipelineCalculator");
auto& geometry_pipeline_options = auto& geometry_pipeline_options =
geometry_pipeline.GetOptions<FaceGeometryPipelineCalculatorOptions>(); geometry_pipeline.GetOptions<FaceGeometryPipelineCalculatorOptions>();
geometry_pipeline_options.set_metadata_path(kGeometryPipelineMetadataPath); geometry_pipeline_options.Swap(
graph_options.mutable_geometry_pipeline_options());
image_size >> geometry_pipeline.In(kImageSizeTag); image_size >> geometry_pipeline.In(kImageSizeTag);
multi_face_landmarks_no_iris >> multi_face_landmarks_no_iris >>
geometry_pipeline.In(kMultiFaceLandmarksTag); geometry_pipeline.In(kMultiFaceLandmarksTag);

View File

@ -20,6 +20,7 @@ limitations under the License.
#include "absl/status/statusor.h" #include "absl/status/statusor.h"
#include "absl/strings/str_format.h" #include "absl/strings/str_format.h"
#include "absl/strings/string_view.h" #include "absl/strings/string_view.h"
#include "absl/strings/substitute.h"
#include "mediapipe/framework/api2/port.h" #include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h" #include "mediapipe/framework/calculator_runner.h"
@ -31,6 +32,7 @@ limitations under the License.
#include "mediapipe/framework/port/gtest.h" #include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/tool/sink.h" #include "mediapipe/framework/tool/sink.h"
#include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h" #include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h" #include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
@ -49,6 +51,9 @@ constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/";
constexpr char kFaceLandmarksFileName[] = constexpr char kFaceLandmarksFileName[] =
"face_blendshapes_in_landmarks.prototxt"; "face_blendshapes_in_landmarks.prototxt";
constexpr char kFaceGeometryFileName[] = "face_geometry_expected_out.pbtxt"; constexpr char kFaceGeometryFileName[] = "face_geometry_expected_out.pbtxt";
constexpr char kGeometryPipelineMetadataPath[] =
"mediapipe/tasks/cc/vision/face_geometry/data/"
"geometry_pipeline_metadata_landmarks.binarypb";
std::vector<NormalizedLandmarkList> GetLandmarks(absl::string_view filename) { std::vector<NormalizedLandmarkList> GetLandmarks(absl::string_view filename) {
NormalizedLandmarkList landmarks; NormalizedLandmarkList landmarks;
@ -89,17 +94,25 @@ void MakeInputPacketsAndRunGraph(CalculatorGraph& graph) {
TEST(FaceGeometryFromLandmarksGraphTest, DefaultEnvironment) { TEST(FaceGeometryFromLandmarksGraphTest, DefaultEnvironment) {
CalculatorGraphConfig graph_config = ParseTextProtoOrDie< CalculatorGraphConfig graph_config = ParseTextProtoOrDie<
CalculatorGraphConfig>(R"pb( CalculatorGraphConfig>(absl::Substitute(
input_stream: "FACE_LANDMARKS:face_landmarks" R"pb(
input_stream: "IMAGE_SIZE:image_size" input_stream: "FACE_LANDMARKS:face_landmarks"
output_stream: "FACE_GEOMETRY:face_geometry" input_stream: "IMAGE_SIZE:image_size"
node { output_stream: "FACE_GEOMETRY:face_geometry"
calculator: "mediapipe.tasks.vision.face_geometry.FaceGeometryFromLandmarksGraph" node {
input_stream: "FACE_LANDMARKS:face_landmarks" calculator: "mediapipe.tasks.vision.face_geometry.FaceGeometryFromLandmarksGraph"
input_stream: "IMAGE_SIZE:image_size" input_stream: "FACE_LANDMARKS:face_landmarks"
output_stream: "FACE_GEOMETRY:face_geometry" input_stream: "IMAGE_SIZE:image_size"
} output_stream: "FACE_GEOMETRY:face_geometry"
)pb"); options: {
[mediapipe.tasks.vision.face_geometry.proto.FaceGeometryGraphOptions
.ext] {
geometry_pipeline_options { metadata_file { file_name: "$0" } }
}
}
}
)pb",
kGeometryPipelineMetadataPath));
std::vector<Packet> output_packets; std::vector<Packet> output_packets;
tool::AddVectorSink("face_geometry", &graph_config, &output_packets); tool::AddVectorSink("face_geometry", &graph_config, &output_packets);
@ -116,19 +129,27 @@ TEST(FaceGeometryFromLandmarksGraphTest, DefaultEnvironment) {
TEST(FaceGeometryFromLandmarksGraphTest, SideInEnvironment) { TEST(FaceGeometryFromLandmarksGraphTest, SideInEnvironment) {
CalculatorGraphConfig graph_config = ParseTextProtoOrDie< CalculatorGraphConfig graph_config = ParseTextProtoOrDie<
CalculatorGraphConfig>(R"pb( CalculatorGraphConfig>(absl::Substitute(
input_stream: "FACE_LANDMARKS:face_landmarks" R"pb(
input_stream: "IMAGE_SIZE:image_size" input_stream: "FACE_LANDMARKS:face_landmarks"
input_side_packet: "ENVIRONMENT:environment" input_stream: "IMAGE_SIZE:image_size"
output_stream: "FACE_GEOMETRY:face_geometry" input_side_packet: "ENVIRONMENT:environment"
node { output_stream: "FACE_GEOMETRY:face_geometry"
calculator: "mediapipe.tasks.vision.face_geometry.FaceGeometryFromLandmarksGraph" node {
input_stream: "FACE_LANDMARKS:face_landmarks" calculator: "mediapipe.tasks.vision.face_geometry.FaceGeometryFromLandmarksGraph"
input_stream: "IMAGE_SIZE:image_size" input_stream: "FACE_LANDMARKS:face_landmarks"
input_side_packet: "ENVIRONMENT:environment" input_stream: "IMAGE_SIZE:image_size"
output_stream: "FACE_GEOMETRY:face_geometry" input_side_packet: "ENVIRONMENT:environment"
} output_stream: "FACE_GEOMETRY:face_geometry"
)pb"); options: {
[mediapipe.tasks.vision.face_geometry.proto.FaceGeometryGraphOptions
.ext] {
geometry_pipeline_options { metadata_file { file_name: "$0" } }
}
}
}
)pb",
kGeometryPipelineMetadataPath));
std::vector<Packet> output_packets; std::vector<Packet> output_packets;
tool::AddVectorSink("face_geometry", &graph_config, &output_packets); tool::AddVectorSink("face_geometry", &graph_config, &output_packets);

View File

@ -44,3 +44,12 @@ mediapipe_proto_library(
name = "mesh_3d_proto", name = "mesh_3d_proto",
srcs = ["mesh_3d.proto"], srcs = ["mesh_3d.proto"],
) )
mediapipe_proto_library(
name = "face_geometry_graph_options_proto",
srcs = ["face_geometry_graph_options.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator_proto",
],
)

View File

@ -0,0 +1,28 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.tasks.vision.face_geometry.proto;
import "mediapipe/framework/calculator_options.proto";
import "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.proto";
message FaceGeometryGraphOptions {
extend mediapipe.CalculatorOptions {
optional FaceGeometryGraphOptions ext = 515723506;
}
optional FaceGeometryPipelineCalculatorOptions geometry_pipeline_options = 1;
}

View File

@ -210,8 +210,10 @@ cc_library(
"//mediapipe/tasks/cc/vision/face_detector:face_detector_graph", "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph",
"//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry:face_geometry_from_landmarks_graph", "//mediapipe/tasks/cc/vision/face_geometry:face_geometry_from_landmarks_graph",
"//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto", "//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto", "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/face_landmarker/proto:face_blendshapes_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_blendshapes_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_cc_proto", "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_cc_proto",

View File

@ -40,8 +40,10 @@ limitations under the License.
#include "mediapipe/tasks/cc/core/utils.h" #include "mediapipe/tasks/cc/core/utils.h"
#include "mediapipe/tasks/cc/metadata/utils/zip_utils.h" #include "mediapipe/tasks/cc/metadata/utils/zip_utils.h"
#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h" #include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h" #include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h" #include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h"
@ -93,6 +95,8 @@ constexpr char kFaceDetectorTFLiteName[] = "face_detector.tflite";
constexpr char kFaceLandmarksDetectorTFLiteName[] = constexpr char kFaceLandmarksDetectorTFLiteName[] =
"face_landmarks_detector.tflite"; "face_landmarks_detector.tflite";
constexpr char kFaceBlendshapeTFLiteName[] = "face_blendshapes.tflite"; constexpr char kFaceBlendshapeTFLiteName[] = "face_blendshapes.tflite";
constexpr char kFaceGeometryPipelineMetadataName[] =
"geometry_pipeline_metadata_landmarks.binarypb";
struct FaceLandmarkerOutputs { struct FaceLandmarkerOutputs {
Source<std::vector<NormalizedLandmarkList>> landmark_lists; Source<std::vector<NormalizedLandmarkList>> landmark_lists;
@ -305,6 +309,7 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
absl::StatusOr<CalculatorGraphConfig> GetConfig( absl::StatusOr<CalculatorGraphConfig> GetConfig(
SubgraphContext* sc) override { SubgraphContext* sc) override {
Graph graph; Graph graph;
bool output_geometry = HasOutput(sc->OriginalNode(), kFaceGeometryTag);
if (sc->Options<FaceLandmarkerGraphOptions>() if (sc->Options<FaceLandmarkerGraphOptions>()
.base_options() .base_options()
.has_model_asset()) { .has_model_asset()) {
@ -318,6 +323,18 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
sc->MutableOptions<FaceLandmarkerGraphOptions>(), sc->MutableOptions<FaceLandmarkerGraphOptions>(),
!sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService) !sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService)
.IsAvailable())); .IsAvailable()));
if (output_geometry) {
// Set the face geometry metdata file for
// FaceGeometryFromLandmarksGraph.
ASSIGN_OR_RETURN(auto face_geometry_pipeline_metadata_file,
model_asset_bundle_resources->GetModelFile(
kFaceGeometryPipelineMetadataName));
SetExternalFile(face_geometry_pipeline_metadata_file,
sc->MutableOptions<FaceLandmarkerGraphOptions>()
->mutable_face_geometry_graph_options()
->mutable_geometry_pipeline_options()
->mutable_metadata_file());
}
} }
std::optional<SidePacket<Environment>> environment; std::optional<SidePacket<Environment>> environment;
if (HasSideInput(sc->OriginalNode(), kEnvironmentTag)) { if (HasSideInput(sc->OriginalNode(), kEnvironmentTag)) {
@ -338,7 +355,6 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
.face_landmarks_detector_graph_options() .face_landmarks_detector_graph_options()
.has_face_blendshapes_graph_options())); .has_face_blendshapes_graph_options()));
} }
bool output_geometry = HasOutput(sc->OriginalNode(), kFaceGeometryTag);
ASSIGN_OR_RETURN( ASSIGN_OR_RETURN(
auto outs, auto outs,
BuildFaceLandmarkerGraph( BuildFaceLandmarkerGraph(
@ -481,6 +497,9 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
auto& face_geometry_from_landmarks = graph.AddNode( auto& face_geometry_from_landmarks = graph.AddNode(
"mediapipe.tasks.vision.face_geometry." "mediapipe.tasks.vision.face_geometry."
"FaceGeometryFromLandmarksGraph"); "FaceGeometryFromLandmarksGraph");
face_geometry_from_landmarks
.GetOptions<face_geometry::proto::FaceGeometryGraphOptions>()
.Swap(tasks_options.mutable_face_geometry_graph_options());
if (environment.has_value()) { if (environment.has_value()) {
*environment >> face_geometry_from_landmarks.SideIn(kEnvironmentTag); *environment >> face_geometry_from_landmarks.SideIn(kEnvironmentTag);
} }

View File

@ -60,5 +60,6 @@ mediapipe_proto_library(
"//mediapipe/framework:calculator_proto", "//mediapipe/framework:calculator_proto",
"//mediapipe/tasks/cc/core/proto:base_options_proto", "//mediapipe/tasks/cc/core/proto:base_options_proto",
"//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_proto", "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_graph_options_proto",
], ],
) )

View File

@ -21,6 +21,7 @@ import "mediapipe/framework/calculator.proto";
import "mediapipe/framework/calculator_options.proto"; import "mediapipe/framework/calculator_options.proto";
import "mediapipe/tasks/cc/core/proto/base_options.proto"; import "mediapipe/tasks/cc/core/proto/base_options.proto";
import "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.proto"; import "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.proto";
import "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry_graph_options.proto";
import "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.proto"; import "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.proto";
option java_package = "com.google.mediapipe.tasks.vision.facelandmarker.proto"; option java_package = "com.google.mediapipe.tasks.vision.facelandmarker.proto";
@ -45,4 +46,8 @@ message FaceLandmarkerGraphOptions {
// Minimum confidence for face landmarks tracking to be considered // Minimum confidence for face landmarks tracking to be considered
// successfully. // successfully.
optional float min_tracking_confidence = 4 [default = 0.5]; optional float min_tracking_confidence = 4 [default = 0.5];
// Options for FaceGeometryGraph to get facial transformation matrix.
optional face_geometry.proto.FaceGeometryGraphOptions
face_geometry_graph_options = 5;
} }

View File

@ -67,7 +67,7 @@ def external_files():
http_file( http_file(
name = "com_google_mediapipe_BUILD", name = "com_google_mediapipe_BUILD",
sha256 = "d2b2a8346202691d7f831887c84e9642e974f64ed67851d9a58cf15c94b1f6b3", sha256 = "d2b2a8346202691d7f831887c84e9642e974f64ed67851d9a58cf15c94b1f6b3",
urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD?generation=1661875663693976"], urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD?generation=16618756636939761678323576393653"],
) )
http_file( http_file(
@ -318,8 +318,8 @@ def external_files():
http_file( http_file(
name = "com_google_mediapipe_face_landmarker_with_blendshapes_task", name = "com_google_mediapipe_face_landmarker_with_blendshapes_task",
sha256 = "a75c1ba70e4b8568000af2ad0b355ed559ab5d5793db50fa9ad241f8dc4fad5f", sha256 = "b44e4cae6f5822456d60f33e7c852640d78c7e342aee7eacc22589451a0b9dc2",
urls = ["https://storage.googleapis.com/mediapipe-assets/face_landmarker_with_blendshapes.task?generation=1678323586260800"], urls = ["https://storage.googleapis.com/mediapipe-assets/face_landmarker_with_blendshapes.task?generation=1678504998301299"],
) )
http_file( http_file(
@ -822,8 +822,8 @@ def external_files():
http_file( http_file(
name = "com_google_mediapipe_portrait_expected_face_geometry_with_attention_pbtxt", name = "com_google_mediapipe_portrait_expected_face_geometry_with_attention_pbtxt",
sha256 = "5cc57b8da3ad0527dce581fe1309f6b36043e5837e3f4f5af5e24005a99dc52a", sha256 = "7ed1eed98e61e0a10811bb611c895d87c8023f398a36db01b6d9ba2e1ab09e16",
urls = ["https://storage.googleapis.com/mediapipe-assets/portrait_expected_face_geometry_with_attention.pbtxt?generation=1678323601064393"], urls = ["https://storage.googleapis.com/mediapipe-assets/portrait_expected_face_geometry_with_attention.pbtxt?generation=1678505004840652"],
) )
http_file( http_file(