cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use ExternalFile to set metadata of GeometryPipelineCalculator.

Browse Source 
PiperOrigin-RevId: 516384491
This commit is contained in:
MediaPipe Team 2023-03-13 18:46:54 -07:00 committed by Copybara-Service
parent 57f106e0a7
commit 46ba1d8051
14 changed files with 137 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mediapipe/tasks/cc/vision/face_geometry/BUILD
View File

@ -19,9 +19,6 @@ package(default_visibility = ["//mediapipe/tasks:internal"])
cc_library(
name = "face_geometry_from_landmarks_graph",
srcs = ["face_geometry_from_landmarks_graph.cc"],
data = [
"//mediapipe/tasks/cc/vision/face_geometry/data:geometry_pipeline_metadata_landmarks",
],
deps = [
"//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:end_loop_calculator",
@ -39,6 +36,7 @@ cc_library(
"//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_graph_options_cc_proto",
"//mediapipe/util:graph_builder_utils",
"@com_google_absl//absl/status:statusor",
],

3
mediapipe/tasks/cc/vision/face_geometry/calculators/BUILD
View File

@ -45,6 +45,7 @@ mediapipe_proto_library(
srcs = ["geometry_pipeline_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/tasks/cc/core/proto:external_file_proto",
],
)
@ -59,6 +60,8 @@ cc_library(
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/tasks/cc/core:external_file_handler",
"//mediapipe/tasks/cc/core/proto:external_file_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/libs:geometry_pipeline",
"//mediapipe/tasks/cc/vision/face_geometry/libs:validation_utils",
"//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto",

33
mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.cc
View File

@ -24,6 +24,8 @@
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/tasks/cc/core/external_file_handler.h"
#include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/libs/geometry_pipeline.h"
#include "mediapipe/tasks/cc/vision/face_geometry/libs/validation_utils.h"
@ -69,8 +71,8 @@ using ::mediapipe::tasks::vision::face_geometry::proto::
// A vector of face geometry data.
//
// Options:
// metadata_path (`string`, optional):
// Defines a path for the geometry pipeline metadata file.
// metadata_file (`ExternalFile`, optional):
// Defines an ExternalFile for the geometry pipeline metadata file.
//
// The geometry pipeline metadata file format must be the binary
// `GeometryPipelineMetadata` proto.
@ -95,7 +97,7 @@ class GeometryPipelineCalculator : public CalculatorBase {
ASSIGN_OR_RETURN(
GeometryPipelineMetadata metadata,
ReadMetadataFromFile(options.metadata_path()),
ReadMetadataFromFile(options.metadata_file()),
_ << "Failed to read the geometry pipeline metadata from file!");
MP_RETURN_IF_ERROR(ValidateGeometryPipelineMetadata(metadata))
@ -155,32 +157,19 @@ class GeometryPipelineCalculator : public CalculatorBase {
private:
static absl::StatusOr<GeometryPipelineMetadata> ReadMetadataFromFile(
const std::string& metadata_path) {
ASSIGN_OR_RETURN(std::string metadata_blob,
ReadContentBlobFromFile(metadata_path),
_ << "Failed to read a metadata blob from file!");
const core::proto::ExternalFile& metadata_file) {
ASSIGN_OR_RETURN(
const auto file_handler,
core::ExternalFileHandler::CreateFromExternalFile(&metadata_file));
GeometryPipelineMetadata metadata;
RET_CHECK(metadata.ParseFromString(metadata_blob))
RET_CHECK(
metadata.ParseFromString(std::string(file_handler->GetFileContent())))
<< "Failed to parse a metadata proto from a binary blob!";
return metadata;
}
static absl::StatusOr<std::string> ReadContentBlobFromFile(
const std::string& unresolved_path) {
ASSIGN_OR_RETURN(std::string resolved_path,
mediapipe::PathToResourceAsFile(unresolved_path),
_ << "Failed to resolve path! Path = " << unresolved_path);
std::string content_blob;
MP_RETURN_IF_ERROR(
mediapipe::GetResourceContents(resolved_path, &content_blob))
<< "Failed to read content blob! Resolved path = " << resolved_path;
return content_blob;
}
std::unique_ptr<GeometryPipeline> geometry_pipeline_;
};

3
mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.proto
View File

@ -17,11 +17,12 @@ syntax = "proto2";
package mediapipe.tasks.vision.face_geometry;
import "mediapipe/framework/calculator_options.proto";
import "mediapipe/tasks/cc/core/proto/external_file.proto";
message FaceGeometryPipelineCalculatorOptions {
extend mediapipe.CalculatorOptions {
optional FaceGeometryPipelineCalculatorOptions ext = 512499200;
}
optional string metadata_path = 1;
optional core.proto.ExternalFile metadata_file = 1;
}

10
mediapipe/tasks/cc/vision/face_geometry/face_geometry_from_landmarks_graph.cc
View File

@ -28,6 +28,7 @@ limitations under the License.
#include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry_graph_options.pb.h"
#include "mediapipe/util/graph_builder_utils.h"
namespace mediapipe::tasks::vision::face_geometry {
@ -49,10 +50,6 @@ constexpr char kIterableTag[] = "ITERABLE";
constexpr char kBatchEndTag[] = "BATCH_END";
constexpr char kItemTag[] = "ITEM";
constexpr char kGeometryPipelineMetadataPath[] =
"mediapipe/tasks/cc/vision/face_geometry/data/"
"geometry_pipeline_metadata_landmarks.binarypb";
struct FaceGeometryOuts {
Stream<std::vector<FaceGeometry>> multi_face_geometry;
};
@ -127,6 +124,7 @@ class FaceGeometryFromLandmarksGraph : public Subgraph {
}
ASSIGN_OR_RETURN(auto outs,
BuildFaceGeometryFromLandmarksGraph(
*sc->MutableOptions<proto::FaceGeometryGraphOptions>(),
graph.In(kFaceLandmarksTag)
.Cast<std::vector<NormalizedLandmarkList>>(),
graph.In(kImageSizeTag).Cast<std::pair<int, int>>(),
@ -138,6 +136,7 @@ class FaceGeometryFromLandmarksGraph : public Subgraph {
private:
absl::StatusOr<FaceGeometryOuts> BuildFaceGeometryFromLandmarksGraph(
proto::FaceGeometryGraphOptions& graph_options,
Stream<std::vector<NormalizedLandmarkList>> multi_face_landmarks,
Stream<std::pair<int, int>> image_size,
std::optional<SidePacket<Environment>> environment, Graph& graph) {
@ -185,7 +184,8 @@ class FaceGeometryFromLandmarksGraph : public Subgraph {
"mediapipe.tasks.vision.face_geometry.FaceGeometryPipelineCalculator");
auto& geometry_pipeline_options =
geometry_pipeline.GetOptions<FaceGeometryPipelineCalculatorOptions>();
geometry_pipeline_options.set_metadata_path(kGeometryPipelineMetadataPath);
geometry_pipeline_options.Swap(
graph_options.mutable_geometry_pipeline_options());
image_size >> geometry_pipeline.In(kImageSizeTag);
multi_face_landmarks_no_iris >>
geometry_pipeline.In(kMultiFaceLandmarksTag);

69
mediapipe/tasks/cc/vision/face_geometry/face_geometry_from_landmarks_graph_test.cc
View File

@ -20,6 +20,7 @@ limitations under the License.
#include "absl/status/statusor.h"
#include "absl/strings/str_format.h"
#include "absl/strings/string_view.h"
#include "absl/strings/substitute.h"
#include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
@ -31,6 +32,7 @@ limitations under the License.
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/tool/sink.h"
#include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
@ -49,6 +51,9 @@ constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/";
constexpr char kFaceLandmarksFileName[] =
"face_blendshapes_in_landmarks.prototxt";
constexpr char kFaceGeometryFileName[] = "face_geometry_expected_out.pbtxt";
constexpr char kGeometryPipelineMetadataPath[] =
"mediapipe/tasks/cc/vision/face_geometry/data/"
"geometry_pipeline_metadata_landmarks.binarypb";
std::vector<NormalizedLandmarkList> GetLandmarks(absl::string_view filename) {
NormalizedLandmarkList landmarks;
@ -89,17 +94,25 @@ void MakeInputPacketsAndRunGraph(CalculatorGraph& graph) {
TEST(FaceGeometryFromLandmarksGraphTest, DefaultEnvironment) {
CalculatorGraphConfig graph_config = ParseTextProtoOrDie<
CalculatorGraphConfig>(R"pb(
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "FACE_GEOMETRY:face_geometry"
node {
calculator: "mediapipe.tasks.vision.face_geometry.FaceGeometryFromLandmarksGraph"
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "FACE_GEOMETRY:face_geometry"
}
)pb");
CalculatorGraphConfig>(absl::Substitute(
R"pb(
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "FACE_GEOMETRY:face_geometry"
node {
calculator: "mediapipe.tasks.vision.face_geometry.FaceGeometryFromLandmarksGraph"
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "FACE_GEOMETRY:face_geometry"
options: {
[mediapipe.tasks.vision.face_geometry.proto.FaceGeometryGraphOptions
.ext] {
geometry_pipeline_options { metadata_file { file_name: "$0" } }
}
}
}
)pb",
kGeometryPipelineMetadataPath));
std::vector<Packet> output_packets;
tool::AddVectorSink("face_geometry", &graph_config, &output_packets);
@ -116,19 +129,27 @@ TEST(FaceGeometryFromLandmarksGraphTest, DefaultEnvironment) {
TEST(FaceGeometryFromLandmarksGraphTest, SideInEnvironment) {
CalculatorGraphConfig graph_config = ParseTextProtoOrDie<
CalculatorGraphConfig>(R"pb(
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
input_side_packet: "ENVIRONMENT:environment"
output_stream: "FACE_GEOMETRY:face_geometry"
node {
calculator: "mediapipe.tasks.vision.face_geometry.FaceGeometryFromLandmarksGraph"
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
input_side_packet: "ENVIRONMENT:environment"
output_stream: "FACE_GEOMETRY:face_geometry"
}
)pb");
CalculatorGraphConfig>(absl::Substitute(
R"pb(
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
input_side_packet: "ENVIRONMENT:environment"
output_stream: "FACE_GEOMETRY:face_geometry"
node {
calculator: "mediapipe.tasks.vision.face_geometry.FaceGeometryFromLandmarksGraph"
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
input_side_packet: "ENVIRONMENT:environment"
output_stream: "FACE_GEOMETRY:face_geometry"
options: {
[mediapipe.tasks.vision.face_geometry.proto.FaceGeometryGraphOptions
.ext] {
geometry_pipeline_options { metadata_file { file_name: "$0" } }
}
}
}
)pb",
kGeometryPipelineMetadataPath));
std::vector<Packet> output_packets;
tool::AddVectorSink("face_geometry", &graph_config, &output_packets);

9
mediapipe/tasks/cc/vision/face_geometry/proto/BUILD
View File

@ -44,3 +44,12 @@ mediapipe_proto_library(
name = "mesh_3d_proto",
srcs = ["mesh_3d.proto"],
)
mediapipe_proto_library(
name = "face_geometry_graph_options_proto",
srcs = ["face_geometry_graph_options.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator_proto",
],
)

28
mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry_graph_options.proto Normal file
View File

@ -0,0 +1,28 @@
// Copyright 2023 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.tasks.vision.face_geometry.proto;
import "mediapipe/framework/calculator_options.proto";
import "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.proto";
message FaceGeometryGraphOptions {
extend mediapipe.CalculatorOptions {
optional FaceGeometryGraphOptions ext = 515723506;
}
optional FaceGeometryPipelineCalculatorOptions geometry_pipeline_options = 1;
}

2
mediapipe/tasks/cc/vision/face_landmarker/BUILD
View File

@ -210,8 +210,10 @@ cc_library(
"//mediapipe/tasks/cc/vision/face_detector:face_detector_graph",
"//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry:face_geometry_from_landmarks_graph",
"//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/face_landmarker/proto:face_blendshapes_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_cc_proto",
"//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_cc_proto",

21
mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc
View File

@ -40,8 +40,10 @@ limitations under the License.
#include "mediapipe/tasks/cc/core/utils.h"
#include "mediapipe/tasks/cc/metadata/utils/zip_utils.h"
#include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.pb.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h"
@ -93,6 +95,8 @@ constexpr char kFaceDetectorTFLiteName[] = "face_detector.tflite";
constexpr char kFaceLandmarksDetectorTFLiteName[] =
"face_landmarks_detector.tflite";
constexpr char kFaceBlendshapeTFLiteName[] = "face_blendshapes.tflite";
constexpr char kFaceGeometryPipelineMetadataName[] =
"geometry_pipeline_metadata_landmarks.binarypb";
struct FaceLandmarkerOutputs {
Source<std::vector<NormalizedLandmarkList>> landmark_lists;
@ -305,6 +309,7 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
absl::StatusOr<CalculatorGraphConfig> GetConfig(
SubgraphContext* sc) override {
Graph graph;
bool output_geometry = HasOutput(sc->OriginalNode(), kFaceGeometryTag);
if (sc->Options<FaceLandmarkerGraphOptions>()
.base_options()
.has_model_asset()) {
@ -318,6 +323,18 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
sc->MutableOptions<FaceLandmarkerGraphOptions>(),
!sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService)
.IsAvailable()));
if (output_geometry) {
// Set the face geometry metdata file for
// FaceGeometryFromLandmarksGraph.
ASSIGN_OR_RETURN(auto face_geometry_pipeline_metadata_file,
model_asset_bundle_resources->GetModelFile(
kFaceGeometryPipelineMetadataName));
SetExternalFile(face_geometry_pipeline_metadata_file,
sc->MutableOptions<FaceLandmarkerGraphOptions>()
->mutable_face_geometry_graph_options()
->mutable_geometry_pipeline_options()
->mutable_metadata_file());
}
}
std::optional<SidePacket<Environment>> environment;
if (HasSideInput(sc->OriginalNode(), kEnvironmentTag)) {
@ -338,7 +355,6 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
.face_landmarks_detector_graph_options()
.has_face_blendshapes_graph_options()));
}
bool output_geometry = HasOutput(sc->OriginalNode(), kFaceGeometryTag);
ASSIGN_OR_RETURN(
auto outs,
BuildFaceLandmarkerGraph(
@ -481,6 +497,9 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
auto& face_geometry_from_landmarks = graph.AddNode(
"mediapipe.tasks.vision.face_geometry."
"FaceGeometryFromLandmarksGraph");
face_geometry_from_landmarks
.GetOptions<face_geometry::proto::FaceGeometryGraphOptions>()
.Swap(tasks_options.mutable_face_geometry_graph_options());
if (environment.has_value()) {
*environment >> face_geometry_from_landmarks.SideIn(kEnvironmentTag);
}

1
mediapipe/tasks/cc/vision/face_landmarker/proto/BUILD
View File

@ -60,5 +60,6 @@ mediapipe_proto_library(
"//mediapipe/framework:calculator_proto",
"//mediapipe/tasks/cc/core/proto:base_options_proto",
"//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_graph_options_proto",
],
)

5
mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.proto
View File

@ -21,6 +21,7 @@ import "mediapipe/framework/calculator.proto";
import "mediapipe/framework/calculator_options.proto";
import "mediapipe/tasks/cc/core/proto/base_options.proto";
import "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.proto";
import "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry_graph_options.proto";
import "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.proto";
option java_package = "com.google.mediapipe.tasks.vision.facelandmarker.proto";
@ -45,4 +46,8 @@ message FaceLandmarkerGraphOptions {
// Minimum confidence for face landmarks tracking to be considered
// successfully.
optional float min_tracking_confidence = 4 [default = 0.5];
// Options for FaceGeometryGraph to get facial transformation matrix.
optional face_geometry.proto.FaceGeometryGraphOptions
face_geometry_graph_options = 5;
}

BIN
mediapipe/tasks/testdata/vision/face_landmarker_with_blendshapes.task vendored
View File

Binary file not shown.

10
third_party/external_files.bzl vendored
View File

@ -67,7 +67,7 @@ def external_files():
http_file(
name = "com_google_mediapipe_BUILD",
sha256 = "d2b2a8346202691d7f831887c84e9642e974f64ed67851d9a58cf15c94b1f6b3",
urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD?generation=1661875663693976"],
urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD?generation=16618756636939761678323576393653"],
)
http_file(
@ -318,8 +318,8 @@ def external_files():
http_file(
name = "com_google_mediapipe_face_landmarker_with_blendshapes_task",
sha256 = "a75c1ba70e4b8568000af2ad0b355ed559ab5d5793db50fa9ad241f8dc4fad5f",
urls = ["https://storage.googleapis.com/mediapipe-assets/face_landmarker_with_blendshapes.task?generation=1678323586260800"],
sha256 = "b44e4cae6f5822456d60f33e7c852640d78c7e342aee7eacc22589451a0b9dc2",
urls = ["https://storage.googleapis.com/mediapipe-assets/face_landmarker_with_blendshapes.task?generation=1678504998301299"],
)
http_file(
@ -822,8 +822,8 @@ def external_files():
http_file(
name = "com_google_mediapipe_portrait_expected_face_geometry_with_attention_pbtxt",
sha256 = "5cc57b8da3ad0527dce581fe1309f6b36043e5837e3f4f5af5e24005a99dc52a",
urls = ["https://storage.googleapis.com/mediapipe-assets/portrait_expected_face_geometry_with_attention.pbtxt?generation=1678323601064393"],
sha256 = "7ed1eed98e61e0a10811bb611c895d87c8023f398a36db01b6d9ba2e1ab09e16",
urls = ["https://storage.googleapis.com/mediapipe-assets/portrait_expected_face_geometry_with_attention.pbtxt?generation=1678505004840652"],
)
http_file(