Use ExternalFile to set metadata of GeometryPipelineCalculator.

PiperOrigin-RevId: 516384491

mediapipe/tasks/cc/vision/face_geometry/BUILD

@@ -19,9 +19,6 @@ package(default_visibility = ["//mediapipe/tasks:internal"])
 cc_library(
     name = "face_geometry_from_landmarks_graph",
     srcs = ["face_geometry_from_landmarks_graph.cc"],
-    data = [
-        "//mediapipe/tasks/cc/vision/face_geometry/data:geometry_pipeline_metadata_landmarks",
-    ],
     deps = [
         "//mediapipe/calculators/core:begin_loop_calculator",
         "//mediapipe/calculators/core:end_loop_calculator",
@@ -39,6 +36,7 @@ cc_library(
         "//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator_cc_proto",
         "//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto",
         "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto",
+        "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_graph_options_cc_proto",
         "//mediapipe/util:graph_builder_utils",
         "@com_google_absl//absl/status:statusor",
     ],

mediapipe/tasks/cc/vision/face_geometry/calculators/BUILD

@@ -45,6 +45,7 @@ mediapipe_proto_library(
     srcs = ["geometry_pipeline_calculator.proto"],
     deps = [
         "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/tasks/cc/core/proto:external_file_proto",
     ],
 )
 
@@ -59,6 +60,8 @@ cc_library(
         "//mediapipe/framework/port:ret_check",
         "//mediapipe/framework/port:status",
         "//mediapipe/framework/port:statusor",
+        "//mediapipe/tasks/cc/core:external_file_handler",
+        "//mediapipe/tasks/cc/core/proto:external_file_cc_proto",
         "//mediapipe/tasks/cc/vision/face_geometry/libs:geometry_pipeline",
         "//mediapipe/tasks/cc/vision/face_geometry/libs:validation_utils",
         "//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto",

mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.cc

@@ -24,6 +24,8 @@
 #include "mediapipe/framework/port/status.h"
 #include "mediapipe/framework/port/status_macros.h"
 #include "mediapipe/framework/port/statusor.h"
+#include "mediapipe/tasks/cc/core/external_file_handler.h"
+#include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
 #include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h"
 #include "mediapipe/tasks/cc/vision/face_geometry/libs/geometry_pipeline.h"
 #include "mediapipe/tasks/cc/vision/face_geometry/libs/validation_utils.h"
@@ -69,8 +71,8 @@ using ::mediapipe::tasks::vision::face_geometry::proto::
 //     A vector of face geometry data.
 //
 // Options:
-//   metadata_path (`string`, optional):
+//   metadata_file (`ExternalFile`, optional):
-//     Defines a path for the geometry pipeline metadata file.
+//     Defines an ExternalFile for the geometry pipeline metadata file.
 //
 //     The geometry pipeline metadata file format must be the binary
 //     `GeometryPipelineMetadata` proto.
@@ -95,7 +97,7 @@ class GeometryPipelineCalculator : public CalculatorBase {
 
     ASSIGN_OR_RETURN(
         GeometryPipelineMetadata metadata,
-        ReadMetadataFromFile(options.metadata_path()),
+        ReadMetadataFromFile(options.metadata_file()),
         _ << "Failed to read the geometry pipeline metadata from file!");
 
     MP_RETURN_IF_ERROR(ValidateGeometryPipelineMetadata(metadata))
@@ -155,32 +157,19 @@ class GeometryPipelineCalculator : public CalculatorBase {
 
  private:
   static absl::StatusOr<GeometryPipelineMetadata> ReadMetadataFromFile(
-      const std::string& metadata_path) {
+      const core::proto::ExternalFile& metadata_file) {
-    ASSIGN_OR_RETURN(std::string metadata_blob,
+    ASSIGN_OR_RETURN(
-                     ReadContentBlobFromFile(metadata_path),
+        const auto file_handler,
-                     _ << "Failed to read a metadata blob from file!");
+        core::ExternalFileHandler::CreateFromExternalFile(&metadata_file));
 
     GeometryPipelineMetadata metadata;
-    RET_CHECK(metadata.ParseFromString(metadata_blob))
+    RET_CHECK(
+        metadata.ParseFromString(std::string(file_handler->GetFileContent())))
         << "Failed to parse a metadata proto from a binary blob!";
 
     return metadata;
   }
 
-  static absl::StatusOr<std::string> ReadContentBlobFromFile(
-      const std::string& unresolved_path) {
-    ASSIGN_OR_RETURN(std::string resolved_path,
-                     mediapipe::PathToResourceAsFile(unresolved_path),
-                     _ << "Failed to resolve path! Path = " << unresolved_path);
-
-    std::string content_blob;
-    MP_RETURN_IF_ERROR(
-        mediapipe::GetResourceContents(resolved_path, &content_blob))
-        << "Failed to read content blob! Resolved path = " << resolved_path;
-
-    return content_blob;
-  }
-
   std::unique_ptr<GeometryPipeline> geometry_pipeline_;
 };
 

mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.proto

@@ -17,11 +17,12 @@ syntax = "proto2";
 package mediapipe.tasks.vision.face_geometry;
 
 import "mediapipe/framework/calculator_options.proto";
+import "mediapipe/tasks/cc/core/proto/external_file.proto";
 
 message FaceGeometryPipelineCalculatorOptions {
   extend mediapipe.CalculatorOptions {
     optional FaceGeometryPipelineCalculatorOptions ext = 512499200;
   }
 
-  optional string metadata_path = 1;
+  optional core.proto.ExternalFile metadata_file = 1;
 }

mediapipe/tasks/cc/vision/face_geometry/face_geometry_from_landmarks_graph.cc

@@ -28,6 +28,7 @@ limitations under the License.
 #include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h"
 #include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
 #include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
+#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry_graph_options.pb.h"
 #include "mediapipe/util/graph_builder_utils.h"
 
 namespace mediapipe::tasks::vision::face_geometry {
@@ -49,10 +50,6 @@ constexpr char kIterableTag[] = "ITERABLE";
 constexpr char kBatchEndTag[] = "BATCH_END";
 constexpr char kItemTag[] = "ITEM";
 
-constexpr char kGeometryPipelineMetadataPath[] =
-    "mediapipe/tasks/cc/vision/face_geometry/data/"
-    "geometry_pipeline_metadata_landmarks.binarypb";
-
 struct FaceGeometryOuts {
   Stream<std::vector<FaceGeometry>> multi_face_geometry;
 };
@@ -127,6 +124,7 @@ class FaceGeometryFromLandmarksGraph : public Subgraph {
     }
     ASSIGN_OR_RETURN(auto outs,
                      BuildFaceGeometryFromLandmarksGraph(
+                         *sc->MutableOptions<proto::FaceGeometryGraphOptions>(),
                          graph.In(kFaceLandmarksTag)
                              .Cast<std::vector<NormalizedLandmarkList>>(),
                          graph.In(kImageSizeTag).Cast<std::pair<int, int>>(),
@@ -138,6 +136,7 @@ class FaceGeometryFromLandmarksGraph : public Subgraph {
 
  private:
   absl::StatusOr<FaceGeometryOuts> BuildFaceGeometryFromLandmarksGraph(
+      proto::FaceGeometryGraphOptions& graph_options,
       Stream<std::vector<NormalizedLandmarkList>> multi_face_landmarks,
       Stream<std::pair<int, int>> image_size,
       std::optional<SidePacket<Environment>> environment, Graph& graph) {
@@ -185,7 +184,8 @@ class FaceGeometryFromLandmarksGraph : public Subgraph {
         "mediapipe.tasks.vision.face_geometry.FaceGeometryPipelineCalculator");
     auto& geometry_pipeline_options =
         geometry_pipeline.GetOptions<FaceGeometryPipelineCalculatorOptions>();
-    geometry_pipeline_options.set_metadata_path(kGeometryPipelineMetadataPath);
+    geometry_pipeline_options.Swap(
+        graph_options.mutable_geometry_pipeline_options());
     image_size >> geometry_pipeline.In(kImageSizeTag);
     multi_face_landmarks_no_iris >>
         geometry_pipeline.In(kMultiFaceLandmarksTag);

mediapipe/tasks/cc/vision/face_geometry/face_geometry_from_landmarks_graph_test.cc

@@ -20,6 +20,7 @@ limitations under the License.
 #include "absl/status/statusor.h"
 #include "absl/strings/str_format.h"
 #include "absl/strings/string_view.h"
+#include "absl/strings/substitute.h"
 #include "mediapipe/framework/api2/port.h"
 #include "mediapipe/framework/calculator_framework.h"
 #include "mediapipe/framework/calculator_runner.h"
@@ -31,6 +32,7 @@ limitations under the License.
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/parse_text_proto.h"
 #include "mediapipe/framework/tool/sink.h"
+#include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
 #include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
 #include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
 
@@ -49,6 +51,9 @@ constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/";
 constexpr char kFaceLandmarksFileName[] =
     "face_blendshapes_in_landmarks.prototxt";
 constexpr char kFaceGeometryFileName[] = "face_geometry_expected_out.pbtxt";
+constexpr char kGeometryPipelineMetadataPath[] =
+    "mediapipe/tasks/cc/vision/face_geometry/data/"
+    "geometry_pipeline_metadata_landmarks.binarypb";
 
 std::vector<NormalizedLandmarkList> GetLandmarks(absl::string_view filename) {
   NormalizedLandmarkList landmarks;
@@ -89,7 +94,8 @@ void MakeInputPacketsAndRunGraph(CalculatorGraph& graph) {
 
 TEST(FaceGeometryFromLandmarksGraphTest, DefaultEnvironment) {
   CalculatorGraphConfig graph_config = ParseTextProtoOrDie<
-      CalculatorGraphConfig>(R"pb(
+      CalculatorGraphConfig>(absl::Substitute(
+      R"pb(
         input_stream: "FACE_LANDMARKS:face_landmarks"
         input_stream: "IMAGE_SIZE:image_size"
         output_stream: "FACE_GEOMETRY:face_geometry"
@@ -98,8 +104,15 @@ TEST(FaceGeometryFromLandmarksGraphTest, DefaultEnvironment) {
           input_stream: "FACE_LANDMARKS:face_landmarks"
           input_stream: "IMAGE_SIZE:image_size"
           output_stream: "FACE_GEOMETRY:face_geometry"
+          options: {
+            [mediapipe.tasks.vision.face_geometry.proto.FaceGeometryGraphOptions
+                 .ext] {
+              geometry_pipeline_options { metadata_file { file_name: "$0" } }
             }
-  )pb");
+          }
+        }
+      )pb",
+      kGeometryPipelineMetadataPath));
   std::vector<Packet> output_packets;
   tool::AddVectorSink("face_geometry", &graph_config, &output_packets);
 
@@ -116,7 +129,8 @@ TEST(FaceGeometryFromLandmarksGraphTest, DefaultEnvironment) {
 
 TEST(FaceGeometryFromLandmarksGraphTest, SideInEnvironment) {
   CalculatorGraphConfig graph_config = ParseTextProtoOrDie<
-      CalculatorGraphConfig>(R"pb(
+      CalculatorGraphConfig>(absl::Substitute(
+      R"pb(
         input_stream: "FACE_LANDMARKS:face_landmarks"
         input_stream: "IMAGE_SIZE:image_size"
         input_side_packet: "ENVIRONMENT:environment"
@@ -127,8 +141,15 @@ TEST(FaceGeometryFromLandmarksGraphTest, SideInEnvironment) {
           input_stream: "IMAGE_SIZE:image_size"
           input_side_packet: "ENVIRONMENT:environment"
           output_stream: "FACE_GEOMETRY:face_geometry"
+          options: {
+            [mediapipe.tasks.vision.face_geometry.proto.FaceGeometryGraphOptions
+                 .ext] {
+              geometry_pipeline_options { metadata_file { file_name: "$0" } }
             }
-  )pb");
+          }
+        }
+      )pb",
+      kGeometryPipelineMetadataPath));
   std::vector<Packet> output_packets;
   tool::AddVectorSink("face_geometry", &graph_config, &output_packets);
 

mediapipe/tasks/cc/vision/face_geometry/proto/BUILD

@@ -44,3 +44,12 @@ mediapipe_proto_library(
     name = "mesh_3d_proto",
     srcs = ["mesh_3d.proto"],
 )
+
+mediapipe_proto_library(
+    name = "face_geometry_graph_options_proto",
+    srcs = ["face_geometry_graph_options.proto"],
+    deps = [
+        "//mediapipe/framework:calculator_options_proto",
+        "//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator_proto",
+    ],
+)

mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry_graph_options.proto

@@ -0,0 +1,28 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+syntax = "proto2";
+
+package mediapipe.tasks.vision.face_geometry.proto;
+
+import "mediapipe/framework/calculator_options.proto";
+import "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.proto";
+
+message FaceGeometryGraphOptions {
+  extend mediapipe.CalculatorOptions {
+    optional FaceGeometryGraphOptions ext = 515723506;
+  }
+
+  optional FaceGeometryPipelineCalculatorOptions geometry_pipeline_options = 1;
+}

mediapipe/tasks/cc/vision/face_landmarker/BUILD

@@ -210,8 +210,10 @@ cc_library(
         "//mediapipe/tasks/cc/vision/face_detector:face_detector_graph",
         "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_cc_proto",
         "//mediapipe/tasks/cc/vision/face_geometry:face_geometry_from_landmarks_graph",
+        "//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator_cc_proto",
         "//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto",
         "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto",
+        "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_graph_options_cc_proto",
         "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_blendshapes_graph_options_cc_proto",
         "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarker_graph_options_cc_proto",
         "//mediapipe/tasks/cc/vision/face_landmarker/proto:face_landmarks_detector_graph_options_cc_proto",

mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc

@@ -40,8 +40,10 @@ limitations under the License.
 #include "mediapipe/tasks/cc/core/utils.h"
 #include "mediapipe/tasks/cc/metadata/utils/zip_utils.h"
 #include "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.pb.h"
+#include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h"
 #include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
 #include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
+#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry_graph_options.pb.h"
 #include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h"
 #include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.pb.h"
 #include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.pb.h"
@@ -93,6 +95,8 @@ constexpr char kFaceDetectorTFLiteName[] = "face_detector.tflite";
 constexpr char kFaceLandmarksDetectorTFLiteName[] =
     "face_landmarks_detector.tflite";
 constexpr char kFaceBlendshapeTFLiteName[] = "face_blendshapes.tflite";
+constexpr char kFaceGeometryPipelineMetadataName[] =
+    "geometry_pipeline_metadata_landmarks.binarypb";
 
 struct FaceLandmarkerOutputs {
   Source<std::vector<NormalizedLandmarkList>> landmark_lists;
@@ -305,6 +309,7 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
   absl::StatusOr<CalculatorGraphConfig> GetConfig(
       SubgraphContext* sc) override {
     Graph graph;
+    bool output_geometry = HasOutput(sc->OriginalNode(), kFaceGeometryTag);
     if (sc->Options<FaceLandmarkerGraphOptions>()
             .base_options()
             .has_model_asset()) {
@@ -318,6 +323,18 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
           sc->MutableOptions<FaceLandmarkerGraphOptions>(),
           !sc->Service(::mediapipe::tasks::core::kModelResourcesCacheService)
                .IsAvailable()));
+      if (output_geometry) {
+        // Set the face geometry metdata file for
+        // FaceGeometryFromLandmarksGraph.
+        ASSIGN_OR_RETURN(auto face_geometry_pipeline_metadata_file,
+                         model_asset_bundle_resources->GetModelFile(
+                             kFaceGeometryPipelineMetadataName));
+        SetExternalFile(face_geometry_pipeline_metadata_file,
+                        sc->MutableOptions<FaceLandmarkerGraphOptions>()
+                            ->mutable_face_geometry_graph_options()
+                            ->mutable_geometry_pipeline_options()
+                            ->mutable_metadata_file());
+      }
     }
     std::optional<SidePacket<Environment>> environment;
     if (HasSideInput(sc->OriginalNode(), kEnvironmentTag)) {
@@ -338,7 +355,6 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
               .face_landmarks_detector_graph_options()
               .has_face_blendshapes_graph_options()));
     }
-    bool output_geometry = HasOutput(sc->OriginalNode(), kFaceGeometryTag);
     ASSIGN_OR_RETURN(
         auto outs,
         BuildFaceLandmarkerGraph(
@@ -481,6 +497,9 @@ class FaceLandmarkerGraph : public core::ModelTaskGraph {
       auto& face_geometry_from_landmarks = graph.AddNode(
           "mediapipe.tasks.vision.face_geometry."
           "FaceGeometryFromLandmarksGraph");
+      face_geometry_from_landmarks
+          .GetOptions<face_geometry::proto::FaceGeometryGraphOptions>()
+          .Swap(tasks_options.mutable_face_geometry_graph_options());
       if (environment.has_value()) {
         *environment >> face_geometry_from_landmarks.SideIn(kEnvironmentTag);
       }

mediapipe/tasks/cc/vision/face_landmarker/proto/BUILD

@@ -60,5 +60,6 @@ mediapipe_proto_library(
         "//mediapipe/framework:calculator_proto",
         "//mediapipe/tasks/cc/core/proto:base_options_proto",
         "//mediapipe/tasks/cc/vision/face_detector/proto:face_detector_graph_options_proto",
+        "//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_graph_options_proto",
     ],
 )

mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarker_graph_options.proto

@@ -21,6 +21,7 @@ import "mediapipe/framework/calculator.proto";
 import "mediapipe/framework/calculator_options.proto";
 import "mediapipe/tasks/cc/core/proto/base_options.proto";
 import "mediapipe/tasks/cc/vision/face_detector/proto/face_detector_graph_options.proto";
+import "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry_graph_options.proto";
 import "mediapipe/tasks/cc/vision/face_landmarker/proto/face_landmarks_detector_graph_options.proto";
 
 option java_package = "com.google.mediapipe.tasks.vision.facelandmarker.proto";
@@ -45,4 +46,8 @@ message FaceLandmarkerGraphOptions {
   // Minimum confidence for face landmarks tracking to be considered
   // successfully.
   optional float min_tracking_confidence = 4 [default = 0.5];
+
+  // Options for FaceGeometryGraph to get facial transformation matrix.
+  optional face_geometry.proto.FaceGeometryGraphOptions
+      face_geometry_graph_options = 5;
 }

third_party/external_files.bzl

@@ -67,7 +67,7 @@ def external_files():
     http_file(
         name = "com_google_mediapipe_BUILD",
         sha256 = "d2b2a8346202691d7f831887c84e9642e974f64ed67851d9a58cf15c94b1f6b3",
-        urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD?generation=1661875663693976"],
+        urls = ["https://storage.googleapis.com/mediapipe-assets/BUILD?generation=16618756636939761678323576393653"],
     )
 
     http_file(
@@ -318,8 +318,8 @@ def external_files():
 
     http_file(
         name = "com_google_mediapipe_face_landmarker_with_blendshapes_task",
-        sha256 = "a75c1ba70e4b8568000af2ad0b355ed559ab5d5793db50fa9ad241f8dc4fad5f",
+        sha256 = "b44e4cae6f5822456d60f33e7c852640d78c7e342aee7eacc22589451a0b9dc2",
-        urls = ["https://storage.googleapis.com/mediapipe-assets/face_landmarker_with_blendshapes.task?generation=1678323586260800"],
+        urls = ["https://storage.googleapis.com/mediapipe-assets/face_landmarker_with_blendshapes.task?generation=1678504998301299"],
     )
 
     http_file(
@@ -822,8 +822,8 @@ def external_files():
 
     http_file(
         name = "com_google_mediapipe_portrait_expected_face_geometry_with_attention_pbtxt",
-        sha256 = "5cc57b8da3ad0527dce581fe1309f6b36043e5837e3f4f5af5e24005a99dc52a",
+        sha256 = "7ed1eed98e61e0a10811bb611c895d87c8023f398a36db01b6d9ba2e1ab09e16",
-        urls = ["https://storage.googleapis.com/mediapipe-assets/portrait_expected_face_geometry_with_attention.pbtxt?generation=1678323601064393"],
+        urls = ["https://storage.googleapis.com/mediapipe-assets/portrait_expected_face_geometry_with_attention.pbtxt?generation=1678505004840652"],
     )
 
     http_file(