cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Internal change

Browse Source 
PiperOrigin-RevId: 514498364
This commit is contained in:
MediaPipe Team 2023-03-06 12:56:01 -08:00 committed by Copybara-Service
parent 77fcaa9597
commit f15244997f
6 changed files with 5471 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
mediapipe/tasks/cc/vision/face_geometry/BUILD Normal file
View File

@ -0,0 +1,46 @@
# Copyright 2023 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"])
package(default_visibility = ["//mediapipe/tasks:internal"])
cc_library(
name = "face_geometry_from_landmarks_graph",
srcs = ["face_geometry_from_landmarks_graph.cc"],
data = [
"//mediapipe/tasks/cc/vision/face_geometry/data:geometry_pipeline_metadata_landmarks",
],
deps = [
"//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:end_loop_calculator",
"//mediapipe/calculators/core:split_proto_list_calculator",
"//mediapipe/calculators/core:split_vector_calculator_cc_proto",
"//mediapipe/framework:calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:subgraph",
"//mediapipe/framework/api2:builder",
"//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/calculators:env_generator_calculator",
"//mediapipe/tasks/cc/vision/face_geometry/calculators:env_generator_calculator_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator",
"//mediapipe/tasks/cc/vision/face_geometry/calculators:geometry_pipeline_calculator_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:environment_cc_proto",
"//mediapipe/tasks/cc/vision/face_geometry/proto:face_geometry_cc_proto",
"//mediapipe/util:graph_builder_utils",
"@com_google_absl//absl/status:statusor",
],
alwayslink = 1,
)

204
mediapipe/tasks/cc/vision/face_geometry/face_geometry_from_landmarks_graph.cc Normal file
View File

@ -0,0 +1,204 @@
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <optional>
#include <string>
#include <vector>
#include "absl/status/statusor.h"
#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
#include "mediapipe/framework/api2/builder.h"
#include "mediapipe/framework/calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/subgraph.h"
#include "mediapipe/tasks/cc/vision/face_geometry/calculators/env_generator_calculator.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/calculators/geometry_pipeline_calculator.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
#include "mediapipe/util/graph_builder_utils.h"
namespace mediapipe::tasks::vision::face_geometry {
namespace {
using ::mediapipe::api2::builder::Graph;
using ::mediapipe::api2::builder::SidePacket;
using ::mediapipe::api2::builder::Stream;
using ::mediapipe::tasks::vision::face_geometry::proto::Environment;
using ::mediapipe::tasks::vision::face_geometry::proto::FaceGeometry;
constexpr char kMultiFaceLandmarksTag[] = "MULTI_FACE_LANDMARKS";
constexpr char kMultiFaceGeometryTag[] = "MULTI_FACE_GEOMETRY";
constexpr char kFaceLandmarksTag[] = "FACE_LANDMARKS";
constexpr char kFaceGeometryTag[] = "FACE_GEOMETRY";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr char kEnvironmentTag[] = "ENVIRONMENT";
constexpr char kIterableTag[] = "ITERABLE";
constexpr char kBatchEndTag[] = "BATCH_END";
constexpr char kItemTag[] = "ITEM";
constexpr char kGeometryPipelineMetadataPath[] =
"mediapipe/tasks/cc/vision/face_geometry/data/"
"geometry_pipeline_metadata_landmarks.binarypb";
struct FaceGeometryOuts {
Stream<std::vector<FaceGeometry>> multi_face_geometry;
};
void ConfigureSplitNormalizedLandmarkListCalculator(
mediapipe::SplitVectorCalculatorOptions& options) {
auto& range = *options.add_ranges();
// Extract the first 468 face landmarks, excluding iris;
range.set_begin(0);
range.set_end(468);
}
void ConfigureFaceGeometryEnvGeneratorCalculator(
FaceGeometryEnvGeneratorCalculatorOptions& options) {
options.mutable_environment()->set_origin_point_location(
proto::OriginPointLocation::TOP_LEFT_CORNER);
auto& perspective_camera =
*options.mutable_environment()->mutable_perspective_camera();
perspective_camera.set_vertical_fov_degrees(63.0 /*degrees*/);
perspective_camera.set_near(1.0 /* 1cm */);
perspective_camera.set_far(10000.0 /* 100m */);
}
} // namespace
// A "mediapipe.tasks.vision.face_landmarker.FaceGeometryFromLandmarksGraph"
// graph to extract 3D transform from the given canonical face to multi face
// landmarks.
//
// It is required that "geometry_pipeline_metadata_from_landmark.binarypb" is
// available at
// "mediapipe/tasks/cc/vision/face_geometry/data/geometry_pipeline_metadata_from_landmarks.binarypb"
// path during execution.
//
//
// Inputs:
// IMAGE_SIZE - std::pair<int,int>
// The size of the image that face landmarks are detected on.
// FACE_LANDMARKS - std::vector<NormalizedLandmarkList>
// A vector of multiple face landmarks that the given canonical face would
// transform to.
//
// SideInputs:
// ENVIRONMENT - ENVIRONMENT
// Environment that describes the current virtual scene. If not provided, a
// default environment will be used which can be applied to common webcam.
//
//
// Outputs:
// FACE_GEOMETRY: - std::vector<FaceGeometry>
// A vector of 3D transform data for each detected face.
//
//
// Example:
// node {
// calculator:
// "mediapipe.tasks.vision.face_landmarker.FaceGeometryFromLandmarksGraph"
// input_stream: "IMAGE_SIZE:image_size"
// input_stream: "FACE_LANDMARKS:face_landmarks"
// input_side_packet: "ENVIRONMENT:environment"
// output_stream: "FACE_GEOMETRY:face_geometry"
// }
class FaceGeometryFromLandmarksGraph : public Subgraph {
public:
absl::StatusOr<CalculatorGraphConfig> GetConfig(
SubgraphContext* sc) override {
Graph graph;
std::optional<SidePacket<Environment>> environment;
if (HasSideInput(sc->OriginalNode(), kEnvironmentTag)) {
environment = std::make_optional<>(
graph.SideIn(kEnvironmentTag).Cast<Environment>());
}
ASSIGN_OR_RETURN(auto outs,
BuildFaceGeometryFromLandmarksGraph(
graph.In(kFaceLandmarksTag)
.Cast<std::vector<NormalizedLandmarkList>>(),
graph.In(kImageSizeTag).Cast<std::pair<int, int>>(),
environment, graph));
outs.multi_face_geometry >>
graph.Out(kFaceGeometryTag).Cast<std::vector<FaceGeometry>>();
return graph.GetConfig();
}
private:
absl::StatusOr<FaceGeometryOuts> BuildFaceGeometryFromLandmarksGraph(
Stream<std::vector<NormalizedLandmarkList>> multi_face_landmarks,
Stream<std::pair<int, int>> image_size,
std::optional<SidePacket<Environment>> environment, Graph& graph) {
if (!environment.has_value()) {
// If there is no provided Environment, use a a default environment which
// is good enough for most general use case.
auto& env_generator = graph.AddNode(
"mediapipe.tasks.vision.face_geometry."
"FaceGeometryEnvGeneratorCalculator");
ConfigureFaceGeometryEnvGeneratorCalculator(
env_generator
.GetOptions<FaceGeometryEnvGeneratorCalculatorOptions>());
environment = std::make_optional<>(
env_generator.SideOut(kEnvironmentTag).Cast<Environment>());
}
// For loop to go over the vector of face landmarks list, and remove the
// iris landmarks.
auto& begin_loop_landmark_list_vector =
graph.AddNode("BeginLoopNormalizedLandmarkListVectorCalculator");
multi_face_landmarks >> begin_loop_landmark_list_vector.In(kIterableTag);
auto batch_end = begin_loop_landmark_list_vector.Out(kBatchEndTag);
auto single_face_landmarks = begin_loop_landmark_list_vector.Out(kItemTag);
// Take first 468 face landmarks and exclude iris landmarks.
auto& split_landmark_list =
graph.AddNode("SplitNormalizedLandmarkListCalculator");
ConfigureSplitNormalizedLandmarkListCalculator(
split_landmark_list
.GetOptions<mediapipe::SplitVectorCalculatorOptions>());
single_face_landmarks >> split_landmark_list.In("");
auto single_face_landmarks_no_iris = split_landmark_list.Out("");
auto& end_loop_landmark_list_vector =
graph.AddNode("EndLoopNormalizedLandmarkListVectorCalculator");
batch_end >> end_loop_landmark_list_vector.In(kBatchEndTag);
single_face_landmarks_no_iris >> end_loop_landmark_list_vector.In(kItemTag);
auto multi_face_landmarks_no_iris =
end_loop_landmark_list_vector.Out(kIterableTag)
.Cast<std::vector<NormalizedLandmarkList>>();
// Find the transformation from the canonical face to the list of multi face
// landmarks.
auto& geometry_pipeline = graph.AddNode(
"mediapipe.tasks.vision.face_geometry.FaceGeometryPipelineCalculator");
auto& geometry_pipeline_options =
geometry_pipeline.GetOptions<FaceGeometryPipelineCalculatorOptions>();
geometry_pipeline_options.set_metadata_path(kGeometryPipelineMetadataPath);
image_size >> geometry_pipeline.In(kImageSizeTag);
multi_face_landmarks_no_iris >>
geometry_pipeline.In(kMultiFaceLandmarksTag);
environment.value() >> geometry_pipeline.SideIn(kEnvironmentTag);
auto multi_face_geometry = geometry_pipeline.Out(kMultiFaceGeometryTag)
.Cast<std::vector<FaceGeometry>>();
return {{/*multi_face_geometry */ multi_face_geometry}};
}
};
// clang-format off
REGISTER_MEDIAPIPE_GRAPH(
::mediapipe::tasks::vision::face_geometry::FaceGeometryFromLandmarksGraph); // NOLINT
// clang-format on
} // namespace mediapipe::tasks::vision::face_geometry

153
mediapipe/tasks/cc/vision/face_geometry/face_geometry_from_landmarks_graph_test.cc Normal file
View File

@ -0,0 +1,153 @@
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <string>
#include <utility>
#include "absl/flags/flag.h"
#include "absl/status/statusor.h"
#include "absl/strings/str_format.h"
#include "absl/strings/string_view.h"
#include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_runner.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/packet.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/framework/tool/sink.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/environment.pb.h"
#include "mediapipe/tasks/cc/vision/face_geometry/proto/face_geometry.pb.h"
namespace mediapipe {
namespace tasks {
namespace vision {
namespace face_geometry {
namespace {
using ::file::Defaults;
using ::mediapipe::tasks::vision::face_geometry::proto::Environment;
// using ::mediapipe::face_geometry::Environment;
using ::mediapipe::tasks::vision::face_geometry::proto::FaceGeometry;
constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/";
constexpr char kFaceLandmarksFileName[] =
"face_blendshapes_in_landmarks.prototxt";
constexpr char kFaceGeometryFileName[] = "face_geometry_expected_out.pbtxt";
std::vector<NormalizedLandmarkList> GetLandmarks(absl::string_view filename) {
NormalizedLandmarkList landmarks;
MP_EXPECT_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, filename),
&landmarks, Defaults()));
return {landmarks};
}
FaceGeometry GetExpectedFaceGeometry(absl::string_view filename) {
FaceGeometry face_geometry;
MP_EXPECT_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, filename),
&face_geometry, Defaults()));
return face_geometry;
}
Environment CreateEnvironment() {
Environment environment;
environment.set_origin_point_location(
proto::OriginPointLocation::TOP_LEFT_CORNER);
auto& perspective_camera = *environment.mutable_perspective_camera();
perspective_camera.set_vertical_fov_degrees(63.0 /*degrees*/);
perspective_camera.set_near(1.0 /* 1cm */);
perspective_camera.set_far(10000.0 /* 100m */);
return environment;
}
void MakeInputPacketsAndRunGraph(CalculatorGraph& graph) {
MP_ASSERT_OK(graph.StartRun({}));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"face_landmarks", MakePacket<std::vector<NormalizedLandmarkList>>(
GetLandmarks(kFaceLandmarksFileName))
.At(Timestamp(0))));
MP_ASSERT_OK(graph.AddPacketToInputStream(
"image_size", MakePacket<std::pair<int, int>>(std::make_pair(820, 1024))
.At(Timestamp(0))));
MP_ASSERT_OK(graph.WaitUntilIdle());
}
TEST(FaceGeometryFromLandmarksGraphTest, DefaultEnvironment) {
CalculatorGraphConfig graph_config = ParseTextProtoOrDie<
CalculatorGraphConfig>(R"pb(
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "FACE_GEOMETRY:face_geometry"
node {
calculator: "mediapipe.tasks.vision.face_geometry.FaceGeometryFromLandmarksGraph"
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "FACE_GEOMETRY:face_geometry"
}
)pb");
std::vector<Packet> output_packets;
tool::AddVectorSink("face_geometry", &graph_config, &output_packets);
// Run the graph.
CalculatorGraph graph;
MP_ASSERT_OK(graph.Initialize(graph_config));
MakeInputPacketsAndRunGraph(graph);
ASSERT_THAT(output_packets, testing::SizeIs(1));
auto& face_geometry = output_packets[0].Get<std::vector<FaceGeometry>>()[0];
EXPECT_THAT(
face_geometry,
testing::EqualsProto(GetExpectedFaceGeometry(kFaceGeometryFileName)));
}
TEST(FaceGeometryFromLandmarksGraphTest, SideInEnvironment) {
CalculatorGraphConfig graph_config = ParseTextProtoOrDie<
CalculatorGraphConfig>(R"pb(
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
input_side_packet: "ENVIRONMENT:environment"
output_stream: "FACE_GEOMETRY:face_geometry"
node {
calculator: "mediapipe.tasks.vision.face_geometry.FaceGeometryFromLandmarksGraph"
input_stream: "FACE_LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
input_side_packet: "ENVIRONMENT:environment"
output_stream: "FACE_GEOMETRY:face_geometry"
}
)pb");
std::vector<Packet> output_packets;
tool::AddVectorSink("face_geometry", &graph_config, &output_packets);
// Run the graph.
CalculatorGraph graph;
std::map<std::string, Packet> input_side_packets;
input_side_packets["environment"] =
MakePacket<Environment>(CreateEnvironment());
MP_ASSERT_OK(graph.Initialize(graph_config, input_side_packets));
MakeInputPacketsAndRunGraph(graph);
ASSERT_THAT(output_packets, testing::SizeIs(1));
auto& face_geometry = output_packets[0].Get<std::vector<FaceGeometry>>()[0];
EXPECT_THAT(
face_geometry,
testing::EqualsProto(GetExpectedFaceGeometry(kFaceGeometryFileName)));
}
} // namespace
} // namespace face_geometry
} // namespace vision
} // namespace tasks
} // namespace mediapipe

2
mediapipe/tasks/testdata/vision/BUILD vendored
View File

@ -85,6 +85,7 @@ exports_files(
"expected_left_up_hand_rotated_landmarks.prototxt", "expected_left_up_hand_rotated_landmarks.prototxt",
"expected_right_down_hand_landmarks.prototxt", "expected_right_down_hand_landmarks.prototxt",
"expected_right_up_hand_landmarks.prototxt", "expected_right_up_hand_landmarks.prototxt",
"face_geometry_expected_out.pbtxt",
"gesture_recognizer.task", "gesture_recognizer.task",
"portrait_expected_detection.pbtxt", "portrait_expected_detection.pbtxt",
"portrait_rotated_expected_detection.pbtxt", "portrait_rotated_expected_detection.pbtxt",
@ -172,6 +173,7 @@ filegroup(
"expected_left_up_hand_rotated_landmarks.prototxt", "expected_left_up_hand_rotated_landmarks.prototxt",
"expected_right_down_hand_landmarks.prototxt", "expected_right_down_hand_landmarks.prototxt",
"expected_right_up_hand_landmarks.prototxt", "expected_right_up_hand_landmarks.prototxt",
"face_geometry_expected_out.pbtxt",
"fist_landmarks.pbtxt", "fist_landmarks.pbtxt",
"hand_detector_result_one_hand.pbtxt", "hand_detector_result_one_hand.pbtxt",
"hand_detector_result_one_hand_rotated.pbtxt", "hand_detector_result_one_hand_rotated.pbtxt",

5060
mediapipe/tasks/testdata/vision/face_geometry_expected_out.pbtxt vendored Normal file
View File

File diff suppressed because it is too large Load Diff

30
third_party/external_files.bzl vendored
View File

@ -268,30 +268,6 @@ def external_files():
urls = ["https://storage.googleapis.com/mediapipe-assets/external_file.txt?generation=1661875736240688"], urls = ["https://storage.googleapis.com/mediapipe-assets/external_file.txt?generation=1661875736240688"],
) )
http_file(
name = "com_google_mediapipe_face_blendshapes_generated_graph_pbtxt",
sha256 = "92e016a08940e1f81752e749e1931f9e551fa9483786b8fba0624257e9c41d3d",
urls = ["https://storage.googleapis.com/mediapipe-assets/face_blendshapes_generated_graph.pbtxt?generation=1677522753449135"],
)
http_file(
name = "com_google_mediapipe_face_blendshapes_in_landmarks_prototxt",
sha256 = "f6a10fa5825f2eee695371a449c605698403c146c270b2cb7574512f3f9e4af8",
urls = ["https://storage.googleapis.com/mediapipe-assets/face_blendshapes_in_landmarks.prototxt?generation=1677522757270549"],
)
http_file(
name = "com_google_mediapipe_face_blendshapes_out_prototxt",
sha256 = "ea7740add8b87c9bd375eaa40a05b509eaca04f025cb6bdc7ca486e9fb32dfba",
urls = ["https://storage.googleapis.com/mediapipe-assets/face_blendshapes_out.prototxt?generation=1677522761312644"],
)
http_file(
name = "com_google_mediapipe_face_blendshapes_tflite",
sha256 = "4f36dded049db18d76048567439b2a7f58f1daabc00d78bfe8f3ad396a2d2082",
urls = ["https://storage.googleapis.com/mediapipe-assets/face_blendshapes.tflite?generation=1677522764748685"],
)
http_file( http_file(
name = "com_google_mediapipe_face_detection_full_range_sparse_tflite", name = "com_google_mediapipe_face_detection_full_range_sparse_tflite",
sha256 = "2c3728e6da56f21e21a320433396fb06d40d9088f2247c05e5635a688d45dfe1", sha256 = "2c3728e6da56f21e21a320433396fb06d40d9088f2247c05e5635a688d45dfe1",
@ -310,6 +286,12 @@ def external_files():
urls = ["https://storage.googleapis.com/mediapipe-assets/face_detection_short_range.tflite?generation=1677044301978921"], urls = ["https://storage.googleapis.com/mediapipe-assets/face_detection_short_range.tflite?generation=1677044301978921"],
) )
http_file(
name = "com_google_mediapipe_face_geometry_expected_out_pbtxt",
sha256 = "611b203bca40e547ae75bf0822fda0695d512d02940e7af08a70068eaa8524f7",
urls = ["https://storage.googleapis.com/mediapipe-assets/face_geometry_expected_out.pbtxt?generation=1677787710308910"],
)
http_file( http_file(
name = "com_google_mediapipe_face_landmark_tflite", name = "com_google_mediapipe_face_landmark_tflite",
sha256 = "1055cb9d4a9ca8b8c688902a3a5194311138ba256bcc94e336d8373a5f30c814", sha256 = "1055cb9d4a9ca8b8c688902a3a5194311138ba256bcc94e336d8373a5f30c814",