Add mediapipe tasks face blendshapes graph
PiperOrigin-RevId: 509995915
This commit is contained in:
MediaPipe Team
Copybara-Service
parent
bdd1c24990
commit
ba10ae8410
|
|
@ -18,6 +18,32 @@ package(default_visibility = [
|
|||
|
||||
licenses(["notice"])
|
||||
|
||||
cc_library(
|
||||
name = "face_blendshapes_graph",
|
||||
srcs = ["face_blendshapes_graph.cc"],
|
||||
deps = [
|
||||
"//mediapipe/calculators/core:split_proto_list_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator",
|
||||
"//mediapipe/calculators/core:split_vector_calculator_cc_proto",
|
||||
"//mediapipe/calculators/tensor:inference_calculator",
|
||||
"//mediapipe/calculators/tensor:landmarks_to_tensor_calculator",
|
||||
"//mediapipe/calculators/tensor:landmarks_to_tensor_calculator_cc_proto",
|
||||
"//mediapipe/calculators/tensor:tensors_to_classification_calculator",
|
||||
"//mediapipe/calculators/tensor:tensors_to_classification_calculator_cc_proto",
|
||||
"//mediapipe/framework/api2:builder",
|
||||
"//mediapipe/framework/api2:port",
|
||||
"//mediapipe/framework/formats:classification_cc_proto",
|
||||
"//mediapipe/framework/formats:landmark_cc_proto",
|
||||
"//mediapipe/framework/formats:tensor",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/tasks/cc/core:model_asset_bundle_resources",
|
||||
"//mediapipe/tasks/cc/core:model_resources_cache",
|
||||
"//mediapipe/tasks/cc/core:model_task_graph",
|
||||
"//mediapipe/tasks/cc/vision/face_landmarker/proto:face_blendshapes_graph_options_cc_proto",
|
||||
],
|
||||
alwayslink = 1,
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "face_landmarks_detector_graph",
|
||||
srcs = ["face_landmarks_detector_graph.cc"],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,326 @@
|
|||
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h"
|
||||
#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.pb.h"
|
||||
#include "mediapipe/calculators/tensor/tensors_to_classification_calculator.pb.h"
|
||||
#include "mediapipe/framework/api2/builder.h"
|
||||
#include "mediapipe/framework/api2/port.h"
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/formats/tensor.h"
|
||||
#include "mediapipe/framework/port/status_macros.h"
|
||||
#include "mediapipe/tasks/cc/core/model_task_graph.h"
|
||||
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace tasks {
|
||||
namespace vision {
|
||||
namespace face_landmarker {
|
||||
|
||||
namespace {
|
||||
|
||||
using ::mediapipe::api2::Input;
|
||||
using ::mediapipe::api2::LandmarksToTensorCalculator;
|
||||
using ::mediapipe::api2::Output;
|
||||
using ::mediapipe::api2::builder::Graph;
|
||||
using ::mediapipe::api2::builder::Stream;
|
||||
using ::mediapipe::tasks::vision::face_landmarker::proto::
|
||||
FaceBlendshapesGraphOptions;
|
||||
|
||||
constexpr char kLandmarksTag[] = "LANDMARKS";
|
||||
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
|
||||
constexpr char kBlendshapesTag[] = "BLENDSHAPES";
|
||||
|
||||
// Subset of 478 landmarks required for the HUND model.
|
||||
static constexpr std::array<int, 146> kLandmarksSubsetIdxs = {
|
||||
0, 1, 4, 5, 6, 7, 8, 10, 13, 14, 17, 21, 33, 37, 39,
|
||||
40, 46, 52, 53, 54, 55, 58, 61, 63, 65, 66, 67, 70, 78, 80,
|
||||
81, 82, 84, 87, 88, 91, 93, 95, 103, 105, 107, 109, 127, 132, 133,
|
||||
136, 144, 145, 146, 148, 149, 150, 152, 153, 154, 155, 157, 158, 159, 160,
|
||||
161, 162, 163, 168, 172, 173, 176, 178, 181, 185, 191, 195, 197, 234, 246,
|
||||
249, 251, 263, 267, 269, 270, 276, 282, 283, 284, 285, 288, 291, 293, 295,
|
||||
296, 297, 300, 308, 310, 311, 312, 314, 317, 318, 321, 323, 324, 332, 334,
|
||||
336, 338, 356, 361, 362, 365, 373, 374, 375, 377, 378, 379, 380, 381, 382,
|
||||
384, 385, 386, 387, 388, 389, 390, 397, 398, 400, 402, 405, 409, 415, 454,
|
||||
466, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477};
|
||||
static constexpr std::array<absl::string_view, 52> kBlendshapeNames = {
|
||||
"_neutral",
|
||||
"browDownLeft",
|
||||
"browDownRight",
|
||||
"browInnerUp",
|
||||
"browOuterUpLeft",
|
||||
"browOuterUpRight",
|
||||
"cheekPuff",
|
||||
"cheekSquintLeft",
|
||||
"cheekSquintRight",
|
||||
"eyeBlinkLeft",
|
||||
"eyeBlinkRight",
|
||||
"eyeLookDownLeft",
|
||||
"eyeLookDownRight",
|
||||
"eyeLookInLeft",
|
||||
"eyeLookInRight",
|
||||
"eyeLookOutLeft",
|
||||
"eyeLookOutRight",
|
||||
"eyeLookUpLeft",
|
||||
"eyeLookUpRight",
|
||||
"eyeSquintLeft",
|
||||
"eyeSquintRight",
|
||||
"eyeWideLeft",
|
||||
"eyeWideRight",
|
||||
"jawForward",
|
||||
"jawLeft",
|
||||
"jawOpen",
|
||||
"jawRight",
|
||||
"mouthClose",
|
||||
"mouthDimpleLeft",
|
||||
"mouthDimpleRight",
|
||||
"mouthFrownLeft",
|
||||
"mouthFrownRight",
|
||||
"mouthFunnel",
|
||||
"mouthLeft",
|
||||
"mouthLowerDownLeft",
|
||||
"mouthLowerDownRight",
|
||||
"mouthPressLeft",
|
||||
"mouthPressRight",
|
||||
"mouthPucker",
|
||||
"mouthRight",
|
||||
"mouthRollLower",
|
||||
"mouthRollUpper",
|
||||
"mouthShrugLower",
|
||||
"mouthShrugUpper",
|
||||
"mouthSmileLeft",
|
||||
"mouthSmileRight",
|
||||
"mouthStretchLeft",
|
||||
"mouthStretchRight",
|
||||
"mouthUpperUpLeft",
|
||||
"mouthUpperUpRight",
|
||||
"noseSneerLeft",
|
||||
"noseSneerRight"};
|
||||
|
||||
struct FaceBlendshapesOuts {
|
||||
Stream<ClassificationList> blendshapes;
|
||||
};
|
||||
|
||||
Stream<NormalizedLandmarkList> GetLandmarksSubset(
|
||||
Stream<NormalizedLandmarkList> landmarks,
|
||||
const std::array<int, 146>& subset_idxs, Graph& graph) {
|
||||
auto& node = graph.AddNode("SplitNormalizedLandmarkListCalculator");
|
||||
auto& opts = node.GetOptions<SplitVectorCalculatorOptions>();
|
||||
for (int i = 0; i < subset_idxs.size(); ++i) {
|
||||
auto* range = opts.add_ranges();
|
||||
range->set_begin(subset_idxs[i]);
|
||||
range->set_end(subset_idxs[i] + 1);
|
||||
}
|
||||
opts.set_combine_outputs(true);
|
||||
landmarks >> node.In("");
|
||||
return node.Out("").Cast<NormalizedLandmarkList>();
|
||||
}
|
||||
|
||||
Stream<std::vector<Tensor>> ConvertLandmarksToTensor(
|
||||
Stream<NormalizedLandmarkList> landmarks,
|
||||
Stream<std::pair<int, int>> img_size, Graph& graph) {
|
||||
auto& node = graph.AddNode<LandmarksToTensorCalculator>();
|
||||
auto& opts = node.GetOptions<LandmarksToTensorCalculatorOptions>();
|
||||
opts.add_attributes(LandmarksToTensorCalculatorOptions::X);
|
||||
opts.add_attributes(LandmarksToTensorCalculatorOptions::Y);
|
||||
opts.set_flatten(false);
|
||||
landmarks >> node[LandmarksToTensorCalculator::kInNormLandmarkList];
|
||||
img_size >> node[LandmarksToTensorCalculator::kImageSize];
|
||||
return node[LandmarksToTensorCalculator::kOutTensors];
|
||||
}
|
||||
|
||||
Stream<std::vector<Tensor>> GetTensorWithBlendshapes(
|
||||
Stream<std::vector<Tensor>> tensors, Graph& graph) {
|
||||
auto& node = graph.AddNode("SplitTensorVectorCalculator");
|
||||
auto& opts = node.GetOptions<SplitVectorCalculatorOptions>();
|
||||
auto* range = opts.add_ranges();
|
||||
range->set_begin(0);
|
||||
range->set_end(1);
|
||||
opts.set_combine_outputs(true);
|
||||
tensors >> node.In(0);
|
||||
return node.Out(0).Cast<std::vector<Tensor>>();
|
||||
}
|
||||
|
||||
Stream<ClassificationList> ConvertTensorToBlendshapes(
|
||||
Stream<std::vector<Tensor>> tensors,
|
||||
const std::array<absl::string_view, 52>& labels, Graph& graph) {
|
||||
auto& node = graph.AddNode("TensorsToClassificationCalculator");
|
||||
auto& opts = node.GetOptions<TensorsToClassificationCalculatorOptions>();
|
||||
// Disable top_k and min_score_threshold logic to return coefficients as is.
|
||||
opts.set_top_k(0);
|
||||
opts.set_min_score_threshold(-1.0);
|
||||
for (int i = 0; i < labels.size(); ++i) {
|
||||
auto* entry = opts.mutable_label_map()->add_entries();
|
||||
entry->set_id(i);
|
||||
// TODO: Replace with entry->set_label(labels[i])
|
||||
entry->mutable_label()->assign(labels[i].data(), labels[i].size());
|
||||
}
|
||||
tensors >> node.In("TENSORS");
|
||||
return node.Out("CLASSIFICATIONS").Cast<ClassificationList>();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
// Predicts face blendshapes from landmarks.
|
||||
//
|
||||
// Inputs:
|
||||
// LANDMARKS - NormalizedLandmarkList
|
||||
// 478 2.5D face landmarks predicted by an Attention Mesh model.
|
||||
// IMAGE_SIZE - std::pair<int, int>
|
||||
// Input frame size.
|
||||
//
|
||||
// Outputs:
|
||||
// FACE_BLENDSHAPES - ClassificationList
|
||||
// if EXTRA_FACE_BLENDSHAPES is provided, we have 53 blendshape coeffs
|
||||
// output; if not, 52 coeffs output.
|
||||
// All 52 blendshape coefficients:
|
||||
// 0 - _neutral (ignore it)
|
||||
// 1 - browDownLeft
|
||||
// 2 - browDownRight
|
||||
// 3 - browInnerUp
|
||||
// 4 - browOuterUpLeft
|
||||
// 5 - browOuterUpRight
|
||||
// 6 - cheekPuff
|
||||
// 7 - cheekSquintLeft
|
||||
// 8 - cheekSquintRight
|
||||
// 9 - eyeBlinkLeft
|
||||
// 10 - eyeBlinkRight
|
||||
// 11 - eyeLookDownLeft
|
||||
// 12 - eyeLookDownRight
|
||||
// 13 - eyeLookInLeft
|
||||
// 14 - eyeLookInRight
|
||||
// 15 - eyeLookOutLeft
|
||||
// 16 - eyeLookOutRight
|
||||
// 17 - eyeLookUpLeft
|
||||
// 18 - eyeLookUpRight
|
||||
// 19 - eyeSquintLeft
|
||||
// 20 - eyeSquintRight
|
||||
// 21 - eyeWideLeft
|
||||
// 22 - eyeWideRight
|
||||
// 23 - jawForward
|
||||
// 24 - jawLeft
|
||||
// 25 - jawOpen
|
||||
// 26 - jawRight
|
||||
// 27 - mouthClose
|
||||
// 28 - mouthDimpleLeft
|
||||
// 29 - mouthDimpleRight
|
||||
// 30 - mouthFrownLeft
|
||||
// 31 - mouthFrownRight
|
||||
// 32 - mouthFunnel
|
||||
// 33 - mouthLeft
|
||||
// 34 - mouthLowerDownLeft
|
||||
// 35 - mouthLowerDownRight
|
||||
// 36 - mouthPressLeft
|
||||
// 37 - mouthPressRight
|
||||
// 38 - mouthPucker
|
||||
// 39 - mouthRight
|
||||
// 40 - mouthRollLower
|
||||
// 41 - mouthRollUpper
|
||||
// 42 - mouthShrugLower
|
||||
// 43 - mouthShrugUpper
|
||||
// 44 - mouthSmileLeft
|
||||
// 45 - mouthSmileRight
|
||||
// 46 - mouthStretchLeft
|
||||
// 47 - mouthStretchRight
|
||||
// 48 - mouthUpperUpLeft
|
||||
// 49 - mouthUpperUpRight
|
||||
// 50 - noseSneerLeft
|
||||
// 51 - noseSneerRight
|
||||
//
|
||||
// Example:
|
||||
// node {
|
||||
// calculator: "mediapipe.tasks.vision.face_landmarker.FaceBlendshapesGraph"
|
||||
// input_stream: "LANDMARKS:face_landmarks"
|
||||
// input_stream: "IMAGE_SIZE:image_size"
|
||||
// output_stream: "BLENDSHAPES:face_blendshapes"
|
||||
// options {
|
||||
// [mediapipe.tasks.vision.face_landmarker.proto.FaceBlendshapesGraphOptions.ext]
|
||||
// {
|
||||
// base_options {
|
||||
// model_asset {
|
||||
// file_name: "face_blendshapes.tflite"
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
class FaceBlendshapesGraph : public core::ModelTaskGraph {
|
||||
public:
|
||||
absl::StatusOr<CalculatorGraphConfig> GetConfig(SubgraphContext* sc) {
|
||||
ASSIGN_OR_RETURN(const auto* model_resources,
|
||||
CreateModelResources<FaceBlendshapesGraphOptions>(sc));
|
||||
Graph graph;
|
||||
ASSIGN_OR_RETURN(
|
||||
auto face_blendshapes_outs,
|
||||
BuildFaceBlendshapesSubgraph(
|
||||
sc->Options<FaceBlendshapesGraphOptions>(), *model_resources,
|
||||
graph[Input<NormalizedLandmarkList>(kLandmarksTag)],
|
||||
graph[Input<std::pair<int, int>>(kImageSizeTag)], graph));
|
||||
face_blendshapes_outs.blendshapes >>
|
||||
graph[Output<ClassificationList>(kBlendshapesTag)];
|
||||
|
||||
return graph.GetConfig();
|
||||
}
|
||||
|
||||
private:
|
||||
// Updates graph to predict face blendshapes from landmarks. Returns list of
|
||||
// blendsahpes.
|
||||
//
|
||||
// subgraph_options: the mediapipe tasks module FaceBlendshapesGraphOptions.
|
||||
// model_resources: the ModelSources object initialized from a face
|
||||
// blendshapes model file with model metadata.
|
||||
// landmarks: 478 normalized face landmarks
|
||||
// img_size: Image size to denormalize landmarks.
|
||||
// graph: the mediapipe builder::Graph instance to be updated.
|
||||
absl::StatusOr<FaceBlendshapesOuts> BuildFaceBlendshapesSubgraph(
|
||||
const FaceBlendshapesGraphOptions& subgraph_options,
|
||||
const core::ModelResources& model_resources,
|
||||
Stream<NormalizedLandmarkList> landmarks,
|
||||
Stream<std::pair<int, int>> img_size, Graph& graph) {
|
||||
// Take required subset of landmarks.
|
||||
landmarks = GetLandmarksSubset(landmarks, kLandmarksSubsetIdxs, graph);
|
||||
|
||||
// Convert landmarks to input tensor.
|
||||
auto tensor_in = ConvertLandmarksToTensor(landmarks, img_size, graph);
|
||||
|
||||
// Run Blendshapes model.
|
||||
auto& inference = AddInference(
|
||||
model_resources, subgraph_options.base_options().acceleration(), graph);
|
||||
tensor_in >> inference.In("TENSORS");
|
||||
auto tensors_out = inference.Out("TENSORS").Cast<std::vector<Tensor>>();
|
||||
|
||||
// Take output tensor with blendshapes and wrap it in vector.
|
||||
auto blendshapes_tensor = GetTensorWithBlendshapes(tensors_out, graph);
|
||||
|
||||
// Convert tensor to ClassificationList.
|
||||
auto face_blendshapes =
|
||||
ConvertTensorToBlendshapes(blendshapes_tensor, kBlendshapeNames, graph);
|
||||
|
||||
return FaceBlendshapesOuts{face_blendshapes};
|
||||
}
|
||||
};
|
||||
|
||||
REGISTER_MEDIAPIPE_GRAPH(
|
||||
::mediapipe::tasks::vision::face_landmarker::FaceBlendshapesGraph);
|
||||
|
||||
} // namespace face_landmarker
|
||||
} // namespace vision
|
||||
} // namespace tasks
|
||||
} // namespace mediapipe
|
||||
|
|
@ -0,0 +1,173 @@
|
|||
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
|
||||
#include "mediapipe/framework/api2/builder.h"
|
||||
#include "mediapipe/framework/api2/port.h"
|
||||
#include "mediapipe/framework/deps/file_path.h"
|
||||
#include "mediapipe/framework/formats/classification.pb.h"
|
||||
#include "mediapipe/framework/formats/landmark.pb.h"
|
||||
#include "mediapipe/framework/port/file_helpers.h"
|
||||
#include "mediapipe/framework/port/gmock.h"
|
||||
#include "mediapipe/framework/port/gtest.h"
|
||||
#include "mediapipe/framework/port/parse_text_proto.h"
|
||||
#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h"
|
||||
#include "mediapipe/tasks/cc/core/proto/base_options.pb.h"
|
||||
#include "mediapipe/tasks/cc/core/task_runner.h"
|
||||
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h"
|
||||
#include "tensorflow/lite/core/shims/cc/shims_test_util.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace tasks {
|
||||
namespace vision {
|
||||
namespace face_landmarker {
|
||||
namespace {
|
||||
|
||||
using ::file::Defaults;
|
||||
using ::mediapipe::api2::Input;
|
||||
using ::mediapipe::api2::Output;
|
||||
using ::mediapipe::api2::builder::Graph;
|
||||
using ::mediapipe::file::JoinPath;
|
||||
using ::mediapipe::tasks::core::TaskRunner;
|
||||
using ::mediapipe::tasks::vision::face_landmarker::proto::
|
||||
FaceBlendshapesGraphOptions;
|
||||
|
||||
constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/";
|
||||
constexpr char kFaceBlendshapesModel[] = "face_blendshapes.tflite";
|
||||
constexpr char kInLandmarks[] = "face_blendshapes_in_landmarks.prototxt";
|
||||
constexpr char kOutBlendshapes[] = "face_blendshapes_out.prototxt";
|
||||
constexpr float kSimilarityThreshold = 0.1;
|
||||
constexpr std::string_view kGeneratedGraph =
|
||||
"face_blendshapes_generated_graph.pbtxt";
|
||||
|
||||
constexpr char kLandmarksTag[] = "LANDMARKS";
|
||||
constexpr char kLandmarksName[] = "landmarks";
|
||||
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
|
||||
constexpr char kImageSizeName[] = "image_size";
|
||||
constexpr char kBlendshapesTag[] = "BLENDSHAPES";
|
||||
constexpr char kBlendshapesName[] = "blendshapes";
|
||||
|
||||
absl::StatusOr<CalculatorGraphConfig> ExpandConfig(
|
||||
const std::string& config_str) {
|
||||
auto config =
|
||||
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(config_str);
|
||||
CalculatorGraph graph;
|
||||
MP_RETURN_IF_ERROR(graph.Initialize(config));
|
||||
return graph.Config();
|
||||
}
|
||||
|
||||
NormalizedLandmarkList GetLandmarks(absl::string_view filename) {
|
||||
NormalizedLandmarkList landmarks;
|
||||
MP_EXPECT_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, filename),
|
||||
&landmarks, Defaults()));
|
||||
return landmarks;
|
||||
}
|
||||
|
||||
ClassificationList GetBlendshapes(absl::string_view filename) {
|
||||
ClassificationList blendshapes;
|
||||
MP_EXPECT_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, filename),
|
||||
&blendshapes, Defaults()));
|
||||
return blendshapes;
|
||||
}
|
||||
|
||||
// Helper function to create a Face Blendshapes TaskRunner.
|
||||
absl::StatusOr<std::unique_ptr<TaskRunner>> CreateTaskRunner() {
|
||||
Graph graph;
|
||||
auto& face_blendshapes_graph = graph.AddNode(
|
||||
"mediapipe.tasks.vision.face_landmarker.FaceBlendshapesGraph");
|
||||
auto& options =
|
||||
face_blendshapes_graph.GetOptions<FaceBlendshapesGraphOptions>();
|
||||
options.mutable_base_options()->mutable_model_asset()->set_file_name(
|
||||
JoinPath("./", kTestDataDirectory, kFaceBlendshapesModel));
|
||||
|
||||
graph[Input<NormalizedLandmarkList>(kLandmarksTag)].SetName(kLandmarksName) >>
|
||||
face_blendshapes_graph.In(kLandmarksTag);
|
||||
graph[Input<std::pair<int, int>>(kImageSizeTag)].SetName(kImageSizeName) >>
|
||||
face_blendshapes_graph.In(kImageSizeTag);
|
||||
face_blendshapes_graph.Out(kBlendshapesTag).SetName(kBlendshapesName) >>
|
||||
graph[Output<ClassificationList>(kBlendshapesTag)];
|
||||
|
||||
return TaskRunner::Create(
|
||||
graph.GetConfig(), absl::make_unique<core::MediaPipeBuiltinOpResolver>());
|
||||
}
|
||||
|
||||
class FaceBlendshapesTest : public tflite_shims::testing::Test {};
|
||||
|
||||
TEST_F(FaceBlendshapesTest, SmokeTest) {
|
||||
// Prepare graph inputs.
|
||||
auto in_landmarks = GetLandmarks(kInLandmarks);
|
||||
std::pair<int, int> in_image_size = {820, 1024};
|
||||
|
||||
// Run graph.
|
||||
MP_ASSERT_OK_AND_ASSIGN(auto task_runner, CreateTaskRunner());
|
||||
auto output_packets = task_runner->Process(
|
||||
{{kLandmarksName,
|
||||
MakePacket<NormalizedLandmarkList>(std::move(in_landmarks))},
|
||||
{kImageSizeName,
|
||||
MakePacket<std::pair<int, int>>(std::move(in_image_size))}});
|
||||
MP_ASSERT_OK(output_packets);
|
||||
|
||||
// Compare with expected result.
|
||||
const auto& actual_blendshapes =
|
||||
(*output_packets)[kBlendshapesName].Get<ClassificationList>();
|
||||
auto expected_blendshapes = GetBlendshapes(kOutBlendshapes);
|
||||
EXPECT_THAT(
|
||||
actual_blendshapes,
|
||||
testing::proto::Approximately(testing::EqualsProto(expected_blendshapes),
|
||||
kSimilarityThreshold));
|
||||
}
|
||||
|
||||
TEST(FaceRigGhumGpuTest, VerifyGraph) {
|
||||
MP_ASSERT_OK_AND_ASSIGN(
|
||||
auto actual_graph,
|
||||
ExpandConfig(
|
||||
R"pb(
|
||||
node {
|
||||
calculator: "mediapipe.tasks.vision.face_landmarker.FaceBlendshapesGraph"
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
output_stream: "BLENDSHAPES:blendshapes"
|
||||
options {
|
||||
[mediapipe.tasks.vision.face_landmarker.proto
|
||||
.FaceBlendshapesGraphOptions.ext] {
|
||||
base_options {
|
||||
model_asset {
|
||||
file_name: "mediapipe/tasks/testdata/vision/face_blendshapes.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
)pb"));
|
||||
|
||||
std::string expected_graph_contents;
|
||||
MP_ASSERT_OK(file::GetContents(
|
||||
file::JoinPath("./", kTestDataDirectory, kGeneratedGraph),
|
||||
&expected_graph_contents));
|
||||
auto expected_graph = mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
|
||||
expected_graph_contents);
|
||||
EXPECT_THAT(actual_graph, testing::proto::IgnoringRepeatedFieldOrdering(
|
||||
testing::EqualsProto(expected_graph)));
|
||||
}
|
||||
|
||||
} // namespace
|
||||
} // namespace face_landmarker
|
||||
} // namespace vision
|
||||
} // namespace tasks
|
||||
} // namespace mediapipe
|
||||
|
|
@ -20,6 +20,16 @@ package(default_visibility = [
|
|||
|
||||
licenses(["notice"])
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "face_blendshapes_graph_options_proto",
|
||||
srcs = ["face_blendshapes_graph_options.proto"],
|
||||
deps = [
|
||||
"//mediapipe/framework:calculator_options_proto",
|
||||
"//mediapipe/framework:calculator_proto",
|
||||
"//mediapipe/tasks/cc/core/proto:base_options_proto",
|
||||
],
|
||||
)
|
||||
|
||||
mediapipe_proto_library(
|
||||
name = "face_landmarks_detector_graph_options_proto",
|
||||
srcs = ["face_landmarks_detector_graph_options.proto"],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,34 @@
|
|||
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
==============================================================================*/
|
||||
|
||||
syntax = "proto2";
|
||||
|
||||
package mediapipe.tasks.vision.face_landmarker.proto;
|
||||
|
||||
import "mediapipe/framework/calculator.proto";
|
||||
import "mediapipe/framework/calculator_options.proto";
|
||||
import "mediapipe/tasks/cc/core/proto/base_options.proto";
|
||||
|
||||
option java_package = "com.google.mediapipe.tasks.vision.facelandmarker.proto";
|
||||
option java_outer_classname = "FaceBlendshapesGraphOptionsProto";
|
||||
|
||||
message FaceBlendshapesGraphOptions {
|
||||
extend mediapipe.CalculatorOptions {
|
||||
optional FaceBlendshapesGraphOptions ext = 508981768;
|
||||
}
|
||||
// Base options for configuring Task library, such as specifying the TfLite
|
||||
// model file with metadata, accelerator options, etc.
|
||||
optional core.proto.BaseOptions base_options = 1;
|
||||
}
|
||||
878
mediapipe/tasks/testdata/vision/face_blendshapes_generated_graph.pbtxt
vendored
Normal file
878
mediapipe/tasks/testdata/vision/face_blendshapes_generated_graph.pbtxt
vendored
Normal file
|
|
@ -0,0 +1,878 @@
|
|||
node {
|
||||
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__SplitNormalizedLandmarkListCalculator"
|
||||
calculator: "SplitNormalizedLandmarkListCalculator"
|
||||
input_stream: "landmarks"
|
||||
output_stream: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_2"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges {
|
||||
begin: 0
|
||||
end: 1
|
||||
}
|
||||
ranges {
|
||||
begin: 1
|
||||
end: 2
|
||||
}
|
||||
ranges {
|
||||
begin: 4
|
||||
end: 5
|
||||
}
|
||||
ranges {
|
||||
begin: 5
|
||||
end: 6
|
||||
}
|
||||
ranges {
|
||||
begin: 6
|
||||
end: 7
|
||||
}
|
||||
ranges {
|
||||
begin: 7
|
||||
end: 8
|
||||
}
|
||||
ranges {
|
||||
begin: 8
|
||||
end: 9
|
||||
}
|
||||
ranges {
|
||||
begin: 10
|
||||
end: 11
|
||||
}
|
||||
ranges {
|
||||
begin: 13
|
||||
end: 14
|
||||
}
|
||||
ranges {
|
||||
begin: 14
|
||||
end: 15
|
||||
}
|
||||
ranges {
|
||||
begin: 17
|
||||
end: 18
|
||||
}
|
||||
ranges {
|
||||
begin: 21
|
||||
end: 22
|
||||
}
|
||||
ranges {
|
||||
begin: 33
|
||||
end: 34
|
||||
}
|
||||
ranges {
|
||||
begin: 37
|
||||
end: 38
|
||||
}
|
||||
ranges {
|
||||
begin: 39
|
||||
end: 40
|
||||
}
|
||||
ranges {
|
||||
begin: 40
|
||||
end: 41
|
||||
}
|
||||
ranges {
|
||||
begin: 46
|
||||
end: 47
|
||||
}
|
||||
ranges {
|
||||
begin: 52
|
||||
end: 53
|
||||
}
|
||||
ranges {
|
||||
begin: 53
|
||||
end: 54
|
||||
}
|
||||
ranges {
|
||||
begin: 54
|
||||
end: 55
|
||||
}
|
||||
ranges {
|
||||
begin: 55
|
||||
end: 56
|
||||
}
|
||||
ranges {
|
||||
begin: 58
|
||||
end: 59
|
||||
}
|
||||
ranges {
|
||||
begin: 61
|
||||
end: 62
|
||||
}
|
||||
ranges {
|
||||
begin: 63
|
||||
end: 64
|
||||
}
|
||||
ranges {
|
||||
begin: 65
|
||||
end: 66
|
||||
}
|
||||
ranges {
|
||||
begin: 66
|
||||
end: 67
|
||||
}
|
||||
ranges {
|
||||
begin: 67
|
||||
end: 68
|
||||
}
|
||||
ranges {
|
||||
begin: 70
|
||||
end: 71
|
||||
}
|
||||
ranges {
|
||||
begin: 78
|
||||
end: 79
|
||||
}
|
||||
ranges {
|
||||
begin: 80
|
||||
end: 81
|
||||
}
|
||||
ranges {
|
||||
begin: 81
|
||||
end: 82
|
||||
}
|
||||
ranges {
|
||||
begin: 82
|
||||
end: 83
|
||||
}
|
||||
ranges {
|
||||
begin: 84
|
||||
end: 85
|
||||
}
|
||||
ranges {
|
||||
begin: 87
|
||||
end: 88
|
||||
}
|
||||
ranges {
|
||||
begin: 88
|
||||
end: 89
|
||||
}
|
||||
ranges {
|
||||
begin: 91
|
||||
end: 92
|
||||
}
|
||||
ranges {
|
||||
begin: 93
|
||||
end: 94
|
||||
}
|
||||
ranges {
|
||||
begin: 95
|
||||
end: 96
|
||||
}
|
||||
ranges {
|
||||
begin: 103
|
||||
end: 104
|
||||
}
|
||||
ranges {
|
||||
begin: 105
|
||||
end: 106
|
||||
}
|
||||
ranges {
|
||||
begin: 107
|
||||
end: 108
|
||||
}
|
||||
ranges {
|
||||
begin: 109
|
||||
end: 110
|
||||
}
|
||||
ranges {
|
||||
begin: 127
|
||||
end: 128
|
||||
}
|
||||
ranges {
|
||||
begin: 132
|
||||
end: 133
|
||||
}
|
||||
ranges {
|
||||
begin: 133
|
||||
end: 134
|
||||
}
|
||||
ranges {
|
||||
begin: 136
|
||||
end: 137
|
||||
}
|
||||
ranges {
|
||||
begin: 144
|
||||
end: 145
|
||||
}
|
||||
ranges {
|
||||
begin: 145
|
||||
end: 146
|
||||
}
|
||||
ranges {
|
||||
begin: 146
|
||||
end: 147
|
||||
}
|
||||
ranges {
|
||||
begin: 148
|
||||
end: 149
|
||||
}
|
||||
ranges {
|
||||
begin: 149
|
||||
end: 150
|
||||
}
|
||||
ranges {
|
||||
begin: 150
|
||||
end: 151
|
||||
}
|
||||
ranges {
|
||||
begin: 152
|
||||
end: 153
|
||||
}
|
||||
ranges {
|
||||
begin: 153
|
||||
end: 154
|
||||
}
|
||||
ranges {
|
||||
begin: 154
|
||||
end: 155
|
||||
}
|
||||
ranges {
|
||||
begin: 155
|
||||
end: 156
|
||||
}
|
||||
ranges {
|
||||
begin: 157
|
||||
end: 158
|
||||
}
|
||||
ranges {
|
||||
begin: 158
|
||||
end: 159
|
||||
}
|
||||
ranges {
|
||||
begin: 159
|
||||
end: 160
|
||||
}
|
||||
ranges {
|
||||
begin: 160
|
||||
end: 161
|
||||
}
|
||||
ranges {
|
||||
begin: 161
|
||||
end: 162
|
||||
}
|
||||
ranges {
|
||||
begin: 162
|
||||
end: 163
|
||||
}
|
||||
ranges {
|
||||
begin: 163
|
||||
end: 164
|
||||
}
|
||||
ranges {
|
||||
begin: 168
|
||||
end: 169
|
||||
}
|
||||
ranges {
|
||||
begin: 172
|
||||
end: 173
|
||||
}
|
||||
ranges {
|
||||
begin: 173
|
||||
end: 174
|
||||
}
|
||||
ranges {
|
||||
begin: 176
|
||||
end: 177
|
||||
}
|
||||
ranges {
|
||||
begin: 178
|
||||
end: 179
|
||||
}
|
||||
ranges {
|
||||
begin: 181
|
||||
end: 182
|
||||
}
|
||||
ranges {
|
||||
begin: 185
|
||||
end: 186
|
||||
}
|
||||
ranges {
|
||||
begin: 191
|
||||
end: 192
|
||||
}
|
||||
ranges {
|
||||
begin: 195
|
||||
end: 196
|
||||
}
|
||||
ranges {
|
||||
begin: 197
|
||||
end: 198
|
||||
}
|
||||
ranges {
|
||||
begin: 234
|
||||
end: 235
|
||||
}
|
||||
ranges {
|
||||
begin: 246
|
||||
end: 247
|
||||
}
|
||||
ranges {
|
||||
begin: 249
|
||||
end: 250
|
||||
}
|
||||
ranges {
|
||||
begin: 251
|
||||
end: 252
|
||||
}
|
||||
ranges {
|
||||
begin: 263
|
||||
end: 264
|
||||
}
|
||||
ranges {
|
||||
begin: 267
|
||||
end: 268
|
||||
}
|
||||
ranges {
|
||||
begin: 269
|
||||
end: 270
|
||||
}
|
||||
ranges {
|
||||
begin: 270
|
||||
end: 271
|
||||
}
|
||||
ranges {
|
||||
begin: 276
|
||||
end: 277
|
||||
}
|
||||
ranges {
|
||||
begin: 282
|
||||
end: 283
|
||||
}
|
||||
ranges {
|
||||
begin: 283
|
||||
end: 284
|
||||
}
|
||||
ranges {
|
||||
begin: 284
|
||||
end: 285
|
||||
}
|
||||
ranges {
|
||||
begin: 285
|
||||
end: 286
|
||||
}
|
||||
ranges {
|
||||
begin: 288
|
||||
end: 289
|
||||
}
|
||||
ranges {
|
||||
begin: 291
|
||||
end: 292
|
||||
}
|
||||
ranges {
|
||||
begin: 293
|
||||
end: 294
|
||||
}
|
||||
ranges {
|
||||
begin: 295
|
||||
end: 296
|
||||
}
|
||||
ranges {
|
||||
begin: 296
|
||||
end: 297
|
||||
}
|
||||
ranges {
|
||||
begin: 297
|
||||
end: 298
|
||||
}
|
||||
ranges {
|
||||
begin: 300
|
||||
end: 301
|
||||
}
|
||||
ranges {
|
||||
begin: 308
|
||||
end: 309
|
||||
}
|
||||
ranges {
|
||||
begin: 310
|
||||
end: 311
|
||||
}
|
||||
ranges {
|
||||
begin: 311
|
||||
end: 312
|
||||
}
|
||||
ranges {
|
||||
begin: 312
|
||||
end: 313
|
||||
}
|
||||
ranges {
|
||||
begin: 314
|
||||
end: 315
|
||||
}
|
||||
ranges {
|
||||
begin: 317
|
||||
end: 318
|
||||
}
|
||||
ranges {
|
||||
begin: 318
|
||||
end: 319
|
||||
}
|
||||
ranges {
|
||||
begin: 321
|
||||
end: 322
|
||||
}
|
||||
ranges {
|
||||
begin: 323
|
||||
end: 324
|
||||
}
|
||||
ranges {
|
||||
begin: 324
|
||||
end: 325
|
||||
}
|
||||
ranges {
|
||||
begin: 332
|
||||
end: 333
|
||||
}
|
||||
ranges {
|
||||
begin: 334
|
||||
end: 335
|
||||
}
|
||||
ranges {
|
||||
begin: 336
|
||||
end: 337
|
||||
}
|
||||
ranges {
|
||||
begin: 338
|
||||
end: 339
|
||||
}
|
||||
ranges {
|
||||
begin: 356
|
||||
end: 357
|
||||
}
|
||||
ranges {
|
||||
begin: 361
|
||||
end: 362
|
||||
}
|
||||
ranges {
|
||||
begin: 362
|
||||
end: 363
|
||||
}
|
||||
ranges {
|
||||
begin: 365
|
||||
end: 366
|
||||
}
|
||||
ranges {
|
||||
begin: 373
|
||||
end: 374
|
||||
}
|
||||
ranges {
|
||||
begin: 374
|
||||
end: 375
|
||||
}
|
||||
ranges {
|
||||
begin: 375
|
||||
end: 376
|
||||
}
|
||||
ranges {
|
||||
begin: 377
|
||||
end: 378
|
||||
}
|
||||
ranges {
|
||||
begin: 378
|
||||
end: 379
|
||||
}
|
||||
ranges {
|
||||
begin: 379
|
||||
end: 380
|
||||
}
|
||||
ranges {
|
||||
begin: 380
|
||||
end: 381
|
||||
}
|
||||
ranges {
|
||||
begin: 381
|
||||
end: 382
|
||||
}
|
||||
ranges {
|
||||
begin: 382
|
||||
end: 383
|
||||
}
|
||||
ranges {
|
||||
begin: 384
|
||||
end: 385
|
||||
}
|
||||
ranges {
|
||||
begin: 385
|
||||
end: 386
|
||||
}
|
||||
ranges {
|
||||
begin: 386
|
||||
end: 387
|
||||
}
|
||||
ranges {
|
||||
begin: 387
|
||||
end: 388
|
||||
}
|
||||
ranges {
|
||||
begin: 388
|
||||
end: 389
|
||||
}
|
||||
ranges {
|
||||
begin: 389
|
||||
end: 390
|
||||
}
|
||||
ranges {
|
||||
begin: 390
|
||||
end: 391
|
||||
}
|
||||
ranges {
|
||||
begin: 397
|
||||
end: 398
|
||||
}
|
||||
ranges {
|
||||
begin: 398
|
||||
end: 399
|
||||
}
|
||||
ranges {
|
||||
begin: 400
|
||||
end: 401
|
||||
}
|
||||
ranges {
|
||||
begin: 402
|
||||
end: 403
|
||||
}
|
||||
ranges {
|
||||
begin: 405
|
||||
end: 406
|
||||
}
|
||||
ranges {
|
||||
begin: 409
|
||||
end: 410
|
||||
}
|
||||
ranges {
|
||||
begin: 415
|
||||
end: 416
|
||||
}
|
||||
ranges {
|
||||
begin: 454
|
||||
end: 455
|
||||
}
|
||||
ranges {
|
||||
begin: 466
|
||||
end: 467
|
||||
}
|
||||
ranges {
|
||||
begin: 468
|
||||
end: 469
|
||||
}
|
||||
ranges {
|
||||
begin: 469
|
||||
end: 470
|
||||
}
|
||||
ranges {
|
||||
begin: 470
|
||||
end: 471
|
||||
}
|
||||
ranges {
|
||||
begin: 471
|
||||
end: 472
|
||||
}
|
||||
ranges {
|
||||
begin: 472
|
||||
end: 473
|
||||
}
|
||||
ranges {
|
||||
begin: 473
|
||||
end: 474
|
||||
}
|
||||
ranges {
|
||||
begin: 474
|
||||
end: 475
|
||||
}
|
||||
ranges {
|
||||
begin: 475
|
||||
end: 476
|
||||
}
|
||||
ranges {
|
||||
begin: 476
|
||||
end: 477
|
||||
}
|
||||
ranges {
|
||||
begin: 477
|
||||
end: 478
|
||||
}
|
||||
combine_outputs: true
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__LandmarksToTensorCalculator"
|
||||
calculator: "LandmarksToTensorCalculator"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
input_stream: "NORM_LANDMARKS:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_2"
|
||||
output_stream: "TENSORS:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_3"
|
||||
options {
|
||||
[mediapipe.LandmarksToTensorCalculatorOptions.ext] {
|
||||
attributes: X
|
||||
attributes: Y
|
||||
flatten: false
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph__ModelResourcesCalculator"
|
||||
calculator: "ModelResourcesCalculator"
|
||||
output_side_packet: "METADATA_EXTRACTOR:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_1"
|
||||
output_side_packet: "MODEL:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_2"
|
||||
output_side_packet: "OP_RESOLVER:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_3"
|
||||
options {
|
||||
[mediapipe.tasks.core.proto.ModelResourcesCalculatorOptions.ext] {
|
||||
model_file {
|
||||
file_name: "mediapipe/tasks/testdata/vision/face_blendshapes.tflite"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph__inferencecalculator__mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph__InferenceCalculator"
|
||||
calculator: "InferenceCalculatorCpu"
|
||||
input_stream: "TENSORS:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_3"
|
||||
output_stream: "TENSORS:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_4"
|
||||
input_side_packet: "MODEL:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_2"
|
||||
input_side_packet: "OP_RESOLVER:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_3"
|
||||
options {
|
||||
[mediapipe.InferenceCalculatorOptions.ext] {
|
||||
delegate {
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__SplitTensorVectorCalculator"
|
||||
calculator: "SplitTensorVectorCalculator"
|
||||
input_stream: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_4"
|
||||
output_stream: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_5"
|
||||
options {
|
||||
[mediapipe.SplitVectorCalculatorOptions.ext] {
|
||||
ranges {
|
||||
begin: 0
|
||||
end: 1
|
||||
}
|
||||
combine_outputs: true
|
||||
}
|
||||
}
|
||||
}
|
||||
node {
|
||||
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__TensorsToClassificationCalculator"
|
||||
calculator: "TensorsToClassificationCalculator"
|
||||
input_stream: "TENSORS:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_5"
|
||||
output_stream: "CLASSIFICATIONS:blendshapes"
|
||||
options {
|
||||
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {
|
||||
min_score_threshold: -1
|
||||
top_k: 0
|
||||
label_map {
|
||||
entries {
|
||||
id: 0
|
||||
label: "_neutral"
|
||||
}
|
||||
entries {
|
||||
id: 1
|
||||
label: "browDownLeft"
|
||||
}
|
||||
entries {
|
||||
id: 2
|
||||
label: "browDownRight"
|
||||
}
|
||||
entries {
|
||||
id: 3
|
||||
label: "browInnerUp"
|
||||
}
|
||||
entries {
|
||||
id: 4
|
||||
label: "browOuterUpLeft"
|
||||
}
|
||||
entries {
|
||||
id: 5
|
||||
label: "browOuterUpRight"
|
||||
}
|
||||
entries {
|
||||
id: 6
|
||||
label: "cheekPuff"
|
||||
}
|
||||
entries {
|
||||
id: 7
|
||||
label: "cheekSquintLeft"
|
||||
}
|
||||
entries {
|
||||
id: 8
|
||||
label: "cheekSquintRight"
|
||||
}
|
||||
entries {
|
||||
id: 9
|
||||
label: "eyeBlinkLeft"
|
||||
}
|
||||
entries {
|
||||
id: 10
|
||||
label: "eyeBlinkRight"
|
||||
}
|
||||
entries {
|
||||
id: 11
|
||||
label: "eyeLookDownLeft"
|
||||
}
|
||||
entries {
|
||||
id: 12
|
||||
label: "eyeLookDownRight"
|
||||
}
|
||||
entries {
|
||||
id: 13
|
||||
label: "eyeLookInLeft"
|
||||
}
|
||||
entries {
|
||||
id: 14
|
||||
label: "eyeLookInRight"
|
||||
}
|
||||
entries {
|
||||
id: 15
|
||||
label: "eyeLookOutLeft"
|
||||
}
|
||||
entries {
|
||||
id: 16
|
||||
label: "eyeLookOutRight"
|
||||
}
|
||||
entries {
|
||||
id: 17
|
||||
label: "eyeLookUpLeft"
|
||||
}
|
||||
entries {
|
||||
id: 18
|
||||
label: "eyeLookUpRight"
|
||||
}
|
||||
entries {
|
||||
id: 19
|
||||
label: "eyeSquintLeft"
|
||||
}
|
||||
entries {
|
||||
id: 20
|
||||
label: "eyeSquintRight"
|
||||
}
|
||||
entries {
|
||||
id: 21
|
||||
label: "eyeWideLeft"
|
||||
}
|
||||
entries {
|
||||
id: 22
|
||||
label: "eyeWideRight"
|
||||
}
|
||||
entries {
|
||||
id: 23
|
||||
label: "jawForward"
|
||||
}
|
||||
entries {
|
||||
id: 24
|
||||
label: "jawLeft"
|
||||
}
|
||||
entries {
|
||||
id: 25
|
||||
label: "jawOpen"
|
||||
}
|
||||
entries {
|
||||
id: 26
|
||||
label: "jawRight"
|
||||
}
|
||||
entries {
|
||||
id: 27
|
||||
label: "mouthClose"
|
||||
}
|
||||
entries {
|
||||
id: 28
|
||||
label: "mouthDimpleLeft"
|
||||
}
|
||||
entries {
|
||||
id: 29
|
||||
label: "mouthDimpleRight"
|
||||
}
|
||||
entries {
|
||||
id: 30
|
||||
label: "mouthFrownLeft"
|
||||
}
|
||||
entries {
|
||||
id: 31
|
||||
label: "mouthFrownRight"
|
||||
}
|
||||
entries {
|
||||
id: 32
|
||||
label: "mouthFunnel"
|
||||
}
|
||||
entries {
|
||||
id: 33
|
||||
label: "mouthLeft"
|
||||
}
|
||||
entries {
|
||||
id: 34
|
||||
label: "mouthLowerDownLeft"
|
||||
}
|
||||
entries {
|
||||
id: 35
|
||||
label: "mouthLowerDownRight"
|
||||
}
|
||||
entries {
|
||||
id: 36
|
||||
label: "mouthPressLeft"
|
||||
}
|
||||
entries {
|
||||
id: 37
|
||||
label: "mouthPressRight"
|
||||
}
|
||||
entries {
|
||||
id: 38
|
||||
label: "mouthPucker"
|
||||
}
|
||||
entries {
|
||||
id: 39
|
||||
label: "mouthRight"
|
||||
}
|
||||
entries {
|
||||
id: 40
|
||||
label: "mouthRollLower"
|
||||
}
|
||||
entries {
|
||||
id: 41
|
||||
label: "mouthRollUpper"
|
||||
}
|
||||
entries {
|
||||
id: 42
|
||||
label: "mouthShrugLower"
|
||||
}
|
||||
entries {
|
||||
id: 43
|
||||
label: "mouthShrugUpper"
|
||||
}
|
||||
entries {
|
||||
id: 44
|
||||
label: "mouthSmileLeft"
|
||||
}
|
||||
entries {
|
||||
id: 45
|
||||
label: "mouthSmileRight"
|
||||
}
|
||||
entries {
|
||||
id: 46
|
||||
label: "mouthStretchLeft"
|
||||
}
|
||||
entries {
|
||||
id: 47
|
||||
label: "mouthStretchRight"
|
||||
}
|
||||
entries {
|
||||
id: 48
|
||||
label: "mouthUpperUpLeft"
|
||||
}
|
||||
entries {
|
||||
id: 49
|
||||
label: "mouthUpperUpRight"
|
||||
}
|
||||
entries {
|
||||
id: 50
|
||||
label: "noseSneerLeft"
|
||||
}
|
||||
entries {
|
||||
id: 51
|
||||
label: "noseSneerRight"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
input_stream: "LANDMARKS:landmarks"
|
||||
input_stream: "IMAGE_SIZE:image_size"
|
||||
executor {
|
||||
}
|
||||
3346
mediapipe/tasks/testdata/vision/face_blendshapes_in_landmarks.prototxt
vendored
Normal file
3346
mediapipe/tasks/testdata/vision/face_blendshapes_in_landmarks.prototxt
vendored
Normal file
File diff suppressed because it is too large
Load Diff
260
mediapipe/tasks/testdata/vision/face_blendshapes_out.prototxt
vendored
Normal file
260
mediapipe/tasks/testdata/vision/face_blendshapes_out.prototxt
vendored
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
classification {
|
||||
index: 0
|
||||
score: 3.0398707e-05
|
||||
label: "_neutral"
|
||||
}
|
||||
classification {
|
||||
index: 1
|
||||
score: 0.9075446
|
||||
label: "browDownLeft"
|
||||
}
|
||||
classification {
|
||||
index: 2
|
||||
score: 0.88084334
|
||||
label: "browDownRight"
|
||||
}
|
||||
classification {
|
||||
index: 3
|
||||
score: 8.574165e-05
|
||||
label: "browInnerUp"
|
||||
}
|
||||
classification {
|
||||
index: 4
|
||||
score: 0.00025488975
|
||||
label: "browOuterUpLeft"
|
||||
}
|
||||
classification {
|
||||
index: 5
|
||||
score: 0.0003774864
|
||||
label: "browOuterUpRight"
|
||||
}
|
||||
classification {
|
||||
index: 6
|
||||
score: 0.0013324458
|
||||
label: "cheekPuff"
|
||||
}
|
||||
classification {
|
||||
index: 7
|
||||
score: 3.3356114e-06
|
||||
label: "cheekSquintLeft"
|
||||
}
|
||||
classification {
|
||||
index: 8
|
||||
score: 3.486574e-06
|
||||
label: "cheekSquintRight"
|
||||
}
|
||||
classification {
|
||||
index: 9
|
||||
score: 0.23493542
|
||||
label: "eyeBlinkLeft"
|
||||
}
|
||||
classification {
|
||||
index: 10
|
||||
score: 0.25156906
|
||||
label: "eyeBlinkRight"
|
||||
}
|
||||
classification {
|
||||
index: 11
|
||||
score: 0.31935102
|
||||
label: "eyeLookDownLeft"
|
||||
}
|
||||
classification {
|
||||
index: 12
|
||||
score: 0.3578726
|
||||
label: "eyeLookDownRight"
|
||||
}
|
||||
classification {
|
||||
index: 13
|
||||
score: 0.040764388
|
||||
label: "eyeLookInLeft"
|
||||
}
|
||||
classification {
|
||||
index: 14
|
||||
score: 0.10176937
|
||||
label: "eyeLookInRight"
|
||||
}
|
||||
classification {
|
||||
index: 15
|
||||
score: 0.13951905
|
||||
label: "eyeLookOutLeft"
|
||||
}
|
||||
classification {
|
||||
index: 16
|
||||
score: 0.08599486
|
||||
label: "eyeLookOutRight"
|
||||
}
|
||||
classification {
|
||||
index: 17
|
||||
score: 0.04357939
|
||||
label: "eyeLookUpLeft"
|
||||
}
|
||||
classification {
|
||||
index: 18
|
||||
score: 0.039808873
|
||||
label: "eyeLookUpRight"
|
||||
}
|
||||
classification {
|
||||
index: 19
|
||||
score: 0.71037096
|
||||
label: "eyeSquintLeft"
|
||||
}
|
||||
classification {
|
||||
index: 20
|
||||
score: 0.5613919
|
||||
label: "eyeSquintRight"
|
||||
}
|
||||
classification {
|
||||
index: 21
|
||||
score: 0.0035762936
|
||||
label: "eyeWideLeft"
|
||||
}
|
||||
classification {
|
||||
index: 22
|
||||
score: 0.0046946923
|
||||
label: "eyeWideRight"
|
||||
}
|
||||
classification {
|
||||
index: 23
|
||||
score: 0.0015334645
|
||||
label: "jawForward"
|
||||
}
|
||||
classification {
|
||||
index: 24
|
||||
score: 0.0015781156
|
||||
label: "jawLeft"
|
||||
}
|
||||
classification {
|
||||
index: 25
|
||||
score: 0.1825222
|
||||
label: "jawOpen"
|
||||
}
|
||||
classification {
|
||||
index: 26
|
||||
score: 0.00022096233
|
||||
label: "jawRight"
|
||||
}
|
||||
classification {
|
||||
index: 27
|
||||
score: 0.04180384
|
||||
label: "mouthClose"
|
||||
}
|
||||
classification {
|
||||
index: 28
|
||||
score: 0.03450827
|
||||
label: "mouthDimpleLeft"
|
||||
}
|
||||
classification {
|
||||
index: 29
|
||||
score: 0.02150588
|
||||
label: "mouthDimpleRight"
|
||||
}
|
||||
classification {
|
||||
index: 30
|
||||
score: 0.0004071877
|
||||
label: "mouthFrownLeft"
|
||||
}
|
||||
classification {
|
||||
index: 31
|
||||
score: 0.00041903765
|
||||
label: "mouthFrownRight"
|
||||
}
|
||||
classification {
|
||||
index: 32
|
||||
score: 0.006162456
|
||||
label: "mouthFunnel"
|
||||
}
|
||||
classification {
|
||||
index: 33
|
||||
score: 0.0007518647
|
||||
label: "mouthLeft"
|
||||
}
|
||||
classification {
|
||||
index: 34
|
||||
score: 0.020578554
|
||||
label: "mouthLowerDownLeft"
|
||||
}
|
||||
classification {
|
||||
index: 35
|
||||
score: 0.06688861
|
||||
label: "mouthLowerDownRight"
|
||||
}
|
||||
classification {
|
||||
index: 36
|
||||
score: 0.21384144
|
||||
label: "mouthPressLeft"
|
||||
}
|
||||
classification {
|
||||
index: 37
|
||||
score: 0.12757207
|
||||
label: "mouthPressRight"
|
||||
}
|
||||
classification {
|
||||
index: 38
|
||||
score: 0.0037631108
|
||||
label: "mouthPucker"
|
||||
}
|
||||
classification {
|
||||
index: 39
|
||||
score: 0.0017571376
|
||||
label: "mouthRight"
|
||||
}
|
||||
classification {
|
||||
index: 40
|
||||
score: 0.017397188
|
||||
label: "mouthRollLower"
|
||||
}
|
||||
classification {
|
||||
index: 41
|
||||
score: 0.073125295
|
||||
label: "mouthRollUpper"
|
||||
}
|
||||
classification {
|
||||
index: 42
|
||||
score: 0.009962141
|
||||
label: "mouthShrugLower"
|
||||
}
|
||||
classification {
|
||||
index: 43
|
||||
score: 0.006196939
|
||||
label: "mouthShrugUpper"
|
||||
}
|
||||
classification {
|
||||
index: 44
|
||||
score: 0.9412662
|
||||
label: "mouthSmileLeft"
|
||||
}
|
||||
classification {
|
||||
index: 45
|
||||
score: 0.9020016
|
||||
label: "mouthSmileRight"
|
||||
}
|
||||
classification {
|
||||
index: 46
|
||||
score: 0.054742802
|
||||
label: "mouthStretchLeft"
|
||||
}
|
||||
classification {
|
||||
index: 47
|
||||
score: 0.080990806
|
||||
label: "mouthStretchRight"
|
||||
}
|
||||
classification {
|
||||
index: 48
|
||||
score: 0.7372297
|
||||
label: "mouthUpperUpLeft"
|
||||
}
|
||||
classification {
|
||||
index: 49
|
||||
score: 0.69999576
|
||||
label: "mouthUpperUpRight"
|
||||
}
|
||||
classification {
|
||||
index: 50
|
||||
score: 3.5180938e-07
|
||||
label: "noseSneerLeft"
|
||||
}
|
||||
classification {
|
||||
index: 51
|
||||
score: 8.41376e-06
|
||||
label: "noseSneerRight"
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user