Add mediapipe tasks face blendshapes graph

PiperOrigin-RevId: 509995915
This commit is contained in:
MediaPipe Team 2023-02-15 19:25:05 -08:00 committed by Copybara-Service
parent bdd1c24990
commit ba10ae8410
8 changed files with 5053 additions and 0 deletions

View File

@ -18,6 +18,32 @@ package(default_visibility = [
licenses(["notice"]) licenses(["notice"])
cc_library(
name = "face_blendshapes_graph",
srcs = ["face_blendshapes_graph.cc"],
deps = [
"//mediapipe/calculators/core:split_proto_list_calculator",
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/core:split_vector_calculator_cc_proto",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:landmarks_to_tensor_calculator",
"//mediapipe/calculators/tensor:landmarks_to_tensor_calculator_cc_proto",
"//mediapipe/calculators/tensor:tensors_to_classification_calculator",
"//mediapipe/calculators/tensor:tensors_to_classification_calculator_cc_proto",
"//mediapipe/framework/api2:builder",
"//mediapipe/framework/api2:port",
"//mediapipe/framework/formats:classification_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:tensor",
"//mediapipe/framework/port:status",
"//mediapipe/tasks/cc/core:model_asset_bundle_resources",
"//mediapipe/tasks/cc/core:model_resources_cache",
"//mediapipe/tasks/cc/core:model_task_graph",
"//mediapipe/tasks/cc/vision/face_landmarker/proto:face_blendshapes_graph_options_cc_proto",
],
alwayslink = 1,
)
cc_library( cc_library(
name = "face_landmarks_detector_graph", name = "face_landmarks_detector_graph",
srcs = ["face_landmarks_detector_graph.cc"], srcs = ["face_landmarks_detector_graph.cc"],

View File

@ -0,0 +1,326 @@
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <utility>
#include <vector>
#include "mediapipe/calculators/core/split_vector_calculator.pb.h"
#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.h"
#include "mediapipe/calculators/tensor/landmarks_to_tensor_calculator.pb.h"
#include "mediapipe/calculators/tensor/tensors_to_classification_calculator.pb.h"
#include "mediapipe/framework/api2/builder.h"
#include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/tensor.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/tasks/cc/core/model_task_graph.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h"
namespace mediapipe {
namespace tasks {
namespace vision {
namespace face_landmarker {
namespace {
using ::mediapipe::api2::Input;
using ::mediapipe::api2::LandmarksToTensorCalculator;
using ::mediapipe::api2::Output;
using ::mediapipe::api2::builder::Graph;
using ::mediapipe::api2::builder::Stream;
using ::mediapipe::tasks::vision::face_landmarker::proto::
FaceBlendshapesGraphOptions;
constexpr char kLandmarksTag[] = "LANDMARKS";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr char kBlendshapesTag[] = "BLENDSHAPES";
// Subset of 478 landmarks required for the HUND model.
static constexpr std::array<int, 146> kLandmarksSubsetIdxs = {
0, 1, 4, 5, 6, 7, 8, 10, 13, 14, 17, 21, 33, 37, 39,
40, 46, 52, 53, 54, 55, 58, 61, 63, 65, 66, 67, 70, 78, 80,
81, 82, 84, 87, 88, 91, 93, 95, 103, 105, 107, 109, 127, 132, 133,
136, 144, 145, 146, 148, 149, 150, 152, 153, 154, 155, 157, 158, 159, 160,
161, 162, 163, 168, 172, 173, 176, 178, 181, 185, 191, 195, 197, 234, 246,
249, 251, 263, 267, 269, 270, 276, 282, 283, 284, 285, 288, 291, 293, 295,
296, 297, 300, 308, 310, 311, 312, 314, 317, 318, 321, 323, 324, 332, 334,
336, 338, 356, 361, 362, 365, 373, 374, 375, 377, 378, 379, 380, 381, 382,
384, 385, 386, 387, 388, 389, 390, 397, 398, 400, 402, 405, 409, 415, 454,
466, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477};
static constexpr std::array<absl::string_view, 52> kBlendshapeNames = {
"_neutral",
"browDownLeft",
"browDownRight",
"browInnerUp",
"browOuterUpLeft",
"browOuterUpRight",
"cheekPuff",
"cheekSquintLeft",
"cheekSquintRight",
"eyeBlinkLeft",
"eyeBlinkRight",
"eyeLookDownLeft",
"eyeLookDownRight",
"eyeLookInLeft",
"eyeLookInRight",
"eyeLookOutLeft",
"eyeLookOutRight",
"eyeLookUpLeft",
"eyeLookUpRight",
"eyeSquintLeft",
"eyeSquintRight",
"eyeWideLeft",
"eyeWideRight",
"jawForward",
"jawLeft",
"jawOpen",
"jawRight",
"mouthClose",
"mouthDimpleLeft",
"mouthDimpleRight",
"mouthFrownLeft",
"mouthFrownRight",
"mouthFunnel",
"mouthLeft",
"mouthLowerDownLeft",
"mouthLowerDownRight",
"mouthPressLeft",
"mouthPressRight",
"mouthPucker",
"mouthRight",
"mouthRollLower",
"mouthRollUpper",
"mouthShrugLower",
"mouthShrugUpper",
"mouthSmileLeft",
"mouthSmileRight",
"mouthStretchLeft",
"mouthStretchRight",
"mouthUpperUpLeft",
"mouthUpperUpRight",
"noseSneerLeft",
"noseSneerRight"};
struct FaceBlendshapesOuts {
Stream<ClassificationList> blendshapes;
};
Stream<NormalizedLandmarkList> GetLandmarksSubset(
Stream<NormalizedLandmarkList> landmarks,
const std::array<int, 146>& subset_idxs, Graph& graph) {
auto& node = graph.AddNode("SplitNormalizedLandmarkListCalculator");
auto& opts = node.GetOptions<SplitVectorCalculatorOptions>();
for (int i = 0; i < subset_idxs.size(); ++i) {
auto* range = opts.add_ranges();
range->set_begin(subset_idxs[i]);
range->set_end(subset_idxs[i] + 1);
}
opts.set_combine_outputs(true);
landmarks >> node.In("");
return node.Out("").Cast<NormalizedLandmarkList>();
}
Stream<std::vector<Tensor>> ConvertLandmarksToTensor(
Stream<NormalizedLandmarkList> landmarks,
Stream<std::pair<int, int>> img_size, Graph& graph) {
auto& node = graph.AddNode<LandmarksToTensorCalculator>();
auto& opts = node.GetOptions<LandmarksToTensorCalculatorOptions>();
opts.add_attributes(LandmarksToTensorCalculatorOptions::X);
opts.add_attributes(LandmarksToTensorCalculatorOptions::Y);
opts.set_flatten(false);
landmarks >> node[LandmarksToTensorCalculator::kInNormLandmarkList];
img_size >> node[LandmarksToTensorCalculator::kImageSize];
return node[LandmarksToTensorCalculator::kOutTensors];
}
Stream<std::vector<Tensor>> GetTensorWithBlendshapes(
Stream<std::vector<Tensor>> tensors, Graph& graph) {
auto& node = graph.AddNode("SplitTensorVectorCalculator");
auto& opts = node.GetOptions<SplitVectorCalculatorOptions>();
auto* range = opts.add_ranges();
range->set_begin(0);
range->set_end(1);
opts.set_combine_outputs(true);
tensors >> node.In(0);
return node.Out(0).Cast<std::vector<Tensor>>();
}
Stream<ClassificationList> ConvertTensorToBlendshapes(
Stream<std::vector<Tensor>> tensors,
const std::array<absl::string_view, 52>& labels, Graph& graph) {
auto& node = graph.AddNode("TensorsToClassificationCalculator");
auto& opts = node.GetOptions<TensorsToClassificationCalculatorOptions>();
// Disable top_k and min_score_threshold logic to return coefficients as is.
opts.set_top_k(0);
opts.set_min_score_threshold(-1.0);
for (int i = 0; i < labels.size(); ++i) {
auto* entry = opts.mutable_label_map()->add_entries();
entry->set_id(i);
// TODO: Replace with entry->set_label(labels[i])
entry->mutable_label()->assign(labels[i].data(), labels[i].size());
}
tensors >> node.In("TENSORS");
return node.Out("CLASSIFICATIONS").Cast<ClassificationList>();
}
} // namespace
// Predicts face blendshapes from landmarks.
//
// Inputs:
// LANDMARKS - NormalizedLandmarkList
// 478 2.5D face landmarks predicted by an Attention Mesh model.
// IMAGE_SIZE - std::pair<int, int>
// Input frame size.
//
// Outputs:
// FACE_BLENDSHAPES - ClassificationList
// if EXTRA_FACE_BLENDSHAPES is provided, we have 53 blendshape coeffs
// output; if not, 52 coeffs output.
// All 52 blendshape coefficients:
// 0 - _neutral (ignore it)
// 1 - browDownLeft
// 2 - browDownRight
// 3 - browInnerUp
// 4 - browOuterUpLeft
// 5 - browOuterUpRight
// 6 - cheekPuff
// 7 - cheekSquintLeft
// 8 - cheekSquintRight
// 9 - eyeBlinkLeft
// 10 - eyeBlinkRight
// 11 - eyeLookDownLeft
// 12 - eyeLookDownRight
// 13 - eyeLookInLeft
// 14 - eyeLookInRight
// 15 - eyeLookOutLeft
// 16 - eyeLookOutRight
// 17 - eyeLookUpLeft
// 18 - eyeLookUpRight
// 19 - eyeSquintLeft
// 20 - eyeSquintRight
// 21 - eyeWideLeft
// 22 - eyeWideRight
// 23 - jawForward
// 24 - jawLeft
// 25 - jawOpen
// 26 - jawRight
// 27 - mouthClose
// 28 - mouthDimpleLeft
// 29 - mouthDimpleRight
// 30 - mouthFrownLeft
// 31 - mouthFrownRight
// 32 - mouthFunnel
// 33 - mouthLeft
// 34 - mouthLowerDownLeft
// 35 - mouthLowerDownRight
// 36 - mouthPressLeft
// 37 - mouthPressRight
// 38 - mouthPucker
// 39 - mouthRight
// 40 - mouthRollLower
// 41 - mouthRollUpper
// 42 - mouthShrugLower
// 43 - mouthShrugUpper
// 44 - mouthSmileLeft
// 45 - mouthSmileRight
// 46 - mouthStretchLeft
// 47 - mouthStretchRight
// 48 - mouthUpperUpLeft
// 49 - mouthUpperUpRight
// 50 - noseSneerLeft
// 51 - noseSneerRight
//
// Example:
// node {
// calculator: "mediapipe.tasks.vision.face_landmarker.FaceBlendshapesGraph"
// input_stream: "LANDMARKS:face_landmarks"
// input_stream: "IMAGE_SIZE:image_size"
// output_stream: "BLENDSHAPES:face_blendshapes"
// options {
// [mediapipe.tasks.vision.face_landmarker.proto.FaceBlendshapesGraphOptions.ext]
// {
// base_options {
// model_asset {
// file_name: "face_blendshapes.tflite"
// }
// }
// }
// }
// }
class FaceBlendshapesGraph : public core::ModelTaskGraph {
public:
absl::StatusOr<CalculatorGraphConfig> GetConfig(SubgraphContext* sc) {
ASSIGN_OR_RETURN(const auto* model_resources,
CreateModelResources<FaceBlendshapesGraphOptions>(sc));
Graph graph;
ASSIGN_OR_RETURN(
auto face_blendshapes_outs,
BuildFaceBlendshapesSubgraph(
sc->Options<FaceBlendshapesGraphOptions>(), *model_resources,
graph[Input<NormalizedLandmarkList>(kLandmarksTag)],
graph[Input<std::pair<int, int>>(kImageSizeTag)], graph));
face_blendshapes_outs.blendshapes >>
graph[Output<ClassificationList>(kBlendshapesTag)];
return graph.GetConfig();
}
private:
// Updates graph to predict face blendshapes from landmarks. Returns list of
// blendsahpes.
//
// subgraph_options: the mediapipe tasks module FaceBlendshapesGraphOptions.
// model_resources: the ModelSources object initialized from a face
// blendshapes model file with model metadata.
// landmarks: 478 normalized face landmarks
// img_size: Image size to denormalize landmarks.
// graph: the mediapipe builder::Graph instance to be updated.
absl::StatusOr<FaceBlendshapesOuts> BuildFaceBlendshapesSubgraph(
const FaceBlendshapesGraphOptions& subgraph_options,
const core::ModelResources& model_resources,
Stream<NormalizedLandmarkList> landmarks,
Stream<std::pair<int, int>> img_size, Graph& graph) {
// Take required subset of landmarks.
landmarks = GetLandmarksSubset(landmarks, kLandmarksSubsetIdxs, graph);
// Convert landmarks to input tensor.
auto tensor_in = ConvertLandmarksToTensor(landmarks, img_size, graph);
// Run Blendshapes model.
auto& inference = AddInference(
model_resources, subgraph_options.base_options().acceleration(), graph);
tensor_in >> inference.In("TENSORS");
auto tensors_out = inference.Out("TENSORS").Cast<std::vector<Tensor>>();
// Take output tensor with blendshapes and wrap it in vector.
auto blendshapes_tensor = GetTensorWithBlendshapes(tensors_out, graph);
// Convert tensor to ClassificationList.
auto face_blendshapes =
ConvertTensorToBlendshapes(blendshapes_tensor, kBlendshapeNames, graph);
return FaceBlendshapesOuts{face_blendshapes};
}
};
REGISTER_MEDIAPIPE_GRAPH(
::mediapipe::tasks::vision::face_landmarker::FaceBlendshapesGraph);
} // namespace face_landmarker
} // namespace vision
} // namespace tasks
} // namespace mediapipe

View File

@ -0,0 +1,173 @@
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include <string_view>
#include <utility>
#include "mediapipe/framework/api2/builder.h"
#include "mediapipe/framework/api2/port.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/classification.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/framework/port/parse_text_proto.h"
#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h"
#include "mediapipe/tasks/cc/core/proto/base_options.pb.h"
#include "mediapipe/tasks/cc/core/task_runner.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/proto/face_blendshapes_graph_options.pb.h"
#include "tensorflow/lite/core/shims/cc/shims_test_util.h"
namespace mediapipe {
namespace tasks {
namespace vision {
namespace face_landmarker {
namespace {
using ::file::Defaults;
using ::mediapipe::api2::Input;
using ::mediapipe::api2::Output;
using ::mediapipe::api2::builder::Graph;
using ::mediapipe::file::JoinPath;
using ::mediapipe::tasks::core::TaskRunner;
using ::mediapipe::tasks::vision::face_landmarker::proto::
FaceBlendshapesGraphOptions;
constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/";
constexpr char kFaceBlendshapesModel[] = "face_blendshapes.tflite";
constexpr char kInLandmarks[] = "face_blendshapes_in_landmarks.prototxt";
constexpr char kOutBlendshapes[] = "face_blendshapes_out.prototxt";
constexpr float kSimilarityThreshold = 0.1;
constexpr std::string_view kGeneratedGraph =
"face_blendshapes_generated_graph.pbtxt";
constexpr char kLandmarksTag[] = "LANDMARKS";
constexpr char kLandmarksName[] = "landmarks";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr char kImageSizeName[] = "image_size";
constexpr char kBlendshapesTag[] = "BLENDSHAPES";
constexpr char kBlendshapesName[] = "blendshapes";
absl::StatusOr<CalculatorGraphConfig> ExpandConfig(
const std::string& config_str) {
auto config =
mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(config_str);
CalculatorGraph graph;
MP_RETURN_IF_ERROR(graph.Initialize(config));
return graph.Config();
}
NormalizedLandmarkList GetLandmarks(absl::string_view filename) {
NormalizedLandmarkList landmarks;
MP_EXPECT_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, filename),
&landmarks, Defaults()));
return landmarks;
}
ClassificationList GetBlendshapes(absl::string_view filename) {
ClassificationList blendshapes;
MP_EXPECT_OK(GetTextProto(file::JoinPath("./", kTestDataDirectory, filename),
&blendshapes, Defaults()));
return blendshapes;
}
// Helper function to create a Face Blendshapes TaskRunner.
absl::StatusOr<std::unique_ptr<TaskRunner>> CreateTaskRunner() {
Graph graph;
auto& face_blendshapes_graph = graph.AddNode(
"mediapipe.tasks.vision.face_landmarker.FaceBlendshapesGraph");
auto& options =
face_blendshapes_graph.GetOptions<FaceBlendshapesGraphOptions>();
options.mutable_base_options()->mutable_model_asset()->set_file_name(
JoinPath("./", kTestDataDirectory, kFaceBlendshapesModel));
graph[Input<NormalizedLandmarkList>(kLandmarksTag)].SetName(kLandmarksName) >>
face_blendshapes_graph.In(kLandmarksTag);
graph[Input<std::pair<int, int>>(kImageSizeTag)].SetName(kImageSizeName) >>
face_blendshapes_graph.In(kImageSizeTag);
face_blendshapes_graph.Out(kBlendshapesTag).SetName(kBlendshapesName) >>
graph[Output<ClassificationList>(kBlendshapesTag)];
return TaskRunner::Create(
graph.GetConfig(), absl::make_unique<core::MediaPipeBuiltinOpResolver>());
}
class FaceBlendshapesTest : public tflite_shims::testing::Test {};
TEST_F(FaceBlendshapesTest, SmokeTest) {
// Prepare graph inputs.
auto in_landmarks = GetLandmarks(kInLandmarks);
std::pair<int, int> in_image_size = {820, 1024};
// Run graph.
MP_ASSERT_OK_AND_ASSIGN(auto task_runner, CreateTaskRunner());
auto output_packets = task_runner->Process(
{{kLandmarksName,
MakePacket<NormalizedLandmarkList>(std::move(in_landmarks))},
{kImageSizeName,
MakePacket<std::pair<int, int>>(std::move(in_image_size))}});
MP_ASSERT_OK(output_packets);
// Compare with expected result.
const auto& actual_blendshapes =
(*output_packets)[kBlendshapesName].Get<ClassificationList>();
auto expected_blendshapes = GetBlendshapes(kOutBlendshapes);
EXPECT_THAT(
actual_blendshapes,
testing::proto::Approximately(testing::EqualsProto(expected_blendshapes),
kSimilarityThreshold));
}
TEST(FaceRigGhumGpuTest, VerifyGraph) {
MP_ASSERT_OK_AND_ASSIGN(
auto actual_graph,
ExpandConfig(
R"pb(
node {
calculator: "mediapipe.tasks.vision.face_landmarker.FaceBlendshapesGraph"
input_stream: "LANDMARKS:landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "BLENDSHAPES:blendshapes"
options {
[mediapipe.tasks.vision.face_landmarker.proto
.FaceBlendshapesGraphOptions.ext] {
base_options {
model_asset {
file_name: "mediapipe/tasks/testdata/vision/face_blendshapes.tflite"
}
}
}
}
}
input_stream: "LANDMARKS:landmarks"
input_stream: "IMAGE_SIZE:image_size"
)pb"));
std::string expected_graph_contents;
MP_ASSERT_OK(file::GetContents(
file::JoinPath("./", kTestDataDirectory, kGeneratedGraph),
&expected_graph_contents));
auto expected_graph = mediapipe::ParseTextProtoOrDie<CalculatorGraphConfig>(
expected_graph_contents);
EXPECT_THAT(actual_graph, testing::proto::IgnoringRepeatedFieldOrdering(
testing::EqualsProto(expected_graph)));
}
} // namespace
} // namespace face_landmarker
} // namespace vision
} // namespace tasks
} // namespace mediapipe

View File

@ -20,6 +20,16 @@ package(default_visibility = [
licenses(["notice"]) licenses(["notice"])
mediapipe_proto_library(
name = "face_blendshapes_graph_options_proto",
srcs = ["face_blendshapes_graph_options.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
"//mediapipe/tasks/cc/core/proto:base_options_proto",
],
)
mediapipe_proto_library( mediapipe_proto_library(
name = "face_landmarks_detector_graph_options_proto", name = "face_landmarks_detector_graph_options_proto",
srcs = ["face_landmarks_detector_graph_options.proto"], srcs = ["face_landmarks_detector_graph_options.proto"],

View File

@ -0,0 +1,34 @@
/* Copyright 2023 The MediaPipe Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
syntax = "proto2";
package mediapipe.tasks.vision.face_landmarker.proto;
import "mediapipe/framework/calculator.proto";
import "mediapipe/framework/calculator_options.proto";
import "mediapipe/tasks/cc/core/proto/base_options.proto";
option java_package = "com.google.mediapipe.tasks.vision.facelandmarker.proto";
option java_outer_classname = "FaceBlendshapesGraphOptionsProto";
message FaceBlendshapesGraphOptions {
extend mediapipe.CalculatorOptions {
optional FaceBlendshapesGraphOptions ext = 508981768;
}
// Base options for configuring Task library, such as specifying the TfLite
// model file with metadata, accelerator options, etc.
optional core.proto.BaseOptions base_options = 1;
}

View File

@ -0,0 +1,878 @@
node {
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__SplitNormalizedLandmarkListCalculator"
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "landmarks"
output_stream: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_2"
options {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges {
begin: 0
end: 1
}
ranges {
begin: 1
end: 2
}
ranges {
begin: 4
end: 5
}
ranges {
begin: 5
end: 6
}
ranges {
begin: 6
end: 7
}
ranges {
begin: 7
end: 8
}
ranges {
begin: 8
end: 9
}
ranges {
begin: 10
end: 11
}
ranges {
begin: 13
end: 14
}
ranges {
begin: 14
end: 15
}
ranges {
begin: 17
end: 18
}
ranges {
begin: 21
end: 22
}
ranges {
begin: 33
end: 34
}
ranges {
begin: 37
end: 38
}
ranges {
begin: 39
end: 40
}
ranges {
begin: 40
end: 41
}
ranges {
begin: 46
end: 47
}
ranges {
begin: 52
end: 53
}
ranges {
begin: 53
end: 54
}
ranges {
begin: 54
end: 55
}
ranges {
begin: 55
end: 56
}
ranges {
begin: 58
end: 59
}
ranges {
begin: 61
end: 62
}
ranges {
begin: 63
end: 64
}
ranges {
begin: 65
end: 66
}
ranges {
begin: 66
end: 67
}
ranges {
begin: 67
end: 68
}
ranges {
begin: 70
end: 71
}
ranges {
begin: 78
end: 79
}
ranges {
begin: 80
end: 81
}
ranges {
begin: 81
end: 82
}
ranges {
begin: 82
end: 83
}
ranges {
begin: 84
end: 85
}
ranges {
begin: 87
end: 88
}
ranges {
begin: 88
end: 89
}
ranges {
begin: 91
end: 92
}
ranges {
begin: 93
end: 94
}
ranges {
begin: 95
end: 96
}
ranges {
begin: 103
end: 104
}
ranges {
begin: 105
end: 106
}
ranges {
begin: 107
end: 108
}
ranges {
begin: 109
end: 110
}
ranges {
begin: 127
end: 128
}
ranges {
begin: 132
end: 133
}
ranges {
begin: 133
end: 134
}
ranges {
begin: 136
end: 137
}
ranges {
begin: 144
end: 145
}
ranges {
begin: 145
end: 146
}
ranges {
begin: 146
end: 147
}
ranges {
begin: 148
end: 149
}
ranges {
begin: 149
end: 150
}
ranges {
begin: 150
end: 151
}
ranges {
begin: 152
end: 153
}
ranges {
begin: 153
end: 154
}
ranges {
begin: 154
end: 155
}
ranges {
begin: 155
end: 156
}
ranges {
begin: 157
end: 158
}
ranges {
begin: 158
end: 159
}
ranges {
begin: 159
end: 160
}
ranges {
begin: 160
end: 161
}
ranges {
begin: 161
end: 162
}
ranges {
begin: 162
end: 163
}
ranges {
begin: 163
end: 164
}
ranges {
begin: 168
end: 169
}
ranges {
begin: 172
end: 173
}
ranges {
begin: 173
end: 174
}
ranges {
begin: 176
end: 177
}
ranges {
begin: 178
end: 179
}
ranges {
begin: 181
end: 182
}
ranges {
begin: 185
end: 186
}
ranges {
begin: 191
end: 192
}
ranges {
begin: 195
end: 196
}
ranges {
begin: 197
end: 198
}
ranges {
begin: 234
end: 235
}
ranges {
begin: 246
end: 247
}
ranges {
begin: 249
end: 250
}
ranges {
begin: 251
end: 252
}
ranges {
begin: 263
end: 264
}
ranges {
begin: 267
end: 268
}
ranges {
begin: 269
end: 270
}
ranges {
begin: 270
end: 271
}
ranges {
begin: 276
end: 277
}
ranges {
begin: 282
end: 283
}
ranges {
begin: 283
end: 284
}
ranges {
begin: 284
end: 285
}
ranges {
begin: 285
end: 286
}
ranges {
begin: 288
end: 289
}
ranges {
begin: 291
end: 292
}
ranges {
begin: 293
end: 294
}
ranges {
begin: 295
end: 296
}
ranges {
begin: 296
end: 297
}
ranges {
begin: 297
end: 298
}
ranges {
begin: 300
end: 301
}
ranges {
begin: 308
end: 309
}
ranges {
begin: 310
end: 311
}
ranges {
begin: 311
end: 312
}
ranges {
begin: 312
end: 313
}
ranges {
begin: 314
end: 315
}
ranges {
begin: 317
end: 318
}
ranges {
begin: 318
end: 319
}
ranges {
begin: 321
end: 322
}
ranges {
begin: 323
end: 324
}
ranges {
begin: 324
end: 325
}
ranges {
begin: 332
end: 333
}
ranges {
begin: 334
end: 335
}
ranges {
begin: 336
end: 337
}
ranges {
begin: 338
end: 339
}
ranges {
begin: 356
end: 357
}
ranges {
begin: 361
end: 362
}
ranges {
begin: 362
end: 363
}
ranges {
begin: 365
end: 366
}
ranges {
begin: 373
end: 374
}
ranges {
begin: 374
end: 375
}
ranges {
begin: 375
end: 376
}
ranges {
begin: 377
end: 378
}
ranges {
begin: 378
end: 379
}
ranges {
begin: 379
end: 380
}
ranges {
begin: 380
end: 381
}
ranges {
begin: 381
end: 382
}
ranges {
begin: 382
end: 383
}
ranges {
begin: 384
end: 385
}
ranges {
begin: 385
end: 386
}
ranges {
begin: 386
end: 387
}
ranges {
begin: 387
end: 388
}
ranges {
begin: 388
end: 389
}
ranges {
begin: 389
end: 390
}
ranges {
begin: 390
end: 391
}
ranges {
begin: 397
end: 398
}
ranges {
begin: 398
end: 399
}
ranges {
begin: 400
end: 401
}
ranges {
begin: 402
end: 403
}
ranges {
begin: 405
end: 406
}
ranges {
begin: 409
end: 410
}
ranges {
begin: 415
end: 416
}
ranges {
begin: 454
end: 455
}
ranges {
begin: 466
end: 467
}
ranges {
begin: 468
end: 469
}
ranges {
begin: 469
end: 470
}
ranges {
begin: 470
end: 471
}
ranges {
begin: 471
end: 472
}
ranges {
begin: 472
end: 473
}
ranges {
begin: 473
end: 474
}
ranges {
begin: 474
end: 475
}
ranges {
begin: 475
end: 476
}
ranges {
begin: 476
end: 477
}
ranges {
begin: 477
end: 478
}
combine_outputs: true
}
}
}
node {
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__LandmarksToTensorCalculator"
calculator: "LandmarksToTensorCalculator"
input_stream: "IMAGE_SIZE:image_size"
input_stream: "NORM_LANDMARKS:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_2"
output_stream: "TENSORS:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_3"
options {
[mediapipe.LandmarksToTensorCalculatorOptions.ext] {
attributes: X
attributes: Y
flatten: false
}
}
}
node {
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph__ModelResourcesCalculator"
calculator: "ModelResourcesCalculator"
output_side_packet: "METADATA_EXTRACTOR:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_1"
output_side_packet: "MODEL:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_2"
output_side_packet: "OP_RESOLVER:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_3"
options {
[mediapipe.tasks.core.proto.ModelResourcesCalculatorOptions.ext] {
model_file {
file_name: "mediapipe/tasks/testdata/vision/face_blendshapes.tflite"
}
}
}
}
node {
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph__inferencecalculator__mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph__InferenceCalculator"
calculator: "InferenceCalculatorCpu"
input_stream: "TENSORS:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_3"
output_stream: "TENSORS:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_4"
input_side_packet: "MODEL:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_2"
input_side_packet: "OP_RESOLVER:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__mediapipe_tasks_core_inferencesubgraph____side_packet_3"
options {
[mediapipe.InferenceCalculatorOptions.ext] {
delegate {
}
}
}
}
node {
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__SplitTensorVectorCalculator"
calculator: "SplitTensorVectorCalculator"
input_stream: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_4"
output_stream: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_5"
options {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges {
begin: 0
end: 1
}
combine_outputs: true
}
}
}
node {
name: "mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph__TensorsToClassificationCalculator"
calculator: "TensorsToClassificationCalculator"
input_stream: "TENSORS:mediapipe_tasks_vision_face_landmarker_faceblendshapesgraph____stream_5"
output_stream: "CLASSIFICATIONS:blendshapes"
options {
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {
min_score_threshold: -1
top_k: 0
label_map {
entries {
id: 0
label: "_neutral"
}
entries {
id: 1
label: "browDownLeft"
}
entries {
id: 2
label: "browDownRight"
}
entries {
id: 3
label: "browInnerUp"
}
entries {
id: 4
label: "browOuterUpLeft"
}
entries {
id: 5
label: "browOuterUpRight"
}
entries {
id: 6
label: "cheekPuff"
}
entries {
id: 7
label: "cheekSquintLeft"
}
entries {
id: 8
label: "cheekSquintRight"
}
entries {
id: 9
label: "eyeBlinkLeft"
}
entries {
id: 10
label: "eyeBlinkRight"
}
entries {
id: 11
label: "eyeLookDownLeft"
}
entries {
id: 12
label: "eyeLookDownRight"
}
entries {
id: 13
label: "eyeLookInLeft"
}
entries {
id: 14
label: "eyeLookInRight"
}
entries {
id: 15
label: "eyeLookOutLeft"
}
entries {
id: 16
label: "eyeLookOutRight"
}
entries {
id: 17
label: "eyeLookUpLeft"
}
entries {
id: 18
label: "eyeLookUpRight"
}
entries {
id: 19
label: "eyeSquintLeft"
}
entries {
id: 20
label: "eyeSquintRight"
}
entries {
id: 21
label: "eyeWideLeft"
}
entries {
id: 22
label: "eyeWideRight"
}
entries {
id: 23
label: "jawForward"
}
entries {
id: 24
label: "jawLeft"
}
entries {
id: 25
label: "jawOpen"
}
entries {
id: 26
label: "jawRight"
}
entries {
id: 27
label: "mouthClose"
}
entries {
id: 28
label: "mouthDimpleLeft"
}
entries {
id: 29
label: "mouthDimpleRight"
}
entries {
id: 30
label: "mouthFrownLeft"
}
entries {
id: 31
label: "mouthFrownRight"
}
entries {
id: 32
label: "mouthFunnel"
}
entries {
id: 33
label: "mouthLeft"
}
entries {
id: 34
label: "mouthLowerDownLeft"
}
entries {
id: 35
label: "mouthLowerDownRight"
}
entries {
id: 36
label: "mouthPressLeft"
}
entries {
id: 37
label: "mouthPressRight"
}
entries {
id: 38
label: "mouthPucker"
}
entries {
id: 39
label: "mouthRight"
}
entries {
id: 40
label: "mouthRollLower"
}
entries {
id: 41
label: "mouthRollUpper"
}
entries {
id: 42
label: "mouthShrugLower"
}
entries {
id: 43
label: "mouthShrugUpper"
}
entries {
id: 44
label: "mouthSmileLeft"
}
entries {
id: 45
label: "mouthSmileRight"
}
entries {
id: 46
label: "mouthStretchLeft"
}
entries {
id: 47
label: "mouthStretchRight"
}
entries {
id: 48
label: "mouthUpperUpLeft"
}
entries {
id: 49
label: "mouthUpperUpRight"
}
entries {
id: 50
label: "noseSneerLeft"
}
entries {
id: 51
label: "noseSneerRight"
}
}
}
}
}
input_stream: "LANDMARKS:landmarks"
input_stream: "IMAGE_SIZE:image_size"
executor {
}

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,260 @@
classification {
index: 0
score: 3.0398707e-05
label: "_neutral"
}
classification {
index: 1
score: 0.9075446
label: "browDownLeft"
}
classification {
index: 2
score: 0.88084334
label: "browDownRight"
}
classification {
index: 3
score: 8.574165e-05
label: "browInnerUp"
}
classification {
index: 4
score: 0.00025488975
label: "browOuterUpLeft"
}
classification {
index: 5
score: 0.0003774864
label: "browOuterUpRight"
}
classification {
index: 6
score: 0.0013324458
label: "cheekPuff"
}
classification {
index: 7
score: 3.3356114e-06
label: "cheekSquintLeft"
}
classification {
index: 8
score: 3.486574e-06
label: "cheekSquintRight"
}
classification {
index: 9
score: 0.23493542
label: "eyeBlinkLeft"
}
classification {
index: 10
score: 0.25156906
label: "eyeBlinkRight"
}
classification {
index: 11
score: 0.31935102
label: "eyeLookDownLeft"
}
classification {
index: 12
score: 0.3578726
label: "eyeLookDownRight"
}
classification {
index: 13
score: 0.040764388
label: "eyeLookInLeft"
}
classification {
index: 14
score: 0.10176937
label: "eyeLookInRight"
}
classification {
index: 15
score: 0.13951905
label: "eyeLookOutLeft"
}
classification {
index: 16
score: 0.08599486
label: "eyeLookOutRight"
}
classification {
index: 17
score: 0.04357939
label: "eyeLookUpLeft"
}
classification {
index: 18
score: 0.039808873
label: "eyeLookUpRight"
}
classification {
index: 19
score: 0.71037096
label: "eyeSquintLeft"
}
classification {
index: 20
score: 0.5613919
label: "eyeSquintRight"
}
classification {
index: 21
score: 0.0035762936
label: "eyeWideLeft"
}
classification {
index: 22
score: 0.0046946923
label: "eyeWideRight"
}
classification {
index: 23
score: 0.0015334645
label: "jawForward"
}
classification {
index: 24
score: 0.0015781156
label: "jawLeft"
}
classification {
index: 25
score: 0.1825222
label: "jawOpen"
}
classification {
index: 26
score: 0.00022096233
label: "jawRight"
}
classification {
index: 27
score: 0.04180384
label: "mouthClose"
}
classification {
index: 28
score: 0.03450827
label: "mouthDimpleLeft"
}
classification {
index: 29
score: 0.02150588
label: "mouthDimpleRight"
}
classification {
index: 30
score: 0.0004071877
label: "mouthFrownLeft"
}
classification {
index: 31
score: 0.00041903765
label: "mouthFrownRight"
}
classification {
index: 32
score: 0.006162456
label: "mouthFunnel"
}
classification {
index: 33
score: 0.0007518647
label: "mouthLeft"
}
classification {
index: 34
score: 0.020578554
label: "mouthLowerDownLeft"
}
classification {
index: 35
score: 0.06688861
label: "mouthLowerDownRight"
}
classification {
index: 36
score: 0.21384144
label: "mouthPressLeft"
}
classification {
index: 37
score: 0.12757207
label: "mouthPressRight"
}
classification {
index: 38
score: 0.0037631108
label: "mouthPucker"
}
classification {
index: 39
score: 0.0017571376
label: "mouthRight"
}
classification {
index: 40
score: 0.017397188
label: "mouthRollLower"
}
classification {
index: 41
score: 0.073125295
label: "mouthRollUpper"
}
classification {
index: 42
score: 0.009962141
label: "mouthShrugLower"
}
classification {
index: 43
score: 0.006196939
label: "mouthShrugUpper"
}
classification {
index: 44
score: 0.9412662
label: "mouthSmileLeft"
}
classification {
index: 45
score: 0.9020016
label: "mouthSmileRight"
}
classification {
index: 46
score: 0.054742802
label: "mouthStretchLeft"
}
classification {
index: 47
score: 0.080990806
label: "mouthStretchRight"
}
classification {
index: 48
score: 0.7372297
label: "mouthUpperUpLeft"
}
classification {
index: 49
score: 0.69999576
label: "mouthUpperUpRight"
}
classification {
index: 50
score: 3.5180938e-07
label: "noseSneerLeft"
}
classification {
index: 51
score: 8.41376e-06
label: "noseSneerRight"
}