cassdlcm/mediapipe-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

git ignore mediapipe and update readme to use symlink

Browse Source
This commit is contained in:
Jules Youngberg 2022-06-20 19:55:03 -07:00
parent bc2c8bb86c
commit c470510a19
212 changed files with 3 additions and 29077 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -9,3 +9,4 @@ Cargo.lock
# These are backup files generated by rustfmt
**/*.rs.bk
/refs/
mediapipe

5
README.md
View File

@ -47,11 +47,10 @@ Add the following to your dependencies list in `Cargo.toml`:
mediapipe = { git = "https://github.com/julesyoungberg/mediapipe-rs" }
```
Mediapipe relies on tflite files which must be available at `./mediapipe/modules/`. The easiest way to do this is
Mediapipe relies on tflite files which must be available at `./mediapipe/modules/`. The easiest way to satisfy this is by creating a symbolic link to mediapipe. Run the following command from the project directory.
```shell
mkdir mediapipe
cp -R ../mediapipe/mediapipe/modules ./mediapipe/modules
ln -s ../mediapipe/mediapipe .
```
The path to mediapipe may be different depending on where you have cloned it to.

18
mediapipe/modules/README.md
View File

@ -1,18 +0,0 @@
# Modules
Each module (represented as a subfolder) provides subgraphs and corresponding resources (e.g. tflite models) to perform domain-specific tasks (e.g. detect faces, detect face landmarks).
*Modules listed below are already used in some of `mediapipe/graphs` and more graphs are being migrated to use existing and upcoming modules.*
| Module | Description |
| :--- | :--- |
| [`face_detection`](face_detection/README.md) | Subgraphs to detect faces. |
| [`face_geometry`](face_geometry/README.md) | Subgraphs to extract face geometry. |
| [`face_landmark`](face_landmark/README.md) | Subgraphs to detect and track face landmarks. |
| [`hand_landmark`](hand_landmark/README.md) | Subgraphs to detect and track hand landmarks. |
| [`holistic_landmark`](holistic_landmark/README.md) | Subgraphs to detect and track holistic pose which consists of pose, face and hand landmarks. |
| [`iris_landmark`](iris_landmark/README.md) | Subgraphs to detect iris landmarks. |
| [`palm_detection`](palm_detection/README.md) | Subgraphs to detect palms/hands. |
| [`pose_detection`](pose_detection/README.md) | Subgraphs to detect poses. |
| [`pose_landmark`](pose_landmark/README.md) | Subgraphs to detect and track pose landmarks. |
| [`objectron`](objectron/README.md) | Subgraphs to detect and track 3D objects. |

150
mediapipe/modules/face_detection/BUILD
View File

@ -1,150 +0,0 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"mediapipe_simple_subgraph",
)
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
mediapipe_simple_subgraph(
name = "face_detection_short_range_by_roi_cpu",
graph = "face_detection_short_range_by_roi_cpu.pbtxt",
register_as = "FaceDetectionShortRangeByRoiCpu",
deps = [
":face_detection_short_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_short_range_by_roi_gpu",
graph = "face_detection_short_range_by_roi_gpu.pbtxt",
register_as = "FaceDetectionShortRangeByRoiGpu",
deps = [
":face_detection_short_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_short_range_cpu",
graph = "face_detection_short_range_cpu.pbtxt",
register_as = "FaceDetectionShortRangeCpu",
deps = [
":face_detection_short_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_short_range_gpu",
graph = "face_detection_short_range_gpu.pbtxt",
register_as = "FaceDetectionShortRangeGpu",
deps = [
":face_detection_short_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_short_range_common",
graph = "face_detection_short_range_common.pbtxt",
register_as = "FaceDetectionShortRangeCommon",
deps = [
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/util:detection_projection_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_full_range_cpu",
graph = "face_detection_full_range_cpu.pbtxt",
register_as = "FaceDetectionFullRangeCpu",
deps = [
":face_detection_full_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_full_range_gpu",
graph = "face_detection_full_range_gpu.pbtxt",
register_as = "FaceDetectionFullRangeGpu",
deps = [
":face_detection_full_range_common",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/util:to_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_full_range_common",
graph = "face_detection_full_range_common.pbtxt",
register_as = "FaceDetectionFullRangeCommon",
deps = [
"//mediapipe/calculators/tensor:tensors_to_detections_calculator",
"//mediapipe/calculators/tflite:ssd_anchors_calculator",
"//mediapipe/calculators/util:detection_projection_calculator",
"//mediapipe/calculators/util:non_max_suppression_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_short_range_image",
graph = "face_detection_short_range_image.pbtxt",
register_as = "FaceDetectionShortRangeImage",
deps = [
":face_detection_short_range_common",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_full_range_image",
graph = "face_detection_full_range_image.pbtxt",
register_as = "FaceDetectionFullRangeImage",
deps = [
":face_detection_full_range_common",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
],
)
exports_files(
srcs = [
"face_detection_full_range.tflite",
"face_detection_full_range_sparse.tflite",
"face_detection_short_range.tflite",
],
)

8
mediapipe/modules/face_detection/README.md
View File

@ -1,8 +0,0 @@
# face_detection
Subgraphs|Details
:--- | :---
[`FaceDetectionFullRangeCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_full_range_cpu.pbtxt)| Detects faces. Works best for faces within 5 meters from the camera. (CPU input, and inference is executed on CPU.)
[`FaceDetectionFullRangeGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_full_range_gpu.pbtxt)| Detects faces. Works best for faces within 5 meters from the camera. (GPU input, and inference is executed on GPU.)
[`FaceDetectionShortRangeCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_short_range_cpu.pbtxt)| Detects faces. Works best for faces within 2 meters from the camera. (CPU input, and inference is executed on CPU.)
[`FaceDetectionShortRangeGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_detection/face_detection_short_range_gpu.pbtxt)| Detects faces. Works best for faces within 2 meters from the camera. (GPU input, and inference is executed on GPU.)

BIN
mediapipe/modules/face_detection/face_detection_full_range.tflite
View File

Binary file not shown.

102
mediapipe/modules/face_detection/face_detection_full_range_common.pbtxt
View File

@ -1,102 +0,0 @@
# MediaPipe graph performing common processing to detect faces using
# face_detection_full_range_sparse.tflite model, currently consisting of tensor
# post processing.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFullRangeCommon"
# input_stream: "TENSORS:detection_tensors"
# input_stream: "MATRIX:transform_matrix"
# output_stream: "DETECTIONS:detections"
# }
type: "FaceDetectionShortRangeCommon"
# Detection tensors. (std::vector<Tensor>)
input_stream: "TENSORS:detection_tensors"
# A 4x4 row-major-order matrix that maps a point represented in the detection
# tensors to a desired coordinate system, e.g., in the original input image
# before scaling/cropping. (std::array<float, 16>)
input_stream: "MATRIX:transform_matrix"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 1
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 192
input_size_width: 192
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 4
aspect_ratios: 1.0
fixed_anchor_size: true
interpolated_scale_aspect_ratio: 0.0
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:unfiltered_detections"
options: {
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 2304
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 192.0
y_scale: 192.0
h_scale: 192.0
w_scale: 192.0
min_score_thresh: 0.6
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "unfiltered_detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
}
}
}
# Projects the detections from input tensor to the corresponding locations on
# the original image (input to the graph).
node {
calculator: "DetectionProjectionCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "PROJECTION_MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}

80
mediapipe/modules/face_detection/face_detection_full_range_cpu.pbtxt
View File

@ -1,80 +0,0 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_full_range_sparse.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFullRangeCpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionFullRangeCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input CPU image (ImageFrame) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_CPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms the input image into a 192x192 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 192
output_tensor_height: 192
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
}
}
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
delegate {
xnnpack {}
}
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFullRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}

80
mediapipe/modules/face_detection/face_detection_full_range_gpu.pbtxt
View File

@ -1,80 +0,0 @@
# MediaPipe graph to detect faces. (GPU input, and inference is executed on
# GPU.)
#
# It is required that "face_detection_full_range_sparse.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionFullRangeGpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionFullRangeGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input GPU image (GpuBuffer) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 192
output_tensor_height: 192
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
gpu_origin: TOP_LEFT
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
#
delegate: { gpu { use_advanced_gpu_api: true } }
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFullRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}

86
mediapipe/modules/face_detection/face_detection_full_range_image.pbtxt
View File

@ -1,86 +0,0 @@
# MediaPipe graph to detect faces. (GPU/CPU input, and inference is executed on
# GPU.)
#
# It is required that "face_detection_full_range_sparse.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
# path during execution.
type: "FaceDetectionFullRangeImage"
# Image. (Image)
input_stream: "IMAGE:image"
# The throttled input image. (Image)
output_stream: "IMAGE:throttled_image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
node {
calculator: "FlowLimiterCalculator"
input_stream: "image"
input_stream: "FINISHED:detections"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_image"
options: {
[mediapipe.FlowLimiterCalculatorOptions.ext] {
max_in_flight: 1
max_in_queue: 1
}
}
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:throttled_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 192
output_tensor_height: 192
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
gpu_origin: CONVENTIONAL
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
# TODO: Use GraphOptions to modify the delegate field to be
# `delegate { xnnpack {} }` for the CPU only use cases.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite"
#
delegate: { gpu { use_advanced_gpu_api: true } }
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionFullRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}

BIN
mediapipe/modules/face_detection/face_detection_full_range_sparse.tflite
View File

Binary file not shown.

BIN
mediapipe/modules/face_detection/face_detection_short_range.tflite
View File

Binary file not shown.

83
mediapipe/modules/face_detection/face_detection_short_range_by_roi_cpu.pbtxt
View File

@ -1,83 +0,0 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRangeByRoiCpu"
# input_stream: "IMAGE:image"
# input_stream: "ROI:roi"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionShortRangeByRoiCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where faces should be
# detected. (NormalizedRect)
input_stream: "ROI:roi"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input CPU image (ImageFrame) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_CPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms specified region of image into 128x128 tensor keeping aspect ratio
# (padding tensor if needed).
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
input_stream: "NORM_RECT:roi"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
output_tensor_height: 128
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
}
}
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
delegate { xnnpack {} }
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}

83
mediapipe/modules/face_detection/face_detection_short_range_by_roi_gpu.pbtxt
View File

@ -1,83 +0,0 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRangeByRoiGpu"
# input_stream: "IMAGE:image"
# input_stream: "ROI:roi"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionShortRangeByRoiGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where faces should be
# detected. (NormalizedRect)
input_stream: "ROI:roi"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input GPU image (GpuBuffer) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms specified region of image into 128x128 tensor keeping aspect ratio
# (padding tensor if needed).
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
input_stream: "NORM_RECT:roi"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
output_tensor_height: 128
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
gpu_origin: TOP_LEFT
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}

103
mediapipe/modules/face_detection/face_detection_short_range_common.pbtxt
View File

@ -1,103 +0,0 @@
# MediaPipe graph performing common processing to detect faces, currently
# consisting of tensor post processing.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRangeCommon"
# input_stream: "TENSORS:detection_tensors"
# input_stream: "MATRIX:transform_matrix"
# output_stream: "DETECTIONS:detections"
# }
type: "FaceDetectionShortRangeCommon"
# Detection tensors. (std::vector<Tensor>)
input_stream: "TENSORS:detection_tensors"
# A 4x4 row-major-order matrix that maps a point represented in the detection
# tensors to a desired coordinate system, e.g., in the original input image
# before scaling/cropping. (std::array<float, 16>)
input_stream: "MATRIX:transform_matrix"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Generates a single side packet containing a vector of SSD anchors based on
# the specification in the options.
node {
calculator: "SsdAnchorsCalculator"
output_side_packet: "anchors"
options: {
[mediapipe.SsdAnchorsCalculatorOptions.ext] {
num_layers: 4
min_scale: 0.1484375
max_scale: 0.75
input_size_height: 128
input_size_width: 128
anchor_offset_x: 0.5
anchor_offset_y: 0.5
strides: 8
strides: 16
strides: 16
strides: 16
aspect_ratios: 1.0
fixed_anchor_size: true
}
}
}
# Decodes the detection tensors generated by the TensorFlow Lite model, based on
# the SSD anchors and the specification in the options, into a vector of
# detections. Each detection describes a detected object.
node {
calculator: "TensorsToDetectionsCalculator"
input_stream: "TENSORS:detection_tensors"
input_side_packet: "ANCHORS:anchors"
output_stream: "DETECTIONS:unfiltered_detections"
options: {
[mediapipe.TensorsToDetectionsCalculatorOptions.ext] {
num_classes: 1
num_boxes: 896
num_coords: 16
box_coord_offset: 0
keypoint_coord_offset: 4
num_keypoints: 6
num_values_per_keypoint: 2
sigmoid_score: true
score_clipping_thresh: 100.0
reverse_output_order: true
x_scale: 128.0
y_scale: 128.0
h_scale: 128.0
w_scale: 128.0
min_score_thresh: 0.5
}
}
}
# Performs non-max suppression to remove excessive detections.
node {
calculator: "NonMaxSuppressionCalculator"
input_stream: "unfiltered_detections"
output_stream: "filtered_detections"
options: {
[mediapipe.NonMaxSuppressionCalculatorOptions.ext] {
min_suppression_threshold: 0.3
overlap_type: INTERSECTION_OVER_UNION
algorithm: WEIGHTED
}
}
}
# Projects the detections from input tensor to the corresponding locations on
# the original image (input to the graph).
node {
calculator: "DetectionProjectionCalculator"
input_stream: "DETECTIONS:filtered_detections"
input_stream: "PROJECTION_MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}

78
mediapipe/modules/face_detection/face_detection_short_range_cpu.pbtxt
View File

@ -1,78 +0,0 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRangeCpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionShortRangeCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input CPU image (ImageFrame) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_CPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
output_tensor_height: 128
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
}
}
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
delegate { xnnpack {} }
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}

78
mediapipe/modules/face_detection/face_detection_short_range_gpu.pbtxt
View File

@ -1,78 +0,0 @@
# MediaPipe graph to detect faces. (CPU input, and inference is executed on
# CPU.)
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRangeGpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionShortRangeGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
# Converts the input GPU image (GpuBuffer) to the multi-backend image type
# (Image).
node: {
calculator: "ToImageCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "IMAGE:multi_backend_image"
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:multi_backend_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
output_tensor_height: 128
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
gpu_origin: TOP_LEFT
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}

94
mediapipe/modules/face_detection/face_detection_short_range_image.pbtxt
View File

@ -1,94 +0,0 @@
# MediaPipe graph to detect faces. (GPU/CPU input, and inference is executed on
# GPU.)
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceDetectionShortRangeCpu"
# input_stream: "IMAGE:image"
# output_stream: "DETECTIONS:face_detections"
# }
type: "FaceDetectionShortRangeCpu"
# Image. (Image)
input_stream: "IMAGE:image"
# The throttled input image. (Image)
output_stream: "IMAGE:throttled_image"
# Detected faces. (std::vector<Detection>)
# NOTE: there will not be an output packet in the DETECTIONS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "DETECTIONS:detections"
node {
calculator: "FlowLimiterCalculator"
input_stream: "image"
input_stream: "FINISHED:detections"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_image"
options: {
[mediapipe.FlowLimiterCalculatorOptions.ext] {
max_in_flight: 1
max_in_queue: 1
}
}
}
# Transforms the input image into a 128x128 tensor while keeping the aspect
# ratio (what is expected by the corresponding face detection model), resulting
# in potential letterboxing in the transformed image.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:throttled_image"
output_stream: "TENSORS:input_tensors"
output_stream: "MATRIX:transform_matrix"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 128
output_tensor_height: 128
keep_aspect_ratio: true
output_tensor_float_range {
min: -1.0
max: 1.0
}
border_mode: BORDER_ZERO
gpu_origin: CONVENTIONAL
}
}
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
# TODO: Use GraphOptions to modify the delegate field to be
# `delegate { xnnpack {} }` for the CPU only use cases.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
output_stream: "TENSORS:detection_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
model_path: "mediapipe/modules/face_detection/face_detection_short_range.tflite"
#
delegate: { gpu { use_advanced_gpu_api: true } }
}
}
}
# Performs tensor post processing to generate face detections.
node {
calculator: "FaceDetectionShortRangeCommon"
input_stream: "TENSORS:detection_tensors"
input_stream: "MATRIX:transform_matrix"
output_stream: "DETECTIONS:detections"
}

137
mediapipe/modules/face_geometry/BUILD
View File

@ -1,137 +0,0 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
load("//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_simple_subgraph")
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
mediapipe_simple_subgraph(
name = "face_geometry",
graph = "face_geometry.pbtxt",
register_as = "FaceGeometry",
deps = [
":geometry_pipeline_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_geometry_from_detection",
graph = "face_geometry_from_detection.pbtxt",
register_as = "FaceGeometryFromDetection",
deps = [
":geometry_pipeline_calculator",
"//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:end_loop_calculator",
"//mediapipe/calculators/util:detection_to_landmarks_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_geometry_from_landmarks",
graph = "face_geometry_from_landmarks.pbtxt",
register_as = "FaceGeometryFromLandmarks",
deps = [
":geometry_pipeline_calculator",
],
)
mediapipe_proto_library(
name = "effect_renderer_calculator_proto",
srcs = ["effect_renderer_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
],
)
cc_library(
name = "effect_renderer_calculator",
srcs = ["effect_renderer_calculator.cc"],
deps = [
":effect_renderer_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgcodecs",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gl_calculator_helper",
"//mediapipe/gpu:gpu_buffer",
"//mediapipe/modules/face_geometry/libs:effect_renderer",
"//mediapipe/modules/face_geometry/libs:validation_utils",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
"//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
"//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
"//mediapipe/util:resource_util",
"@com_google_absl//absl/types:optional",
],
alwayslink = 1,
)
mediapipe_proto_library(
name = "env_generator_calculator_proto",
srcs = ["env_generator_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/modules/face_geometry/protos:environment_proto",
],
)
cc_library(
name = "env_generator_calculator",
srcs = ["env_generator_calculator.cc"],
deps = [
":env_generator_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/port:status",
"//mediapipe/modules/face_geometry/libs:validation_utils",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
],
alwayslink = 1,
)
mediapipe_proto_library(
name = "geometry_pipeline_calculator_proto",
srcs = ["geometry_pipeline_calculator.proto"],
deps = [
"//mediapipe/framework:calculator_options_proto",
],
)
cc_library(
name = "geometry_pipeline_calculator",
srcs = ["geometry_pipeline_calculator.cc"],
deps = [
":geometry_pipeline_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/modules/face_geometry/libs:geometry_pipeline",
"//mediapipe/modules/face_geometry/libs:validation_utils",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
"//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_cc_proto",
"//mediapipe/util:resource_util",
"@com_google_absl//absl/memory",
],
alwayslink = 1,
)

20
mediapipe/modules/face_geometry/README.md
View File

@ -1,20 +0,0 @@
# face_geometry
Protos|Details
:--- | :---
[`face_geometry.Environment`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/environment.proto)| Describes an environment; includes the camera frame origin point location as well as virtual camera parameters.
[`face_geometry.GeometryPipelineMetadata`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.proto)| Describes metadata needed to estimate face geometry based on the face landmark module result.
[`face_geometry.FaceGeometry`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/face_geometry.proto)| Describes geometry data for a single face; includes a face mesh surface and a face pose in a given environment.
[`face_geometry.Mesh3d`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/protos/mesh_3d.proto)| Describes a 3D mesh surface.
Calculators|Details
:--- | :---
[`FaceGeometryEnvGeneratorCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/env_generator_calculator.cc)| Generates an environment that describes a virtual scene.
[`FaceGeometryPipelineCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc)| Extracts face geometry for multiple faces from a vector of landmark lists.
[`FaceGeometryEffectRendererCalculator`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/effect_renderer_calculator.cc)| Renders a face effect.
Subgraphs|Details
:--- | :---
[`FaceGeometryFromDetection`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_from_detection.pbtxt)| Extracts geometry from face detection for multiple faces.
[`FaceGeometryFromLandmarks`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry_from_landmarks.pbtxt)| Extracts geometry from face landmarks for multiple faces.
[`FaceGeometry`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_geometry/face_geometry.pbtxt)| Extracts geometry from face landmarks for multiple faces. Deprecated, please use `FaceGeometryFromLandmarks` in the new code.

59
mediapipe/modules/face_geometry/data/BUILD
View File

@ -1,59 +0,0 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework:encode_binary_proto.bzl", "encode_binary_proto")
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
encode_binary_proto(
name = "geometry_pipeline_metadata_detection",
input = "geometry_pipeline_metadata_detection.pbtxt",
message_type = "mediapipe.face_geometry.GeometryPipelineMetadata",
output = "geometry_pipeline_metadata_detection.binarypb",
deps = [
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_proto",
],
)
encode_binary_proto(
name = "geometry_pipeline_metadata_landmarks",
input = "geometry_pipeline_metadata_landmarks.pbtxt",
message_type = "mediapipe.face_geometry.GeometryPipelineMetadata",
output = "geometry_pipeline_metadata_landmarks.binarypb",
deps = [
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_proto",
],
)
# For backward-compatibility reasons, generate `geometry_pipeline_metadata.binarypb` from
# the `geometry_pipeline_metadata_landmarks.pbtxt` definition.
encode_binary_proto(
name = "geometry_pipeline_metadata",
input = "geometry_pipeline_metadata_landmarks.pbtxt",
message_type = "mediapipe.face_geometry.GeometryPipelineMetadata",
output = "geometry_pipeline_metadata.binarypb",
deps = [
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_proto",
],
)
# These canonical face model files are not meant to be used in runtime, but rather for asset
# creation and/or reference.
exports_files([
"canonical_face_model.fbx",
"canonical_face_model.obj",
"canonical_face_model_uv_visualization.png",
])

BIN
mediapipe/modules/face_geometry/data/canonical_face_model.fbx
View File

Binary file not shown.

1834
mediapipe/modules/face_geometry/data/canonical_face_model.obj
View File

File diff suppressed because it is too large Load Diff

BIN
mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 731 KiB

78
mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_detection.pbtxt
View File

@ -1,78 +0,0 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
input_source: FACE_DETECTION_PIPELINE
procrustes_landmark_basis { landmark_id: 0 weight: 1.0 }
procrustes_landmark_basis { landmark_id: 1 weight: 1.0 }
procrustes_landmark_basis { landmark_id: 2 weight: 1.0 }
procrustes_landmark_basis { landmark_id: 3 weight: 1.0 }
procrustes_landmark_basis { landmark_id: 4 weight: 1.0 }
procrustes_landmark_basis { landmark_id: 5 weight: 1.0 }
# NOTE: the triangular topology of the face meshes is only useful when derived
# from the 468 face landmarks, not from the 6 face detection landmarks
# (keypoints). The former don't cover the entire face and this mesh is
# defined here only to comply with the API. It should be considered as
# a placeholder and/or for debugging purposes.
#
# Use the face geometry derived from the face detection landmarks
# (keypoints) for the face pose transformation matrix, not the mesh.
canonical_mesh: {
vertex_type: VERTEX_PT
primitive_type: TRIANGLE
vertex_buffer: -3.1511454582214355
vertex_buffer: 2.6246179342269897
vertex_buffer: 3.4656630754470825
vertex_buffer: 0.349575996398926
vertex_buffer: 0.38137748837470997
vertex_buffer: 3.1511454582214355
vertex_buffer: 2.6246179342269897
vertex_buffer: 3.4656630754470825
vertex_buffer: 0.650443494319916
vertex_buffer: 0.38137999176979054
vertex_buffer: 0.0
vertex_buffer: -1.126865029335022
vertex_buffer: 7.475604057312012
vertex_buffer: 0.500025987625122
vertex_buffer: 0.547487020492554
vertex_buffer: 0.0
vertex_buffer: -4.304508209228516
vertex_buffer: 4.162498950958252
vertex_buffer: 0.499989986419678
vertex_buffer: 0.694203019142151
vertex_buffer: -7.664182186126709
vertex_buffer: 0.673132002353668
vertex_buffer: -2.435867071151733
vertex_buffer: 0.007561000064015
vertex_buffer: 0.480777025222778
vertex_buffer: 7.664182186126709
vertex_buffer: 0.673132002353668
vertex_buffer: -2.435867071151733
vertex_buffer: 0.992439985275269
vertex_buffer: 0.480777025222778
index_buffer: 0
index_buffer: 1
index_buffer: 2
index_buffer: 1
index_buffer: 5
index_buffer: 2
index_buffer: 4
index_buffer: 0
index_buffer: 2
index_buffer: 4
index_buffer: 2
index_buffer: 3
index_buffer: 2
index_buffer: 5
index_buffer: 3
}

5086
mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_landmarks.pbtxt
View File

File diff suppressed because it is too large Load Diff

284
mediapipe/modules/face_geometry/effect_renderer_calculator.cc
View File

@ -1,284 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include <vector>
#include "absl/types/optional.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/image_frame_opencv.h"
#include "mediapipe/framework/port/opencv_core_inc.h" // NOTYPO
#include "mediapipe/framework/port/opencv_imgcodecs_inc.h" // NOTYPO
#include "mediapipe/framework/port/opencv_imgproc_inc.h" // NOTYPO
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/modules/face_geometry/effect_renderer_calculator.pb.h"
#include "mediapipe/modules/face_geometry/libs/effect_renderer.h"
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
#include "mediapipe/util/resource_util.h"
namespace mediapipe {
namespace {
static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
static constexpr char kImageGpuTag[] = "IMAGE_GPU";
static constexpr char kMultiFaceGeometryTag[] = "MULTI_FACE_GEOMETRY";
// A calculator that renders a visual effect for multiple faces.
//
// Inputs:
// IMAGE_GPU (`GpuBuffer`, required):
// A buffer containing input image.
//
// MULTI_FACE_GEOMETRY (`std::vector<face_geometry::FaceGeometry>`, optional):
// A vector of face geometry data.
//
// If absent, the input GPU buffer is copied over into the output GPU buffer
// without any effect being rendered.
//
// Input side packets:
// ENVIRONMENT (`face_geometry::Environment`, required)
// Describes an environment; includes the camera frame origin point location
// as well as virtual camera parameters.
//
// Output:
// IMAGE_GPU (`GpuBuffer`, required):
// A buffer with a visual effect being rendered for multiple faces.
//
// Options:
// effect_texture_path (`string`, required):
// Defines a path for the visual effect texture file. The effect texture is
// later rendered on top of the effect mesh.
//
// The texture file format must be supported by the OpenCV image decoder. It
// must also define either an RGB or an RGBA texture.
//
// effect_mesh_3d_path (`string`, optional):
// Defines a path for the visual effect mesh 3D file. The effect mesh is
// later "attached" to the face and is driven by the face pose
// transformation matrix.
//
// The mesh 3D file format must be the binary `face_geometry.Mesh3d` proto.
//
// If is not present, the runtime face mesh will be used as the effect mesh
// - this mode is handy for facepaint effects.
//
class EffectRendererCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc) {
MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc))
<< "Failed to update contract for the GPU helper!";
cc->InputSidePackets()
.Tag(kEnvironmentTag)
.Set<face_geometry::Environment>();
cc->Inputs().Tag(kImageGpuTag).Set<GpuBuffer>();
cc->Inputs()
.Tag(kMultiFaceGeometryTag)
.Set<std::vector<face_geometry::FaceGeometry>>();
cc->Outputs().Tag(kImageGpuTag).Set<GpuBuffer>();
return mediapipe::GlCalculatorHelper::UpdateContract(cc);
}
absl::Status Open(CalculatorContext* cc) override {
cc->SetOffset(mediapipe::TimestampDiff(0));
MP_RETURN_IF_ERROR(gpu_helper_.Open(cc))
<< "Failed to open the GPU helper!";
return gpu_helper_.RunInGlContext([&]() -> absl::Status {
const auto& options =
cc->Options<FaceGeometryEffectRendererCalculatorOptions>();
const auto& environment = cc->InputSidePackets()
.Tag(kEnvironmentTag)
.Get<face_geometry::Environment>();
MP_RETURN_IF_ERROR(face_geometry::ValidateEnvironment(environment))
<< "Invalid environment!";
absl::optional<face_geometry::Mesh3d> effect_mesh_3d;
if (options.has_effect_mesh_3d_path()) {
ASSIGN_OR_RETURN(effect_mesh_3d,
ReadMesh3dFromFile(options.effect_mesh_3d_path()),
_ << "Failed to read the effect 3D mesh from file!");
MP_RETURN_IF_ERROR(face_geometry::ValidateMesh3d(*effect_mesh_3d))
<< "Invalid effect 3D mesh!";
}
ASSIGN_OR_RETURN(ImageFrame effect_texture,
ReadTextureFromFile(options.effect_texture_path()),
_ << "Failed to read the effect texture from file!");
ASSIGN_OR_RETURN(effect_renderer_,
CreateEffectRenderer(environment, effect_mesh_3d,
std::move(effect_texture)),
_ << "Failed to create the effect renderer!");
return absl::OkStatus();
});
}
absl::Status Process(CalculatorContext* cc) override {
// The `IMAGE_GPU` stream is required to have a non-empty packet. In case
// this requirement is not met, there's nothing to be processed at the
// current timestamp.
if (cc->Inputs().Tag(kImageGpuTag).IsEmpty()) {
return absl::OkStatus();
}
return gpu_helper_.RunInGlContext([this, cc]() -> absl::Status {
const auto& input_gpu_buffer =
cc->Inputs().Tag(kImageGpuTag).Get<GpuBuffer>();
GlTexture input_gl_texture =
gpu_helper_.CreateSourceTexture(input_gpu_buffer);
GlTexture output_gl_texture = gpu_helper_.CreateDestinationTexture(
input_gl_texture.width(), input_gl_texture.height());
std::vector<face_geometry::FaceGeometry> empty_multi_face_geometry;
const auto& multi_face_geometry =
cc->Inputs().Tag(kMultiFaceGeometryTag).IsEmpty()
? empty_multi_face_geometry
: cc->Inputs()
.Tag(kMultiFaceGeometryTag)
.Get<std::vector<face_geometry::FaceGeometry>>();
// Validate input multi face geometry data.
for (const face_geometry::FaceGeometry& face_geometry :
multi_face_geometry) {
MP_RETURN_IF_ERROR(face_geometry::ValidateFaceGeometry(face_geometry))
<< "Invalid face geometry!";
}
MP_RETURN_IF_ERROR(effect_renderer_->RenderEffect(
multi_face_geometry, input_gl_texture.width(),
input_gl_texture.height(), input_gl_texture.target(),
input_gl_texture.name(), output_gl_texture.target(),
output_gl_texture.name()))
<< "Failed to render the effect!";
std::unique_ptr<GpuBuffer> output_gpu_buffer =
output_gl_texture.GetFrame<GpuBuffer>();
cc->Outputs()
.Tag(kImageGpuTag)
.AddPacket(mediapipe::Adopt<GpuBuffer>(output_gpu_buffer.release())
.At(cc->InputTimestamp()));
output_gl_texture.Release();
input_gl_texture.Release();
return absl::OkStatus();
});
}
~EffectRendererCalculator() {
gpu_helper_.RunInGlContext([this]() { effect_renderer_.reset(); });
}
private:
static absl::StatusOr<ImageFrame> ReadTextureFromFile(
const std::string& texture_path) {
ASSIGN_OR_RETURN(std::string texture_blob,
ReadContentBlobFromFile(texture_path),
_ << "Failed to read texture blob from file!");
// Use OpenCV image decoding functionality to finish reading the texture.
std::vector<char> texture_blob_vector(texture_blob.begin(),
texture_blob.end());
cv::Mat decoded_mat =
cv::imdecode(texture_blob_vector, cv::IMREAD_UNCHANGED);
RET_CHECK(decoded_mat.type() == CV_8UC3 || decoded_mat.type() == CV_8UC4)
<< "Texture must have `char` as the underlying type and "
"must have either 3 or 4 channels!";
ImageFormat::Format image_format = ImageFormat::UNKNOWN;
cv::Mat output_mat;
switch (decoded_mat.channels()) {
case 3:
image_format = ImageFormat::SRGB;
cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGR2RGB);
break;
case 4:
image_format = ImageFormat::SRGBA;
cv::cvtColor(decoded_mat, output_mat, cv::COLOR_BGRA2RGBA);
break;
default:
RET_CHECK_FAIL()
<< "Unexpected number of channels; expected 3 or 4, got "
<< decoded_mat.channels() << "!";
}
ImageFrame output_image_frame(image_format, output_mat.size().width,
output_mat.size().height,
ImageFrame::kGlDefaultAlignmentBoundary);
output_mat.copyTo(formats::MatView(&output_image_frame));
return output_image_frame;
}
static absl::StatusOr<face_geometry::Mesh3d> ReadMesh3dFromFile(
const std::string& mesh_3d_path) {
ASSIGN_OR_RETURN(std::string mesh_3d_blob,
ReadContentBlobFromFile(mesh_3d_path),
_ << "Failed to read mesh 3D blob from file!");
face_geometry::Mesh3d mesh_3d;
RET_CHECK(mesh_3d.ParseFromString(mesh_3d_blob))
<< "Failed to parse a mesh 3D proto from a binary blob!";
return mesh_3d;
}
static absl::StatusOr<std::string> ReadContentBlobFromFile(
const std::string& unresolved_path) {
ASSIGN_OR_RETURN(std::string resolved_path,
mediapipe::PathToResourceAsFile(unresolved_path),
_ << "Failed to resolve path! Path = " << unresolved_path);
std::string content_blob;
MP_RETURN_IF_ERROR(
mediapipe::GetResourceContents(resolved_path, &content_blob))
<< "Failed to read content blob! Resolved path = " << resolved_path;
return content_blob;
}
mediapipe::GlCalculatorHelper gpu_helper_;
std::unique_ptr<face_geometry::EffectRenderer> effect_renderer_;
};
} // namespace
using FaceGeometryEffectRendererCalculator = EffectRendererCalculator;
REGISTER_CALCULATOR(FaceGeometryEffectRendererCalculator);
} // namespace mediapipe

46
mediapipe/modules/face_geometry/effect_renderer_calculator.proto
View File

@ -1,46 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator_options.proto";
message FaceGeometryEffectRendererCalculatorOptions {
extend CalculatorOptions {
optional FaceGeometryEffectRendererCalculatorOptions ext = 323693808;
}
// Defines a path for the visual effect texture file. The effect texture is
// later rendered on top of the effect mesh.
//
// Please be aware about the difference between the CPU texture memory layout
// and the GPU texture sampler coordinate space. This renderer follows
// conventions discussed here: https://open.gl/textures
//
// The texture file format must be supported by the OpenCV image decoder. It
// must also define either an RGB or an RGBA texture.
optional string effect_texture_path = 1;
// Defines a path for the visual effect mesh 3D file. The effect mesh is later
// "attached" to the face and is driven by the face pose transformation
// matrix.
//
// The mesh 3D file format must be the binary `face_system.Mesh3d` proto.
//
// If is not present, the runtime face mesh will be used as the effect mesh
// - this mode is handy for facepaint effects.
optional string effect_mesh_3d_path = 2;
}

81
mediapipe/modules/face_geometry/env_generator_calculator.cc
View File

@ -1,81 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/modules/face_geometry/env_generator_calculator.pb.h"
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
namespace mediapipe {
namespace {
static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
// A calculator that generates an environment, which describes a virtual scene.
//
// Output side packets:
// ENVIRONMENT (`face_geometry::Environment`, required)
// Describes an environment; includes the camera frame origin point location
// as well as virtual camera parameters.
//
// Options:
// environment (`face_geometry.Environment`, required):
// Defines an environment to be packed as the output side packet.
//
// Must be valid (for details, please refer to the proto message definition
// comments and/or `modules/face_geometry/libs/validation_utils.h/cc`)
//
class EnvGeneratorCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc) {
cc->OutputSidePackets()
.Tag(kEnvironmentTag)
.Set<face_geometry::Environment>();
return absl::OkStatus();
}
absl::Status Open(CalculatorContext* cc) override {
cc->SetOffset(mediapipe::TimestampDiff(0));
const face_geometry::Environment& environment =
cc->Options<FaceGeometryEnvGeneratorCalculatorOptions>().environment();
MP_RETURN_IF_ERROR(face_geometry::ValidateEnvironment(environment))
<< "Invalid environment!";
cc->OutputSidePackets()
.Tag(kEnvironmentTag)
.Set(mediapipe::MakePacket<face_geometry::Environment>(environment));
return absl::OkStatus();
}
absl::Status Process(CalculatorContext* cc) override {
return absl::OkStatus();
}
absl::Status Close(CalculatorContext* cc) override {
return absl::OkStatus();
}
};
} // namespace
using FaceGeometryEnvGeneratorCalculator = EnvGeneratorCalculator;
REGISTER_CALCULATOR(FaceGeometryEnvGeneratorCalculator);
} // namespace mediapipe

32
mediapipe/modules/face_geometry/env_generator_calculator.proto
View File

@ -1,32 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator_options.proto";
import "mediapipe/modules/face_geometry/protos/environment.proto";
message FaceGeometryEnvGeneratorCalculatorOptions {
extend CalculatorOptions {
optional FaceGeometryEnvGeneratorCalculatorOptions ext = 323693810;
}
// Defines an environment to be packed as the output side packet.
//
// Must be valid (for details, please refer to the proto message definition
// comments and/or `modules/face_geometry/libs/validation_utils.h/cc`)
optional face_geometry.Environment environment = 1;
}

48
mediapipe/modules/face_geometry/face_geometry.pbtxt
View File

@ -1,48 +0,0 @@
# MediaPipe graph to extract geometry from face landmarks for multiple faces.
#
# It is required that "geometry_pipeline_metadata.binarypb" is available at
# "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata.binarypb"
# path during execution.
#
# This is a deprecated subgraph kept for backward-compatibility reasons. Please,
# be explicit and use the `FaceGeometryFromLandmarks` subgraph in the new code
# to enable the same runtime behaviour.
type: "FaceGeometry"
# The size of the input frame. The first element of the pair is the frame width;
# the other one is the frame height.
#
# The face landmarks should have been detected on a frame with the same
# ratio. If used as-is, the resulting face geometry visualization should be
# happening on a frame with the same ratio as well.
#
# (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# Collection of detected/predicted faces, each represented as a list of face
# landmarks. (std::vector<NormalizedLandmarkList>)
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
# Environment that describes the current virtual scene.
# (face_geometry::Environment)
input_side_packet: "ENVIRONMENT:environment"
# A list of geometry data for each detected face.
# (std::vector<face_geometry::FaceGeometry>)
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# Extracts face geometry for multiple faces from a vector of face landmark
# lists.
node {
calculator: "FaceGeometryPipelineCalculator"
input_side_packet: "ENVIRONMENT:environment"
input_stream: "IMAGE_SIZE:image_size"
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
options: {
[mediapipe.FaceGeometryPipelineCalculatorOptions.ext] {
metadata_path: "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata.binarypb"
}
}
}

87
mediapipe/modules/face_geometry/face_geometry_from_detection.pbtxt
View File

@ -1,87 +0,0 @@
# MediaPipe graph to extract geometry from face detection for multiple faces.
#
# It is required that "geometry_pipeline_metadata_detection.binarypb" is
# available at
# "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_detection.binarypb"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceGeometryFromDetection"
# input_stream: "IMAGE_SIZE:image_size"
# input_stream: "MULTI_FACE_DETECTION:multi_face_detection"
# input_side_packet: "ENVIRONMENT:environment"
# output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# }
type: "FaceGeometryFromDetection"
# The size of the input frame. The first element of the pair is the frame width;
# the other one is the frame height.
#
# The face landmarks should have been detected on a frame with the same
# ratio. If used as-is, the resulting face geometry visualization should be
# happening on a frame with the same ratio as well.
#
# (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# Collection of detected/predicted faces, each represented as a detection.
# (std::vector<DETECTION>)
input_stream: "MULTI_FACE_DETECTION:multi_face_detection"
# Environment that describes the current virtual scene.
# (face_geometry::Environment)
input_side_packet: "ENVIRONMENT:environment"
# A list of geometry data for each detected face.
# (std::vector<face_geometry::FaceGeometry>)
#
# NOTE: the triangular topology of the face meshes is only useful when derived
# from the 468 face landmarks, not from the 6 face detection landmarks
# (keypoints). The former don't cover the entire face and this mesh is
# defined here only to comply with the API. It should be considered as
# a placeholder and/or for debugging purposes.
#
# Use the face geometry derived from the face detection landmarks
# (keypoints) for the face pose transformation matrix, not the mesh.
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# Begin iterating over a vector of the face detections.
node {
calculator: "BeginLoopDetectionCalculator"
input_stream: "ITERABLE:multi_face_detection"
output_stream: "ITEM:face_detection"
output_stream: "BATCH_END:detection_timestamp"
}
# Extracts face detection keypoints as a normalized landmarks.
node {
calculator: "DetectionToLandmarksCalculator"
input_stream: "DETECTION:face_detection"
output_stream: "LANDMARKS:face_landmarks"
}
# End iterating over a vector of the face detections and receive a vector of
# face landmark lists as a result.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:face_landmarks"
input_stream: "BATCH_END:detection_timestamp"
output_stream: "ITERABLE:multi_face_landmarks"
}
# Extracts face geometry for multiple faces from a vector of face detection
# landmark lists.
node {
calculator: "FaceGeometryPipelineCalculator"
input_side_packet: "ENVIRONMENT:environment"
input_stream: "IMAGE_SIZE:image_size"
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
options: {
[mediapipe.FaceGeometryPipelineCalculatorOptions.ext] {
metadata_path: "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_detection.binarypb"
}
}
}

54
mediapipe/modules/face_geometry/face_geometry_from_landmarks.pbtxt
View File

@ -1,54 +0,0 @@
# MediaPipe graph to extract geometry from face landmarks for multiple faces.
#
# It is required that "geometry_pipeline_metadata_from_landmark.binarypb" is
# available at
# "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_from_landmarks.binarypb"
# path during execution.
#
# EXAMPLE:
# node {
# calculator: "FaceGeometryFromLandmarks"
# input_stream: "IMAGE_SIZE:image_size"
# input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
# input_side_packet: "ENVIRONMENT:environment"
# output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# }
type: "FaceGeometryFromLandmarks"
# The size of the input frame. The first element of the pair is the frame width;
# the other one is the frame height.
#
# The face landmarks should have been detected on a frame with the same
# ratio. If used as-is, the resulting face geometry visualization should be
# happening on a frame with the same ratio as well.
#
# (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# Collection of detected/predicted faces, each represented as a list of face
# landmarks. (std::vector<NormalizedLandmarkList>)
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
# Environment that describes the current virtual scene.
# (face_geometry::Environment)
input_side_packet: "ENVIRONMENT:environment"
# A list of geometry data for each detected face.
# (std::vector<face_geometry::FaceGeometry>)
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
# Extracts face geometry for multiple faces from a vector of face landmark
# lists.
node {
calculator: "FaceGeometryPipelineCalculator"
input_side_packet: "ENVIRONMENT:environment"
input_stream: "IMAGE_SIZE:image_size"
input_stream: "MULTI_FACE_LANDMARKS:multi_face_landmarks"
output_stream: "MULTI_FACE_GEOMETRY:multi_face_geometry"
options: {
[mediapipe.FaceGeometryPipelineCalculatorOptions.ext] {
metadata_path: "mediapipe/modules/face_geometry/data/geometry_pipeline_metadata_landmarks.binarypb"
}
}
}

197
mediapipe/modules/face_geometry/geometry_pipeline_calculator.cc
View File

@ -1,197 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "absl/memory/memory.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/modules/face_geometry/geometry_pipeline_calculator.pb.h"
#include "mediapipe/modules/face_geometry/libs/geometry_pipeline.h"
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
#include "mediapipe/util/resource_util.h"
namespace mediapipe {
namespace {
static constexpr char kEnvironmentTag[] = "ENVIRONMENT";
static constexpr char kImageSizeTag[] = "IMAGE_SIZE";
static constexpr char kMultiFaceGeometryTag[] = "MULTI_FACE_GEOMETRY";
static constexpr char kMultiFaceLandmarksTag[] = "MULTI_FACE_LANDMARKS";
// A calculator that renders a visual effect for multiple faces.
//
// Inputs:
// IMAGE_SIZE (`std::pair<int, int>`, required):
// The size of the current frame. The first element of the pair is the frame
// width; the other one is the frame height.
//
// The face landmarks should have been detected on a frame with the same
// ratio. If used as-is, the resulting face geometry visualization should be
// happening on a frame with the same ratio as well.
//
// MULTI_FACE_LANDMARKS (`std::vector<NormalizedLandmarkList>`, required):
// A vector of face landmark lists.
//
// Input side packets:
// ENVIRONMENT (`face_geometry::Environment`, required)
// Describes an environment; includes the camera frame origin point location
// as well as virtual camera parameters.
//
// Output:
// MULTI_FACE_GEOMETRY (`std::vector<face_geometry::FaceGeometry>`, required):
// A vector of face geometry data.
//
// Options:
// metadata_path (`string`, optional):
// Defines a path for the geometry pipeline metadata file.
//
// The geometry pipeline metadata file format must be the binary
// `face_geometry.GeometryPipelineMetadata` proto.
//
class GeometryPipelineCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc) {
cc->InputSidePackets()
.Tag(kEnvironmentTag)
.Set<face_geometry::Environment>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Inputs()
.Tag(kMultiFaceLandmarksTag)
.Set<std::vector<NormalizedLandmarkList>>();
cc->Outputs()
.Tag(kMultiFaceGeometryTag)
.Set<std::vector<face_geometry::FaceGeometry>>();
return absl::OkStatus();
}
absl::Status Open(CalculatorContext* cc) override {
cc->SetOffset(mediapipe::TimestampDiff(0));
const auto& options = cc->Options<FaceGeometryPipelineCalculatorOptions>();
ASSIGN_OR_RETURN(
face_geometry::GeometryPipelineMetadata metadata,
ReadMetadataFromFile(options.metadata_path()),
_ << "Failed to read the geometry pipeline metadata from file!");
MP_RETURN_IF_ERROR(
face_geometry::ValidateGeometryPipelineMetadata(metadata))
<< "Invalid geometry pipeline metadata!";
const face_geometry::Environment& environment =
cc->InputSidePackets()
.Tag(kEnvironmentTag)
.Get<face_geometry::Environment>();
MP_RETURN_IF_ERROR(face_geometry::ValidateEnvironment(environment))
<< "Invalid environment!";
ASSIGN_OR_RETURN(
geometry_pipeline_,
face_geometry::CreateGeometryPipeline(environment, metadata),
_ << "Failed to create a geometry pipeline!");
return absl::OkStatus();
}
absl::Status Process(CalculatorContext* cc) override {
// Both the `IMAGE_SIZE` and the `MULTI_FACE_LANDMARKS` streams are required
// to have a non-empty packet. In case this requirement is not met, there's
// nothing to be processed at the current timestamp.
if (cc->Inputs().Tag(kImageSizeTag).IsEmpty() ||
cc->Inputs().Tag(kMultiFaceLandmarksTag).IsEmpty()) {
return absl::OkStatus();
}
const auto& image_size =
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
const auto& multi_face_landmarks =
cc->Inputs()
.Tag(kMultiFaceLandmarksTag)
.Get<std::vector<NormalizedLandmarkList>>();
auto multi_face_geometry =
absl::make_unique<std::vector<face_geometry::FaceGeometry>>();
ASSIGN_OR_RETURN(
*multi_face_geometry,
geometry_pipeline_->EstimateFaceGeometry(
multi_face_landmarks, //
/*frame_width*/ image_size.first,
/*frame_height*/ image_size.second),
_ << "Failed to estimate face geometry for multiple faces!");
cc->Outputs()
.Tag(kMultiFaceGeometryTag)
.AddPacket(mediapipe::Adopt<std::vector<face_geometry::FaceGeometry>>(
multi_face_geometry.release())
.At(cc->InputTimestamp()));
return absl::OkStatus();
}
absl::Status Close(CalculatorContext* cc) override {
return absl::OkStatus();
}
private:
static absl::StatusOr<face_geometry::GeometryPipelineMetadata>
ReadMetadataFromFile(const std::string& metadata_path) {
ASSIGN_OR_RETURN(std::string metadata_blob,
ReadContentBlobFromFile(metadata_path),
_ << "Failed to read a metadata blob from file!");
face_geometry::GeometryPipelineMetadata metadata;
RET_CHECK(metadata.ParseFromString(metadata_blob))
<< "Failed to parse a metadata proto from a binary blob!";
return metadata;
}
static absl::StatusOr<std::string> ReadContentBlobFromFile(
const std::string& unresolved_path) {
ASSIGN_OR_RETURN(std::string resolved_path,
mediapipe::PathToResourceAsFile(unresolved_path),
_ << "Failed to resolve path! Path = " << unresolved_path);
std::string content_blob;
MP_RETURN_IF_ERROR(
mediapipe::GetResourceContents(resolved_path, &content_blob))
<< "Failed to read content blob! Resolved path = " << resolved_path;
return content_blob;
}
std::unique_ptr<face_geometry::GeometryPipeline> geometry_pipeline_;
};
} // namespace
using FaceGeometryPipelineCalculator = GeometryPipelineCalculator;
REGISTER_CALCULATOR(FaceGeometryPipelineCalculator);
} // namespace mediapipe

27
mediapipe/modules/face_geometry/geometry_pipeline_calculator.proto
View File

@ -1,27 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator_options.proto";
message FaceGeometryPipelineCalculatorOptions {
extend CalculatorOptions {
optional FaceGeometryPipelineCalculatorOptions ext = 323693812;
}
optional string metadata_path = 1;
}

103
mediapipe/modules/face_geometry/libs/BUILD
View File

@ -1,103 +0,0 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
cc_library(
name = "effect_renderer",
srcs = ["effect_renderer.cc"],
hdrs = ["effect_renderer.h"],
deps = [
":mesh_3d_utils",
":validation_utils",
"//mediapipe/framework/formats:image_format_cc_proto",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/gpu:gl_base",
"//mediapipe/gpu:shader_util",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
"//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
"//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
"@com_google_absl//absl/memory",
"@com_google_absl//absl/types:optional",
],
)
cc_library(
name = "geometry_pipeline",
srcs = ["geometry_pipeline.cc"],
hdrs = ["geometry_pipeline.h"],
deps = [
":mesh_3d_utils",
":procrustes_solver",
":validation_utils",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
"//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_cc_proto",
"//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
"@com_google_absl//absl/memory",
"@eigen_archive//:eigen3",
],
)
cc_library(
name = "mesh_3d_utils",
srcs = ["mesh_3d_utils.cc"],
hdrs = ["mesh_3d_utils.h"],
deps = [
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:statusor",
"//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
],
)
cc_library(
name = "procrustes_solver",
srcs = ["procrustes_solver.cc"],
hdrs = ["procrustes_solver.h"],
deps = [
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/framework/port:statusor",
"@com_google_absl//absl/memory",
"@eigen_archive//:eigen3",
],
)
cc_library(
name = "validation_utils",
srcs = ["validation_utils.cc"],
hdrs = ["validation_utils.h"],
deps = [
":mesh_3d_utils",
"//mediapipe/framework/formats:matrix_data_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
"//mediapipe/modules/face_geometry/protos:environment_cc_proto",
"//mediapipe/modules/face_geometry/protos:face_geometry_cc_proto",
"//mediapipe/modules/face_geometry/protos:geometry_pipeline_metadata_cc_proto",
"//mediapipe/modules/face_geometry/protos:mesh_3d_cc_proto",
],
)

733
mediapipe/modules/face_geometry/libs/effect_renderer.cc
View File

@ -1,733 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/modules/face_geometry/libs/effect_renderer.h"
#include <cmath>
#include <cstdint>
#include <limits>
#include <memory>
#include <utility>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/types/optional.h"
#include "mediapipe/framework/formats/image_format.pb.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/formats/matrix_data.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/gpu/shader_util.h"
#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
namespace {
struct RenderableMesh3d {
static absl::StatusOr<RenderableMesh3d> CreateFromProtoMesh3d(
const Mesh3d& proto_mesh_3d) {
Mesh3d::VertexType vertex_type = proto_mesh_3d.vertex_type();
RenderableMesh3d renderable_mesh_3d;
renderable_mesh_3d.vertex_size = GetVertexSize(vertex_type);
ASSIGN_OR_RETURN(
renderable_mesh_3d.vertex_position_size,
GetVertexComponentSize(vertex_type, VertexComponent::POSITION),
_ << "Failed to get the position vertex size!");
ASSIGN_OR_RETURN(
renderable_mesh_3d.tex_coord_position_size,
GetVertexComponentSize(vertex_type, VertexComponent::TEX_COORD),
_ << "Failed to get the tex coord vertex size!");
ASSIGN_OR_RETURN(
renderable_mesh_3d.vertex_position_offset,
GetVertexComponentOffset(vertex_type, VertexComponent::POSITION),
_ << "Failed to get the position vertex offset!");
ASSIGN_OR_RETURN(
renderable_mesh_3d.tex_coord_position_offset,
GetVertexComponentOffset(vertex_type, VertexComponent::TEX_COORD),
_ << "Failed to get the tex coord vertex offset!");
switch (proto_mesh_3d.primitive_type()) {
case Mesh3d::TRIANGLE:
renderable_mesh_3d.primitive_type = GL_TRIANGLES;
break;
default:
RET_CHECK_FAIL() << "Only triangle primitive types are supported!";
}
renderable_mesh_3d.vertex_buffer.reserve(
proto_mesh_3d.vertex_buffer_size());
for (float vertex_element : proto_mesh_3d.vertex_buffer()) {
renderable_mesh_3d.vertex_buffer.push_back(vertex_element);
}
renderable_mesh_3d.index_buffer.reserve(proto_mesh_3d.index_buffer_size());
for (uint32_t index_element : proto_mesh_3d.index_buffer()) {
RET_CHECK_LE(index_element, std::numeric_limits<uint16_t>::max())
<< "Index buffer elements must fit into the `uint16` type in order "
"to be renderable!";
renderable_mesh_3d.index_buffer.push_back(
static_cast<uint16_t>(index_element));
}
return renderable_mesh_3d;
}
uint32_t vertex_size;
uint32_t vertex_position_size;
uint32_t tex_coord_position_size;
uint32_t vertex_position_offset;
uint32_t tex_coord_position_offset;
uint32_t primitive_type;
std::vector<float> vertex_buffer;
std::vector<uint16_t> index_buffer;
};
class Texture {
public:
static absl::StatusOr<std::unique_ptr<Texture>> WrapExternalTexture(
GLuint handle, GLenum target, int width, int height) {
RET_CHECK(handle) << "External texture must have a non-null handle!";
return absl::WrapUnique(new Texture(handle, target, width, height,
/*is_owned*/ false));
}
static absl::StatusOr<std::unique_ptr<Texture>> CreateFromImageFrame(
const ImageFrame& image_frame) {
RET_CHECK(image_frame.IsAligned(ImageFrame::kGlDefaultAlignmentBoundary))
<< "Image frame memory must be aligned for GL usage!";
RET_CHECK(image_frame.Width() > 0 && image_frame.Height() > 0)
<< "Image frame must have positive dimensions!";
RET_CHECK(image_frame.Format() == ImageFormat::SRGB ||
image_frame.Format() == ImageFormat::SRGBA)
<< "Image frame format must be either SRGB or SRGBA!";
GLint image_format;
switch (image_frame.NumberOfChannels()) {
case 3:
image_format = GL_RGB;
break;
case 4:
image_format = GL_RGBA;
break;
default:
RET_CHECK_FAIL()
<< "Unexpected number of channels; expected 3 or 4, got "
<< image_frame.NumberOfChannels() << "!";
}
GLuint handle;
glGenTextures(1, &handle);
RET_CHECK(handle) << "Failed to initialize an OpenGL texture!";
glBindTexture(GL_TEXTURE_2D, handle);
glTexParameteri(GL_TEXTURE_2D, GL_NEAREST_MIPMAP_LINEAR, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexImage2D(GL_TEXTURE_2D, 0, image_format, image_frame.Width(),
image_frame.Height(), 0, image_format, GL_UNSIGNED_BYTE,
image_frame.PixelData());
glGenerateMipmap(GL_TEXTURE_2D);
glBindTexture(GL_TEXTURE_2D, 0);
return absl::WrapUnique(new Texture(
handle, GL_TEXTURE_2D, image_frame.Width(), image_frame.Height(),
/*is_owned*/ true));
}
~Texture() {
if (is_owned_) {
glDeleteProgram(handle_);
}
}
GLuint handle() const { return handle_; }
GLenum target() const { return target_; }
int width() const { return width_; }
int height() const { return height_; }
private:
Texture(GLuint handle, GLenum target, int width, int height, bool is_owned)
: handle_(handle),
target_(target),
width_(width),
height_(height),
is_owned_(is_owned) {}
GLuint handle_;
GLenum target_;
int width_;
int height_;
bool is_owned_;
};
class RenderTarget {
public:
static absl::StatusOr<std::unique_ptr<RenderTarget>> Create() {
GLuint framebuffer_handle;
glGenFramebuffers(1, &framebuffer_handle);
RET_CHECK(framebuffer_handle)
<< "Failed to initialize an OpenGL framebuffer!";
return absl::WrapUnique(new RenderTarget(framebuffer_handle));
}
~RenderTarget() {
glDeleteFramebuffers(1, &framebuffer_handle_);
// Renderbuffer handle might have never been created if this render target
// is destroyed before `SetColorbuffer()` is called for the first time.
if (renderbuffer_handle_) {
glDeleteFramebuffers(1, &renderbuffer_handle_);
}
}
absl::Status SetColorbuffer(const Texture& colorbuffer_texture) {
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_handle_);
glViewport(0, 0, colorbuffer_texture.width(), colorbuffer_texture.height());
glActiveTexture(GL_TEXTURE0);
glBindTexture(colorbuffer_texture.target(), colorbuffer_texture.handle());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
colorbuffer_texture.target(),
colorbuffer_texture.handle(),
/*level*/ 0);
glBindTexture(colorbuffer_texture.target(), 0);
// If the existing depth buffer has different dimensions, delete it.
if (renderbuffer_handle_ &&
(viewport_width_ != colorbuffer_texture.width() ||
viewport_height_ != colorbuffer_texture.height())) {
glDeleteRenderbuffers(1, &renderbuffer_handle_);
renderbuffer_handle_ = 0;
}
// If there is no depth buffer, create one.
if (!renderbuffer_handle_) {
glGenRenderbuffers(1, &renderbuffer_handle_);
RET_CHECK(renderbuffer_handle_)
<< "Failed to initialize an OpenGL renderbuffer!";
glBindRenderbuffer(GL_RENDERBUFFER, renderbuffer_handle_);
glRenderbufferStorage(GL_RENDERBUFFER, GL_DEPTH_COMPONENT16,
colorbuffer_texture.width(),
colorbuffer_texture.height());
glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT,
GL_RENDERBUFFER, renderbuffer_handle_);
glBindRenderbuffer(GL_RENDERBUFFER, 0);
}
viewport_width_ = colorbuffer_texture.width();
viewport_height_ = colorbuffer_texture.height();
glBindFramebuffer(GL_FRAMEBUFFER, 0);
glFlush();
return absl::OkStatus();
}
void Bind() const {
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_handle_);
glViewport(0, 0, viewport_width_, viewport_height_);
}
void Unbind() const { glBindFramebuffer(GL_FRAMEBUFFER, 0); }
void Clear() const {
Bind();
glEnable(GL_DEPTH_TEST);
glDepthMask(GL_TRUE);
glClearColor(0.f, 0.f, 0.f, 0.f);
glClearDepthf(1.f);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glDepthMask(GL_FALSE);
glDisable(GL_DEPTH_TEST);
Unbind();
glFlush();
}
private:
explicit RenderTarget(GLuint framebuffer_handle)
: framebuffer_handle_(framebuffer_handle),
renderbuffer_handle_(0),
viewport_width_(-1),
viewport_height_(-1) {}
GLuint framebuffer_handle_;
GLuint renderbuffer_handle_;
int viewport_width_;
int viewport_height_;
};
class Renderer {
public:
enum class RenderMode { OPAQUE, OVERDRAW, OCCLUSION };
static absl::StatusOr<std::unique_ptr<Renderer>> Create() {
static const GLint kAttrLocation[NUM_ATTRIBUTES] = {
ATTRIB_VERTEX,
ATTRIB_TEXTURE_POSITION,
};
static const GLchar* kAttrName[NUM_ATTRIBUTES] = {
"position",
"tex_coord",
};
static const GLchar* kVertSrc = R"(
uniform mat4 projection_mat;
uniform mat4 model_mat;
attribute vec4 position;
attribute vec4 tex_coord;
varying vec2 v_tex_coord;
void main() {
v_tex_coord = tex_coord.xy;
gl_Position = projection_mat * model_mat * position;
}
)";
static const GLchar* kFragSrc = R"(
precision mediump float;
varying vec2 v_tex_coord;
uniform sampler2D texture;
void main() {
gl_FragColor = texture2D(texture, v_tex_coord);
}
)";
GLuint program_handle = 0;
GlhCreateProgram(kVertSrc, kFragSrc, NUM_ATTRIBUTES,
(const GLchar**)&kAttrName[0], kAttrLocation,
&program_handle);
RET_CHECK(program_handle) << "Problem initializing the texture program!";
GLint projection_mat_uniform =
glGetUniformLocation(program_handle, "projection_mat");
GLint model_mat_uniform = glGetUniformLocation(program_handle, "model_mat");
GLint texture_uniform = glGetUniformLocation(program_handle, "texture");
RET_CHECK_NE(projection_mat_uniform, -1)
<< "Failed to find `projection_mat` uniform!";
RET_CHECK_NE(model_mat_uniform, -1)
<< "Failed to find `model_mat` uniform!";
RET_CHECK_NE(texture_uniform, -1) << "Failed to find `texture` uniform!";
return absl::WrapUnique(new Renderer(program_handle, projection_mat_uniform,
model_mat_uniform, texture_uniform));
}
~Renderer() { glDeleteProgram(program_handle_); }
absl::Status Render(const RenderTarget& render_target, const Texture& texture,
const RenderableMesh3d& mesh_3d,
const std::array<float, 16>& projection_mat,
const std::array<float, 16>& model_mat,
RenderMode render_mode) const {
glUseProgram(program_handle_);
// Set up the GL state.
glEnable(GL_BLEND);
glFrontFace(GL_CCW);
switch (render_mode) {
case RenderMode::OPAQUE:
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glEnable(GL_DEPTH_TEST);
glDepthMask(GL_TRUE);
break;
case RenderMode::OVERDRAW:
glBlendFunc(GL_ONE, GL_ZERO);
glDisable(GL_DEPTH_TEST);
glDepthMask(GL_FALSE);
break;
case RenderMode::OCCLUSION:
glBlendFunc(GL_ZERO, GL_ONE);
glEnable(GL_DEPTH_TEST);
glDepthMask(GL_TRUE);
break;
}
render_target.Bind();
// Set up vertex attributes.
glVertexAttribPointer(
ATTRIB_VERTEX, mesh_3d.vertex_position_size, GL_FLOAT, 0,
mesh_3d.vertex_size * sizeof(float),
mesh_3d.vertex_buffer.data() + mesh_3d.vertex_position_offset);
glEnableVertexAttribArray(ATTRIB_VERTEX);
glVertexAttribPointer(
ATTRIB_TEXTURE_POSITION, mesh_3d.tex_coord_position_size, GL_FLOAT, 0,
mesh_3d.vertex_size * sizeof(float),
mesh_3d.vertex_buffer.data() + mesh_3d.tex_coord_position_offset);
glEnableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
// Set up textures and uniforms.
glActiveTexture(GL_TEXTURE1);
glBindTexture(texture.target(), texture.handle());
glUniform1i(texture_uniform_, 1);
glUniformMatrix4fv(projection_mat_uniform_, 1, GL_FALSE,
projection_mat.data());
glUniformMatrix4fv(model_mat_uniform_, 1, GL_FALSE, model_mat.data());
// Draw the mesh.
glDrawElements(mesh_3d.primitive_type, mesh_3d.index_buffer.size(),
GL_UNSIGNED_SHORT, mesh_3d.index_buffer.data());
// Unbind textures and uniforms.
glActiveTexture(GL_TEXTURE1);
glBindTexture(texture.target(), 0);
render_target.Unbind();
// Unbind vertex attributes.
glDisableVertexAttribArray(ATTRIB_TEXTURE_POSITION);
glDisableVertexAttribArray(ATTRIB_VERTEX);
// Restore the GL state.
glDepthMask(GL_FALSE);
glDisable(GL_DEPTH_TEST);
glDisable(GL_BLEND);
glUseProgram(0);
glFlush();
return absl::OkStatus();
}
private:
enum { ATTRIB_VERTEX, ATTRIB_TEXTURE_POSITION, NUM_ATTRIBUTES };
Renderer(GLuint program_handle, GLint projection_mat_uniform,
GLint model_mat_uniform, GLint texture_uniform)
: program_handle_(program_handle),
projection_mat_uniform_(projection_mat_uniform),
model_mat_uniform_(model_mat_uniform),
texture_uniform_(texture_uniform) {}
GLuint program_handle_;
GLint projection_mat_uniform_;
GLint model_mat_uniform_;
GLint texture_uniform_;
};
class EffectRendererImpl : public EffectRenderer {
public:
EffectRendererImpl(
const Environment& environment,
std::unique_ptr<RenderTarget> render_target,
std::unique_ptr<Renderer> renderer,
RenderableMesh3d&& renderable_quad_mesh_3d,
absl::optional<RenderableMesh3d>&& renderable_effect_mesh_3d,
std::unique_ptr<Texture> empty_color_texture,
std::unique_ptr<Texture> effect_texture)
: environment_(environment),
render_target_(std::move(render_target)),
renderer_(std::move(renderer)),
renderable_quad_mesh_3d_(std::move(renderable_quad_mesh_3d)),
renderable_effect_mesh_3d_(std::move(renderable_effect_mesh_3d)),
empty_color_texture_(std::move(empty_color_texture)),
effect_texture_(std::move(effect_texture)),
identity_matrix_(Create4x4IdentityMatrix()) {}
absl::Status RenderEffect(
const std::vector<FaceGeometry>& multi_face_geometry,
int frame_width, //
int frame_height, //
GLenum src_texture_target, //
GLuint src_texture_name, //
GLenum dst_texture_target, //
GLuint dst_texture_name) {
// Validate input arguments.
MP_RETURN_IF_ERROR(ValidateFrameDimensions(frame_width, frame_height))
<< "Invalid frame dimensions!";
RET_CHECK(src_texture_name > 0 && dst_texture_name > 0)
<< "Both source and destination texture names must be non-null!";
RET_CHECK_NE(src_texture_name, dst_texture_name)
<< "Source and destination texture names must be different!";
// Validate all input face geometries.
for (const FaceGeometry& face_geometry : multi_face_geometry) {
MP_RETURN_IF_ERROR(ValidateFaceGeometry(face_geometry))
<< "Invalid face geometry!";
}
// Wrap both source and destination textures.
ASSIGN_OR_RETURN(
std::unique_ptr<Texture> src_texture,
Texture::WrapExternalTexture(src_texture_name, src_texture_target,
frame_width, frame_height),
_ << "Failed to wrap the external source texture");
ASSIGN_OR_RETURN(
std::unique_ptr<Texture> dst_texture,
Texture::WrapExternalTexture(dst_texture_name, dst_texture_target,
frame_width, frame_height),
_ << "Failed to wrap the external destination texture");
// Set the destination texture as the color buffer. Then, clear both the
// color and the depth buffers for the render target.
MP_RETURN_IF_ERROR(render_target_->SetColorbuffer(*dst_texture))
<< "Failed to set the destination texture as the colorbuffer!";
render_target_->Clear();
// Render the source texture on top of the quad mesh (i.e. make a copy)
// into the render target.
MP_RETURN_IF_ERROR(renderer_->Render(
*render_target_, *src_texture, renderable_quad_mesh_3d_,
identity_matrix_, identity_matrix_, Renderer::RenderMode::OVERDRAW))
<< "Failed to render the source texture on top of the quad mesh!";
// Extract pose transform matrices and meshes from the face geometry data;
const int num_faces = multi_face_geometry.size();
std::vector<std::array<float, 16>> face_pose_transform_matrices(num_faces);
std::vector<RenderableMesh3d> renderable_face_meshes(num_faces);
for (int i = 0; i < num_faces; ++i) {
const FaceGeometry& face_geometry = multi_face_geometry[i];
// Extract the face pose transformation matrix.
ASSIGN_OR_RETURN(
face_pose_transform_matrices[i],
Convert4x4MatrixDataToArrayFormat(
face_geometry.pose_transform_matrix()),
_ << "Failed to extract the face pose transformation matrix!");
// Extract the face mesh as a renderable.
ASSIGN_OR_RETURN(
renderable_face_meshes[i],
RenderableMesh3d::CreateFromProtoMesh3d(face_geometry.mesh()),
_ << "Failed to extract a renderable face mesh!");
}
// Create a perspective matrix using the frame aspect ratio.
std::array<float, 16> perspective_matrix = CreatePerspectiveMatrix(
/*aspect_ratio*/ static_cast<float>(frame_width) / frame_height);
// Render a face mesh occluder for each face.
for (int i = 0; i < num_faces; ++i) {
const std::array<float, 16>& face_pose_transform_matrix =
face_pose_transform_matrices[i];
const RenderableMesh3d& renderable_face_mesh = renderable_face_meshes[i];
// Render the face mesh using the empty color texture, i.e. the face
// mesh occluder.
//
// For occlusion, the pose transformation is moved ~1mm away from camera
// in order to allow the face mesh texture to be rendered without
// failing the depth test.
std::array<float, 16> occlusion_face_pose_transform_matrix =
face_pose_transform_matrix;
occlusion_face_pose_transform_matrix[14] -= 0.1f; // ~ 1mm
MP_RETURN_IF_ERROR(renderer_->Render(
*render_target_, *empty_color_texture_, renderable_face_mesh,
perspective_matrix, occlusion_face_pose_transform_matrix,
Renderer::RenderMode::OCCLUSION))
<< "Failed to render the face mesh occluder!";
}
// Render the main face mesh effect component for each face.
for (int i = 0; i < num_faces; ++i) {
const std::array<float, 16>& face_pose_transform_matrix =
face_pose_transform_matrices[i];
// If there is no effect 3D mesh provided, then the face mesh itself is
// used as a topology for rendering (for example, this can be used for
// facepaint effects or AR makeup).
const RenderableMesh3d& main_effect_mesh_3d =
renderable_effect_mesh_3d_ ? *renderable_effect_mesh_3d_
: renderable_face_meshes[i];
MP_RETURN_IF_ERROR(renderer_->Render(
*render_target_, *effect_texture_, main_effect_mesh_3d,
perspective_matrix, face_pose_transform_matrix,
Renderer::RenderMode::OPAQUE))
<< "Failed to render the main effect pass!";
}
// At this point in the code, the destination texture must contain the
// correctly renderer effect, so we should just return.
return absl::OkStatus();
}
private:
std::array<float, 16> CreatePerspectiveMatrix(float aspect_ratio) const {
static constexpr float kDegreesToRadians = M_PI / 180.f;
std::array<float, 16> perspective_matrix;
perspective_matrix.fill(0.f);
const auto& env_camera = environment_.perspective_camera();
// Standard perspective projection matrix calculations.
const float f = 1.0f / std::tan(kDegreesToRadians *
env_camera.vertical_fov_degrees() / 2.f);
const float denom = 1.0f / (env_camera.near() - env_camera.far());
perspective_matrix[0] = f / aspect_ratio;
perspective_matrix[5] = f;
perspective_matrix[10] = (env_camera.near() + env_camera.far()) * denom;
perspective_matrix[11] = -1.f;
perspective_matrix[14] = 2.f * env_camera.far() * env_camera.near() * denom;
// If the environment's origin point location is in the top left corner,
// then skip additional flip along Y-axis is required to render correctly.
if (environment_.origin_point_location() ==
OriginPointLocation::TOP_LEFT_CORNER) {
perspective_matrix[5] *= -1.f;
}
return perspective_matrix;
}
static std::array<float, 16> Create4x4IdentityMatrix() {
return {1.f, 0.f, 0.f, 0.f, //
0.f, 1.f, 0.f, 0.f, //
0.f, 0.f, 1.f, 0.f, //
0.f, 0.f, 0.f, 1.f};
}
static absl::StatusOr<std::array<float, 16>>
Convert4x4MatrixDataToArrayFormat(const MatrixData& matrix_data) {
RET_CHECK(matrix_data.rows() == 4 && //
matrix_data.cols() == 4 && //
matrix_data.packed_data_size() == 16)
<< "The matrix data must define a 4x4 matrix!";
std::array<float, 16> matrix_array;
for (int i = 0; i < 16; i++) {
matrix_array[i] = matrix_data.packed_data(i);
}
// Matrix array must be in the OpenGL-friendly column-major order. If
// `matrix_data` is in the row-major order, then transpose.
if (matrix_data.layout() == MatrixData::ROW_MAJOR) {
std::swap(matrix_array[1], matrix_array[4]);
std::swap(matrix_array[2], matrix_array[8]);
std::swap(matrix_array[3], matrix_array[12]);
std::swap(matrix_array[6], matrix_array[9]);
std::swap(matrix_array[7], matrix_array[13]);
std::swap(matrix_array[11], matrix_array[14]);
}
return matrix_array;
}
Environment environment_;
std::unique_ptr<RenderTarget> render_target_;
std::unique_ptr<Renderer> renderer_;
RenderableMesh3d renderable_quad_mesh_3d_;
absl::optional<RenderableMesh3d> renderable_effect_mesh_3d_;
std::unique_ptr<Texture> empty_color_texture_;
std::unique_ptr<Texture> effect_texture_;
std::array<float, 16> identity_matrix_;
};
Mesh3d CreateQuadMesh3d() {
static constexpr float kQuadMesh3dVertexBuffer[] = {
-1.f, -1.f, 0.f, 0.f, 0.f, //
1.f, -1.f, 0.f, 1.f, 0.f, //
-1.f, 1.f, 0.f, 0.f, 1.f, //
1.f, 1.f, 0.f, 1.f, 1.f, //
};
static constexpr uint16_t kQuadMesh3dIndexBuffer[] = {0, 1, 2, 1, 3, 2};
static constexpr int kQuadMesh3dVertexBufferSize =
sizeof(kQuadMesh3dVertexBuffer) / sizeof(float);
static constexpr int kQuadMesh3dIndexBufferSize =
sizeof(kQuadMesh3dIndexBuffer) / sizeof(uint16_t);
Mesh3d quad_mesh_3d;
quad_mesh_3d.set_vertex_type(Mesh3d::VERTEX_PT);
quad_mesh_3d.set_primitive_type(Mesh3d::TRIANGLE);
for (int i = 0; i < kQuadMesh3dVertexBufferSize; ++i) {
quad_mesh_3d.add_vertex_buffer(kQuadMesh3dVertexBuffer[i]);
}
for (int i = 0; i < kQuadMesh3dIndexBufferSize; ++i) {
quad_mesh_3d.add_index_buffer(kQuadMesh3dIndexBuffer[i]);
}
return quad_mesh_3d;
}
ImageFrame CreateEmptyColorTexture() {
static constexpr ImageFormat::Format kEmptyColorTextureFormat =
ImageFormat::SRGBA;
static constexpr int kEmptyColorTextureWidth = 1;
static constexpr int kEmptyColorTextureHeight = 1;
ImageFrame empty_color_texture(
kEmptyColorTextureFormat, kEmptyColorTextureWidth,
kEmptyColorTextureHeight, ImageFrame::kGlDefaultAlignmentBoundary);
empty_color_texture.SetToZero();
return empty_color_texture;
}
} // namespace
absl::StatusOr<std::unique_ptr<EffectRenderer>> CreateEffectRenderer(
const Environment& environment, //
const absl::optional<Mesh3d>& effect_mesh_3d, //
ImageFrame&& effect_texture) {
MP_RETURN_IF_ERROR(ValidateEnvironment(environment))
<< "Invalid environment!";
if (effect_mesh_3d) {
MP_RETURN_IF_ERROR(ValidateMesh3d(*effect_mesh_3d))
<< "Invalid effect 3D mesh!";
}
ASSIGN_OR_RETURN(std::unique_ptr<RenderTarget> render_target,
RenderTarget::Create(),
_ << "Failed to create a render target!");
ASSIGN_OR_RETURN(std::unique_ptr<Renderer> renderer, Renderer::Create(),
_ << "Failed to create a renderer!");
ASSIGN_OR_RETURN(RenderableMesh3d renderable_quad_mesh_3d,
RenderableMesh3d::CreateFromProtoMesh3d(CreateQuadMesh3d()),
_ << "Failed to create a renderable quad mesh!");
absl::optional<RenderableMesh3d> renderable_effect_mesh_3d;
if (effect_mesh_3d) {
ASSIGN_OR_RETURN(renderable_effect_mesh_3d,
RenderableMesh3d::CreateFromProtoMesh3d(*effect_mesh_3d),
_ << "Failed to create a renderable effect mesh!");
}
ASSIGN_OR_RETURN(std::unique_ptr<Texture> empty_color_gl_texture,
Texture::CreateFromImageFrame(CreateEmptyColorTexture()),
_ << "Failed to create an empty color texture!");
ASSIGN_OR_RETURN(std::unique_ptr<Texture> effect_gl_texture,
Texture::CreateFromImageFrame(effect_texture),
_ << "Failed to create an effect texture!");
std::unique_ptr<EffectRenderer> result =
absl::make_unique<EffectRendererImpl>(
environment, std::move(render_target), std::move(renderer),
std::move(renderable_quad_mesh_3d),
std::move(renderable_effect_mesh_3d),
std::move(empty_color_gl_texture), std::move(effect_gl_texture));
return result;
}
} // namespace mediapipe::face_geometry

92
mediapipe/modules/face_geometry/libs/effect_renderer.h
View File

@ -1,92 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_MODULES_FACE_GEOMETRY_LIBS_EFFECT_RENDERER_H_
#define MEDIAPIPE_MODULES_FACE_GEOMETRY_LIBS_EFFECT_RENDERER_H_
#include <memory>
#include <vector>
#include "absl/types/optional.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/gpu/gl_base.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
// Encapsulates a stateful face effect renderer.
class EffectRenderer {
public:
virtual ~EffectRenderer() = default;
// Renders a face effect based on the multiple facial geometries.
//
// Must be called in the same GL context as was used upon initialization.
//
// Each of the `multi_face_geometry` must be valid (for details, please refer
// to the proto message definition comments and/or `validation_utils.h/cc`).
// Additionally, all face mesh index buffer elements must fit into the
// `uint16` type in order to be renderable.
//
// Both `frame_width` and `frame_height` must be positive.
//
// Both `src_texture_name` and `dst_texture_name` must be positive and
// reference existing OpenGL textures in the current context. They should also
// reference different textures as the in-place effect rendering is not yet
// supported.
virtual absl::Status RenderEffect(
const std::vector<FaceGeometry>& multi_face_geometry,
int frame_width, //
int frame_height, //
GLenum src_texture_target, //
GLuint src_texture_name, //
GLenum dst_texture_target, //
GLuint dst_texture_name) = 0;
};
// Creates an instance of `EffectRenderer`.
//
// `effect_mesh_3d` defines a rigid 3d mesh which is "attached" to the face and
// is driven by the face pose transformation matrix. If is not present, the
// runtime face mesh will be used as the effect mesh - this mode is handy for
// facepaint effects. In both rendering modes, the face mesh is first rendered
// as an occluder straight into the depth buffer. This step helps to create a
// more believable effect via hiding invisible elements behind the face surface.
//
// `effect_texture` defines the color texture to be rendered on top of the
// effect mesh. Please be aware about the difference between the CPU texture
// memory layout and the GPU texture sampler coordinate space. This renderer
// follows conventions discussed here: https://open.gl/textures
//
// Must be called in the same GL context as will be used for rendering.
//
// Both `environment` and `effect_mesh_3d` (is present) must be valid (for
// details, please refer to the proto message definition comments and/or
// `validation_utils.h/cc`). Additionally, `effect_mesh_3d`s index buffer
// elements must fit into the `uint16` type in order to be renderable.
//
// `effect_texture` must have positive dimensions. Its format must be either
// `SRGB` or `SRGBA`. Its memory must be aligned for GL usage.
absl::StatusOr<std::unique_ptr<EffectRenderer>> CreateEffectRenderer(
const Environment& environment, //
const absl::optional<Mesh3d>& effect_mesh_3d, //
ImageFrame&& effect_texture);
} // namespace mediapipe::face_geometry
#endif // MEDIAPIPE_MODULES_FACE_GEOMETRY_LIBS_EFFECT_RENDERER_H_

466
mediapipe/modules/face_geometry/libs/geometry_pipeline.cc
View File

@ -1,466 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/modules/face_geometry/libs/geometry_pipeline.h"
#include <cmath>
#include <cstdint>
#include <memory>
#include <utility>
#include <vector>
#include "Eigen/Core"
#include "absl/memory/memory.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/formats/matrix_data.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
#include "mediapipe/modules/face_geometry/libs/procrustes_solver.h"
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
namespace {
struct PerspectiveCameraFrustum {
// NOTE: all arguments must be validated prior to calling this constructor.
PerspectiveCameraFrustum(const PerspectiveCamera& perspective_camera,
int frame_width, int frame_height) {
static constexpr float kDegreesToRadians = 3.14159265358979323846f / 180.f;
const float height_at_near =
2.f * perspective_camera.near() *
std::tan(0.5f * kDegreesToRadians *
perspective_camera.vertical_fov_degrees());
const float width_at_near = frame_width * height_at_near / frame_height;
left = -0.5f * width_at_near;
right = 0.5f * width_at_near;
bottom = -0.5f * height_at_near;
top = 0.5f * height_at_near;
near = perspective_camera.near();
far = perspective_camera.far();
}
float left;
float right;
float bottom;
float top;
float near;
float far;
};
class ScreenToMetricSpaceConverter {
public:
ScreenToMetricSpaceConverter(
OriginPointLocation origin_point_location, //
InputSource input_source, //
Eigen::Matrix3Xf&& canonical_metric_landmarks, //
Eigen::VectorXf&& landmark_weights, //
std::unique_ptr<ProcrustesSolver> procrustes_solver)
: origin_point_location_(origin_point_location),
input_source_(input_source),
canonical_metric_landmarks_(std::move(canonical_metric_landmarks)),
landmark_weights_(std::move(landmark_weights)),
procrustes_solver_(std::move(procrustes_solver)) {}
// Converts `screen_landmark_list` into `metric_landmark_list` and estimates
// the `pose_transform_mat`.
//
// Here's the algorithm summary:
//
// (1) Project X- and Y- screen landmark coordinates at the Z near plane.
//
// (2) Estimate a canonical-to-runtime landmark set scale by running the
// Procrustes solver using the screen runtime landmarks.
//
// On this iteration, screen landmarks are used instead of unprojected
// metric landmarks as it is not safe to unproject due to the relative
// nature of the input screen landmark Z coordinate.
//
// (3) Use the canonical-to-runtime scale from (2) to unproject the screen
// landmarks. The result is referenced as "intermediate landmarks" because
// they are the first estimation of the resuling metric landmarks, but are
// not quite there yet.
//
// (4) Estimate a canonical-to-runtime landmark set scale by running the
// Procrustes solver using the intermediate runtime landmarks.
//
// (5) Use the product of the scale factors from (2) and (4) to unproject
// the screen landmarks the second time. This is the second and the final
// estimation of the metric landmarks.
//
// (6) Multiply each of the metric landmarks by the inverse pose
// transformation matrix to align the runtime metric face landmarks with
// the canonical metric face landmarks.
//
// Note: the input screen landmarks are in the left-handed coordinate system,
// however any metric landmarks - including the canonical metric
// landmarks, the final runtime metric landmarks and any intermediate
// runtime metric landmarks - are in the right-handed coordinate system.
//
// To keep the logic correct, the landmark set handedness is changed any
// time the screen-to-metric semantic barrier is passed.
absl::Status Convert(const NormalizedLandmarkList& screen_landmark_list, //
const PerspectiveCameraFrustum& pcf, //
LandmarkList& metric_landmark_list, //
Eigen::Matrix4f& pose_transform_mat) const {
RET_CHECK_EQ(screen_landmark_list.landmark_size(),
canonical_metric_landmarks_.cols())
<< "The number of landmarks doesn't match the number passed upon "
"initialization!";
Eigen::Matrix3Xf screen_landmarks;
ConvertLandmarkListToEigenMatrix(screen_landmark_list, screen_landmarks);
ProjectXY(pcf, screen_landmarks);
const float depth_offset = screen_landmarks.row(2).mean();
// 1st iteration: don't unproject XY because it's unsafe to do so due to
// the relative nature of the Z coordinate. Instead, run the
// first estimation on the projected XY and use that scale to
// unproject for the 2nd iteration.
Eigen::Matrix3Xf intermediate_landmarks(screen_landmarks);
ChangeHandedness(intermediate_landmarks);
ASSIGN_OR_RETURN(const float first_iteration_scale,
EstimateScale(intermediate_landmarks),
_ << "Failed to estimate first iteration scale!");
// 2nd iteration: unproject XY using the scale from the 1st iteration.
intermediate_landmarks = screen_landmarks;
MoveAndRescaleZ(pcf, depth_offset, first_iteration_scale,
intermediate_landmarks);
UnprojectXY(pcf, intermediate_landmarks);
ChangeHandedness(intermediate_landmarks);
// For face detection input landmarks, re-write Z-coord from the canonical
// landmarks.
if (input_source_ == InputSource::FACE_DETECTION_PIPELINE) {
Eigen::Matrix4f intermediate_pose_transform_mat;
MP_RETURN_IF_ERROR(procrustes_solver_->SolveWeightedOrthogonalProblem(
canonical_metric_landmarks_, intermediate_landmarks,
landmark_weights_, intermediate_pose_transform_mat))
<< "Failed to estimate pose transform matrix!";
intermediate_landmarks.row(2) =
(intermediate_pose_transform_mat *
canonical_metric_landmarks_.colwise().homogeneous())
.row(2);
}
ASSIGN_OR_RETURN(const float second_iteration_scale,
EstimateScale(intermediate_landmarks),
_ << "Failed to estimate second iteration scale!");
// Use the total scale to unproject the screen landmarks.
const float total_scale = first_iteration_scale * second_iteration_scale;
MoveAndRescaleZ(pcf, depth_offset, total_scale, screen_landmarks);
UnprojectXY(pcf, screen_landmarks);
ChangeHandedness(screen_landmarks);
// At this point, screen landmarks are converted into metric landmarks.
Eigen::Matrix3Xf& metric_landmarks = screen_landmarks;
MP_RETURN_IF_ERROR(procrustes_solver_->SolveWeightedOrthogonalProblem(
canonical_metric_landmarks_, metric_landmarks, landmark_weights_,
pose_transform_mat))
<< "Failed to estimate pose transform matrix!";
// For face detection input landmarks, re-write Z-coord from the canonical
// landmarks and run the pose transform estimation again.
if (input_source_ == InputSource::FACE_DETECTION_PIPELINE) {
metric_landmarks.row(2) =
(pose_transform_mat *
canonical_metric_landmarks_.colwise().homogeneous())
.row(2);
MP_RETURN_IF_ERROR(procrustes_solver_->SolveWeightedOrthogonalProblem(
canonical_metric_landmarks_, metric_landmarks, landmark_weights_,
pose_transform_mat))
<< "Failed to estimate pose transform matrix!";
}
// Multiply each of the metric landmarks by the inverse pose
// transformation matrix to align the runtime metric face landmarks with
// the canonical metric face landmarks.
metric_landmarks = (pose_transform_mat.inverse() *
metric_landmarks.colwise().homogeneous())
.topRows(3);
ConvertEigenMatrixToLandmarkList(metric_landmarks, metric_landmark_list);
return absl::OkStatus();
}
private:
void ProjectXY(const PerspectiveCameraFrustum& pcf,
Eigen::Matrix3Xf& landmarks) const {
float x_scale = pcf.right - pcf.left;
float y_scale = pcf.top - pcf.bottom;
float x_translation = pcf.left;
float y_translation = pcf.bottom;
if (origin_point_location_ == OriginPointLocation::TOP_LEFT_CORNER) {
landmarks.row(1) = 1.f - landmarks.row(1).array();
}
landmarks =
landmarks.array().colwise() * Eigen::Array3f(x_scale, y_scale, x_scale);
landmarks.colwise() += Eigen::Vector3f(x_translation, y_translation, 0.f);
}
absl::StatusOr<float> EstimateScale(Eigen::Matrix3Xf& landmarks) const {
Eigen::Matrix4f transform_mat;
MP_RETURN_IF_ERROR(procrustes_solver_->SolveWeightedOrthogonalProblem(
canonical_metric_landmarks_, landmarks, landmark_weights_,
transform_mat))
<< "Failed to estimate canonical-to-runtime landmark set transform!";
return transform_mat.col(0).norm();
}
static void MoveAndRescaleZ(const PerspectiveCameraFrustum& pcf,
float depth_offset, float scale,
Eigen::Matrix3Xf& landmarks) {
landmarks.row(2) =
(landmarks.array().row(2) - depth_offset + pcf.near) / scale;
}
static void UnprojectXY(const PerspectiveCameraFrustum& pcf,
Eigen::Matrix3Xf& landmarks) {
landmarks.row(0) =
landmarks.row(0).cwiseProduct(landmarks.row(2)) / pcf.near;
landmarks.row(1) =
landmarks.row(1).cwiseProduct(landmarks.row(2)) / pcf.near;
}
static void ChangeHandedness(Eigen::Matrix3Xf& landmarks) {
landmarks.row(2) *= -1.f;
}
static void ConvertLandmarkListToEigenMatrix(
const NormalizedLandmarkList& landmark_list,
Eigen::Matrix3Xf& eigen_matrix) {
eigen_matrix = Eigen::Matrix3Xf(3, landmark_list.landmark_size());
for (int i = 0; i < landmark_list.landmark_size(); ++i) {
const auto& landmark = landmark_list.landmark(i);
eigen_matrix(0, i) = landmark.x();
eigen_matrix(1, i) = landmark.y();
eigen_matrix(2, i) = landmark.z();
}
}
static void ConvertEigenMatrixToLandmarkList(
const Eigen::Matrix3Xf& eigen_matrix, LandmarkList& landmark_list) {
landmark_list.Clear();
for (int i = 0; i < eigen_matrix.cols(); ++i) {
auto& landmark = *landmark_list.add_landmark();
landmark.set_x(eigen_matrix(0, i));
landmark.set_y(eigen_matrix(1, i));
landmark.set_z(eigen_matrix(2, i));
}
}
const OriginPointLocation origin_point_location_;
const InputSource input_source_;
Eigen::Matrix3Xf canonical_metric_landmarks_;
Eigen::VectorXf landmark_weights_;
std::unique_ptr<ProcrustesSolver> procrustes_solver_;
};
class GeometryPipelineImpl : public GeometryPipeline {
public:
GeometryPipelineImpl(
const PerspectiveCamera& perspective_camera, //
const Mesh3d& canonical_mesh, //
uint32_t canonical_mesh_vertex_size, //
uint32_t canonical_mesh_num_vertices,
uint32_t canonical_mesh_vertex_position_offset,
std::unique_ptr<ScreenToMetricSpaceConverter> space_converter)
: perspective_camera_(perspective_camera),
canonical_mesh_(canonical_mesh),
canonical_mesh_vertex_size_(canonical_mesh_vertex_size),
canonical_mesh_num_vertices_(canonical_mesh_num_vertices),
canonical_mesh_vertex_position_offset_(
canonical_mesh_vertex_position_offset),
space_converter_(std::move(space_converter)) {}
absl::StatusOr<std::vector<FaceGeometry>> EstimateFaceGeometry(
const std::vector<NormalizedLandmarkList>& multi_face_landmarks,
int frame_width, int frame_height) const override {
MP_RETURN_IF_ERROR(ValidateFrameDimensions(frame_width, frame_height))
<< "Invalid frame dimensions!";
// Create a perspective camera frustum to be shared for geometry estimation
// per each face.
PerspectiveCameraFrustum pcf(perspective_camera_, frame_width,
frame_height);
std::vector<FaceGeometry> multi_face_geometry;
// From this point, the meaning of "face landmarks" is clarified further as
// "screen face landmarks". This is done do distinguish from "metric face
// landmarks" that are derived during the face geometry estimation process.
for (const NormalizedLandmarkList& screen_face_landmarks :
multi_face_landmarks) {
// Having a too compact screen landmark list will result in numerical
// instabilities, therefore such faces are filtered.
if (IsScreenLandmarkListTooCompact(screen_face_landmarks)) {
continue;
}
// Convert the screen landmarks into the metric landmarks and get the pose
// transformation matrix.
LandmarkList metric_face_landmarks;
Eigen::Matrix4f pose_transform_mat;
MP_RETURN_IF_ERROR(space_converter_->Convert(screen_face_landmarks, pcf,
metric_face_landmarks,
pose_transform_mat))
<< "Failed to convert landmarks from the screen to the metric space!";
// Pack geometry data for this face.
FaceGeometry face_geometry;
Mesh3d* mutable_mesh = face_geometry.mutable_mesh();
// Copy the canonical face mesh as the face geometry mesh.
mutable_mesh->CopyFrom(canonical_mesh_);
// Replace XYZ vertex mesh coodinates with the metric landmark positions.
for (int i = 0; i < canonical_mesh_num_vertices_; ++i) {
uint32_t vertex_buffer_offset = canonical_mesh_vertex_size_ * i +
canonical_mesh_vertex_position_offset_;
mutable_mesh->set_vertex_buffer(vertex_buffer_offset,
metric_face_landmarks.landmark(i).x());
mutable_mesh->set_vertex_buffer(vertex_buffer_offset + 1,
metric_face_landmarks.landmark(i).y());
mutable_mesh->set_vertex_buffer(vertex_buffer_offset + 2,
metric_face_landmarks.landmark(i).z());
}
// Populate the face pose transformation matrix.
mediapipe::MatrixDataProtoFromMatrix(
pose_transform_mat, face_geometry.mutable_pose_transform_matrix());
multi_face_geometry.push_back(face_geometry);
}
return multi_face_geometry;
}
private:
static bool IsScreenLandmarkListTooCompact(
const NormalizedLandmarkList& screen_landmarks) {
float mean_x = 0.f;
float mean_y = 0.f;
for (int i = 0; i < screen_landmarks.landmark_size(); ++i) {
const auto& landmark = screen_landmarks.landmark(i);
mean_x += (landmark.x() - mean_x) / static_cast<float>(i + 1);
mean_y += (landmark.y() - mean_y) / static_cast<float>(i + 1);
}
float max_sq_dist = 0.f;
for (const auto& landmark : screen_landmarks.landmark()) {
const float d_x = landmark.x() - mean_x;
const float d_y = landmark.y() - mean_y;
max_sq_dist = std::max(max_sq_dist, d_x * d_x + d_y * d_y);
}
static constexpr float kIsScreenLandmarkListTooCompactThreshold = 1e-3f;
return std::sqrt(max_sq_dist) <= kIsScreenLandmarkListTooCompactThreshold;
}
const PerspectiveCamera perspective_camera_;
const Mesh3d canonical_mesh_;
const uint32_t canonical_mesh_vertex_size_;
const uint32_t canonical_mesh_num_vertices_;
const uint32_t canonical_mesh_vertex_position_offset_;
std::unique_ptr<ScreenToMetricSpaceConverter> space_converter_;
};
} // namespace
absl::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
const Environment& environment, const GeometryPipelineMetadata& metadata) {
MP_RETURN_IF_ERROR(ValidateEnvironment(environment))
<< "Invalid environment!";
MP_RETURN_IF_ERROR(ValidateGeometryPipelineMetadata(metadata))
<< "Invalid geometry pipeline metadata!";
const auto& canonical_mesh = metadata.canonical_mesh();
RET_CHECK(HasVertexComponent(canonical_mesh.vertex_type(),
VertexComponent::POSITION))
<< "Canonical face mesh must have the `POSITION` vertex component!";
RET_CHECK(HasVertexComponent(canonical_mesh.vertex_type(),
VertexComponent::TEX_COORD))
<< "Canonical face mesh must have the `TEX_COORD` vertex component!";
uint32_t canonical_mesh_vertex_size =
GetVertexSize(canonical_mesh.vertex_type());
uint32_t canonical_mesh_num_vertices =
canonical_mesh.vertex_buffer_size() / canonical_mesh_vertex_size;
uint32_t canonical_mesh_vertex_position_offset =
GetVertexComponentOffset(canonical_mesh.vertex_type(),
VertexComponent::POSITION)
.value();
// Put the Procrustes landmark basis into Eigen matrices for an easier access.
Eigen::Matrix3Xf canonical_metric_landmarks =
Eigen::Matrix3Xf::Zero(3, canonical_mesh_num_vertices);
Eigen::VectorXf landmark_weights =
Eigen::VectorXf::Zero(canonical_mesh_num_vertices);
for (int i = 0; i < canonical_mesh_num_vertices; ++i) {
uint32_t vertex_buffer_offset =
canonical_mesh_vertex_size * i + canonical_mesh_vertex_position_offset;
canonical_metric_landmarks(0, i) =
canonical_mesh.vertex_buffer(vertex_buffer_offset);
canonical_metric_landmarks(1, i) =
canonical_mesh.vertex_buffer(vertex_buffer_offset + 1);
canonical_metric_landmarks(2, i) =
canonical_mesh.vertex_buffer(vertex_buffer_offset + 2);
}
for (const WeightedLandmarkRef& wlr : metadata.procrustes_landmark_basis()) {
uint32_t landmark_id = wlr.landmark_id();
landmark_weights(landmark_id) = wlr.weight();
}
std::unique_ptr<GeometryPipeline> result =
absl::make_unique<GeometryPipelineImpl>(
environment.perspective_camera(), canonical_mesh,
canonical_mesh_vertex_size, canonical_mesh_num_vertices,
canonical_mesh_vertex_position_offset,
absl::make_unique<ScreenToMetricSpaceConverter>(
environment.origin_point_location(),
metadata.input_source() == InputSource::DEFAULT
? InputSource::FACE_LANDMARK_PIPELINE
: metadata.input_source(),
std::move(canonical_metric_landmarks),
std::move(landmark_weights),
CreateFloatPrecisionProcrustesSolver()));
return result;
}
} // namespace mediapipe::face_geometry

67
mediapipe/modules/face_geometry/libs/geometry_pipeline.h
View File

@ -1,67 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_GEOMETRY_PIPELINE_H_
#define MEDIAPIPE_FACE_GEOMETRY_LIBS_GEOMETRY_PIPELINE_H_
#include <memory>
#include <vector>
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
namespace mediapipe::face_geometry {
// Encapsulates a stateless estimator of facial geometry in a Metric space based
// on the normalized face landmarks in the Screen space.
class GeometryPipeline {
public:
virtual ~GeometryPipeline() = default;
// Estimates geometry data for multiple faces.
//
// Returns an error status if any of the passed arguments is invalid.
//
// The result includes face geometry data for a subset of the input faces,
// however geometry data for some faces might be missing. This may happen if
// it'd be unstable to estimate the facial geometry based on a corresponding
// face landmark list for any reason (for example, if the landmark list is too
// compact).
//
// Each face landmark list must have the same number of landmarks as was
// passed upon initialization via the canonical face mesh (as a part of the
// geometry pipeline metadata).
//
// Both `frame_width` and `frame_height` must be positive.
virtual absl::StatusOr<std::vector<FaceGeometry>> EstimateFaceGeometry(
const std::vector<NormalizedLandmarkList>& multi_face_landmarks,
int frame_width, int frame_height) const = 0;
};
// Creates an instance of `GeometryPipeline`.
//
// Both `environment` and `metadata` must be valid (for details, please refer to
// the proto message definition comments and/or `validation_utils.h/cc`).
//
// Canonical face mesh (defined as a part of `metadata`) must have the
// `POSITION` and the `TEX_COORD` vertex components.
absl::StatusOr<std::unique_ptr<GeometryPipeline>> CreateGeometryPipeline(
const Environment& environment, const GeometryPipelineMetadata& metadata);
} // namespace mediapipe::face_geometry
#endif // MEDIAPIPE_FACE_GEOMETRY_LIBS_GEOMETRY_PIPELINE_H_

103
mediapipe/modules/face_geometry/libs/mesh_3d_utils.cc
View File

@ -1,103 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
#include <cstdint>
#include <cstdlib>
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
namespace {
bool HasVertexComponentVertexPT(VertexComponent vertex_component) {
switch (vertex_component) {
case VertexComponent::POSITION:
case VertexComponent::TEX_COORD:
return true;
default:
return false;
}
}
uint32_t GetVertexComponentSizeVertexPT(VertexComponent vertex_component) {
switch (vertex_component) {
case VertexComponent::POSITION:
return 3;
case VertexComponent::TEX_COORD:
return 2;
}
}
uint32_t GetVertexComponentOffsetVertexPT(VertexComponent vertex_component) {
switch (vertex_component) {
case VertexComponent::POSITION:
return 0;
case VertexComponent::TEX_COORD:
return GetVertexComponentSizeVertexPT(VertexComponent::POSITION);
}
}
} // namespace
std::size_t GetVertexSize(Mesh3d::VertexType vertex_type) {
switch (vertex_type) {
case Mesh3d::VERTEX_PT:
return GetVertexComponentSizeVertexPT(VertexComponent::POSITION) +
GetVertexComponentSizeVertexPT(VertexComponent::TEX_COORD);
}
}
std::size_t GetPrimitiveSize(Mesh3d::PrimitiveType primitive_type) {
switch (primitive_type) {
case Mesh3d::TRIANGLE:
return 3;
}
}
bool HasVertexComponent(Mesh3d::VertexType vertex_type,
VertexComponent vertex_component) {
switch (vertex_type) {
case Mesh3d::VERTEX_PT:
return HasVertexComponentVertexPT(vertex_component);
}
}
absl::StatusOr<uint32_t> GetVertexComponentOffset(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component) {
RET_CHECK(HasVertexComponentVertexPT(vertex_component))
<< "A given vertex type doesn't have the requested component!";
switch (vertex_type) {
case Mesh3d::VERTEX_PT:
return GetVertexComponentOffsetVertexPT(vertex_component);
}
}
absl::StatusOr<uint32_t> GetVertexComponentSize(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component) {
RET_CHECK(HasVertexComponentVertexPT(vertex_component))
<< "A given vertex type doesn't have the requested component!";
switch (vertex_type) {
case Mesh3d::VERTEX_PT:
return GetVertexComponentSizeVertexPT(vertex_component);
}
}
} // namespace mediapipe::face_geometry

51
mediapipe/modules/face_geometry/libs/mesh_3d_utils.h
View File

@ -1,51 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_MESH_3D_UTILS_H_
#define MEDIAPIPE_FACE_GEOMETRY_LIBS_MESH_3D_UTILS_H_
#include <cstdint>
#include <cstdlib>
#include "mediapipe/framework/port/statusor.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
enum class VertexComponent { POSITION, TEX_COORD };
std::size_t GetVertexSize(Mesh3d::VertexType vertex_type);
std::size_t GetPrimitiveSize(Mesh3d::PrimitiveType primitive_type);
bool HasVertexComponent(Mesh3d::VertexType vertex_type,
VertexComponent vertex_component);
// Computes the vertex component offset.
//
// Returns an error status if a given vertex type doesn't have the requested
// component.
absl::StatusOr<uint32_t> GetVertexComponentOffset(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component);
// Computes the vertex component size.
//
// Returns an error status if a given vertex type doesn't have the requested
// component.
absl::StatusOr<uint32_t> GetVertexComponentSize(
Mesh3d::VertexType vertex_type, VertexComponent vertex_component);
} // namespace mediapipe::face_geometry
#endif // MEDIAPIPE_FACE_GEOMETRY_LIBS_MESH_3D_UTILS_H_

266
mediapipe/modules/face_geometry/libs/procrustes_solver.cc
View File

@ -1,266 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/modules/face_geometry/libs/procrustes_solver.h"
#include <cmath>
#include <memory>
#include "Eigen/Dense"
#include "absl/memory/memory.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/framework/port/statusor.h"
namespace mediapipe {
namespace face_geometry {
namespace {
class FloatPrecisionProcrustesSolver : public ProcrustesSolver {
public:
FloatPrecisionProcrustesSolver() = default;
absl::Status SolveWeightedOrthogonalProblem(
const Eigen::Matrix3Xf& source_points, //
const Eigen::Matrix3Xf& target_points, //
const Eigen::VectorXf& point_weights,
Eigen::Matrix4f& transform_mat) const override {
// Validate inputs.
MP_RETURN_IF_ERROR(ValidateInputPoints(source_points, target_points))
<< "Failed to validate weighted orthogonal problem input points!";
MP_RETURN_IF_ERROR(
ValidatePointWeights(source_points.cols(), point_weights))
<< "Failed to validate weighted orthogonal problem point weights!";
// Extract square root from the point weights.
Eigen::VectorXf sqrt_weights = ExtractSquareRoot(point_weights);
// Try to solve the WEOP problem.
MP_RETURN_IF_ERROR(InternalSolveWeightedOrthogonalProblem(
source_points, target_points, sqrt_weights, transform_mat))
<< "Failed to solve the WEOP problem!";
return absl::OkStatus();
}
private:
static constexpr float kAbsoluteErrorEps = 1e-9f;
static absl::Status ValidateInputPoints(
const Eigen::Matrix3Xf& source_points,
const Eigen::Matrix3Xf& target_points) {
RET_CHECK_GT(source_points.cols(), 0)
<< "The number of source points must be positive!";
RET_CHECK_EQ(source_points.cols(), target_points.cols())
<< "The number of source and target points must be equal!";
return absl::OkStatus();
}
static absl::Status ValidatePointWeights(
int num_points, const Eigen::VectorXf& point_weights) {
RET_CHECK_GT(point_weights.size(), 0)
<< "The number of point weights must be positive!";
RET_CHECK_EQ(point_weights.size(), num_points)
<< "The number of points and point weights must be equal!";
float total_weight = 0.f;
for (int i = 0; i < num_points; ++i) {
RET_CHECK_GE(point_weights(i), 0.f)
<< "Each point weight must be non-negative!";
total_weight += point_weights(i);
}
RET_CHECK_GT(total_weight, kAbsoluteErrorEps)
<< "The total point weight is too small!";
return absl::OkStatus();
}
static Eigen::VectorXf ExtractSquareRoot(
const Eigen::VectorXf& point_weights) {
Eigen::VectorXf sqrt_weights(point_weights);
for (int i = 0; i < sqrt_weights.size(); ++i) {
sqrt_weights(i) = std::sqrt(sqrt_weights(i));
}
return sqrt_weights;
}
// Combines a 3x3 rotation-and-scale matrix and a 3x1 translation vector into
// a single 4x4 transformation matrix.
static Eigen::Matrix4f CombineTransformMatrix(const Eigen::Matrix3f& r_and_s,
const Eigen::Vector3f& t) {
Eigen::Matrix4f result = Eigen::Matrix4f::Identity();
result.leftCols(3).topRows(3) = r_and_s;
result.col(3).topRows(3) = t;
return result;
}
// The weighted problem is thoroughly addressed in Section 2.4 of:
// D. Akca, Generalized Procrustes analysis and its applications
// in photogrammetry, 2003, https://doi.org/10.3929/ethz-a-004656648
//
// Notable differences in the code presented here are:
//
// * In the paper, the weights matrix W_p is Cholesky-decomposed as Q^T Q.
// Our W_p is diagonal (equal to diag(sqrt_weights^2)),
// so we can just set Q = diag(sqrt_weights) instead.
//
// * In the paper, the problem is presented as
// (for W_k = I and W_p = tranposed(Q) Q):
// || Q (c A T + j tranposed(t) - B) || -> min.
//
// We reformulate it as an equivalent minimization of the transpose's
// norm:
// || (c tranposed(T) tranposed(A) - tranposed(B)) tranposed(Q) || -> min,
// where tranposed(A) and tranposed(B) are the source and the target point
// clouds, respectively, c tranposed(T) is the rotation+scaling R sought
// for, and Q is diag(sqrt_weights).
//
// Most of the derivations are therefore transposed.
//
// Note: the output `transform_mat` argument is used instead of `StatusOr<>`
// return type in order to avoid Eigen memory alignment issues. Details:
// https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
static absl::Status InternalSolveWeightedOrthogonalProblem(
const Eigen::Matrix3Xf& sources, const Eigen::Matrix3Xf& targets,
const Eigen::VectorXf& sqrt_weights, Eigen::Matrix4f& transform_mat) {
// tranposed(A_w).
Eigen::Matrix3Xf weighted_sources =
sources.array().rowwise() * sqrt_weights.array().transpose();
// tranposed(B_w).
Eigen::Matrix3Xf weighted_targets =
targets.array().rowwise() * sqrt_weights.array().transpose();
// w = tranposed(j_w) j_w.
float total_weight = sqrt_weights.cwiseProduct(sqrt_weights).sum();
// Let C = (j_w tranposed(j_w)) / (tranposed(j_w) j_w).
// Note that C = tranposed(C), hence (I - C) = tranposed(I - C).
//
// tranposed(A_w) C = tranposed(A_w) j_w tranposed(j_w) / w =
// (tranposed(A_w) j_w) tranposed(j_w) / w = c_w tranposed(j_w),
//
// where c_w = tranposed(A_w) j_w / w is a k x 1 vector calculated here:
Eigen::Matrix3Xf twice_weighted_sources =
weighted_sources.array().rowwise() * sqrt_weights.array().transpose();
Eigen::Vector3f source_center_of_mass =
twice_weighted_sources.rowwise().sum() / total_weight;
// tranposed((I - C) A_w) = tranposed(A_w) (I - C) =
// tranposed(A_w) - tranposed(A_w) C = tranposed(A_w) - c_w tranposed(j_w).
Eigen::Matrix3Xf centered_weighted_sources =
weighted_sources - source_center_of_mass * sqrt_weights.transpose();
Eigen::Matrix3f rotation;
MP_RETURN_IF_ERROR(ComputeOptimalRotation(
weighted_targets * centered_weighted_sources.transpose(), rotation))
<< "Failed to compute the optimal rotation!";
ASSIGN_OR_RETURN(
float scale,
ComputeOptimalScale(centered_weighted_sources, weighted_sources,
weighted_targets, rotation),
_ << "Failed to compute the optimal scale!");
// R = c tranposed(T).
Eigen::Matrix3f rotation_and_scale = scale * rotation;
// Compute optimal translation for the weighted problem.
// tranposed(B_w - c A_w T) = tranposed(B_w) - R tranposed(A_w) in (54).
const auto pointwise_diffs =
weighted_targets - rotation_and_scale * weighted_sources;
// Multiplication by j_w is a respectively weighted column sum.
// (54) from the paper.
const auto weighted_pointwise_diffs =
pointwise_diffs.array().rowwise() * sqrt_weights.array().transpose();
Eigen::Vector3f translation =
weighted_pointwise_diffs.rowwise().sum() / total_weight;
transform_mat = CombineTransformMatrix(rotation_and_scale, translation);
return absl::OkStatus();
}
// `design_matrix` is a transposed LHS of (51) in the paper.
//
// Note: the output `rotation` argument is used instead of `StatusOr<>`
// return type in order to avoid Eigen memory alignment issues. Details:
// https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
static absl::Status ComputeOptimalRotation(
const Eigen::Matrix3f& design_matrix, Eigen::Matrix3f& rotation) {
RET_CHECK_GT(design_matrix.norm(), kAbsoluteErrorEps)
<< "Design matrix norm is too small!";
Eigen::JacobiSVD<Eigen::Matrix3f> svd(
design_matrix, Eigen::ComputeFullU | Eigen::ComputeFullV);
Eigen::Matrix3f postrotation = svd.matrixU();
Eigen::Matrix3f prerotation = svd.matrixV().transpose();
// Disallow reflection by ensuring that det(`rotation`) = +1 (and not -1),
// see "4.6 Constrained orthogonal Procrustes problems"
// in the Gower & Dijksterhuis's book "Procrustes Analysis".
// We flip the sign of the least singular value along with a column in W.
//
// Note that now the sum of singular values doesn't work for scale
// estimation due to this sign flip.
if (postrotation.determinant() * prerotation.determinant() <
static_cast<float>(0)) {
postrotation.col(2) *= static_cast<float>(-1);
}
// Transposed (52) from the paper.
rotation = postrotation * prerotation;
return absl::OkStatus();
}
static absl::StatusOr<float> ComputeOptimalScale(
const Eigen::Matrix3Xf& centered_weighted_sources,
const Eigen::Matrix3Xf& weighted_sources,
const Eigen::Matrix3Xf& weighted_targets,
const Eigen::Matrix3f& rotation) {
// tranposed(T) tranposed(A_w) (I - C).
const auto rotated_centered_weighted_sources =
rotation * centered_weighted_sources;
// Use the identity trace(A B) = sum(A * B^T)
// to avoid building large intermediate matrices (* is Hadamard product).
// (53) from the paper.
float numerator =
rotated_centered_weighted_sources.cwiseProduct(weighted_targets).sum();
float denominator =
centered_weighted_sources.cwiseProduct(weighted_sources).sum();
RET_CHECK_GT(denominator, kAbsoluteErrorEps)
<< "Scale expression denominator is too small!";
RET_CHECK_GT(numerator / denominator, kAbsoluteErrorEps)
<< "Scale is too small!";
return numerator / denominator;
}
};
} // namespace
std::unique_ptr<ProcrustesSolver> CreateFloatPrecisionProcrustesSolver() {
return absl::make_unique<FloatPrecisionProcrustesSolver>();
}
} // namespace face_geometry
} // namespace mediapipe

70
mediapipe/modules/face_geometry/libs/procrustes_solver.h
View File

@ -1,70 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_PROCRUSTES_SOLVER_H_
#define MEDIAPIPE_FACE_GEOMETRY_LIBS_PROCRUSTES_SOLVER_H_
#include <memory>
#include "Eigen/Dense"
#include "mediapipe/framework/port/status.h"
namespace mediapipe::face_geometry {
// Encapsulates a stateless solver for the Weighted Extended Orthogonal
// Procrustes (WEOP) Problem, as defined in Section 2.4 of
// https://doi.org/10.3929/ethz-a-004656648.
//
// Given the source and the target point clouds, the algorithm estimates
// a 4x4 transformation matrix featuring the following semantic components:
//
// * Uniform scale
// * Rotation
// * Translation
//
// The matrix maps the source point cloud into the target point cloud minimizing
// the Mean Squared Error.
class ProcrustesSolver {
public:
virtual ~ProcrustesSolver() = default;
// Solves the Weighted Extended Orthogonal Procrustes (WEOP) Problem.
//
// All `source_points`, `target_points` and `point_weights` must define the
// same number of points. Elements of `point_weights` must be non-negative.
//
// A too small diameter of either of the point clouds will likely lead to
// numerical instabilities and failure to estimate the transformation.
//
// A too small point cloud total weight will likely lead to numerical
// instabilities and failure to estimate the transformation too.
//
// Small point coordinate deviation for either of the point cloud will likely
// result in a failure as it will make the solution very unstable if possible.
//
// Note: the output `transform_mat` argument is used instead of `StatusOr<>`
// return type in order to avoid Eigen memory alignment issues. Details:
// https://eigen.tuxfamily.org/dox/group__TopicStructHavingEigenMembers.html
virtual absl::Status SolveWeightedOrthogonalProblem(
const Eigen::Matrix3Xf& source_points, //
const Eigen::Matrix3Xf& target_points, //
const Eigen::VectorXf& point_weights, //
Eigen::Matrix4f& transform_mat) const = 0;
};
std::unique_ptr<ProcrustesSolver> CreateFloatPrecisionProcrustesSolver();
} // namespace mediapipe::face_geometry
#endif // MEDIAPIPE_FACE_GEOMETRY_LIBS_PROCRUSTES_SOLVER_H_

126
mediapipe/modules/face_geometry/libs/validation_utils.cc
View File

@ -1,126 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "mediapipe/modules/face_geometry/libs/validation_utils.h"
#include <cstdint>
#include <cstdlib>
#include "mediapipe/framework/formats/matrix_data.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/modules/face_geometry/libs/mesh_3d_utils.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
absl::Status ValidatePerspectiveCamera(
const PerspectiveCamera& perspective_camera) {
static constexpr float kAbsoluteErrorEps = 1e-9f;
RET_CHECK_GT(perspective_camera.near(), kAbsoluteErrorEps)
<< "Near Z must be greater than 0 with a margin of 10^{-9}!";
RET_CHECK_GT(perspective_camera.far(),
perspective_camera.near() + kAbsoluteErrorEps)
<< "Far Z must be greater than Near Z with a margin of 10^{-9}!";
RET_CHECK_GT(perspective_camera.vertical_fov_degrees(), kAbsoluteErrorEps)
<< "Vertical FOV must be positive with a margin of 10^{-9}!";
RET_CHECK_LT(perspective_camera.vertical_fov_degrees() + kAbsoluteErrorEps,
180.f)
<< "Vertical FOV must be less than 180 degrees with a margin of 10^{-9}";
return absl::OkStatus();
}
absl::Status ValidateEnvironment(const Environment& environment) {
MP_RETURN_IF_ERROR(
ValidatePerspectiveCamera(environment.perspective_camera()))
<< "Invalid perspective camera!";
return absl::OkStatus();
}
absl::Status ValidateMesh3d(const Mesh3d& mesh_3d) {
const std::size_t vertex_size = GetVertexSize(mesh_3d.vertex_type());
const std::size_t primitive_type = GetPrimitiveSize(mesh_3d.primitive_type());
RET_CHECK_EQ(mesh_3d.vertex_buffer_size() % vertex_size, 0)
<< "Vertex buffer size must a multiple of the vertex size!";
RET_CHECK_EQ(mesh_3d.index_buffer_size() % primitive_type, 0)
<< "Index buffer size must a multiple of the primitive size!";
const int num_vertices = mesh_3d.vertex_buffer_size() / vertex_size;
for (uint32_t idx : mesh_3d.index_buffer()) {
RET_CHECK_LT(idx, num_vertices)
<< "All mesh indices must refer to an existing vertex!";
}
return absl::OkStatus();
}
absl::Status ValidateFaceGeometry(const FaceGeometry& face_geometry) {
MP_RETURN_IF_ERROR(ValidateMesh3d(face_geometry.mesh())) << "Invalid mesh!";
static constexpr char kInvalid4x4MatrixMessage[] =
"Pose transformation matrix must be a 4x4 matrix!";
const MatrixData& pose_transform_matrix =
face_geometry.pose_transform_matrix();
RET_CHECK_EQ(pose_transform_matrix.rows(), 4) << kInvalid4x4MatrixMessage;
RET_CHECK_EQ(pose_transform_matrix.rows(), 4) << kInvalid4x4MatrixMessage;
RET_CHECK_EQ(pose_transform_matrix.packed_data_size(), 16)
<< kInvalid4x4MatrixMessage;
return absl::OkStatus();
}
absl::Status ValidateGeometryPipelineMetadata(
const GeometryPipelineMetadata& metadata) {
MP_RETURN_IF_ERROR(ValidateMesh3d(metadata.canonical_mesh()))
<< "Invalid canonical mesh!";
RET_CHECK_GT(metadata.procrustes_landmark_basis_size(), 0)
<< "Procrustes landmark basis must be non-empty!";
const int num_vertices =
metadata.canonical_mesh().vertex_buffer_size() /
GetVertexSize(metadata.canonical_mesh().vertex_type());
for (const WeightedLandmarkRef& wlr : metadata.procrustes_landmark_basis()) {
RET_CHECK_LT(wlr.landmark_id(), num_vertices)
<< "All Procrustes basis indices must refer to an existing canonical "
"mesh vertex!";
RET_CHECK_GE(wlr.weight(), 0.f)
<< "All Procrustes basis landmarks must have a non-negative weight!";
}
return absl::OkStatus();
}
absl::Status ValidateFrameDimensions(int frame_width, int frame_height) {
RET_CHECK_GT(frame_width, 0) << "Frame width must be positive!";
RET_CHECK_GT(frame_height, 0) << "Frame height must be positive!";
return absl::OkStatus();
}
} // namespace mediapipe::face_geometry

70
mediapipe/modules/face_geometry/libs/validation_utils.h
View File

@ -1,70 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_FACE_GEOMETRY_LIBS_VALIDATION_UTILS_H_
#define MEDIAPIPE_FACE_GEOMETRY_LIBS_VALIDATION_UTILS_H_
#include "mediapipe/framework/port/status.h"
#include "mediapipe/modules/face_geometry/protos/environment.pb.h"
#include "mediapipe/modules/face_geometry/protos/face_geometry.pb.h"
#include "mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.pb.h"
#include "mediapipe/modules/face_geometry/protos/mesh_3d.pb.h"
namespace mediapipe::face_geometry {
// Validates `perspective_camera`.
//
// Near Z must be greater than 0 with a margin of `1e-9`.
// Far Z must be greater than Near Z with a margin of `1e-9`.
// Vertical FOV must be in range (0, 180) with a margin of `1e-9` on the range
// edges.
absl::Status ValidatePerspectiveCamera(
const PerspectiveCamera& perspective_camera);
// Validates `environment`.
//
// Environment's perspective camera must be valid.
absl::Status ValidateEnvironment(const Environment& environment);
// Validates `mesh_3d`.
//
// Mesh vertex buffer size must a multiple of the vertex size.
// Mesh index buffer size must a multiple of the primitive size.
// All mesh indices must reference an existing mesh vertex.
absl::Status ValidateMesh3d(const Mesh3d& mesh_3d);
// Validates `face_geometry`.
//
// Face mesh must be valid.
// Face pose transformation matrix must be a 4x4 matrix.
absl::Status ValidateFaceGeometry(const FaceGeometry& face_geometry);
// Validates `metadata`.
//
// Canonical face mesh must be valid.
// Procrustes landmark basis must be non-empty.
// All Procrustes basis indices must reference an existing canonical mesh
// vertex.
// All Procrustes basis landmarks must have a non-negative weight.
absl::Status ValidateGeometryPipelineMetadata(
const GeometryPipelineMetadata& metadata);
// Validates frame dimensions.
//
// Both frame width and frame height must be positive.
absl::Status ValidateFrameDimensions(int frame_width, int frame_height);
} // namespace mediapipe::face_geometry
#endif // MEDIAPIPE_FACE_GEOMETRY_LIBS_VALIDATION_UTILS_H_

46
mediapipe/modules/face_geometry/protos/BUILD
View File

@ -1,46 +0,0 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
mediapipe_proto_library(
name = "environment_proto",
srcs = ["environment.proto"],
)
mediapipe_proto_library(
name = "face_geometry_proto",
srcs = ["face_geometry.proto"],
deps = [
":mesh_3d_proto",
"//mediapipe/framework/formats:matrix_data_proto",
],
)
mediapipe_proto_library(
name = "geometry_pipeline_metadata_proto",
srcs = ["geometry_pipeline_metadata.proto"],
deps = [
":mesh_3d_proto",
],
)
mediapipe_proto_library(
name = "mesh_3d_proto",
srcs = ["mesh_3d.proto"],
)

84
mediapipe/modules/face_geometry/protos/environment.proto
View File

@ -1,84 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.face_geometry;
option java_package = "com.google.mediapipe.modules.facegeometry";
option java_outer_classname = "EnvironmentProto";
// Defines the (0, 0) origin point location of the environment.
//
// The variation in the origin point location can be traced back to the memory
// layout of the camera video frame buffers.
//
// Usually, the memory layout for most CPU (and also some GPU) camera video
// frame buffers results in having the (0, 0) origin point located in the
// Top Left corner.
//
// On the contrary, the memory layout for most GPU camera video frame buffers
// results in having the (0, 0) origin point located in the Bottom Left corner.
//
// Let's consider the following example:
//
// (A) ---------------+
// ___ |
// | (1) | | |
// | / \ | | |
// | |---|===|-| |
// | |---| | | |
// | / \ | | |
// | | | | | |
// | | (2) |=| | |
// | | | | | |
// | |_______| |_| |
// | |@| |@| | | |
// | ___________|_|_ |
// |
// (B) ---------------+
//
// On this example, (1) and (2) have the same X coordinate regardless of the
// origin point location. However, having the origin point located at (A)
// (Top Left corner) results in (1) having a smaller Y coordinate if compared to
// (2). Similarly, having the origin point located at (B) (Bottom Left corner)
// results in (1) having a greater Y coordinate if compared to (2).
//
// Providing the correct origin point location for your environment and making
// sure all the input landmarks are in-sync with this location is crucial
// for receiving the correct output face geometry and visual renders.
enum OriginPointLocation {
BOTTOM_LEFT_CORNER = 1;
TOP_LEFT_CORNER = 2;
}
// The perspective camera is defined through its vertical FOV angle and the
// Z-clipping planes. The aspect ratio is a runtime variable for the face
// geometry module and should be provided alongside the face landmarks in order
// to estimate the face geometry on a given frame.
//
// More info on Perspective Cameras:
// http://www.songho.ca/opengl/gl_projectionmatrix.html#perspective
message PerspectiveCamera {
// `0 < vertical_fov_degrees < 180`.
optional float vertical_fov_degrees = 1;
// `0 < near < far`.
optional float near = 2;
optional float far = 3;
}
message Environment {
optional OriginPointLocation origin_point_location = 1;
optional PerspectiveCamera perspective_camera = 2;
}

60
mediapipe/modules/face_geometry/protos/face_geometry.proto
View File

@ -1,60 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.face_geometry;
import "mediapipe/framework/formats/matrix_data.proto";
import "mediapipe/modules/face_geometry/protos/mesh_3d.proto";
option java_package = "com.google.mediapipe.modules.facegeometry";
option java_outer_classname = "FaceGeometryProto";
// Defines the face geometry pipeline estimation result format.
message FaceGeometry {
// Defines a mesh surface for a face. The face mesh vertex IDs are the same as
// the face landmark IDs.
//
// XYZ coordinates exist in the right-handed Metric 3D space configured by an
// environment. UV coodinates are taken from the canonical face mesh model.
//
// XY coordinates are guaranteed to match the screen positions of
// the input face landmarks after (1) being multiplied by the face pose
// transformation matrix and then (2) being projected with a perspective
// camera matrix of the same environment.
//
// NOTE: the triangular topology of the face mesh is only useful when derived
// from the 468 face landmarks, not from the 6 face detection landmarks
// (keypoints). The former don't cover the entire face and this mesh is
// defined here only to comply with the API. It should be considered as
// a placeholder and/or for debugging purposes.
//
// Use the face geometry derived from the face detection landmarks
// (keypoints) for the face pose transformation matrix, not the mesh.
optional Mesh3d mesh = 1;
// Defines a face pose transformation matrix, which provides mapping from
// the static canonical face model to the runtime face. Tries to distinguish
// a head pose change from a facial expression change and to only reflect the
// former.
//
// Is a 4x4 matrix and contains only the following components:
// * Uniform scale
// * Rotation
// * Translation
//
// The last row is guaranteed to be `[0 0 0 1]`.
optional MatrixData pose_transform_matrix = 2;
}

63
mediapipe/modules/face_geometry/protos/geometry_pipeline_metadata.proto
View File

@ -1,63 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.face_geometry;
import "mediapipe/modules/face_geometry/protos/mesh_3d.proto";
option java_package = "com.google.mediapipe.modules.facegeometry";
option java_outer_classname = "GeometryPipelineMetadataProto";
enum InputSource {
DEFAULT = 0; // FACE_LANDMARK_PIPELINE
FACE_LANDMARK_PIPELINE = 1;
FACE_DETECTION_PIPELINE = 2;
}
message WeightedLandmarkRef {
// Defines the landmark ID. References an existing face landmark ID.
optional uint32 landmark_id = 1;
// Defines the landmark weight. The larger the weight the more influence this
// landmark has in the basis.
//
// Is positive.
optional float weight = 2;
}
// Next field ID: 4
message GeometryPipelineMetadata {
// Defines the source of the input landmarks to let the underlying geometry
// pipeline to adjust in order to produce the best results.
//
// Face landmark pipeline is expected to produce 3D landmarks with relative Z
// coordinate, which is scaled as the X coordinate assuming the weak
// perspective projection camera model.
//
// Face landmark pipeline is expected to produce 2D landmarks with Z
// coordinate being equal to 0.
optional InputSource input_source = 3;
// Defines a mesh surface for a canonical face. The canonical face mesh vertex
// IDs are the same as the face landmark IDs.
//
// XYZ coordinates are defined in centimeter units.
optional Mesh3d canonical_mesh = 1;
// Defines a weighted landmark basis for running the Procrustes solver
// algorithm inside the geometry pipeline.
//
// A good basis sets face landmark weights in way to distinguish a head pose
// change from a facial expression change and to only respond to the former.
repeated WeightedLandmarkRef procrustes_landmark_basis = 2;
}

41
mediapipe/modules/face_geometry/protos/mesh_3d.proto
View File

@ -1,41 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe.face_geometry;
option java_package = "com.google.mediapipe.modules.facegeometry";
option java_outer_classname = "Mesh3dProto";
message Mesh3d {
enum VertexType {
// Is defined by 5 coordinates: Position (XYZ) + Texture coordinate (UV).
VERTEX_PT = 0;
}
enum PrimitiveType {
// Is defined by 3 indices: triangle vertex IDs.
TRIANGLE = 0;
}
optional VertexType vertex_type = 1;
optional PrimitiveType primitive_type = 2;
// Vertex buffer size is a multiple of the vertex size (e.g., 5 for
// VERTEX_PT).
repeated float vertex_buffer = 3;
// Index buffer size is a multiple of the primitive size (e.g., 3 for
// TRIANGLE).
repeated uint32 index_buffer = 4;
}

190
mediapipe/modules/face_landmark/BUILD
View File

@ -1,190 +0,0 @@
# Copyright 2019 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"mediapipe_simple_subgraph",
)
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
mediapipe_simple_subgraph(
name = "face_landmark_cpu",
graph = "face_landmark_cpu.pbtxt",
register_as = "FaceLandmarkCpu",
deps = [
":face_landmarks_model_loader",
":tensors_to_face_landmarks",
":tensors_to_face_landmarks_with_attention",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
"//mediapipe/calculators/util:landmark_projection_calculator",
"//mediapipe/calculators/util:thresholding_calculator",
"//mediapipe/framework/tool:switch_container",
],
)
mediapipe_simple_subgraph(
name = "face_landmark_gpu",
graph = "face_landmark_gpu.pbtxt",
register_as = "FaceLandmarkGpu",
deps = [
":face_landmarks_model_loader",
":tensors_to_face_landmarks",
":tensors_to_face_landmarks_with_attention",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
"//mediapipe/calculators/tflite:tflite_custom_op_resolver_calculator",
"//mediapipe/calculators/util:landmark_projection_calculator",
"//mediapipe/calculators/util:thresholding_calculator",
"//mediapipe/framework/tool:switch_container",
],
)
mediapipe_simple_subgraph(
name = "face_landmark_front_cpu",
graph = "face_landmark_front_cpu.pbtxt",
register_as = "FaceLandmarkFrontCpu",
deps = [
":face_detection_front_detection_to_roi",
":face_landmark_cpu",
":face_landmark_landmarks_to_roi",
"//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:clip_vector_size_calculator",
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/core:end_loop_calculator",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:previous_loopback_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/util:association_norm_rect_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator",
"//mediapipe/modules/face_detection:face_detection_short_range_cpu",
],
)
mediapipe_simple_subgraph(
name = "face_landmark_front_gpu",
graph = "face_landmark_front_gpu.pbtxt",
register_as = "FaceLandmarkFrontGpu",
deps = [
":face_detection_front_detection_to_roi",
":face_landmark_gpu",
":face_landmark_landmarks_to_roi",
"//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:clip_vector_size_calculator",
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/core:end_loop_calculator",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:previous_loopback_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/util:association_norm_rect_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator",
"//mediapipe/modules/face_detection:face_detection_short_range_gpu",
],
)
mediapipe_simple_subgraph(
name = "face_landmark_front_cpu_image",
graph = "face_landmark_front_cpu_image.pbtxt",
register_as = "FaceLandmarkFrontCpuImage",
deps = [
":face_landmark_front_cpu",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/calculators/util:from_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_landmark_front_gpu_image",
graph = "face_landmark_front_gpu_image.pbtxt",
register_as = "FaceLandmarkFrontGpuImage",
deps = [
":face_landmark_front_gpu",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/calculators/util:from_image_calculator",
],
)
exports_files(
srcs = [
"face_landmark.tflite",
"face_landmark_with_attention.tflite",
],
)
mediapipe_simple_subgraph(
name = "face_detection_front_detection_to_roi",
graph = "face_detection_front_detection_to_roi.pbtxt",
register_as = "FaceDetectionFrontDetectionToRoi",
deps = [
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_landmark_landmarks_to_roi",
graph = "face_landmark_landmarks_to_roi.pbtxt",
register_as = "FaceLandmarkLandmarksToRoi",
deps = [
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_landmarks_model_loader",
graph = "face_landmarks_model_loader.pbtxt",
register_as = "FaceLandmarksModelLoader",
deps = [
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/tflite:tflite_model_calculator",
"//mediapipe/calculators/util:local_file_contents_calculator",
"//mediapipe/framework/tool:switch_container",
],
)
mediapipe_simple_subgraph(
name = "tensors_to_face_landmarks",
graph = "tensors_to_face_landmarks.pbtxt",
register_as = "TensorsToFaceLandmarks",
deps = [
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
],
)
mediapipe_simple_subgraph(
name = "tensors_to_face_landmarks_with_attention",
graph = "tensors_to_face_landmarks_with_attention.pbtxt",
register_as = "TensorsToFaceLandmarksWithAttention",
deps = [
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
"//mediapipe/calculators/util:landmarks_refinement_calculator",
],
)

9
mediapipe/modules/face_landmark/README.md
View File

@ -1,9 +0,0 @@
# face_landmark
Subgraphs|Details
:--- | :---
[`FaceLandmarkCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_cpu.pbtxt)| Detects landmarks on a single face. (CPU input, and inference is executed on CPU.)
[`FaceLandmarkGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_gpu.pbtxt)| Detects landmarks on a single face. (GPU input, and inference is executed on GPU)
[`FaceLandmarkFrontCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt)| Detects and tracks landmarks on multiple faces. (CPU input, and inference is executed on CPU)
[`FaceLandmarkFrontGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt)| Detects and tracks landmarks on multiple faces. (GPU input, and inference is executed on GPU.)

47
mediapipe/modules/face_landmark/face_detection_front_detection_to_roi.pbtxt
View File

@ -1,47 +0,0 @@
# MediaPipe graph to calculate face region of interest (ROI) from the very
# first face detection in the vector of detections provided by
# "FaceDetectionShortRangeCpu" or "FaceDetectionShortRangeGpu"
#
# NOTE: this graph is subject to change and should not be used directly.
type: "FaceDetectionFrontDetectionToRoi"
# Face detection. (Detection)
input_stream: "DETECTION:detection"
# Frame size (width and height). (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# ROI according to the first detection of input detections. (NormalizedRect)
output_stream: "ROI:roi"
# Converts results of face detection into a rectangle (normalized by image size)
# that encloses the face and is rotated such that the line connecting left eye
# and right eye is aligned with the X-axis of the rectangle.
node {
calculator: "DetectionsToRectsCalculator"
input_stream: "DETECTION:detection"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECT:initial_roi"
options: {
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
rotation_vector_start_keypoint_index: 0 # Left eye.
rotation_vector_end_keypoint_index: 1 # Right eye.
rotation_vector_target_angle_degrees: 0
}
}
}
# Expands and shifts the rectangle that contains the face so that it's likely
# to cover the entire face.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:initial_roi"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "roi"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 1.5
scale_y: 1.5
square_long: true
}
}
}

BIN
mediapipe/modules/face_landmark/face_landmark.tflite
View File

Binary file not shown.

184
mediapipe/modules/face_landmark/face_landmark_cpu.pbtxt
View File

@ -1,184 +0,0 @@
# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is
# executed on CPU.)
#
# It is required that "face_landmark.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark.tflite"
# path during execution if `with_attention` is not set or set to `false`.
#
# It is required that "face_landmark_with_attention.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark_with_attention.tflite"
# path during execution if `with_attention` is set to `true`.
#
# EXAMPLE:
# node {
# calculator: "FaceLandmarkCpu"
# input_stream: "IMAGE:image"
# input_stream: "ROI:face_roi"
# input_side_packet: "WITH_ATTENTION:with_attention"
# output_stream: "LANDMARKS:face_landmarks"
# }
type: "FaceLandmarkCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where a face is located.
# (NormalizedRect)
input_stream: "ROI:roi"
# Whether to run face mesh model with attention on lips and eyes. (bool)
# Attention provides more accuracy on lips and eye regions as well as iris
# landmarks.
input_side_packet: "WITH_ATTENTION:with_attention"
# 468 or 478 facial landmarks within the given ROI. (NormalizedLandmarkList)
#
# Number of landmarks depends on the WITH_ATTENTION flag. If it's `true` - then
# there will be 478 landmarks with refined lips, eyes and irises (10 extra
# landmarks are for irises), otherwise 468 non-refined landmarks are returned.
#
# NOTE: if a face is not present within the given ROI, for this particular
# timestamp there will not be an output packet in the LANDMARKS stream. However,
# the MediaPipe framework will internally inform the downstream calculators of
# the absence of this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:face_landmarks"
# Transforms the input image into a 192x192 tensor.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:image"
input_stream: "NORM_RECT:roi"
output_stream: "TENSORS:input_tensors"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 192
output_tensor_height: 192
output_tensor_float_range {
min: 0.0
max: 1.0
}
}
}
}
# Loads the face landmarks TF Lite model.
node {
calculator: "FaceLandmarksModelLoader"
input_side_packet: "WITH_ATTENTION:with_attention"
output_side_packet: "MODEL:model"
}
# Generates a single side packet containing a TensorFlow Lite op resolver that
# supports custom ops needed by the model used in this graph.
node {
calculator: "TfLiteCustomOpResolverCalculator"
output_side_packet: "op_resolver"
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
input_side_packet: "MODEL:model"
input_side_packet: "CUSTOM_OP_RESOLVER:op_resolver"
output_stream: "TENSORS:output_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
delegate { xnnpack {} }
}
}
}
# Splits a vector of tensors into landmark tensors and face flag tensor.
node {
calculator: "SwitchContainer"
input_side_packet: "ENABLE:with_attention"
input_stream: "output_tensors"
output_stream: "landmark_tensors"
output_stream: "face_flag_tensor"
options: {
[mediapipe.SwitchContainerOptions.ext] {
contained_node: {
calculator: "SplitTensorVectorCalculator"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 1 }
ranges: { begin: 1 end: 2 }
}
}
}
contained_node: {
calculator: "SplitTensorVectorCalculator"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 6 }
ranges: { begin: 6 end: 7 }
}
}
}
}
}
}
# Converts the face-flag tensor into a float that represents the confidence
# score of face presence.
node {
calculator: "TensorsToFloatsCalculator"
input_stream: "TENSORS:face_flag_tensor"
output_stream: "FLOAT:face_presence_score"
options {
[mediapipe.TensorsToFloatsCalculatorOptions.ext] {
activation: SIGMOID
}
}
}
# Applies a threshold to the confidence score to determine whether a face is
# present.
node {
calculator: "ThresholdingCalculator"
input_stream: "FLOAT:face_presence_score"
output_stream: "FLAG:face_presence"
options: {
[mediapipe.ThresholdingCalculatorOptions.ext] {
threshold: 0.5
}
}
}
# Drop landmarks tensors if face is not present.
node {
calculator: "GateCalculator"
input_stream: "landmark_tensors"
input_stream: "ALLOW:face_presence"
output_stream: "ensured_landmark_tensors"
}
# Decodes the landmark tensors into a vector of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "SwitchContainer"
input_side_packet: "ENABLE:with_attention"
input_stream: "TENSORS:ensured_landmark_tensors"
output_stream: "LANDMARKS:landmarks"
options: {
[mediapipe.SwitchContainerOptions.ext] {
contained_node: {
calculator: "TensorsToFaceLandmarks"
}
contained_node: {
calculator: "TensorsToFaceLandmarksWithAttention"
}
}
}
}
# Projects the landmarks from the cropped face image to the corresponding
# locations on the full image before cropping (input to the graph).
node {
calculator: "LandmarkProjectionCalculator"
input_stream: "NORM_LANDMARKS:landmarks"
input_stream: "NORM_RECT:roi"
output_stream: "NORM_LANDMARKS:face_landmarks"
}

247
mediapipe/modules/face_landmark/face_landmark_front_cpu.pbtxt
View File

@ -1,247 +0,0 @@
# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is
# executed on CPU.) This graph tries to skip face detection as much as possible
# by using previously detected/predicted landmarks for new images.
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# It is required that "face_landmark.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark.tflite"
# path during execution if `with_attention` is not set or set to `false`.
#
# It is required that "face_landmark_with_attention.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark_with_attention.tflite"
# path during execution if `with_attention` is set to `true`.
#
# EXAMPLE:
# node {
# calculator: "FaceLandmarkFrontCpu"
# input_stream: "IMAGE:image"
# input_side_packet: "NUM_FACES:num_faces"
# input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
# input_side_packet: "WITH_ATTENTION:with_attention"
# output_stream: "LANDMARKS:multi_face_landmarks"
# }
type: "FaceLandmarkFrontCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
# Max number of faces to detect/track. (int)
input_side_packet: "NUM_FACES:num_faces"
# Whether landmarks on the previous image should be used to help localize
# landmarks on the current image. (bool)
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
# Whether to run face mesh model with attention on lips and eyes. (bool)
# Attention provides more accuracy on lips and eye regions as well as iris
# landmarks.
input_side_packet: "WITH_ATTENTION:with_attention"
# Collection of detected/predicted faces, each represented as a list of 468 face
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_face_landmarks"
# Extra outputs (for debugging, for instance).
# Detected faces. (std::vector<Detection>)
output_stream: "DETECTIONS:face_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
# Regions of interest calculated based on face detections.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
# When the optional input side packet "use_prev_landmarks" is either absent or
# set to true, uses the landmarks on the previous image to help localize
# landmarks on the current image.
node {
calculator: "GateCalculator"
input_side_packet: "ALLOW:use_prev_landmarks"
input_stream: "prev_face_rects_from_landmarks"
output_stream: "gated_prev_face_rects_from_landmarks"
options: {
[mediapipe.GateCalculatorOptions.ext] {
allow: true
}
}
}
# Determines if an input vector of NormalizedRect has a size greater than or
# equal to the provided num_faces.
node {
calculator: "NormalizedRectVectorHasMinSizeCalculator"
input_stream: "ITERABLE:gated_prev_face_rects_from_landmarks"
input_side_packet: "num_faces"
output_stream: "prev_has_enough_faces"
}
# Drops the incoming image if enough faces have already been identified from the
# previous image. Otherwise, passes the incoming image through to trigger a new
# round of face detection.
node {
calculator: "GateCalculator"
input_stream: "image"
input_stream: "DISALLOW:prev_has_enough_faces"
output_stream: "gated_image"
options: {
[mediapipe.GateCalculatorOptions.ext] {
empty_packets_as_allow: true
}
}
}
# Detects faces.
node {
calculator: "FaceDetectionShortRangeCpu"
input_stream: "IMAGE:gated_image"
output_stream: "DETECTIONS:all_face_detections"
}
# Makes sure there are no more detections than the provided num_faces.
node {
calculator: "ClipDetectionVectorSizeCalculator"
input_stream: "all_face_detections"
output_stream: "face_detections"
input_side_packet: "num_faces"
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:gated_image"
output_stream: "SIZE:gated_image_size"
}
# Outputs each element of face_detections at a fake timestamp for the rest of
# the graph to process. Clones the image size packet for each face_detection at
# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp
# for downstream calculators to inform them that all elements in the vector have
# been processed.
node {
calculator: "BeginLoopDetectionCalculator"
input_stream: "ITERABLE:face_detections"
input_stream: "CLONE:gated_image_size"
output_stream: "ITEM:face_detection"
output_stream: "CLONE:detections_loop_image_size"
output_stream: "BATCH_END:detections_loop_end_timestamp"
}
# Calculates region of interest based on face detections, so that can be used
# to detect landmarks.
node {
calculator: "FaceDetectionFrontDetectionToRoi"
input_stream: "DETECTION:face_detection"
input_stream: "IMAGE_SIZE:detections_loop_image_size"
output_stream: "ROI:face_rect_from_detection"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_detection"
input_stream: "BATCH_END:detections_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_detections"
}
# Performs association between NormalizedRect vector elements from previous
# image and rects based on face detections from the current image. This
# calculator ensures that the output face_rects vector doesn't contain
# overlapping regions based on the specified min_similarity_threshold.
node {
calculator: "AssociationNormRectCalculator"
input_stream: "face_rects_from_detections"
input_stream: "gated_prev_face_rects_from_landmarks"
output_stream: "face_rects"
options: {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.5
}
}
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:image"
output_stream: "SIZE:image_size"
}
# Outputs each element of face_rects at a fake timestamp for the rest of the
# graph to process. Clones image and image size packets for each
# single_face_rect at the fake timestamp. At the end of the loop, outputs the
# BATCH_END timestamp for downstream calculators to inform them that all
# elements in the vector have been processed.
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:face_rects"
input_stream: "CLONE:0:image"
input_stream: "CLONE:1:image_size"
output_stream: "ITEM:face_rect"
output_stream: "CLONE:0:landmarks_loop_image"
output_stream: "CLONE:1:landmarks_loop_image_size"
output_stream: "BATCH_END:landmarks_loop_end_timestamp"
}
# Detects face landmarks within specified region of interest of the image.
node {
calculator: "FaceLandmarkCpu"
input_stream: "IMAGE:landmarks_loop_image"
input_stream: "ROI:face_rect"
input_side_packet: "WITH_ATTENTION:with_attention"
output_stream: "LANDMARKS:face_landmarks"
}
# Calculates region of interest based on face landmarks, so that can be reused
# for subsequent image.
node {
calculator: "FaceLandmarkLandmarksToRoi"
input_stream: "LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:landmarks_loop_image_size"
output_stream: "ROI:face_rect_from_landmarks"
}
# Collects a set of landmarks for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:face_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:multi_face_landmarks"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_landmarks"
}
# Caches face rects calculated from landmarks, and upon the arrival of the next
# input image, sends out the cached rects with timestamps replaced by that of
# the input image, essentially generating a packet that carries the previous
# face rects. Note that upon the arrival of the very first input image, a
# timestamp bound update occurs to jump start the feedback loop.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:image"
input_stream: "LOOP:face_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
}
output_stream: "PREV_LOOP:prev_face_rects_from_landmarks"
}

87
mediapipe/modules/face_landmark/face_landmark_front_cpu_image.pbtxt
View File

@ -1,87 +0,0 @@
# MediaPipe graph to detect/predict face landmarks on CPU.
type: "FaceLandmarkFrontCpuImage"
# Input image. (Image)
input_stream: "IMAGE:image"
# Max number of faces to detect/track. (int)
input_side_packet: "NUM_FACES:num_faces"
# Whether landmarks on the previous image should be used to help localize
# landmarks on the current image. (bool)
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
# Whether to run face mesh model with attention on lips and eyes. (bool)
# Attention provides more accuracy on lips and eye regions as well as iris
# landmarks.
input_side_packet: "WITH_ATTENTION:with_attention"
# The throttled input image. (Image)
output_stream: "IMAGE:throttled_image"
# Collection of detected/predicted faces, each represented as a list of 468 face
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_face_landmarks"
# Extra outputs (for debugging, for instance).
# Detected faces. (std::vector<Detection>)
output_stream: "DETECTIONS:face_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
# Regions of interest calculated based on face detections.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
node {
calculator: "FlowLimiterCalculator"
input_stream: "image"
input_stream: "FINISHED:multi_face_landmarks"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_image"
options: {
[mediapipe.FlowLimiterCalculatorOptions.ext] {
max_in_flight: 1
max_in_queue: 1
}
}
}
# Converts Image to ImageFrame for FaceLandmarkFrontCpu to consume.
node {
calculator: "FromImageCalculator"
input_stream: "IMAGE:throttled_image"
output_stream: "IMAGE_CPU:raw_image_frame"
output_stream: "SOURCE_ON_GPU:is_gpu_image"
}
# TODO: Remove the extra flipping once adopting MlImage.
# If the source images are on gpu, flip the data vertically before sending them
# into FaceLandmarkFrontCpu. This maybe needed because OpenGL represents images
# assuming the image origin is at the bottom-left corner, whereas MediaPipe in
# general assumes the image origin is at the top-left corner.
node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE:raw_image_frame"
input_stream: "FLIP_VERTICALLY:is_gpu_image"
output_stream: "IMAGE:image_frame"
}
node {
calculator: "FaceLandmarkFrontCpu"
input_stream: "IMAGE:image_frame"
input_side_packet: "NUM_FACES:num_faces"
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
input_side_packet: "WITH_ATTENTION:with_attention"
output_stream: "LANDMARKS:multi_face_landmarks"
output_stream: "DETECTIONS:face_detections"
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
}

247
mediapipe/modules/face_landmark/face_landmark_front_gpu.pbtxt
View File

@ -1,247 +0,0 @@
# MediaPipe graph to detect/predict face landmarks. (GPU input, and inference is
# executed on GPU.) This graph tries to skip face detection as much as possible
# by using previously detected/predicted landmarks for new images.
#
# It is required that "face_detection_short_range.tflite" is available at
# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
# path during execution.
#
# It is required that "face_landmark.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark.tflite"
# path during execution if `with_attention` is not set or set to `false`.
#
# It is required that "face_landmark_with_attention.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark_with_attention.tflite"
# path during execution if `with_attention` is set to `true`.
#
# EXAMPLE:
# node {
# calculator: "FaceLandmarkFrontGpu"
# input_stream: "IMAGE:image"
# input_side_packet: "NUM_FACES:num_faces"
# input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
# input_side_packet: "WITH_ATTENTION:with_attention"
# output_stream: "LANDMARKS:multi_face_landmarks"
# }
type: "FaceLandmarkFrontGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
# Max number of faces to detect/track. (int)
input_side_packet: "NUM_FACES:num_faces"
# Whether landmarks on the previous image should be used to help localize
# landmarks on the current image. (bool)
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
# Whether to run face mesh model with attention on lips and eyes. (bool)
# Attention provides more accuracy on lips and eye regions as well as iris
# landmarks.
input_side_packet: "WITH_ATTENTION:with_attention"
# Collection of detected/predicted faces, each represented as a list of 468 face
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_face_landmarks"
# Extra outputs (for debugging, for instance).
# Detected faces. (std::vector<Detection>)
output_stream: "DETECTIONS:face_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
# Regions of interest calculated based on face detections.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
# When the optional input side packet "use_prev_landmarks" is either absent or
# set to true, uses the landmarks on the previous image to help localize
# landmarks on the current image.
node {
calculator: "GateCalculator"
input_side_packet: "ALLOW:use_prev_landmarks"
input_stream: "prev_face_rects_from_landmarks"
output_stream: "gated_prev_face_rects_from_landmarks"
options: {
[mediapipe.GateCalculatorOptions.ext] {
allow: true
}
}
}
# Determines if an input vector of NormalizedRect has a size greater than or
# equal to the provided num_faces.
node {
calculator: "NormalizedRectVectorHasMinSizeCalculator"
input_stream: "ITERABLE:gated_prev_face_rects_from_landmarks"
input_side_packet: "num_faces"
output_stream: "prev_has_enough_faces"
}
# Drops the incoming image if enough faces have already been identified from the
# previous image. Otherwise, passes the incoming image through to trigger a new
# round of face detection.
node {
calculator: "GateCalculator"
input_stream: "image"
input_stream: "DISALLOW:prev_has_enough_faces"
output_stream: "gated_image"
options: {
[mediapipe.GateCalculatorOptions.ext] {
empty_packets_as_allow: true
}
}
}
# Detects faces.
node {
calculator: "FaceDetectionShortRangeGpu"
input_stream: "IMAGE:gated_image"
output_stream: "DETECTIONS:all_face_detections"
}
# Makes sure there are no more detections than the provided num_faces.
node {
calculator: "ClipDetectionVectorSizeCalculator"
input_stream: "all_face_detections"
output_stream: "face_detections"
input_side_packet: "num_faces"
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:gated_image"
output_stream: "SIZE:gated_image_size"
}
# Outputs each element of face_detections at a fake timestamp for the rest of
# the graph to process. Clones the image size packet for each face_detection at
# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp
# for downstream calculators to inform them that all elements in the vector have
# been processed.
node {
calculator: "BeginLoopDetectionCalculator"
input_stream: "ITERABLE:face_detections"
input_stream: "CLONE:gated_image_size"
output_stream: "ITEM:face_detection"
output_stream: "CLONE:detections_loop_image_size"
output_stream: "BATCH_END:detections_loop_end_timestamp"
}
# Calculates region of interest based on face detections, so that can be used
# to detect landmarks.
node {
calculator: "FaceDetectionFrontDetectionToRoi"
input_stream: "DETECTION:face_detection"
input_stream: "IMAGE_SIZE:detections_loop_image_size"
output_stream: "ROI:face_rect_from_detection"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_detection"
input_stream: "BATCH_END:detections_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_detections"
}
# Performs association between NormalizedRect vector elements from previous
# image and rects based on face detections from the current image. This
# calculator ensures that the output face_rects vector doesn't contain
# overlapping regions based on the specified min_similarity_threshold.
node {
calculator: "AssociationNormRectCalculator"
input_stream: "face_rects_from_detections"
input_stream: "gated_prev_face_rects_from_landmarks"
output_stream: "face_rects"
options: {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.5
}
}
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "SIZE:image_size"
}
# Outputs each element of face_rects at a fake timestamp for the rest of the
# graph to process. Clones image and image size packets for each
# single_face_rect at the fake timestamp. At the end of the loop, outputs the
# BATCH_END timestamp for downstream calculators to inform them that all
# elements in the vector have been processed.
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:face_rects"
input_stream: "CLONE:0:image"
input_stream: "CLONE:1:image_size"
output_stream: "ITEM:face_rect"
output_stream: "CLONE:0:landmarks_loop_image"
output_stream: "CLONE:1:landmarks_loop_image_size"
output_stream: "BATCH_END:landmarks_loop_end_timestamp"
}
# Detects face landmarks within specified region of interest of the image.
node {
calculator: "FaceLandmarkGpu"
input_stream: "IMAGE:landmarks_loop_image"
input_stream: "ROI:face_rect"
input_side_packet: "WITH_ATTENTION:with_attention"
output_stream: "LANDMARKS:face_landmarks"
}
# Calculates region of interest based on face landmarks, so that can be reused
# for subsequent image.
node {
calculator: "FaceLandmarkLandmarksToRoi"
input_stream: "LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:landmarks_loop_image_size"
output_stream: "ROI:face_rect_from_landmarks"
}
# Collects a set of landmarks for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:face_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:multi_face_landmarks"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_landmarks"
}
# Caches face rects calculated from landmarks, and upon the arrival of the next
# input image, sends out the cached rects with timestamps replaced by that of
# the input image, essentially generating a packet that carries the previous
# face rects. Note that upon the arrival of the very first input image, a
# timestamp bound update occurs to jump start the feedback loop.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:image"
input_stream: "LOOP:face_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
}
output_stream: "PREV_LOOP:prev_face_rects_from_landmarks"
}

87
mediapipe/modules/face_landmark/face_landmark_front_gpu_image.pbtxt
View File

@ -1,87 +0,0 @@
# MediaPipe graph to detect/predict face landmarks on GPU.
type: "FaceLandmarkFrontGpuImage"
# Input image. (Image)
input_stream: "IMAGE:image"
# Max number of faces to detect/track. (int)
input_side_packet: "NUM_FACES:num_faces"
# Whether landmarks on the previous image should be used to help localize
# landmarks on the current image. (bool)
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
# Whether to run face mesh model with attention on lips and eyes. (bool)
# Attention provides more accuracy on lips and eye regions as well as iris
# landmarks.
input_side_packet: "WITH_ATTENTION:with_attention"
# The throttled input image. (Image)
output_stream: "IMAGE:throttled_image"
# Collection of detected/predicted faces, each represented as a list of 468 face
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_face_landmarks"
# Extra outputs (for debugging, for instance).
# Detected faces. (std::vector<Detection>)
output_stream: "DETECTIONS:face_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
# Regions of interest calculated based on face detections.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
node {
calculator: "FlowLimiterCalculator"
input_stream: "image"
input_stream: "FINISHED:multi_face_landmarks"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_image"
options: {
[mediapipe.FlowLimiterCalculatorOptions.ext] {
max_in_flight: 1
max_in_queue: 1
}
}
}
# Converts Image to GpuBuffer for FaceLandmarkFrontGpu to consume.
node {
calculator: "FromImageCalculator"
input_stream: "IMAGE:throttled_image"
output_stream: "IMAGE_GPU:raw_gpu_buffer"
output_stream: "SOURCE_ON_GPU:is_gpu_image"
}
# TODO: Remove the extra flipping once adopting MlImage.
# If the source images are on gpu, flip the data vertically before sending them
# into FaceLandmarkFrontGpu. This maybe needed because OpenGL represents images
# assuming the image origin is at the bottom-left corner, whereas MediaPipe in
# general assumes the image origin is at the top-left corner.
node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:raw_gpu_buffer"
input_stream: "FLIP_VERTICALLY:is_gpu_image"
output_stream: "IMAGE_GPU:gpu_buffer"
}
node {
calculator: "FaceLandmarkFrontGpu"
input_stream: "IMAGE:gpu_buffer"
input_side_packet: "NUM_FACES:num_faces"
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
input_side_packet: "WITH_ATTENTION:with_attention"
output_stream: "LANDMARKS:multi_face_landmarks"
output_stream: "DETECTIONS:face_detections"
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
}

224
mediapipe/modules/face_landmark/face_landmark_front_side_model_cpu.pbtxt
View File

@ -1,224 +0,0 @@
# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is
# executed on CPU.) This graph tries to skip face detection as much as possible
# by using previously detected/predicted landmarks for new images.
#
# EXAMPLE:
# node {
# calculator: "FaceLandmarkFrontSideModelCpu"
# input_stream: "IMAGE:image"
# input_side_packet: "NUM_FACES:num_faces"
# input_side_packet: "MODEL:0:face_detection_model"
# input_side_packet: "MODEL:1:face_landmark_model"
# output_stream: "LANDMARKS:multi_face_landmarks"
# }
type: "FaceLandmarkFrontSideModelCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
# Max number of faces to detect/track. (int)
input_side_packet: "NUM_FACES:num_faces"
# TfLite model to detect faces.
# (std::unique_ptr<tflite::FlatBufferModel,
# std::function<void(tflite::FlatBufferModel*)>>)
# NOTE: mediapipe/modules/face_detection/face_detection_short_range.tflite
# model only, can be passed here, otherwise - results are undefined.
input_side_packet: "MODEL:0:face_detection_model"
# TfLite model to detect face landmarks.
# (std::unique_ptr<tflite::FlatBufferModel,
# std::function<void(tflite::FlatBufferModel*)>>)
# NOTE: mediapipe/modules/face_landmark/face_landmark.tflite model
# only, can be passed here, otherwise - results are undefined.
input_side_packet: "MODEL:1:face_landmark_model"
# Collection of detected/predicted faces, each represented as a list of 468 face
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_face_landmarks"
# Extra outputs (for debugging, for instance).
# Detected faces. (std::vector<Detection>)
output_stream: "DETECTIONS:face_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
# Regions of interest calculated based on face detections.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
# Determines if an input vector of NormalizedRect has a size greater than or
# equal to the provided num_faces.
node {
calculator: "NormalizedRectVectorHasMinSizeCalculator"
input_stream: "ITERABLE:prev_face_rects_from_landmarks"
input_side_packet: "num_faces"
output_stream: "prev_has_enough_faces"
}
# Drops the incoming image if FaceLandmarkCpu was able to identify face presence
# in the previous image. Otherwise, passes the incoming image through to trigger
# a new round of face detection in FaceDetectionShortRangeCpu.
node {
calculator: "GateCalculator"
input_stream: "image"
input_stream: "DISALLOW:prev_has_enough_faces"
output_stream: "gated_image"
options: {
[mediapipe.GateCalculatorOptions.ext] {
empty_packets_as_allow: true
}
}
}
# Detects faces.
node {
calculator: "FaceDetectionShortRangeSideModelCpu"
input_stream: "IMAGE:gated_image"
input_side_packet: "MODEL:face_detection_model"
output_stream: "DETECTIONS:all_face_detections"
}
# Makes sure there are no more detections than the provided num_faces.
node {
calculator: "ClipDetectionVectorSizeCalculator"
input_stream: "all_face_detections"
output_stream: "face_detections"
input_side_packet: "num_faces"
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:gated_image"
output_stream: "SIZE:gated_image_size"
}
# Outputs each element of face_detections at a fake timestamp for the rest of
# the graph to process. Clones the image size packet for each face_detection at
# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp
# for downstream calculators to inform them that all elements in the vector have
# been processed.
node {
calculator: "BeginLoopDetectionCalculator"
input_stream: "ITERABLE:face_detections"
input_stream: "CLONE:gated_image_size"
output_stream: "ITEM:face_detection"
output_stream: "CLONE:detections_loop_image_size"
output_stream: "BATCH_END:detections_loop_end_timestamp"
}
# Calculates region of interest based on face detections, so that can be used
# to detect landmarks.
node {
calculator: "FaceDetectionFrontDetectionToRoi"
input_stream: "DETECTION:face_detection"
input_stream: "IMAGE_SIZE:detections_loop_image_size"
output_stream: "ROI:face_rect_from_detection"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_detection"
input_stream: "BATCH_END:detections_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_detections"
}
# Performs association between NormalizedRect vector elements from previous
# image and rects based on face detections from the current image. This
# calculator ensures that the output face_rects vector doesn't contain
# overlapping regions based on the specified min_similarity_threshold.
node {
calculator: "AssociationNormRectCalculator"
input_stream: "face_rects_from_detections"
input_stream: "prev_face_rects_from_landmarks"
output_stream: "face_rects"
options: {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.5
}
}
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:image"
output_stream: "SIZE:image_size"
}
# Outputs each element of face_rects at a fake timestamp for the rest of the
# graph to process. Clones image and image size packets for each
# single_face_rect at the fake timestamp. At the end of the loop, outputs the
# BATCH_END timestamp for downstream calculators to inform them that all
# elements in the vector have been processed.
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:face_rects"
input_stream: "CLONE:0:image"
input_stream: "CLONE:1:image_size"
output_stream: "ITEM:face_rect"
output_stream: "CLONE:0:landmarks_loop_image"
output_stream: "CLONE:1:landmarks_loop_image_size"
output_stream: "BATCH_END:landmarks_loop_end_timestamp"
}
# Detects face landmarks within specified region of interest of the image.
node {
calculator: "FaceLandmarkSideModelCpu"
input_stream: "IMAGE:landmarks_loop_image"
input_stream: "ROI:face_rect"
input_side_packet: "MODEL:face_landmark_model"
output_stream: "LANDMARKS:face_landmarks"
}
# Calculates region of interest based on face landmarks, so that can be reused
# for subsequent image.
node {
calculator: "FaceLandmarkLandmarksToRoi"
input_stream: "LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:landmarks_loop_image_size"
output_stream: "ROI:face_rect_from_landmarks"
}
# Collects a set of landmarks for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:face_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:multi_face_landmarks"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_landmarks"
}
# Caches face rects calculated from landmarks, and upon the arrival of the next
# input image, sends out the cached rects with timestamps replaced by that of
# the input image, essentially generating a packet that carries the previous
# face rects. Note that upon the arrival of the very first input image, a
# timestamp bound update occurs to jump start the feedback loop.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:image"
input_stream: "LOOP:face_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
}
output_stream: "PREV_LOOP:prev_face_rects_from_landmarks"
}

224
mediapipe/modules/face_landmark/face_landmark_front_side_model_gpu.pbtxt
View File

@ -1,224 +0,0 @@
# MediaPipe graph to detect/predict face landmarks. (GPU input, and inference is
# executed on GPU.) This graph tries to skip face detection as much as possible
# by using previously detected/predicted landmarks for new images.
#
# EXAMPLE:
# node {
# calculator: "FaceLandmarkFrontSideModelGpu"
# input_stream: "IMAGE:image"
# input_side_packet: "NUM_FACES:num_faces"
# input_side_packet: "MODEL:0:face_detection_model"
# input_side_packet: "MODEL:1:face_landmark_model"
# output_stream: "LANDMARKS:multi_face_landmarks"
# }
type: "FaceLandmarkFrontSideModelGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
# Max number of faces to detect/track. (int)
input_side_packet: "NUM_FACES:num_faces"
# TfLite model to detect faces.
# (std::unique_ptr<tflite::FlatBufferModel,
# std::function<void(tflite::FlatBufferModel*)>>)
# NOTE: mediapipe/modules/face_detection/face_detection_short_range.tflite
# model only, can be passed here, otherwise - results are undefined.
input_side_packet: "MODEL:0:face_detection_model"
# TfLite model to detect face landmarks.
# (std::unique_ptr<tflite::FlatBufferModel,
# std::function<void(tflite::FlatBufferModel*)>>)
# NOTE: mediapipe/modules/face_landmark/face_landmark.tflite model
# only, can be passed here, otherwise - results are undefined.
input_side_packet: "MODEL:1:face_landmark_model"
# Collection of detected/predicted faces, each represented as a list of 468 face
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of faces detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_face_landmarks"
# Extra outputs (for debugging, for instance).
# Detected faces. (std::vector<Detection>)
output_stream: "DETECTIONS:face_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
# Regions of interest calculated based on face detections.
# (std::vector<NormalizedRect>)
output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
# Determines if an input vector of NormalizedRect has a size greater than or
# equal to the provided num_faces.
node {
calculator: "NormalizedRectVectorHasMinSizeCalculator"
input_stream: "ITERABLE:prev_face_rects_from_landmarks"
input_side_packet: "num_faces"
output_stream: "prev_has_enough_faces"
}
# Drops the incoming image if FaceLandmarkGpu was able to identify face presence
# in the previous image. Otherwise, passes the incoming image through to trigger
# a new round of face detection in FaceDetectionShortRangeGpu.
node {
calculator: "GateCalculator"
input_stream: "image"
input_stream: "DISALLOW:prev_has_enough_faces"
output_stream: "gated_image"
options: {
[mediapipe.GateCalculatorOptions.ext] {
empty_packets_as_allow: true
}
}
}
# Detects faces.
node {
calculator: "FaceDetectionShortRangeSideModelGpu"
input_stream: "IMAGE:gated_image"
input_side_packet: "MODEL:face_detection_model"
output_stream: "DETECTIONS:all_face_detections"
}
# Makes sure there are no more detections than the provided num_faces.
node {
calculator: "ClipDetectionVectorSizeCalculator"
input_stream: "all_face_detections"
output_stream: "face_detections"
input_side_packet: "num_faces"
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:gated_image"
output_stream: "SIZE:gated_image_size"
}
# Outputs each element of face_detections at a fake timestamp for the rest of
# the graph to process. Clones the image size packet for each face_detection at
# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp
# for downstream calculators to inform them that all elements in the vector have
# been processed.
node {
calculator: "BeginLoopDetectionCalculator"
input_stream: "ITERABLE:face_detections"
input_stream: "CLONE:gated_image_size"
output_stream: "ITEM:face_detection"
output_stream: "CLONE:detections_loop_image_size"
output_stream: "BATCH_END:detections_loop_end_timestamp"
}
# Calculates region of interest based on face detections, so that can be used
# to detect landmarks.
node {
calculator: "FaceDetectionFrontDetectionToRoi"
input_stream: "DETECTION:face_detection"
input_stream: "IMAGE_SIZE:detections_loop_image_size"
output_stream: "ROI:face_rect_from_detection"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_detection"
input_stream: "BATCH_END:detections_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_detections"
}
# Performs association between NormalizedRect vector elements from previous
# image and rects based on face detections from the current image. This
# calculator ensures that the output face_rects vector doesn't contain
# overlapping regions based on the specified min_similarity_threshold.
node {
calculator: "AssociationNormRectCalculator"
input_stream: "face_rects_from_detections"
input_stream: "prev_face_rects_from_landmarks"
output_stream: "face_rects"
options: {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.5
}
}
}
# Calculate size of the image.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "SIZE:image_size"
}
# Outputs each element of face_rects at a fake timestamp for the rest of the
# graph to process. Clones image and image size packets for each
# single_face_rect at the fake timestamp. At the end of the loop, outputs the
# BATCH_END timestamp for downstream calculators to inform them that all
# elements in the vector have been processed.
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:face_rects"
input_stream: "CLONE:0:image"
input_stream: "CLONE:1:image_size"
output_stream: "ITEM:face_rect"
output_stream: "CLONE:0:landmarks_loop_image"
output_stream: "CLONE:1:landmarks_loop_image_size"
output_stream: "BATCH_END:landmarks_loop_end_timestamp"
}
# Detects face landmarks within specified region of interest of the image.
node {
calculator: "FaceLandmarkSideModelGpu"
input_stream: "IMAGE:landmarks_loop_image"
input_stream: "ROI:face_rect"
input_side_packet: "MODEL:face_landmark_model"
output_stream: "LANDMARKS:face_landmarks"
}
# Calculates region of interest based on face landmarks, so that can be reused
# for subsequent image.
node {
calculator: "FaceLandmarkLandmarksToRoi"
input_stream: "LANDMARKS:face_landmarks"
input_stream: "IMAGE_SIZE:landmarks_loop_image_size"
output_stream: "ROI:face_rect_from_landmarks"
}
# Collects a set of landmarks for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:face_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:multi_face_landmarks"
}
# Collects a NormalizedRect for each face into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:face_rect_from_landmarks"
input_stream: "BATCH_END:landmarks_loop_end_timestamp"
output_stream: "ITERABLE:face_rects_from_landmarks"
}
# Caches face rects calculated from landmarks, and upon the arrival of the next
# input image, sends out the cached rects with timestamps replaced by that of
# the input image, essentially generating a packet that carries the previous
# face rects. Note that upon the arrival of the very first input image, a
# timestamp bound update occurs to jump start the feedback loop.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:image"
input_stream: "LOOP:face_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
}
output_stream: "PREV_LOOP:prev_face_rects_from_landmarks"
}

185
mediapipe/modules/face_landmark/face_landmark_gpu.pbtxt
View File

@ -1,185 +0,0 @@
# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is
# executed on CPU.)
#
# It is required that "face_landmark.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark.tflite"
# path during execution if `with_attention` is not set or set to `false`.
#
# It is required that "face_landmark_with_attention.tflite" is available at
# "mediapipe/modules/face_landmark/face_landmark_with_attention.tflite"
# path during execution if `with_attention` is set to `true`.
#
# EXAMPLE:
# node {
# calculator: "FaceLandmarkGpu"
# input_stream: "IMAGE:image"
# input_stream: "ROI:face_roi"
# input_side_packet: "WITH_ATTENTION:with_attention"
# output_stream: "LANDMARKS:face_landmarks"
# }
type: "FaceLandmarkGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where a face is located.
# (NormalizedRect)
input_stream: "ROI:roi"
# Whether to run face mesh model with attention on lips and eyes. (bool)
# Attention provides more accuracy on lips and eye regions as well as iris
# landmarks.
input_side_packet: "WITH_ATTENTION:with_attention"
# 468 or 478 facial landmarks within the given ROI. (NormalizedLandmarkList)
#
# Number of landmarks depends on the WITH_ATTENTION flag. If it's `true` - then
# there will be 478 landmarks with refined lips, eyes and irises (10 extra
# landmarks are for irises), otherwise 468 non-refined landmarks are returned.
#
# NOTE: if a face is not present within the given ROI, for this particular
# timestamp there will not be an output packet in the LANDMARKS stream. However,
# the MediaPipe framework will internally inform the downstream calculators of
# the absence of this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:face_landmarks"
# Transforms the input image into a 192x192 tensor.
node: {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE_GPU:image"
input_stream: "NORM_RECT:roi"
output_stream: "TENSORS:input_tensors"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 192
output_tensor_height: 192
output_tensor_float_range {
min: 0.0
max: 1.0
}
gpu_origin: TOP_LEFT
}
}
}
# Loads the face landmarks TF Lite model.
node {
calculator: "FaceLandmarksModelLoader"
input_side_packet: "WITH_ATTENTION:with_attention"
output_side_packet: "MODEL:model"
}
# Generates a single side packet containing a TensorFlow Lite op resolver that
# supports custom ops needed by the model used in this graph.
node {
calculator: "TfLiteCustomOpResolverCalculator"
output_side_packet: "op_resolver"
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of GPU tensors representing, for instance, detection boxes/keypoints
# and scores.
node {
calculator: "InferenceCalculator"
input_stream: "TENSORS:input_tensors"
input_side_packet: "MODEL:model"
input_side_packet: "CUSTOM_OP_RESOLVER:op_resolver"
output_stream: "TENSORS:output_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
# Do not remove. Used for generation of XNNPACK/NNAPI graphs.
}
}
}
# Splits a vector of tensors into landmark tensors and face flag tensor.
node {
calculator: "SwitchContainer"
input_side_packet: "ENABLE:with_attention"
input_stream: "output_tensors"
output_stream: "landmark_tensors"
output_stream: "face_flag_tensor"
options {
[mediapipe.SwitchContainerOptions.ext] {
contained_node: {
calculator: "SplitTensorVectorCalculator"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 1 }
ranges: { begin: 1 end: 2 }
}
}
}
contained_node: {
calculator: "SplitTensorVectorCalculator"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 6 }
ranges: { begin: 6 end: 7 }
}
}
}
}
}
}
# Converts the face-flag tensor into a float that represents the confidence
# score of face presence.
node {
calculator: "TensorsToFloatsCalculator"
input_stream: "TENSORS:face_flag_tensor"
output_stream: "FLOAT:face_presence_score"
options: {
[mediapipe.TensorsToFloatsCalculatorOptions.ext] {
activation: SIGMOID
}
}
}
# Applies a threshold to the confidence score to determine whether a face is
# present.
node {
calculator: "ThresholdingCalculator"
input_stream: "FLOAT:face_presence_score"
output_stream: "FLAG:face_presence"
options: {
[mediapipe.ThresholdingCalculatorOptions.ext] {
threshold: 0.5
}
}
}
# Drop landmarks tensors if face is not present.
node {
calculator: "GateCalculator"
input_stream: "landmark_tensors"
input_stream: "ALLOW:face_presence"
output_stream: "ensured_landmark_tensors"
}
# Decodes the landmark tensors into a vector of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "SwitchContainer"
input_side_packet: "ENABLE:with_attention"
input_stream: "TENSORS:ensured_landmark_tensors"
output_stream: "LANDMARKS:landmarks"
options: {
[mediapipe.SwitchContainerOptions.ext] {
contained_node: {
calculator: "TensorsToFaceLandmarks"
}
contained_node: {
calculator: "TensorsToFaceLandmarksWithAttention"
}
}
}
}
# Projects the landmarks from the cropped face image to the corresponding
# locations on the full image before cropping (input to the graph).
node {
calculator: "LandmarkProjectionCalculator"
input_stream: "NORM_LANDMARKS:landmarks"
input_stream: "NORM_RECT:roi"
output_stream: "NORM_LANDMARKS:face_landmarks"
}

54
mediapipe/modules/face_landmark/face_landmark_landmarks_to_roi.pbtxt
View File

@ -1,54 +0,0 @@
# MediaPipe graph to calculate face region of interest (ROI) from landmarks
# detected by "FaceLandmarkCpu" or "FaceLandmarkGpu".
#
# NOTE: this graph is subject to change and should not be used directly.
type: "FaceLandmarkLandmarksToRoi"
# Normalized landmarks. (NormalizedLandmarkList)
input_stream: "LANDMARKS:landmarks"
# Frame size (width & height). (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# ROI according to landmarks. (NormalizedRect)
output_stream: "ROI:roi"
# Converts face landmarks to a detection that tightly encloses all landmarks.
node {
calculator: "LandmarksToDetectionCalculator"
input_stream: "NORM_LANDMARKS:landmarks"
output_stream: "DETECTION:face_detection"
}
# Converts the face detection into a rectangle (normalized by image size)
# that encloses the face and is rotated such that the line connecting left side
# of the left eye and right side of the right eye is aligned with the X-axis of
# the rectangle.
node {
calculator: "DetectionsToRectsCalculator"
input_stream: "DETECTION:face_detection"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECT:face_rect_from_landmarks"
options: {
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
rotation_vector_start_keypoint_index: 33 # Left side of left eye.
rotation_vector_end_keypoint_index: 263 # Right side of right eye.
rotation_vector_target_angle_degrees: 0
}
}
}
# Expands the face rectangle so that in the next video image it's likely to
# still contain the face even with some motion.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:face_rect_from_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "roi"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 1.5
scale_y: 1.5
square_long: true
}
}
}

BIN
mediapipe/modules/face_landmark/face_landmark_with_attention.tflite
View File

Binary file not shown.

58
mediapipe/modules/face_landmark/face_landmarks_model_loader.pbtxt
View File

@ -1,58 +0,0 @@
# MediaPipe graph to load a selected face landmarks TF Lite model.
type: "FaceLandmarksModelLoader"
# Whether to run face mesh model with attention on lips and eyes. (bool)
# Attention provides more accuracy on lips and eye regions as well as iris
# landmarks.
input_side_packet: "WITH_ATTENTION:with_attention"
# TF Lite model represented as a FlatBuffer.
# (std::unique_ptr<tflite::FlatBufferModel, std::function<void(tflite::FlatBufferModel*)>>)
output_side_packet: "MODEL:model"
# Determines path to the desired face landmark model file based on specification
# in the input side packet.
node {
calculator: "SwitchContainer"
input_side_packet: "ENABLE:with_attention"
output_side_packet: "PACKET:model_path"
options: {
[mediapipe.SwitchContainerOptions.ext] {
contained_node: {
calculator: "ConstantSidePacketCalculator"
options: {
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
packet {
string_value: "mediapipe/modules/face_landmark/face_landmark.tflite"
}
}
}
}
contained_node: {
calculator: "ConstantSidePacketCalculator"
options: {
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
packet {
string_value: "mediapipe/modules/face_landmark/face_landmark_with_attention.tflite"
}
}
}
}
}
}
}
# Loads the file in the specified path into a blob.
node {
calculator: "LocalFileContentsCalculator"
input_side_packet: "FILE_PATH:model_path"
output_side_packet: "CONTENTS:model_blob"
}
# Converts the input blob into a TF Lite model.
node {
calculator: "TfLiteModelCalculator"
input_side_packet: "MODEL_BLOB:model_blob"
output_side_packet: "MODEL:model"
}

24
mediapipe/modules/face_landmark/tensors_to_face_landmarks.pbtxt
View File

@ -1,24 +0,0 @@
# MediaPipe graph to transform single tensor into 468 facial landmarks.
type: "TensorsToFaceLandmarks"
# Vector with a single tensor that contains 468 landmarks. (std::vector<Tensor>)
input_stream: "TENSORS:tensors"
# 468 facial landmarks (NormalizedLandmarkList)
output_stream: "LANDMARKS:landmarks"
# Decodes the landmark tensors into a vector of lanmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:tensors"
output_stream: "NORM_LANDMARKS:landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 468
input_image_width: 192
input_image_height: 192
}
}
}

299
mediapipe/modules/face_landmark/tensors_to_face_landmarks_with_attention.pbtxt
View File

@ -1,299 +0,0 @@
# MediaPipe graph to transform model output tensors into 478 facial landmarks
# with refined lips, eyes and irises.
type: "TensorsToFaceLandmarksWithAttention"
# Vector with a six tensors to parse landmarks from. (std::vector<Tensor>)
# Landmark tensors order:
# - mesh_tensor
# - lips_tensor
# - left_eye_tensor
# - right_eye_tensor
# - left_iris_tensor
# - right_iris_tensor
input_stream: "TENSORS:tensors"
# 478 facial landmarks (NormalizedLandmarkList)
output_stream: "LANDMARKS:landmarks"
# Splits a vector of tensors into multiple vectors.
node {
calculator: "SplitTensorVectorCalculator"
input_stream: "tensors"
output_stream: "mesh_tensor"
output_stream: "lips_tensor"
output_stream: "left_eye_tensor"
output_stream: "right_eye_tensor"
output_stream: "left_iris_tensor"
output_stream: "right_iris_tensor"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 1 }
ranges: { begin: 1 end: 2 }
ranges: { begin: 2 end: 3 }
ranges: { begin: 3 end: 4 }
ranges: { begin: 4 end: 5 }
ranges: { begin: 5 end: 6 }
}
}
}
# Decodes mesh landmarks tensor into a vector of normalized lanmarks.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:mesh_tensor"
output_stream: "NORM_LANDMARKS:mesh_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 468
input_image_width: 192
input_image_height: 192
}
}
}
# Decodes lips landmarks tensor into a vector of normalized lanmarks.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:lips_tensor"
output_stream: "NORM_LANDMARKS:lips_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 80
input_image_width: 192
input_image_height: 192
}
}
}
# Decodes left eye landmarks tensor into a vector of normalized lanmarks.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:left_eye_tensor"
output_stream: "NORM_LANDMARKS:left_eye_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 71
input_image_width: 192
input_image_height: 192
}
}
}
# Decodes right eye landmarks tensor into a vector of normalized lanmarks.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:right_eye_tensor"
output_stream: "NORM_LANDMARKS:right_eye_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 71
input_image_width: 192
input_image_height: 192
}
}
}
# Decodes left iris landmarks tensor into a vector of normalized lanmarks.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:left_iris_tensor"
output_stream: "NORM_LANDMARKS:left_iris_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 5
input_image_width: 192
input_image_height: 192
}
}
}
# Decodes right iris landmarks tensor into a vector of normalized lanmarks.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:right_iris_tensor"
output_stream: "NORM_LANDMARKS:right_iris_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 5
input_image_width: 192
input_image_height: 192
}
}
}
# Refine mesh landmarks with lips, eyes and irises.
node {
calculator: "LandmarksRefinementCalculator"
input_stream: "LANDMARKS:0:mesh_landmarks"
input_stream: "LANDMARKS:1:lips_landmarks"
input_stream: "LANDMARKS:2:left_eye_landmarks"
input_stream: "LANDMARKS:3:right_eye_landmarks"
input_stream: "LANDMARKS:4:left_iris_landmarks"
input_stream: "LANDMARKS:5:right_iris_landmarks"
output_stream: "REFINED_LANDMARKS:landmarks"
options: {
[mediapipe.LandmarksRefinementCalculatorOptions.ext] {
# 0 - mesh
refinement: {
indexes_mapping: [
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70,
71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87,
88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103,
104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117,
118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131,
132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145,
146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159,
160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173,
174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187,
188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201,
202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215,
216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229,
230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243,
244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255, 256, 257,
258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271,
272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285,
286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299,
300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313,
314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327,
328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341,
342, 343, 344, 345, 346, 347, 348, 349, 350, 351, 352, 353, 354, 355,
356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369,
370, 371, 372, 373, 374, 375, 376, 377, 378, 379, 380, 381, 382, 383,
384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 397,
398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411,
412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425,
426, 427, 428, 429, 430, 431, 432, 433, 434, 435, 436, 437, 438, 439,
440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 452, 453,
454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467
]
z_refinement: { copy {} }
}
# 1 - lips
refinement: {
indexes_mapping: [
# Lower outer.
61, 146, 91, 181, 84, 17, 314, 405, 321, 375, 291,
# Upper outer (excluding corners).
185, 40, 39, 37, 0, 267, 269, 270, 409,
# Lower inner.
78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308,
# Upper inner (excluding corners).
191, 80, 81, 82, 13, 312, 311, 310, 415,
# Lower semi-outer.
76, 77, 90, 180, 85, 16, 315, 404, 320, 307, 306,
# Upper semi-outer (excluding corners).
184, 74, 73, 72, 11, 302, 303, 304, 408,
# Lower semi-inner.
62, 96, 89, 179, 86, 15, 316, 403, 319, 325, 292,
# Upper semi-inner (excluding corners).
183, 42, 41, 38, 12, 268, 271, 272, 407
]
z_refinement: { none {} }
}
# 2 - left eye
refinement: {
indexes_mapping: [
# Lower contour.
33, 7, 163, 144, 145, 153, 154, 155, 133,
# upper contour (excluding corners).
246, 161, 160, 159, 158, 157, 173,
# Halo x2 lower contour.
130, 25, 110, 24, 23, 22, 26, 112, 243,
# Halo x2 upper contour (excluding corners).
247, 30, 29, 27, 28, 56, 190,
# Halo x3 lower contour.
226, 31, 228, 229, 230, 231, 232, 233, 244,
# Halo x3 upper contour (excluding corners).
113, 225, 224, 223, 222, 221, 189,
# Halo x4 upper contour (no lower because of mesh structure) or
# eyebrow inner contour.
35, 124, 46, 53, 52, 65,
# Halo x5 lower contour.
143, 111, 117, 118, 119, 120, 121, 128, 245,
# Halo x5 upper contour (excluding corners) or eyebrow outer contour.
156, 70, 63, 105, 66, 107, 55, 193
]
z_refinement: { none {} }
}
# 3 - right eye
refinement: {
indexes_mapping: [
# Lower contour.
263, 249, 390, 373, 374, 380, 381, 382, 362,
# Upper contour (excluding corners).
466, 388, 387, 386, 385, 384, 398,
# Halo x2 lower contour.
359, 255, 339, 254, 253, 252, 256, 341, 463,
# Halo x2 upper contour (excluding corners).
467, 260, 259, 257, 258, 286, 414,
# Halo x3 lower contour.
446, 261, 448, 449, 450, 451, 452, 453, 464,
# Halo x3 upper contour (excluding corners).
342, 445, 444, 443, 442, 441, 413,
# Halo x4 upper contour (no lower because of mesh structure) or
# eyebrow inner contour.
265, 353, 276, 283, 282, 295,
# Halo x5 lower contour.
372, 340, 346, 347, 348, 349, 350, 357, 465,
# Halo x5 upper contour (excluding corners) or eyebrow outer contour.
383, 300, 293, 334, 296, 336, 285, 417
]
z_refinement: { none {} }
}
# 4 - left iris
refinement: {
indexes_mapping: [
# Center.
468,
# Iris right edge.
469,
# Iris top edge.
470,
# Iris left edge.
471,
# Iris bottom edge.
472
]
z_refinement: {
assign_average: {
indexes_for_average: [
# Lower contour.
33, 7, 163, 144, 145, 153, 154, 155, 133,
# Upper contour (excluding corners).
246, 161, 160, 159, 158, 157, 173
]
}
}
}
# 5 - right iris
refinement: {
indexes_mapping: [
# Center.
473,
# Iris right edge.
474,
# Iris top edge.
475,
# Iris left edge.
476,
# Iris bottom edge.
477
]
z_refinement: {
assign_average: {
indexes_for_average: [
# Lower contour.
263, 249, 390, 373, 374, 380, 381, 382, 362,
# Upper contour (excluding corners).
466, 388, 387, 386, 385, 384, 398
]
}
}
}
}
}
}

171
mediapipe/modules/hand_landmark/BUILD
View File

@ -1,171 +0,0 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load(
"//mediapipe/framework/tool:mediapipe_graph.bzl",
"mediapipe_simple_subgraph",
)
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
exports_files([
"hand_landmark_full.tflite",
"hand_landmark_lite.tflite",
"handedness.txt",
])
mediapipe_simple_subgraph(
name = "hand_landmark_model_loader",
graph = "hand_landmark_model_loader.pbtxt",
register_as = "HandLandmarkModelLoader",
deps = [
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/tflite:tflite_model_calculator",
"//mediapipe/calculators/util:local_file_contents_calculator",
"//mediapipe/framework/tool:switch_container",
],
)
mediapipe_simple_subgraph(
name = "hand_landmark_cpu",
graph = "hand_landmark_cpu.pbtxt",
register_as = "HandLandmarkCpu",
deps = [
":hand_landmark_model_loader",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_classification_calculator",
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
"//mediapipe/calculators/util:landmark_projection_calculator",
"//mediapipe/calculators/util:thresholding_calculator",
"//mediapipe/calculators/util:world_landmark_projection_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_landmark_gpu",
graph = "hand_landmark_gpu.pbtxt",
register_as = "HandLandmarkGpu",
deps = [
":hand_landmark_model_loader",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:split_vector_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_classification_calculator",
"//mediapipe/calculators/tensor:tensors_to_floats_calculator",
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
"//mediapipe/calculators/util:landmark_projection_calculator",
"//mediapipe/calculators/util:thresholding_calculator",
"//mediapipe/calculators/util:world_landmark_projection_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_landmark_tracking_gpu",
graph = "hand_landmark_tracking_gpu.pbtxt",
register_as = "HandLandmarkTrackingGpu",
deps = [
":hand_landmark_gpu",
":hand_landmark_landmarks_to_roi",
":palm_detection_detection_to_roi",
"//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:clip_vector_size_calculator",
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/core:end_loop_calculator",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:previous_loopback_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/util:association_norm_rect_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator",
"//mediapipe/calculators/util:filter_collection_calculator",
"//mediapipe/modules/palm_detection:palm_detection_gpu",
],
)
mediapipe_simple_subgraph(
name = "hand_landmark_tracking_cpu_image",
graph = "hand_landmark_tracking_cpu_image.pbtxt",
register_as = "HandLandmarkTrackingCpuImage",
deps = [
":hand_landmark_tracking_cpu",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/calculators/util:from_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_landmark_tracking_gpu_image",
graph = "hand_landmark_tracking_gpu_image.pbtxt",
register_as = "HandLandmarkTrackingGpuImage",
deps = [
":hand_landmark_tracking_gpu",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/calculators/util:from_image_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_landmark_tracking_cpu",
graph = "hand_landmark_tracking_cpu.pbtxt",
register_as = "HandLandmarkTrackingCpu",
deps = [
":hand_landmark_cpu",
":hand_landmark_landmarks_to_roi",
":palm_detection_detection_to_roi",
"//mediapipe/calculators/core:begin_loop_calculator",
"//mediapipe/calculators/core:clip_vector_size_calculator",
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/core:end_loop_calculator",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/core:previous_loopback_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/util:association_norm_rect_calculator",
"//mediapipe/calculators/util:collection_has_min_size_calculator",
"//mediapipe/calculators/util:filter_collection_calculator",
"//mediapipe/modules/palm_detection:palm_detection_cpu",
],
)
mediapipe_simple_subgraph(
name = "palm_detection_detection_to_roi",
graph = "palm_detection_detection_to_roi.pbtxt",
register_as = "PalmDetectionDetectionToRoi",
deps = [
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_landmark_landmarks_to_roi",
graph = "hand_landmark_landmarks_to_roi.pbtxt",
register_as = "HandLandmarkLandmarksToRoi",
deps = [
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
"//mediapipe/modules/hand_landmark/calculators:hand_landmarks_to_rect_calculator",
],
)

8
mediapipe/modules/hand_landmark/README.md
View File

@ -1,8 +0,0 @@
# hand_landmark
Subgraphs|Details
:--- | :---
[`HandLandmarkCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_cpu.pbtxt)| Detects landmarks of a single hand. (CPU input.)
[`HandLandmarkGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_gpu.pbtxt)| Detects landmarks of a single hand. (GPU input.)
[`HandLandmarkTrackingCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu.pbtxt)| Detects and tracks landmarks of multiple hands. (CPU input.)
[`HandLandmarkTrackingGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu.pbtxt)| Detects and tracks landmarks of multiple hands. (GPU input.)

33
mediapipe/modules/hand_landmark/calculators/BUILD
View File

@ -1,33 +0,0 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
cc_library(
name = "hand_landmarks_to_rect_calculator",
srcs = ["hand_landmarks_to_rect_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:location_data_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)

167
mediapipe/modules/hand_landmark/calculators/hand_landmarks_to_rect_calculator.cc
View File

@ -1,167 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <cmath>
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
namespace {
constexpr char kNormalizedLandmarksTag[] = "NORM_LANDMARKS";
constexpr char kNormRectTag[] = "NORM_RECT";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr int kWristJoint = 0;
constexpr int kMiddleFingerPIPJoint = 6;
constexpr int kIndexFingerPIPJoint = 4;
constexpr int kRingFingerPIPJoint = 8;
constexpr float kTargetAngle = M_PI * 0.5f;
inline float NormalizeRadians(float angle) {
return angle - 2 * M_PI * std::floor((angle - (-M_PI)) / (2 * M_PI));
}
float ComputeRotation(const NormalizedLandmarkList& landmarks,
const std::pair<int, int>& image_size) {
const float x0 = landmarks.landmark(kWristJoint).x() * image_size.first;
const float y0 = landmarks.landmark(kWristJoint).y() * image_size.second;
float x1 = (landmarks.landmark(kIndexFingerPIPJoint).x() +
landmarks.landmark(kRingFingerPIPJoint).x()) /
2.f;
float y1 = (landmarks.landmark(kIndexFingerPIPJoint).y() +
landmarks.landmark(kRingFingerPIPJoint).y()) /
2.f;
x1 = (x1 + landmarks.landmark(kMiddleFingerPIPJoint).x()) / 2.f *
image_size.first;
y1 = (y1 + landmarks.landmark(kMiddleFingerPIPJoint).y()) / 2.f *
image_size.second;
const float rotation =
NormalizeRadians(kTargetAngle - std::atan2(-(y1 - y0), x1 - x0));
return rotation;
}
absl::Status NormalizedLandmarkListToRect(
const NormalizedLandmarkList& landmarks,
const std::pair<int, int>& image_size, NormalizedRect* rect) {
const float rotation = ComputeRotation(landmarks, image_size);
const float reverse_angle = NormalizeRadians(-rotation);
// Find boundaries of landmarks.
float max_x = std::numeric_limits<float>::min();
float max_y = std::numeric_limits<float>::min();
float min_x = std::numeric_limits<float>::max();
float min_y = std::numeric_limits<float>::max();
for (int i = 0; i < landmarks.landmark_size(); ++i) {
max_x = std::max(max_x, landmarks.landmark(i).x());
max_y = std::max(max_y, landmarks.landmark(i).y());
min_x = std::min(min_x, landmarks.landmark(i).x());
min_y = std::min(min_y, landmarks.landmark(i).y());
}
const float axis_aligned_center_x = (max_x + min_x) / 2.f;
const float axis_aligned_center_y = (max_y + min_y) / 2.f;
// Find boundaries of rotated landmarks.
max_x = std::numeric_limits<float>::min();
max_y = std::numeric_limits<float>::min();
min_x = std::numeric_limits<float>::max();
min_y = std::numeric_limits<float>::max();
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const float original_x =
(landmarks.landmark(i).x() - axis_aligned_center_x) * image_size.first;
const float original_y =
(landmarks.landmark(i).y() - axis_aligned_center_y) * image_size.second;
const float projected_x = original_x * std::cos(reverse_angle) -
original_y * std::sin(reverse_angle);
const float projected_y = original_x * std::sin(reverse_angle) +
original_y * std::cos(reverse_angle);
max_x = std::max(max_x, projected_x);
max_y = std::max(max_y, projected_y);
min_x = std::min(min_x, projected_x);
min_y = std::min(min_y, projected_y);
}
const float projected_center_x = (max_x + min_x) / 2.f;
const float projected_center_y = (max_y + min_y) / 2.f;
const float center_x = projected_center_x * std::cos(rotation) -
projected_center_y * std::sin(rotation) +
image_size.first * axis_aligned_center_x;
const float center_y = projected_center_x * std::sin(rotation) +
projected_center_y * std::cos(rotation) +
image_size.second * axis_aligned_center_y;
const float width = (max_x - min_x) / image_size.first;
const float height = (max_y - min_y) / image_size.second;
rect->set_x_center(center_x / image_size.first);
rect->set_y_center(center_y / image_size.second);
rect->set_width(width);
rect->set_height(height);
rect->set_rotation(rotation);
return absl::OkStatus();
}
} // namespace
// A calculator that converts subset of hand landmarks to a bounding box
// NormalizedRect. The rotation angle of the bounding box is computed based on
// 1) the wrist joint and 2) the average of PIP joints of index finger, middle
// finger and ring finger. After rotation, the vector from the wrist to the mean
// of PIP joints is expected to be vertical with wrist at the bottom and the
// mean of PIP joints at the top.
class HandLandmarksToRectCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc) {
cc->Inputs().Tag(kNormalizedLandmarksTag).Set<NormalizedLandmarkList>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Outputs().Tag(kNormRectTag).Set<NormalizedRect>();
return absl::OkStatus();
}
absl::Status Open(CalculatorContext* cc) override {
cc->SetOffset(TimestampDiff(0));
return absl::OkStatus();
}
absl::Status Process(CalculatorContext* cc) override {
if (cc->Inputs().Tag(kNormalizedLandmarksTag).IsEmpty()) {
return absl::OkStatus();
}
RET_CHECK(!cc->Inputs().Tag(kImageSizeTag).IsEmpty());
std::pair<int, int> image_size =
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
const auto& landmarks =
cc->Inputs().Tag(kNormalizedLandmarksTag).Get<NormalizedLandmarkList>();
auto output_rect = absl::make_unique<NormalizedRect>();
MP_RETURN_IF_ERROR(
NormalizedLandmarkListToRect(landmarks, image_size, output_rect.get()));
cc->Outputs()
.Tag(kNormRectTag)
.Add(output_rect.release(), cc->InputTimestamp());
return absl::OkStatus();
}
};
REGISTER_CALCULATOR(HandLandmarksToRectCalculator);
} // namespace mediapipe

219
mediapipe/modules/hand_landmark/hand_landmark_cpu.pbtxt
View File

@ -1,219 +0,0 @@
# MediaPipe graph to detect/predict hand landmarks on CPU.
type: "HandLandmarkCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where a palm/hand is located.
# (NormalizedRect)
input_stream: "ROI:hand_rect"
# Complexity of the hand landmark model: 0 or 1. Landmark accuracy as well as
# inference latency generally go up with the model complexity. If unspecified,
# functions as set to 1. (int)
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
# 21 hand landmarks within the given ROI. (NormalizedLandmarkList)
# NOTE: if a hand is not present within the given ROI, for this particular
# timestamp there will not be an output packet in the LANDMARKS stream. However,
# the MediaPipe framework will internally inform the downstream calculators of
# the absence of this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:hand_landmarks"
# Hand world landmarks within the given ROI. (LandmarkList)
# World landmarks are real-world 3D coordinates in meters with the origin in the
# center of the given ROI.
#
# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
# the 3D object itself.
output_stream: "WORLD_LANDMARKS:hand_world_landmarks"
# Handedness of the detected hand (i.e. is hand left or right).
# (ClassificationList)
output_stream: "HANDEDNESS:handedness"
# Transforms a region of image into a 224x224 tensor while keeping the aspect
# ratio, and therefore may result in potential letterboxing.
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE:image"
input_stream: "NORM_RECT:hand_rect"
output_stream: "TENSORS:input_tensor"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 224
output_tensor_height: 224
keep_aspect_ratio: true
output_tensor_float_range {
min: 0.0
max: 1.0
}
}
}
}
# Loads the hand landmark TF Lite model.
node {
calculator: "HandLandmarkModelLoader"
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
output_side_packet: "MODEL:model"
}
# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_side_packet: "MODEL:model"
input_stream: "TENSORS:input_tensor"
output_stream: "TENSORS:output_tensors"
options: {
[mediapipe.InferenceCalculatorOptions.ext] {
delegate {
xnnpack {}
}
}
}
}
# Splits a vector of tensors to multiple vectors according to the ranges
# specified in option.
node {
calculator: "SplitTensorVectorCalculator"
input_stream: "output_tensors"
output_stream: "landmark_tensors"
output_stream: "hand_flag_tensor"
output_stream: "handedness_tensor"
output_stream: "world_landmark_tensor"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 1 }
ranges: { begin: 1 end: 2 }
ranges: { begin: 2 end: 3 }
ranges: { begin: 3 end: 4 }
}
}
}
# Converts the hand-flag tensor into a float that represents the confidence
# score of hand presence.
node {
calculator: "TensorsToFloatsCalculator"
input_stream: "TENSORS:hand_flag_tensor"
output_stream: "FLOAT:hand_presence_score"
}
# Applies a threshold to the confidence score to determine whether a hand is
# present.
node {
calculator: "ThresholdingCalculator"
input_stream: "FLOAT:hand_presence_score"
output_stream: "FLAG:hand_presence"
options: {
[mediapipe.ThresholdingCalculatorOptions.ext] {
threshold: 0.5
}
}
}
# Drops handedness tensor if hand is not present.
node {
calculator: "GateCalculator"
input_stream: "handedness_tensor"
input_stream: "ALLOW:hand_presence"
output_stream: "ensured_handedness_tensor"
}
# Converts the handedness tensor into a float that represents the classification
# score of handedness.
node {
calculator: "TensorsToClassificationCalculator"
input_stream: "TENSORS:ensured_handedness_tensor"
output_stream: "CLASSIFICATIONS:handedness"
options: {
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {
top_k: 1
label_map_path: "mediapipe/modules/hand_landmark/handedness.txt"
binary_classification: true
}
}
}
# Drops landmarks tensors if hand is not present.
node {
calculator: "GateCalculator"
input_stream: "landmark_tensors"
input_stream: "ALLOW:hand_presence"
output_stream: "ensured_landmark_tensors"
}
# Decodes the landmark tensors into a list of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:ensured_landmark_tensors"
output_stream: "NORM_LANDMARKS:landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 21
input_image_width: 224
input_image_height: 224
# The additional scaling factor is used to account for the Z coordinate
# distribution in the training data.
normalize_z: 0.4
}
}
}
# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed hand
# image (after image transformation with the FIT scale mode) to the
# corresponding locations on the same image with the letterbox removed (hand
# image before image transformation).
node {
calculator: "LandmarkLetterboxRemovalCalculator"
input_stream: "LANDMARKS:landmarks"
input_stream: "LETTERBOX_PADDING:letterbox_padding"
output_stream: "LANDMARKS:scaled_landmarks"
}
# Projects the landmarks from the cropped hand image to the corresponding
# locations on the full image before cropping (input to the graph).
node {
calculator: "LandmarkProjectionCalculator"
input_stream: "NORM_LANDMARKS:scaled_landmarks"
input_stream: "NORM_RECT:hand_rect"
output_stream: "NORM_LANDMARKS:hand_landmarks"
}
# Drops world landmarks tensors if hand is not present.
node {
calculator: "GateCalculator"
input_stream: "world_landmark_tensor"
input_stream: "ALLOW:hand_presence"
output_stream: "ensured_world_landmark_tensor"
}
# Decodes the landmark tensors into a list of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:ensured_world_landmark_tensor"
output_stream: "LANDMARKS:unprojected_world_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 21
}
}
}
# Projects the world landmarks from the cropped hand image to the corresponding
# locations on the full image before cropping (input to the graph).
node {
calculator: "WorldLandmarkProjectionCalculator"
input_stream: "LANDMARKS:unprojected_world_landmarks"
input_stream: "NORM_RECT:hand_rect"
output_stream: "LANDMARKS:hand_world_landmarks"
}

BIN
mediapipe/modules/hand_landmark/hand_landmark_full.tflite
View File

Binary file not shown.

213
mediapipe/modules/hand_landmark/hand_landmark_gpu.pbtxt
View File

@ -1,213 +0,0 @@
# MediaPipe graph to detect/predict hand landmarks on CPU.
type: "HandLandmarkGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
# ROI (region of interest) within the given image where a palm/hand is located.
# (NormalizedRect)
input_stream: "ROI:hand_rect"
# Complexity of the hand landmark model: 0 or 1. Landmark accuracy as well as
# inference latency generally go up with the model complexity. If unspecified,
# functions as set to 1. (int)
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
# 21 hand landmarks within the given ROI. (NormalizedLandmarkList)
# NOTE: if a hand is not present within the given ROI, for this particular
# timestamp there will not be an output packet in the LANDMARKS stream. However,
# the MediaPipe framework will internally inform the downstream calculators of
# the absence of this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:hand_landmarks"
# Hand world landmarks within the given ROI. (LandmarkList)
# World landmarks are real-world 3D coordinates in meters with the origin in the
# center of the given ROI.
#
# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
# the 3D object itself.
output_stream: "WORLD_LANDMARKS:hand_world_landmarks"
# Handedness of the detected hand (i.e. is hand left or right).
# (ClassificationList)
output_stream: "HANDEDNESS:handedness"
# Transforms a region of image into a 224x224 tensor while keeping the aspect
# ratio, and therefore may result in potential letterboxing.
node {
calculator: "ImageToTensorCalculator"
input_stream: "IMAGE_GPU:image"
input_stream: "NORM_RECT:hand_rect"
output_stream: "TENSORS:input_tensor"
output_stream: "LETTERBOX_PADDING:letterbox_padding"
options: {
[mediapipe.ImageToTensorCalculatorOptions.ext] {
output_tensor_width: 224
output_tensor_height: 224
keep_aspect_ratio: true
output_tensor_float_range {
min: 0.0
max: 1.0
}
gpu_origin: TOP_LEFT
}
}
}
# Loads the hand landmark TF Lite model.
node {
calculator: "HandLandmarkModelLoader"
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
output_side_packet: "MODEL:model"
}
# Runs a TensorFlow Lite model on GPU that takes an image tensor and outputs a
# vector of tensors representing, for instance, detection boxes/keypoints and
# scores.
node {
calculator: "InferenceCalculator"
input_side_packet: "MODEL:model"
input_stream: "TENSORS:input_tensor"
output_stream: "TENSORS:output_tensors"
}
# Splits a vector of tensors to multiple vectors according to the ranges
# specified in option.
node {
calculator: "SplitTensorVectorCalculator"
input_stream: "output_tensors"
output_stream: "landmark_tensors"
output_stream: "hand_flag_tensor"
output_stream: "handedness_tensor"
output_stream: "world_landmark_tensor"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 1 }
ranges: { begin: 1 end: 2 }
ranges: { begin: 2 end: 3 }
ranges: { begin: 3 end: 4 }
}
}
}
# Converts the hand-flag tensor into a float that represents the confidence
# score of hand presence.
node {
calculator: "TensorsToFloatsCalculator"
input_stream: "TENSORS:hand_flag_tensor"
output_stream: "FLOAT:hand_presence_score"
}
# Applies a threshold to the confidence score to determine whether a hand is
# present.
node {
calculator: "ThresholdingCalculator"
input_stream: "FLOAT:hand_presence_score"
output_stream: "FLAG:hand_presence"
options: {
[mediapipe.ThresholdingCalculatorOptions.ext] {
threshold: 0.5
}
}
}
# Drops handedness tensor if hand is not present.
node {
calculator: "GateCalculator"
input_stream: "handedness_tensor"
input_stream: "ALLOW:hand_presence"
output_stream: "ensured_handedness_tensor"
}
# Converts the handedness tensor into a float that represents the classification
# score of handedness.
node {
calculator: "TensorsToClassificationCalculator"
input_stream: "TENSORS:ensured_handedness_tensor"
output_stream: "CLASSIFICATIONS:handedness"
options: {
[mediapipe.TensorsToClassificationCalculatorOptions.ext] {
top_k: 1
label_map_path: "mediapipe/modules/hand_landmark/handedness.txt"
binary_classification: true
}
}
}
# Drops landmarks tensors if hand is not present.
node {
calculator: "GateCalculator"
input_stream: "landmark_tensors"
input_stream: "ALLOW:hand_presence"
output_stream: "ensured_landmark_tensors"
}
# Decodes the landmark tensors into a list of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:ensured_landmark_tensors"
output_stream: "NORM_LANDMARKS:landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 21
input_image_width: 224
input_image_height: 224
# The additional scaling factor is used to account for the Z coordinate
# distribution in the training data.
normalize_z: 0.4
}
}
}
# Adjusts landmarks (already normalized to [0.f, 1.f]) on the letterboxed hand
# image (after image transformation with the FIT scale mode) to the
# corresponding locations on the same image with the letterbox removed (hand
# image before image transformation).
node {
calculator: "LandmarkLetterboxRemovalCalculator"
input_stream: "LANDMARKS:landmarks"
input_stream: "LETTERBOX_PADDING:letterbox_padding"
output_stream: "LANDMARKS:scaled_landmarks"
}
# Projects the landmarks from the cropped hand image to the corresponding
# locations on the full image before cropping (input to the graph).
node {
calculator: "LandmarkProjectionCalculator"
input_stream: "NORM_LANDMARKS:scaled_landmarks"
input_stream: "NORM_RECT:hand_rect"
output_stream: "NORM_LANDMARKS:hand_landmarks"
}
# Drops world landmarks tensors if hand is not present.
node {
calculator: "GateCalculator"
input_stream: "world_landmark_tensor"
input_stream: "ALLOW:hand_presence"
output_stream: "ensured_world_landmark_tensor"
}
# Decodes the landmark tensors into a list of landmarks, where the landmark
# coordinates are normalized by the size of the input image to the model.
node {
calculator: "TensorsToLandmarksCalculator"
input_stream: "TENSORS:ensured_world_landmark_tensor"
output_stream: "LANDMARKS:unprojected_world_landmarks"
options: {
[mediapipe.TensorsToLandmarksCalculatorOptions.ext] {
num_landmarks: 21
}
}
}
# Projects the world landmarks from the cropped hand image to the corresponding
# locations on the full image before cropping (input to the graph).
node {
calculator: "WorldLandmarkProjectionCalculator"
input_stream: "LANDMARKS:unprojected_world_landmarks"
input_stream: "NORM_RECT:hand_rect"
output_stream: "LANDMARKS:hand_world_landmarks"
}

63
mediapipe/modules/hand_landmark/hand_landmark_landmarks_to_roi.pbtxt
View File

@ -1,63 +0,0 @@
# MediaPipe graph to calculate hand region of interest (ROI) from landmarks
# detected by "HandLandmarkCpu" or "HandLandmarkGpu".
type: "HandLandmarkLandmarksToRoi"
# Normalized landmarks. (NormalizedLandmarkList)
input_stream: "LANDMARKS:landmarks"
# Image size (width & height). (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# ROI according to landmarks. (NormalizedRect)
output_stream: "ROI:roi"
# Extracts a subset of the hand landmarks that are relatively more stable across
# frames (e.g. comparing to finger tips) for computing the bounding box. The box
# will later be expanded to contain the entire hand. In this approach, it is
# more robust to drastically changing hand size.
# The landmarks extracted are: wrist, MCP/PIP of five fingers.
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "landmarks"
output_stream: "partial_landmarks"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 0 end: 4 }
ranges: { begin: 5 end: 7 }
ranges: { begin: 9 end: 11 }
ranges: { begin: 13 end: 15 }
ranges: { begin: 17 end: 19 }
combine_outputs: true
}
}
}
# Converts the hand landmarks into a rectangle (normalized by image size)
# that encloses the hand. The calculator uses a subset of all hand landmarks
# extracted from SplitNormalizedLandmarkListCalculator above to
# calculate the bounding box and the rotation of the output rectangle. Please
# see the comments in the calculator for more detail.
node {
calculator: "HandLandmarksToRectCalculator"
input_stream: "NORM_LANDMARKS:partial_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECT:hand_rect_from_landmarks"
}
# Expands the hand rectangle so that the box contains the entire hand and it's
# big enough so that it's likely to still contain the hand even with some motion
# in the next video frame .
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:hand_rect_from_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "roi"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 2.0
scale_y: 2.0
shift_y: -0.1
square_long: true
}
}
}

BIN
mediapipe/modules/hand_landmark/hand_landmark_lite.tflite
View File

Binary file not shown.

63
mediapipe/modules/hand_landmark/hand_landmark_model_loader.pbtxt
View File

@ -1,63 +0,0 @@
# MediaPipe graph to load a selected hand landmark TF Lite model.
type: "HandLandmarkModelLoader"
# Complexity of the hand landmark model: 0 or 1. Landmark accuracy as well as
# inference latency generally go up with the model complexity. If unspecified,
# functions as set to 1. (int)
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
# TF Lite model represented as a FlatBuffer.
# (std::unique_ptr<tflite::FlatBufferModel, std::function<void(tflite::FlatBufferModel*)>>)
output_side_packet: "MODEL:model"
# Determines path to the desired pose landmark model file.
node {
calculator: "SwitchContainer"
input_side_packet: "SELECT:model_complexity"
output_side_packet: "PACKET:model_path"
options: {
[mediapipe.SwitchContainerOptions.ext] {
select: 1
contained_node: {
calculator: "ConstantSidePacketCalculator"
options: {
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
packet {
string_value: "mediapipe/modules/hand_landmark/hand_landmark_lite.tflite"
}
}
}
}
contained_node: {
calculator: "ConstantSidePacketCalculator"
options: {
[mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
packet {
string_value: "mediapipe/modules/hand_landmark/hand_landmark_full.tflite"
}
}
}
}
}
}
}
# Loads the file in the specified path into a blob.
node {
calculator: "LocalFileContentsCalculator"
input_side_packet: "FILE_PATH:model_path"
output_side_packet: "CONTENTS:model_blob"
options: {
[mediapipe.LocalFileContentsCalculatorOptions.ext]: {
text_mode: false
}
}
}
# Converts the input blob into a TF Lite model.
node {
calculator: "TfLiteModelCalculator"
input_side_packet: "MODEL_BLOB:model_blob"
output_side_packet: "MODEL:model"
}

271
mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu.pbtxt
View File

@ -1,271 +0,0 @@
# MediaPipe graph to detect/predict hand landmarks on CPU.
#
# The procedure is done in two steps:
# - locate palms/hands
# - detect landmarks for each palm/hand.
# This graph tries to skip palm detection as much as possible by reusing
# previously detected/predicted landmarks for new images.
type: "HandLandmarkTrackingCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:image"
# Max number of hands to detect/track. (int)
input_side_packet: "NUM_HANDS:num_hands"
# Complexity of hand landmark and palm detection models: 0 or 1. Accuracy as
# well as inference latency generally go up with the model complexity. If
# unspecified, functions as set to 1. (int)
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
# Whether landmarks on the previous image should be used to help localize
# landmarks on the current image. (bool)
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
# Collection of detected/predicted hands, each represented as a list of
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of hands detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_hand_landmarks"
# Collection of detected/predicted hand world landmarks.
# (std::vector<LandmarkList>)
#
# World landmarks are real-world 3D coordinates in meters with the origin in the
# center of the hand bounding box calculated from the landmarks.
#
# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
# the 3D object itself.
output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
# Collection of handedness of the detected hands (i.e. is hand left or right),
# each represented as a ClassificationList proto with a single Classification
# entry. (std::vector<ClassificationList>)
# Note that handedness is determined assuming the input image is mirrored,
# i.e., taken with a front-facing/selfie camera with images flipped
# horizontally.
output_stream: "HANDEDNESS:multi_handedness"
# Extra outputs (for debugging, for instance).
# Detected palms. (std::vector<Detection>)
output_stream: "PALM_DETECTIONS:palm_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
# Regions of interest calculated based on palm detections.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
# When the optional input side packet "use_prev_landmarks" is either absent or
# set to true, uses the landmarks on the previous image to help localize
# landmarks on the current image.
node {
calculator: "GateCalculator"
input_side_packet: "ALLOW:use_prev_landmarks"
input_stream: "prev_hand_rects_from_landmarks"
output_stream: "gated_prev_hand_rects_from_landmarks"
options: {
[mediapipe.GateCalculatorOptions.ext] {
allow: true
}
}
}
# Determines if an input vector of NormalizedRect has a size greater than or
# equal to the provided num_hands.
node {
calculator: "NormalizedRectVectorHasMinSizeCalculator"
input_stream: "ITERABLE:gated_prev_hand_rects_from_landmarks"
input_side_packet: "num_hands"
output_stream: "prev_has_enough_hands"
}
# Drops the incoming image if enough hands have already been identified from the
# previous image. Otherwise, passes the incoming image through to trigger a new
# round of palm detection.
node {
calculator: "GateCalculator"
input_stream: "image"
input_stream: "DISALLOW:prev_has_enough_hands"
output_stream: "palm_detection_image"
options: {
[mediapipe.GateCalculatorOptions.ext] {
empty_packets_as_allow: true
}
}
}
# Detects palms.
node {
calculator: "PalmDetectionCpu"
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
input_stream: "IMAGE:palm_detection_image"
output_stream: "DETECTIONS:all_palm_detections"
}
# Makes sure there are no more detections than the provided num_hands.
node {
calculator: "ClipDetectionVectorSizeCalculator"
input_stream: "all_palm_detections"
output_stream: "palm_detections"
input_side_packet: "num_hands"
}
# Extracts image size.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:palm_detection_image"
output_stream: "SIZE:palm_detection_image_size"
}
# Outputs each element of palm_detections at a fake timestamp for the rest of
# the graph to process. Clones the image size packet for each palm_detection at
# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp
# for downstream calculators to inform them that all elements in the vector have
# been processed.
node {
calculator: "BeginLoopDetectionCalculator"
input_stream: "ITERABLE:palm_detections"
input_stream: "CLONE:palm_detection_image_size"
output_stream: "ITEM:palm_detection"
output_stream: "CLONE:image_size_for_palms"
output_stream: "BATCH_END:palm_detections_timestamp"
}
# Calculates region of interest (ROI) based on the specified palm.
node {
calculator: "PalmDetectionDetectionToRoi"
input_stream: "DETECTION:palm_detection"
input_stream: "IMAGE_SIZE:image_size_for_palms"
output_stream: "ROI:hand_rect_from_palm_detection"
}
# Collects a NormalizedRect for each hand into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:hand_rect_from_palm_detection"
input_stream: "BATCH_END:palm_detections_timestamp"
output_stream: "ITERABLE:hand_rects_from_palm_detections"
}
# Performs association between NormalizedRect vector elements from previous
# image and rects based on palm detections from the current image. This
# calculator ensures that the output hand_rects vector doesn't contain
# overlapping regions based on the specified min_similarity_threshold.
node {
calculator: "AssociationNormRectCalculator"
input_stream: "hand_rects_from_palm_detections"
input_stream: "gated_prev_hand_rects_from_landmarks"
output_stream: "hand_rects"
options: {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.5
}
}
}
# Extracts image size.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_CPU:image"
output_stream: "SIZE:image_size"
}
# Outputs each element of hand_rects at a fake timestamp for the rest of the
# graph to process. Clones image and image size packets for each
# single_hand_rect at the fake timestamp. At the end of the loop, outputs the
# BATCH_END timestamp for downstream calculators to inform them that all
# elements in the vector have been processed.
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:hand_rects"
input_stream: "CLONE:0:image"
input_stream: "CLONE:1:image_size"
output_stream: "ITEM:single_hand_rect"
output_stream: "CLONE:0:image_for_landmarks"
output_stream: "CLONE:1:image_size_for_landmarks"
output_stream: "BATCH_END:hand_rects_timestamp"
}
# Detect hand landmarks for the specific hand rect.
node {
calculator: "HandLandmarkCpu"
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
input_stream: "IMAGE:image_for_landmarks"
input_stream: "ROI:single_hand_rect"
output_stream: "LANDMARKS:single_hand_landmarks"
output_stream: "WORLD_LANDMARKS:single_hand_world_landmarks"
output_stream: "HANDEDNESS:single_handedness"
}
# Collects the handedness for each single hand into a vector. Upon receiving the
# BATCH_END timestamp, outputs a vector of ClassificationList at the BATCH_END
# timestamp.
node {
calculator: "EndLoopClassificationListCalculator"
input_stream: "ITEM:single_handedness"
input_stream: "BATCH_END:hand_rects_timestamp"
output_stream: "ITERABLE:multi_handedness"
}
# Calculate region of interest (ROI) based on detected hand landmarks to reuse
# on the subsequent runs of the graph.
node {
calculator: "HandLandmarkLandmarksToRoi"
input_stream: "IMAGE_SIZE:image_size_for_landmarks"
input_stream: "LANDMARKS:single_hand_landmarks"
output_stream: "ROI:single_hand_rect_from_landmarks"
}
# Collects a set of landmarks for each hand into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:single_hand_landmarks"
input_stream: "BATCH_END:hand_rects_timestamp"
output_stream: "ITERABLE:multi_hand_landmarks"
}
# Collects a set of world landmarks for each hand into a vector. Upon receiving
# the BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopLandmarkListVectorCalculator"
input_stream: "ITEM:single_hand_world_landmarks"
input_stream: "BATCH_END:hand_rects_timestamp"
output_stream: "ITERABLE:multi_hand_world_landmarks"
}
# Collects a NormalizedRect for each hand into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:single_hand_rect_from_landmarks"
input_stream: "BATCH_END:hand_rects_timestamp"
output_stream: "ITERABLE:hand_rects_from_landmarks"
}
# Caches hand rects calculated from landmarks, and upon the arrival of the next
# input image, sends out the cached rects with timestamps replaced by that of
# the input image, essentially generating a packet that carries the previous
# hand rects. Note that upon the arrival of the very first input image, a
# timestamp bound update occurs to jump start the feedback loop.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:image"
input_stream: "LOOP:hand_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
}
output_stream: "PREV_LOOP:prev_hand_rects_from_landmarks"
}

116
mediapipe/modules/hand_landmark/hand_landmark_tracking_cpu_image.pbtxt
View File

@ -1,116 +0,0 @@
# MediaPipe graph to detect/predict hand landmarks on CPU.
#
# The procedure is done in two steps:
# - locate palms/hands
# - detect landmarks for each palm/hand.
# This graph tries to skip palm detection as much as possible by reusing
# previously detected/predicted landmarks for new images.
type: "HandLandmarkTrackingCpuImage"
# Input image. (Image)
input_stream: "IMAGE:image"
# Max number of hands to detect/track. (int)
input_side_packet: "NUM_HANDS:num_hands"
# Complexity of hand landmark and palm detection models: 0 or 1. Accuracy as
# well as inference latency generally go up with the model complexity. If
# unspecified, functions as set to 1. (int)
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
# Whether landmarks on the previous image should be used to help localize
# landmarks on the current image. (bool)
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
# The throttled input image. (Image)
output_stream: "IMAGE:throttled_image"
# Collection of detected/predicted hands, each represented as a list of
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of hands detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_hand_landmarks"
# Collection of detected/predicted hand world landmarks.
# (std::vector<LandmarkList>)
#
# World landmarks are real-world 3D coordinates in meters with the origin in the
# center of the hand bounding box calculated from the landmarks.
#
# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
# the 3D object itself.
output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
# Collection of handedness of the detected hands (i.e. is hand left or right),
# each represented as a ClassificationList proto with a single Classification
# entry. (std::vector<ClassificationList>)
# Note that handedness is determined assuming the input image is mirrored,
# i.e., taken with a front-facing/selfie camera with images flipped
# horizontally.
output_stream: "HANDEDNESS:multi_handedness"
# Extra outputs (for debugging, for instance).
# Detected palms. (std::vector<Detection>)
output_stream: "PALM_DETECTIONS:palm_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
# Regions of interest calculated based on palm detections.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
node {
calculator: "FlowLimiterCalculator"
input_stream: "image"
input_stream: "FINISHED:multi_hand_landmarks"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_image"
options: {
[mediapipe.FlowLimiterCalculatorOptions.ext] {
max_in_flight: 1
max_in_queue: 1
}
}
}
# Converts Image to ImageFrame for HandLandmarkTrackingCpu to consume.
node {
calculator: "FromImageCalculator"
input_stream: "IMAGE:throttled_image"
output_stream: "IMAGE_CPU:raw_image_frame"
output_stream: "SOURCE_ON_GPU:is_gpu_image"
}
# TODO: Remove the extra flipping once adopting MlImage.
# If the source images are on gpu, flip the data vertically before sending them
# into HandLandmarkTrackingCpu. This maybe needed because OpenGL represents
# images assuming the image origin is at the bottom-left corner, whereas
# MediaPipe in general assumes the image origin is at the top-left corner.
node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE:raw_image_frame"
input_stream: "FLIP_VERTICALLY:is_gpu_image"
output_stream: "IMAGE:image_frame"
}
node {
calculator: "HandLandmarkTrackingCpu"
input_stream: "IMAGE:image_frame"
input_side_packet: "NUM_HANDS:num_hands"
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
output_stream: "LANDMARKS:multi_hand_landmarks"
output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
output_stream: "HANDEDNESS:multi_handedness"
output_stream: "PALM_DETECTIONS:palm_detections"
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
}

272
mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu.pbtxt
View File

@ -1,272 +0,0 @@
# MediaPipe graph to detect/predict hand landmarks on GPU.
#
# The procedure is done in two steps:
# - locate palms/hands
# - detect landmarks for each palm/hand.
# This graph tries to skip palm detection as much as possible by reusing
# previously detected/predicted landmarks for new images.
type: "HandLandmarkTrackingGpu"
# GPU image. (GpuBuffer)
input_stream: "IMAGE:image"
# Max number of hands to detect/track. (int)
input_side_packet: "NUM_HANDS:num_hands"
# Complexity of hand landmark and palm detection models: 0 or 1. Accuracy as
# well as inference latency generally go up with the model complexity. If
# unspecified, functions as set to 1. (int)
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
# Whether landmarks on the previous image should be used to help localize
# landmarks on the current image. (bool)
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
# Collection of detected/predicted hands, each represented as a list of
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of hands detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_hand_landmarks"
# Collection of detected/predicted hand world landmarks.
# (std::vector<LandmarkList>)
#
# World landmarks are real-world 3D coordinates in meters with the origin in the
# center of the hand bounding box calculated from the landmarks.
#
# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
# the 3D object itself.
output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
# Collection of handedness of the detected hands (i.e. is hand left or right),
# each represented as a ClassificationList proto with a single Classification
# entry. (std::vector<ClassificationList>)
# Note that handedness is determined assuming the input image is mirrored,
# i.e., taken with a front-facing/selfie camera with images flipped
# horizontally.
output_stream: "HANDEDNESS:multi_handedness"
# Extra outputs (for debugging, for instance).
# Detected palms. (std::vector<Detection>)
output_stream: "PALM_DETECTIONS:palm_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
# Regions of interest calculated based on palm detections.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
# When the optional input side packet "use_prev_landmarks" is either absent or
# set to true, uses the landmarks on the previous image to help localize
# landmarks on the current image.
node {
calculator: "GateCalculator"
input_side_packet: "ALLOW:use_prev_landmarks"
input_stream: "prev_hand_rects_from_landmarks"
output_stream: "gated_prev_hand_rects_from_landmarks"
options: {
[mediapipe.GateCalculatorOptions.ext] {
allow: true
}
}
}
# Determines if an input vector of NormalizedRect has a size greater than or
# equal to the provided num_hands.
node {
calculator: "NormalizedRectVectorHasMinSizeCalculator"
input_stream: "ITERABLE:gated_prev_hand_rects_from_landmarks"
input_side_packet: "num_hands"
output_stream: "prev_has_enough_hands"
}
# Drops the incoming image if enough hands have already been identified from the
# previous image. Otherwise, passes the incoming image through to trigger a new
# round of palm detection.
node {
calculator: "GateCalculator"
input_stream: "image"
input_stream: "DISALLOW:prev_has_enough_hands"
output_stream: "palm_detection_image"
options: {
[mediapipe.GateCalculatorOptions.ext] {
empty_packets_as_allow: true
}
}
}
# Detects palms.
node {
calculator: "PalmDetectionGpu"
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
input_stream: "IMAGE:palm_detection_image"
output_stream: "DETECTIONS:all_palm_detections"
}
# Makes sure there are no more detections than provided num_hands.
node {
calculator: "ClipDetectionVectorSizeCalculator"
input_stream: "all_palm_detections"
output_stream: "palm_detections"
input_side_packet: "num_hands"
}
# Extracts image size.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:palm_detection_image"
output_stream: "SIZE:palm_detection_image_size"
}
# Outputs each element of palm_detections at a fake timestamp for the rest of
# the graph to process. Clones the image_size packet for each palm_detection at
# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp
# for downstream calculators to inform them that all elements in the vector have
# been processed.
node {
calculator: "BeginLoopDetectionCalculator"
input_stream: "ITERABLE:palm_detections"
input_stream: "CLONE:palm_detection_image_size"
output_stream: "ITEM:palm_detection"
output_stream: "CLONE:image_size_for_palms"
output_stream: "BATCH_END:palm_detections_timestamp"
}
# Calculates region of interest (ROI) base on the specified palm.
node {
calculator: "PalmDetectionDetectionToRoi"
input_stream: "DETECTION:palm_detection"
input_stream: "IMAGE_SIZE:image_size_for_palms"
output_stream: "ROI:hand_rect_from_palm_detection"
}
# Collects a NormalizedRect for each hand into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
name: "EndLoopForPalmDetections"
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:hand_rect_from_palm_detection"
input_stream: "BATCH_END:palm_detections_timestamp"
output_stream: "ITERABLE:hand_rects_from_palm_detections"
}
# Performs association between NormalizedRect vector elements from previous
# image and rects based on palm detections from the current image. This
# calculator ensures that the output hand_rects vector doesn't contain
# overlapping regions based on the specified min_similarity_threshold.
node {
calculator: "AssociationNormRectCalculator"
input_stream: "hand_rects_from_palm_detections"
input_stream: "gated_prev_hand_rects_from_landmarks"
output_stream: "hand_rects"
options: {
[mediapipe.AssociationCalculatorOptions.ext] {
min_similarity_threshold: 0.5
}
}
}
# Extracts image size.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:image"
output_stream: "SIZE:image_size"
}
# Outputs each element of hand_rects at a fake timestamp for the rest of the
# graph to process. Clones image and image size packets for each
# single_hand_rect at the fake timestamp. At the end of the loop, outputs the
# BATCH_END timestamp for downstream calculators to inform them that all
# elements in the vector have been processed.
node {
calculator: "BeginLoopNormalizedRectCalculator"
input_stream: "ITERABLE:hand_rects"
input_stream: "CLONE:0:image"
input_stream: "CLONE:1:image_size"
output_stream: "ITEM:single_hand_rect"
output_stream: "CLONE:0:image_for_landmarks"
output_stream: "CLONE:1:image_size_for_landmarks"
output_stream: "BATCH_END:hand_rects_timestamp"
}
# Detect hand landmarks for the specific hand rect.
node {
calculator: "HandLandmarkGpu"
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
input_stream: "IMAGE:image_for_landmarks"
input_stream: "ROI:single_hand_rect"
output_stream: "LANDMARKS:single_hand_landmarks"
output_stream: "WORLD_LANDMARKS:single_hand_world_landmarks"
output_stream: "HANDEDNESS:single_handedness"
}
# Collects the handedness for each single hand into a vector. Upon receiving the
# BATCH_END timestamp, outputs a vector of ClassificationList at the BATCH_END
# timestamp.
node {
calculator: "EndLoopClassificationListCalculator"
input_stream: "ITEM:single_handedness"
input_stream: "BATCH_END:hand_rects_timestamp"
output_stream: "ITERABLE:multi_handedness"
}
# Calculate region of interest (ROI) based on detected hand landmarks to reuse
# on the subsequent runs of the graph.
node {
calculator: "HandLandmarkLandmarksToRoi"
input_stream: "IMAGE_SIZE:image_size_for_landmarks"
input_stream: "LANDMARKS:single_hand_landmarks"
output_stream: "ROI:single_hand_rect_from_landmarks"
}
# Collects a set of landmarks for each hand into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
input_stream: "ITEM:single_hand_landmarks"
input_stream: "BATCH_END:hand_rects_timestamp"
output_stream: "ITERABLE:multi_hand_landmarks"
}
# Collects a set of world landmarks for each hand into a vector. Upon receiving
# the BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
# timestamp.
node {
calculator: "EndLoopLandmarkListVectorCalculator"
input_stream: "ITEM:single_hand_world_landmarks"
input_stream: "BATCH_END:hand_rects_timestamp"
output_stream: "ITERABLE:multi_hand_world_landmarks"
}
# Collects a NormalizedRect for each hand into a vector. Upon receiving the
# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
# timestamp.
node {
calculator: "EndLoopNormalizedRectCalculator"
input_stream: "ITEM:single_hand_rect_from_landmarks"
input_stream: "BATCH_END:hand_rects_timestamp"
output_stream: "ITERABLE:hand_rects_from_landmarks"
}
# Caches hand rects calculated from landmarks, and upon the arrival of the next
# input image, sends out the cached rects with timestamps replaced by that of
# the input image, essentially generating a packet that carries the previous
# hand rects. Note that upon the arrival of the very first input image, a
# timestamp bound update occurs to jump start the feedback loop.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:image"
input_stream: "LOOP:hand_rects_from_landmarks"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
}
output_stream: "PREV_LOOP:prev_hand_rects_from_landmarks"
}

115
mediapipe/modules/hand_landmark/hand_landmark_tracking_gpu_image.pbtxt
View File

@ -1,115 +0,0 @@
# MediaPipe graph to detect/predict hand landmarks on GPU.
#
# The procedure is done in two steps:
# - locate palms/hands
# - detect landmarks for each palm/hand.
# This graph tries to skip palm detection as much as possible by reusing
# previously detected/predicted landmarks for new images.
type: "HandLandmarkTrackingGpuImage"
# Input image. (Image)
input_stream: "IMAGE:image"
# Max number of hands to detect/track. (int)
input_side_packet: "NUM_HANDS:num_hands"
# Complexity of hand landmark and palm detection models: 0 or 1. Accuracy as
# well as inference latency generally go up with the model complexity. If
# unspecified, functions as set to 1. (int)
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
# Whether landmarks on the previous image should be used to help localize
# landmarks on the current image. (bool)
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
# Collection of detected/predicted hands, each represented as a list of
# landmarks. (std::vector<NormalizedLandmarkList>)
# NOTE: there will not be an output packet in the LANDMARKS stream for this
# particular timestamp if none of hands detected. However, the MediaPipe
# framework will internally inform the downstream calculators of the absence of
# this packet so that they don't wait for it unnecessarily.
output_stream: "LANDMARKS:multi_hand_landmarks"
# Collection of detected/predicted hand world landmarks.
# (std::vector<LandmarkList>)
#
# World landmarks are real-world 3D coordinates in meters with the origin in the
# center of the hand bounding box calculated from the landmarks.
#
# WORLD_LANDMARKS shares the same landmark topology as LANDMARKS. However,
# LANDMARKS provides coordinates (in pixels) of a 3D object projected onto the
# 2D image surface, while WORLD_LANDMARKS provides coordinates (in meters) of
# the 3D object itself.
output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
# Collection of handedness of the detected hands (i.e. is hand left or right),
# each represented as a ClassificationList proto with a single Classification
# entry. (std::vector<ClassificationList>)
# Note that handedness is determined assuming the input image is mirrored,
# i.e., taken with a front-facing/selfie camera with images flipped
# horizontally.
output_stream: "HANDEDNESS:multi_handedness"
# The throttled input image. (Image)
output_stream: "IMAGE:throttled_image"
# Extra outputs (for debugging, for instance).
# Detected palms. (std::vector<Detection>)
output_stream: "PALM_DETECTIONS:palm_detections"
# Regions of interest calculated based on landmarks.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
# Regions of interest calculated based on palm detections.
# (std::vector<NormalizedRect>)
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
node {
calculator: "FlowLimiterCalculator"
input_stream: "image"
input_stream: "FINISHED:multi_hand_landmarks"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true
}
output_stream: "throttled_image"
options: {
[mediapipe.FlowLimiterCalculatorOptions.ext] {
max_in_flight: 1
max_in_queue: 1
}
}
}
# Converts Image to GpuBuffer for HandLandmarkTrackingGpu to consume.
node {
calculator: "FromImageCalculator"
input_stream: "IMAGE:throttled_image"
output_stream: "IMAGE_GPU:raw_gpu_buffer"
output_stream: "SOURCE_ON_GPU:is_gpu_image"
}
# TODO: Remove the extra flipping once adopting MlImage.
# If the source images are on gpu, flip the data vertically before sending them
# into HandLandmarkTrackingGpu. This maybe needed because OpenGL represents
# images assuming the image origin is at the bottom-left corner, whereas
# MediaPipe in general assumes the image origin is at the top-left corner.
node: {
calculator: "ImageTransformationCalculator"
input_stream: "IMAGE_GPU:raw_gpu_buffer"
input_stream: "FLIP_VERTICALLY:is_gpu_image"
output_stream: "IMAGE_GPU:gpu_buffer"
}
node {
calculator: "HandLandmarkTrackingGpu"
input_stream: "IMAGE:gpu_buffer"
input_side_packet: "NUM_HANDS:num_hands"
input_side_packet: "MODEL_COMPLEXITY:model_complexity"
input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks"
output_stream: "LANDMARKS:multi_hand_landmarks"
output_stream: "WORLD_LANDMARKS:multi_hand_world_landmarks"
output_stream: "HANDEDNESS:multi_handedness"
output_stream: "PALM_DETECTIONS:palm_detections"
output_stream: "HAND_ROIS_FROM_LANDMARKS:hand_rects"
output_stream: "HAND_ROIS_FROM_PALM_DETECTIONS:hand_rects_from_palm_detections"
}

2
mediapipe/modules/hand_landmark/handedness.txt
View File

@ -1,2 +0,0 @@
Left
Right

47
mediapipe/modules/hand_landmark/palm_detection_detection_to_roi.pbtxt
View File

@ -1,47 +0,0 @@
# MediaPipe subgraph that calculates hand ROI from palm detection.
type: "PalmDetectionDetectionToRoi"
# Palm detection. (Detection)
input_stream: "DETECTION:detection"
# Frame size. (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# ROI (region of interest) according to landmarks, represented as normalized
# rect. (NormalizedRect)
output_stream: "ROI:roi"
# Converts results of palm detection into a rectangle (normalized by image size)
# that encloses the palm and is rotated such that the line connecting center of
# the wrist and MCP of the middle finger is aligned with the Y-axis of the
# rectangle.
node {
calculator: "DetectionsToRectsCalculator"
input_stream: "DETECTION:detection"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECT:raw_roi"
options: {
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
rotation_vector_start_keypoint_index: 0 # Center of wrist.
rotation_vector_end_keypoint_index: 2 # MCP of middle finger.
rotation_vector_target_angle_degrees: 90
}
}
}
# Expands and shifts the rectangle that contains the palm so that it's likely
# to cover the entire hand.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:raw_roi"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "roi"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 2.6
scale_y: 2.6
shift_y: -0.5
square_long: true
}
}
}

267
mediapipe/modules/holistic_landmark/BUILD
View File

@ -1,267 +0,0 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/tool:mediapipe_graph.bzl", "mediapipe_simple_subgraph")
# TODO: revert to private.
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
exports_files([
"hand_recrop.tflite",
])
mediapipe_simple_subgraph(
name = "face_landmarks_from_pose_gpu",
graph = "face_landmarks_from_pose_gpu.pbtxt",
register_as = "FaceLandmarksFromPoseGpu",
deps = [
":face_detection_front_detections_to_roi",
":face_landmarks_from_pose_to_recrop_roi",
":face_tracking",
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/modules/face_detection:face_detection_short_range_by_roi_gpu",
"//mediapipe/modules/face_landmark:face_landmark_gpu",
],
)
mediapipe_simple_subgraph(
name = "face_landmarks_from_pose_cpu",
graph = "face_landmarks_from_pose_cpu.pbtxt",
register_as = "FaceLandmarksFromPoseCpu",
deps = [
":face_detection_front_detections_to_roi",
":face_landmarks_from_pose_to_recrop_roi",
":face_tracking",
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/modules/face_detection:face_detection_short_range_by_roi_cpu",
"//mediapipe/modules/face_landmark:face_landmark_cpu",
],
)
mediapipe_simple_subgraph(
name = "face_landmarks_to_roi",
graph = "face_landmarks_to_roi.pbtxt",
register_as = "FaceLandmarksToRoi",
deps = [
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_detection_front_detections_to_roi",
graph = "face_detection_front_detections_to_roi.pbtxt",
register_as = "FaceDetectionFrontDetectionsToRoi",
deps = [
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_tracking",
graph = "face_tracking.pbtxt",
register_as = "FaceTracking",
deps = [
":face_landmarks_to_roi",
"//mediapipe/calculators/core:previous_loopback_calculator",
"//mediapipe/modules/holistic_landmark/calculators:roi_tracking_calculator",
],
)
mediapipe_simple_subgraph(
name = "face_landmarks_from_pose_to_recrop_roi",
graph = "face_landmarks_from_pose_to_recrop_roi.pbtxt",
register_as = "FaceLandmarksFromPoseToRecropRoi",
deps = [
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_landmarks_from_pose_gpu",
graph = "hand_landmarks_from_pose_gpu.pbtxt",
register_as = "HandLandmarksFromPoseGpu",
deps = [
":hand_landmarks_from_pose_to_recrop_roi",
":hand_recrop_by_roi_gpu",
":hand_tracking",
":hand_visibility_from_hand_landmarks_from_pose",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/modules/hand_landmark:hand_landmark_gpu",
],
)
mediapipe_simple_subgraph(
name = "hand_landmarks_from_pose_cpu",
graph = "hand_landmarks_from_pose_cpu.pbtxt",
register_as = "HandLandmarksFromPoseCpu",
deps = [
":hand_landmarks_from_pose_to_recrop_roi",
":hand_recrop_by_roi_cpu",
":hand_tracking",
":hand_visibility_from_hand_landmarks_from_pose",
"//mediapipe/calculators/core:gate_calculator",
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/modules/hand_landmark:hand_landmark_cpu",
],
)
mediapipe_simple_subgraph(
name = "hand_landmarks_to_roi",
graph = "hand_landmarks_to_roi.pbtxt",
register_as = "HandLandmarksToRoi",
deps = [
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
"//mediapipe/modules/hand_landmark/calculators:hand_landmarks_to_rect_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_recrop_by_roi_gpu",
graph = "hand_recrop_by_roi_gpu.pbtxt",
register_as = "HandRecropByRoiGpu",
deps = [
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
"//mediapipe/calculators/util:alignment_points_to_rects_calculator",
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
"//mediapipe/calculators/util:landmark_projection_calculator",
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_recrop_by_roi_cpu",
graph = "hand_recrop_by_roi_cpu.pbtxt",
register_as = "HandRecropByRoiCpu",
deps = [
"//mediapipe/calculators/image:image_properties_calculator",
"//mediapipe/calculators/tensor:image_to_tensor_calculator",
"//mediapipe/calculators/tensor:inference_calculator",
"//mediapipe/calculators/tensor:tensors_to_landmarks_calculator",
"//mediapipe/calculators/util:alignment_points_to_rects_calculator",
"//mediapipe/calculators/util:landmark_letterbox_removal_calculator",
"//mediapipe/calculators/util:landmark_projection_calculator",
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_tracking",
graph = "hand_tracking.pbtxt",
register_as = "HandTracking",
deps = [
":hand_landmarks_to_roi",
"//mediapipe/calculators/core:previous_loopback_calculator",
"//mediapipe/modules/holistic_landmark/calculators:roi_tracking_calculator",
],
)
# TODO: parametrize holistic_landmark graph with visibility and make private.
mediapipe_simple_subgraph(
name = "hand_wrist_for_pose",
graph = "hand_wrist_for_pose.pbtxt",
register_as = "HandWristForPose",
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/core:constant_side_packet_calculator",
"//mediapipe/calculators/core:side_packet_to_stream_calculator",
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/util:set_landmark_visibility_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_landmarks_left_and_right_gpu",
graph = "hand_landmarks_left_and_right_gpu.pbtxt",
register_as = "HandLandmarksLeftAndRightGpu",
deps = [
":hand_landmarks_from_pose_gpu",
"//mediapipe/calculators/core:split_landmarks_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_landmarks_left_and_right_cpu",
graph = "hand_landmarks_left_and_right_cpu.pbtxt",
register_as = "HandLandmarksLeftAndRightCpu",
deps = [
":hand_landmarks_from_pose_cpu",
"//mediapipe/calculators/core:split_landmarks_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_landmarks_from_pose_to_recrop_roi",
graph = "hand_landmarks_from_pose_to_recrop_roi.pbtxt",
register_as = "HandLandmarksFromPoseToRecropRoi",
deps = [
"//mediapipe/calculators/util:landmarks_to_detection_calculator",
"//mediapipe/calculators/util:rect_transformation_calculator",
"//mediapipe/modules/holistic_landmark/calculators:hand_detections_from_pose_to_rects_calculator",
],
)
mediapipe_simple_subgraph(
name = "hand_visibility_from_hand_landmarks_from_pose",
graph = "hand_visibility_from_hand_landmarks_from_pose.pbtxt",
register_as = "HandVisibilityFromHandLandmarksFromPose",
deps = [
"//mediapipe/calculators/core:split_landmarks_calculator",
"//mediapipe/calculators/util:landmark_visibility_calculator",
"//mediapipe/calculators/util:thresholding_calculator",
],
)
mediapipe_simple_subgraph(
name = "holistic_landmark_gpu",
graph = "holistic_landmark_gpu.pbtxt",
register_as = "HolisticLandmarkGpu",
visibility = ["//visibility:public"],
deps = [
":face_landmarks_from_pose_gpu",
":hand_landmarks_left_and_right_gpu",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/modules/pose_landmark:pose_landmark_gpu",
],
)
mediapipe_simple_subgraph(
name = "holistic_landmark_cpu",
graph = "holistic_landmark_cpu.pbtxt",
register_as = "HolisticLandmarkCpu",
visibility = ["//visibility:public"],
deps = [
":face_landmarks_from_pose_cpu",
":hand_landmarks_left_and_right_cpu",
"//mediapipe/calculators/core:flow_limiter_calculator",
"//mediapipe/calculators/image:image_transformation_calculator",
"//mediapipe/modules/pose_landmark:pose_landmark_cpu",
],
)

6
mediapipe/modules/holistic_landmark/README.md
View File

@ -1,6 +0,0 @@
# holistic_landmark
Subgraphs|Details
:--- | :---
[`HolisticLandmarkCpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/holistic_landmark/holistic_landmark_cpu.pbtxt)| Predicts pose + left/right hand + face landmarks. (CPU input)
[`HolisticLandmarkGpu`](https://github.com/google/mediapipe/tree/master/mediapipe/modules/holistic_landmark/holistic_landmark_gpu.pbtxt)| Predicts pose + left/right hand + face landmarks. (GPU input.)

63
mediapipe/modules/holistic_landmark/calculators/BUILD
View File

@ -1,63 +0,0 @@
# Copyright 2020 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library")
licenses(["notice"])
package(default_visibility = ["//visibility:public"])
cc_library(
name = "hand_detections_from_pose_to_rects_calculator",
srcs = ["hand_detections_from_pose_to_rects_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/calculators/util:detections_to_rects_calculator",
"//mediapipe/calculators/util:detections_to_rects_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework:calculator_options_cc_proto",
"//mediapipe/framework/formats:detection_cc_proto",
"//mediapipe/framework/formats:location_data_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:ret_check",
"//mediapipe/framework/port:status",
],
alwayslink = 1,
)
mediapipe_proto_library(
name = "roi_tracking_calculator_proto",
srcs = ["roi_tracking_calculator.proto"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/framework:calculator_options_proto",
"//mediapipe/framework:calculator_proto",
],
)
cc_library(
name = "roi_tracking_calculator",
srcs = ["roi_tracking_calculator.cc"],
visibility = ["//visibility:public"],
deps = [
":roi_tracking_calculator_cc_proto",
"//mediapipe/framework:calculator_framework",
"//mediapipe/framework/formats:landmark_cc_proto",
"//mediapipe/framework/formats:rect_cc_proto",
"//mediapipe/framework/port:logging",
"//mediapipe/framework/port:rectangle",
"@com_google_absl//absl/strings:str_format",
],
alwayslink = 1,
)

156
mediapipe/modules/holistic_landmark/calculators/hand_detections_from_pose_to_rects_calculator.cc
View File

@ -1,156 +0,0 @@
#include <cmath>
#include "mediapipe/calculators/util/detections_to_rects_calculator.h"
#include "mediapipe/calculators/util/detections_to_rects_calculator.pb.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/calculator_options.pb.h"
#include "mediapipe/framework/formats/detection.pb.h"
#include "mediapipe/framework/formats/location_data.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
namespace mediapipe {
namespace {} // namespace
// Generates a hand ROI based on a hand detection derived from hand-related pose
// landmarks.
//
// Inputs:
// DETECTION - Detection.
// Detection to convert to ROI. Must contain 3 key points indicating: wrist,
// pinky and index fingers.
//
// IMAGE_SIZE - std::pair<int, int>
// Image width and height.
//
// Outputs:
// NORM_RECT - NormalizedRect.
// ROI based on passed input.
//
// Examples
// node {
// calculator: "HandDetectionsFromPoseToRectsCalculator"
// input_stream: "DETECTION:hand_detection_from_pose"
// input_stream: "IMAGE_SIZE:image_size"
// output_stream: "NORM_RECT:hand_roi_from_pose"
// }
class HandDetectionsFromPoseToRectsCalculator
: public DetectionsToRectsCalculator {
public:
absl::Status Open(CalculatorContext* cc) override;
private:
::absl::Status DetectionToNormalizedRect(const Detection& detection,
const DetectionSpec& detection_spec,
NormalizedRect* rect) override;
absl::Status ComputeRotation(const Detection& detection,
const DetectionSpec& detection_spec,
float* rotation) override;
};
REGISTER_CALCULATOR(HandDetectionsFromPoseToRectsCalculator);
namespace {
constexpr int kWrist = 0;
constexpr int kPinky = 1;
constexpr int kIndex = 2;
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
} // namespace
::absl::Status HandDetectionsFromPoseToRectsCalculator::Open(
CalculatorContext* cc) {
RET_CHECK(cc->Inputs().HasTag(kImageSizeTag))
<< "Image size is required to calculate rotated rect.";
cc->SetOffset(TimestampDiff(0));
target_angle_ = M_PI * 0.5f;
rotate_ = true;
options_ = cc->Options<DetectionsToRectsCalculatorOptions>();
output_zero_rect_for_empty_detections_ =
options_.output_zero_rect_for_empty_detections();
return ::absl::OkStatus();
}
::absl::Status
HandDetectionsFromPoseToRectsCalculator ::DetectionToNormalizedRect(
const Detection& detection, const DetectionSpec& detection_spec,
NormalizedRect* rect) {
const auto& location_data = detection.location_data();
const auto& image_size = detection_spec.image_size;
RET_CHECK(image_size) << "Image size is required to calculate rotation";
const float x_wrist =
location_data.relative_keypoints(kWrist).x() * image_size->first;
const float y_wrist =
location_data.relative_keypoints(kWrist).y() * image_size->second;
const float x_index =
location_data.relative_keypoints(kIndex).x() * image_size->first;
const float y_index =
location_data.relative_keypoints(kIndex).y() * image_size->second;
const float x_pinky =
location_data.relative_keypoints(kPinky).x() * image_size->first;
const float y_pinky =
location_data.relative_keypoints(kPinky).y() * image_size->second;
// Estimate middle finger.
const float x_middle = (2.f * x_index + x_pinky) / 3.f;
const float y_middle = (2.f * y_index + y_pinky) / 3.f;
// Crop center as middle finger.
const float center_x = x_middle;
const float center_y = y_middle;
// Bounding box size as double distance from middle finger to wrist.
const float box_size =
std::sqrt((x_middle - x_wrist) * (x_middle - x_wrist) +
(y_middle - y_wrist) * (y_middle - y_wrist)) *
2.0;
// Set resulting bounding box.
rect->set_x_center(center_x / image_size->first);
rect->set_y_center(center_y / image_size->second);
rect->set_width(box_size / image_size->first);
rect->set_height(box_size / image_size->second);
return ::absl::OkStatus();
}
absl::Status HandDetectionsFromPoseToRectsCalculator::ComputeRotation(
const Detection& detection, const DetectionSpec& detection_spec,
float* rotation) {
const auto& location_data = detection.location_data();
const auto& image_size = detection_spec.image_size;
RET_CHECK(image_size) << "Image size is required to calculate rotation";
const float x_wrist =
location_data.relative_keypoints(kWrist).x() * image_size->first;
const float y_wrist =
location_data.relative_keypoints(kWrist).y() * image_size->second;
const float x_index =
location_data.relative_keypoints(kIndex).x() * image_size->first;
const float y_index =
location_data.relative_keypoints(kIndex).y() * image_size->second;
const float x_pinky =
location_data.relative_keypoints(kPinky).x() * image_size->first;
const float y_pinky =
location_data.relative_keypoints(kPinky).y() * image_size->second;
// Estimate middle finger.
const float x_middle = (2.f * x_index + x_pinky) / 3.f;
const float y_middle = (2.f * y_index + y_pinky) / 3.f;
*rotation = NormalizeRadians(
target_angle_ - std::atan2(-(y_middle - y_wrist), x_middle - x_wrist));
return ::absl::OkStatus();
}
} // namespace mediapipe

358
mediapipe/modules/holistic_landmark/calculators/roi_tracking_calculator.cc
View File

@ -1,358 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <math.h>
#include <cstdlib>
#include "absl/strings/str_format.h"
#include "mediapipe/framework/calculator_framework.h"
#include "mediapipe/framework/formats/landmark.pb.h"
#include "mediapipe/framework/formats/rect.pb.h"
#include "mediapipe/framework/port/logging.h"
#include "mediapipe/framework/port/rectangle.h"
#include "mediapipe/modules/holistic_landmark/calculators/roi_tracking_calculator.pb.h"
namespace mediapipe {
namespace {
constexpr char kPrevLandmarksTag[] = "PREV_LANDMARKS";
constexpr char kPrevLandmarksRectTag[] = "PREV_LANDMARKS_RECT";
constexpr char kRecropRectTag[] = "RECROP_RECT";
constexpr char kImageSizeTag[] = "IMAGE_SIZE";
constexpr char kTrackingRectTag[] = "TRACKING_RECT";
// TODO: Use rect rotation.
// Verifies that Intersection over Union of previous frame rect and current
// frame re-crop rect is less than threshold.
bool IouRequirementsSatisfied(const NormalizedRect& prev_rect,
const NormalizedRect& recrop_rect,
const std::pair<int, int>& image_size,
const float min_iou) {
auto r1 = Rectangle_f(prev_rect.x_center() * image_size.first,
prev_rect.y_center() * image_size.second,
prev_rect.width() * image_size.first,
prev_rect.height() * image_size.second);
auto r2 = Rectangle_f(recrop_rect.x_center() * image_size.first,
recrop_rect.y_center() * image_size.second,
recrop_rect.width() * image_size.first,
recrop_rect.height() * image_size.second);
const float intersection_area = r1.Intersect(r2).Area();
const float union_area = r1.Area() + r2.Area() - intersection_area;
const float intersection_threshold = union_area * min_iou;
if (intersection_area < intersection_threshold) {
VLOG(1) << absl::StrFormat("Lost tracking: IoU intersection %f < %f",
intersection_area, intersection_threshold);
return false;
}
return true;
}
// Verifies that current frame re-crop rect rotation/translation/scale didn't
// change much comparing to the previous frame rect. Translation and scale are
// normalized by current frame re-crop rect.
bool RectRequirementsSatisfied(const NormalizedRect& prev_rect,
const NormalizedRect& recrop_rect,
const std::pair<int, int> image_size,
const float rotation_degrees,
const float translation, const float scale) {
// Rotate both rects so that re-crop rect edges are parallel to XY axes. That
// will allow to compute x/y translation of the previous frame rect along axes
// of the current frame re-crop rect.
const float rotation = -recrop_rect.rotation();
const float cosa = cos(rotation);
const float sina = sin(rotation);
// Rotate previous frame rect and get its parameters.
const float prev_rect_x = prev_rect.x_center() * image_size.first * cosa -
prev_rect.y_center() * image_size.second * sina;
const float prev_rect_y = prev_rect.x_center() * image_size.first * sina +
prev_rect.y_center() * image_size.second * cosa;
const float prev_rect_width = prev_rect.width() * image_size.first;
const float prev_rect_height = prev_rect.height() * image_size.second;
const float prev_rect_rotation = prev_rect.rotation() / M_PI * 180.f;
// Rotate current frame re-crop rect and get its parameters.
const float recrop_rect_x = recrop_rect.x_center() * image_size.first * cosa -
recrop_rect.y_center() * image_size.second * sina;
const float recrop_rect_y = recrop_rect.x_center() * image_size.first * sina +
recrop_rect.y_center() * image_size.second * cosa;
const float recrop_rect_width = recrop_rect.width() * image_size.first;
const float recrop_rect_height = recrop_rect.height() * image_size.second;
const float recrop_rect_rotation = recrop_rect.rotation() / M_PI * 180.f;
// Rect requirements are satisfied unless one of the checks below fails.
bool satisfied = true;
// Ensure that rotation diff is in [0, 180] range.
float rotation_diff = prev_rect_rotation - recrop_rect_rotation;
if (rotation_diff > 180.f) {
rotation_diff -= 360.f;
}
if (rotation_diff < -180.f) {
rotation_diff += 360.f;
}
rotation_diff = abs(rotation_diff);
if (rotation_diff > rotation_degrees) {
satisfied = false;
VLOG(1) << absl::StrFormat("Lost tracking: rect rotation %f > %f",
rotation_diff, rotation_degrees);
}
const float x_diff = abs(prev_rect_x - recrop_rect_x);
const float x_threshold = recrop_rect_width * translation;
if (x_diff > x_threshold) {
satisfied = false;
VLOG(1) << absl::StrFormat("Lost tracking: rect x translation %f > %f",
x_diff, x_threshold);
}
const float y_diff = abs(prev_rect_y - recrop_rect_y);
const float y_threshold = recrop_rect_height * translation;
if (y_diff > y_threshold) {
satisfied = false;
VLOG(1) << absl::StrFormat("Lost tracking: rect y translation %f > %f",
y_diff, y_threshold);
}
const float width_diff = abs(prev_rect_width - recrop_rect_width);
const float width_threshold = recrop_rect_width * scale;
if (width_diff > width_threshold) {
satisfied = false;
VLOG(1) << absl::StrFormat("Lost tracking: rect width %f > %f", width_diff,
width_threshold);
}
const float height_diff = abs(prev_rect_height - recrop_rect_height);
const float height_threshold = recrop_rect_height * scale;
if (height_diff > height_threshold) {
satisfied = false;
VLOG(1) << absl::StrFormat("Lost tracking: rect height %f > %f",
height_diff, height_threshold);
}
return satisfied;
}
// Verifies that landmarks from the previous frame are within re-crop rectangle
// bounds on the current frame.
bool LandmarksRequirementsSatisfied(const NormalizedLandmarkList& landmarks,
const NormalizedRect& recrop_rect,
const std::pair<int, int> image_size,
const float recrop_rect_margin) {
// Rotate both re-crop rectangle and landmarks so that re-crop rectangle edges
// are parallel to XY axes. It will allow to easily check if landmarks are
// within re-crop rect bounds along re-crop rect axes.
//
// Rect rotation is specified clockwise. To apply cos/sin functions we
// transform it into counterclockwise.
const float rotation = -recrop_rect.rotation();
const float cosa = cos(rotation);
const float sina = sin(rotation);
// Rotate rect.
const float rect_x = recrop_rect.x_center() * image_size.first * cosa -
recrop_rect.y_center() * image_size.second * sina;
const float rect_y = recrop_rect.x_center() * image_size.first * sina +
recrop_rect.y_center() * image_size.second * cosa;
const float rect_width =
recrop_rect.width() * image_size.first * (1.f + recrop_rect_margin);
const float rect_height =
recrop_rect.height() * image_size.second * (1.f + recrop_rect_margin);
// Get rect bounds.
const float rect_left = rect_x - rect_width * 0.5f;
const float rect_right = rect_x + rect_width * 0.5f;
const float rect_top = rect_y - rect_height * 0.5f;
const float rect_bottom = rect_y + rect_height * 0.5f;
for (int i = 0; i < landmarks.landmark_size(); ++i) {
const auto& landmark = landmarks.landmark(i);
const float x = landmark.x() * image_size.first * cosa -
landmark.y() * image_size.second * sina;
const float y = landmark.x() * image_size.first * sina +
landmark.y() * image_size.second * cosa;
if (!(rect_left < x && x < rect_right && rect_top < y && y < rect_bottom)) {
VLOG(1) << "Lost tracking: landmarks out of re-crop rect";
return false;
}
}
return true;
}
} // namespace
// A calculator to track object rectangle between frames.
//
// Calculator checks that all requirements for tracking are satisfied and uses
// rectangle from the previous frame in this case, otherwise - uses current
// frame re-crop rectangle.
//
// There are several types of tracking requirements that can be configured via
// options:
// IoU: Verifies that IoU of the previous frame rectangle and current frame
// re-crop rectangle is less than a given threshold.
// Rect parameters: Verifies that rotation/translation/scale of the re-crop
// rectangle on the current frame is close to the rectangle from the
// previous frame within given thresholds.
// Landmarks: Verifies that landmarks from the previous frame are within
// the re-crop rectangle on the current frame.
//
// Inputs:
// PREV_LANDMARKS: Object landmarks from the previous frame.
// PREV_LANDMARKS_RECT: Object rectangle based on the landmarks from the
// previous frame.
// RECROP_RECT: Object re-crop rectangle from the current frame.
// IMAGE_SIZE: Image size to transform normalized coordinates to absolute.
//
// Outputs:
// TRACKING_RECT: Rectangle to use for object prediction on the current frame.
// It will be either object rectangle from the previous frame (if all
// tracking requirements are satisfied) or re-crop rectangle from the
// current frame (if tracking lost the object).
//
// Example config:
// node {
// calculator: "RoiTrackingCalculator"
// input_stream: "PREV_LANDMARKS:prev_hand_landmarks"
// input_stream: "PREV_LANDMARKS_RECT:prev_hand_landmarks_rect"
// input_stream: "RECROP_RECT:hand_recrop_rect"
// input_stream: "IMAGE_SIZE:image_size"
// output_stream: "TRACKING_RECT:hand_tracking_rect"
// options: {
// [mediapipe.RoiTrackingCalculatorOptions.ext] {
// rect_requirements: {
// rotation_degrees: 40.0
// translation: 0.2
// scale: 0.4
// }
// landmarks_requirements: {
// recrop_rect_margin: -0.1
// }
// }
// }
// }
class RoiTrackingCalculator : public CalculatorBase {
public:
static absl::Status GetContract(CalculatorContract* cc);
absl::Status Open(CalculatorContext* cc) override;
absl::Status Process(CalculatorContext* cc) override;
private:
RoiTrackingCalculatorOptions options_;
};
REGISTER_CALCULATOR(RoiTrackingCalculator);
absl::Status RoiTrackingCalculator::GetContract(CalculatorContract* cc) {
cc->Inputs().Tag(kPrevLandmarksTag).Set<NormalizedLandmarkList>();
cc->Inputs().Tag(kPrevLandmarksRectTag).Set<NormalizedRect>();
cc->Inputs().Tag(kRecropRectTag).Set<NormalizedRect>();
cc->Inputs().Tag(kImageSizeTag).Set<std::pair<int, int>>();
cc->Outputs().Tag(kTrackingRectTag).Set<NormalizedRect>();
return absl::OkStatus();
}
absl::Status RoiTrackingCalculator::Open(CalculatorContext* cc) {
cc->SetOffset(TimestampDiff(0));
options_ = cc->Options<RoiTrackingCalculatorOptions>();
return absl::OkStatus();
}
absl::Status RoiTrackingCalculator::Process(CalculatorContext* cc) {
// If there is no current frame re-crop rect (i.e. object is not present on
// the current frame) - return empty packet.
if (cc->Inputs().Tag(kRecropRectTag).IsEmpty()) {
return absl::OkStatus();
}
// If there is no previous rect, but there is current re-crop rect - return
// current re-crop rect as is.
if (cc->Inputs().Tag(kPrevLandmarksRectTag).IsEmpty()) {
cc->Outputs()
.Tag(kTrackingRectTag)
.AddPacket(cc->Inputs().Tag(kRecropRectTag).Value());
return absl::OkStatus();
}
// At this point we have both previous rect (which also means we have previous
// landmarks) and currrent re-crop rect.
const auto& prev_landmarks =
cc->Inputs().Tag(kPrevLandmarksTag).Get<NormalizedLandmarkList>();
const auto& prev_rect =
cc->Inputs().Tag(kPrevLandmarksRectTag).Get<NormalizedRect>();
const auto& recrop_rect =
cc->Inputs().Tag(kRecropRectTag).Get<NormalizedRect>();
const auto& image_size =
cc->Inputs().Tag(kImageSizeTag).Get<std::pair<int, int>>();
// Keep tracking unless one of the requirements below is not satisfied.
bool keep_tracking = true;
// If IoU of the previous rect and current re-crop rect is lower than allowed
// threshold - use current re-crop rect.
if (options_.has_iou_requirements() &&
!IouRequirementsSatisfied(prev_rect, recrop_rect, image_size,
options_.iou_requirements().min_iou())) {
keep_tracking = false;
}
// If previous rect and current re-crop rect differ more than it is allowed by
// the augmentations (used during the model training) - use current re-crop
// rect.
if (options_.has_rect_requirements() &&
!RectRequirementsSatisfied(
prev_rect, recrop_rect, image_size,
options_.rect_requirements().rotation_degrees(),
options_.rect_requirements().translation(),
options_.rect_requirements().scale())) {
keep_tracking = false;
}
// If landmarks from the previous frame are not in the current re-crop rect
// (i.e. object moved too fast and using previous frame rect won't cover
// landmarks on the current frame) - use current re-crop rect.
if (options_.has_landmarks_requirements() &&
!LandmarksRequirementsSatisfied(
prev_landmarks, recrop_rect, image_size,
options_.landmarks_requirements().recrop_rect_margin())) {
keep_tracking = false;
}
// If object didn't move a lot comparing to the previous frame - we'll keep
// tracking it and will return rect from the previous frame, otherwise -
// return re-crop rect from the current frame.
if (keep_tracking) {
cc->Outputs()
.Tag(kTrackingRectTag)
.AddPacket(cc->Inputs().Tag(kPrevLandmarksRectTag).Value());
} else {
cc->Outputs()
.Tag(kTrackingRectTag)
.AddPacket(cc->Inputs().Tag(kRecropRectTag).Value());
VLOG(1) << "Lost tracking: check messages above for details";
}
return absl::OkStatus();
}
} // namespace mediapipe

59
mediapipe/modules/holistic_landmark/calculators/roi_tracking_calculator.proto
View File

@ -1,59 +0,0 @@
// Copyright 2020 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package mediapipe;
import "mediapipe/framework/calculator.proto";
message RoiTrackingCalculatorOptions {
extend CalculatorOptions {
optional RoiTrackingCalculatorOptions ext = 329994630;
}
// Verifies that Intersection over Union of previous frame rect and current
// frame re-crop rect is less than threshold.
message IouRequirements {
optional float min_iou = 1 [default = 0.5];
}
// Verifies that current frame re-crop rect rotation/translation/scale didn't
// change much comparing to the previous frame rect.
message RectRequirements {
// Allowed rotation change defined in degrees.
optional float rotation_degrees = 1 [default = 10.0];
// Allowed translation change defined as absolute translation normalized by
// re-crop rectangle size.
optional float translation = 2 [default = 0.1];
// Allowed scale change defined as absolute translation normalized by
// re-crop rectangle size.
optional float scale = 3 [default = 0.1];
}
// Verifies that landmarks from the previous frame are within re-crop
// rectangle bounds on the current frame.
message LandmarksRequirements {
// Margin to apply to re-crop rectangle before checking verifing landmarks.
optional float recrop_rect_margin = 1 [default = 0.0];
}
optional IouRequirements iou_requirements = 1;
optional RectRequirements rect_requirements = 2;
optional LandmarksRequirements landmarks_requirements = 3;
}

48
mediapipe/modules/holistic_landmark/face_detection_front_detections_to_roi.pbtxt
View File

@ -1,48 +0,0 @@
# Calculates ROI from detections provided by `face_detection_short_range.tflite`
# model.
type: "FaceDetectionFrontDetectionsToRoi"
# Detected faces. (std::vector<Detection>)
input_stream: "DETECTIONS:detections"
# Image size (width & height). (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# Refined (more accurate) ROI to use for face landmarks prediction.
# (NormalizedRect)
output_stream: "ROI:roi"
# Converts the face detection into a rectangle (normalized by image size)
# that encloses the face and is rotated such that the line connecting right side
# of the right eye and left side of the left eye is aligned with the X-axis of
# the rectangle.
node {
calculator: "DetectionsToRectsCalculator"
input_stream: "DETECTIONS:detections"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECT:raw_roi"
options: {
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
rotation_vector_start_keypoint_index: 0 # Right eye.
rotation_vector_end_keypoint_index: 1 # Left eye.
rotation_vector_target_angle_degrees: 0
conversion_mode: USE_KEYPOINTS
}
}
}
# Expands and shifts the rectangle that contains the face so that it's likely
# to cover the entire face.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:raw_roi"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "roi"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 2.0
scale_y: 2.0
shift_y: -0.1
square_long: true
}
}
}

82
mediapipe/modules/holistic_landmark/face_landmarks_from_pose_cpu.pbtxt
View File

@ -1,82 +0,0 @@
# Predicts face landmarks within an ROI derived from face-related pose
# landmarks.
type: "FaceLandmarksFromPoseCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:input_video"
# Face-related pose landmarks. (NormalizedLandmarkList)
input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
# Whether to run the face landmark model with attention on lips and eyes to
# provide more accuracy, and additionally output iris landmarks. If unspecified,
# functions as set to false. (bool)
input_side_packet: "REFINE_LANDMARKS:refine_landmarks"
# Face landmarks. (NormalizedLandmarkList)
output_stream: "FACE_LANDMARKS:face_landmarks"
# Debug outputs.
# Face ROI derived from face-related pose landmarks, which defines the search
# region for the face detection model. (NormalizedRect)
output_stream: "FACE_ROI_FROM_POSE:face_roi_from_pose"
# Refined face crop rectangle predicted by face detection model.
# (NormalizedRect)
output_stream: "FACE_ROI_FROM_DETECTION:face_roi_from_detection"
# Rectangle used to predict face landmarks. (NormalizedRect)
output_stream: "FACE_TRACKING_ROI:face_tracking_roi"
# TODO: do not predict face when most of the face landmarks from
# pose are invisible.
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:input_video"
output_stream: "SIZE:image_size"
}
# Gets ROI for re-crop model from face-related pose landmarks.
node {
calculator: "FaceLandmarksFromPoseToRecropRoi"
input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "ROI:face_roi_from_pose"
}
# Detects faces within the face ROI calculated from pose landmarks. This is done
# to refine face ROI for further landmark detection as ROI calculated from
# pose landmarks may be inaccurate.
node {
calculator: "FaceDetectionShortRangeByRoiCpu"
input_stream: "IMAGE:input_video"
input_stream: "ROI:face_roi_from_pose"
output_stream: "DETECTIONS:face_detections"
}
# Calculates refined face ROI.
node {
calculator: "FaceDetectionFrontDetectionsToRoi"
input_stream: "DETECTIONS:face_detections"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "ROI:face_roi_from_detection"
}
# Gets face tracking rectangle (either face rectangle from the previous
# frame or face re-crop rectangle from the current frame) for face prediction.
node {
calculator: "FaceTracking"
input_stream: "LANDMARKS:face_landmarks"
input_stream: "FACE_RECROP_ROI:face_roi_from_detection"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "FACE_TRACKING_ROI:face_tracking_roi"
}
# Predicts face landmarks from the tracking rectangle.
node {
calculator: "FaceLandmarkCpu"
input_stream: "IMAGE:input_video"
input_stream: "ROI:face_tracking_roi"
input_side_packet: "WITH_ATTENTION:refine_landmarks"
output_stream: "LANDMARKS:face_landmarks"
}

82
mediapipe/modules/holistic_landmark/face_landmarks_from_pose_gpu.pbtxt
View File

@ -1,82 +0,0 @@
# Predicts face landmarks within an ROI derived from face-related pose
# landmarks.
type: "FaceLandmarksFromPoseGpu"
# GPU image. (ImageFrame)
input_stream: "IMAGE:input_video"
# Face-related pose landmarks. (NormalizedLandmarkList)
input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
# Whether to run the face landmark model with attention on lips and eyes to
# provide more accuracy, and additionally output iris landmarks. If unspecified,
# functions as set to false. (bool)
input_side_packet: "REFINE_LANDMARKS:refine_landmarks"
# Face landmarks. (NormalizedLandmarkList)
output_stream: "FACE_LANDMARKS:face_landmarks"
# Debug outputs.
# Face ROI derived from face-related pose landmarks, which defines the search
# region for the face detection model. (NormalizedRect)
output_stream: "FACE_ROI_FROM_POSE:face_roi_from_pose"
# Refined face crop rectangle predicted by face detection model.
# (NormalizedRect)
output_stream: "FACE_ROI_FROM_DETECTION:face_roi_from_detection"
# Rectangle used to predict face landmarks. (NormalizedRect)
output_stream: "FACE_TRACKING_ROI:face_tracking_roi"
# TODO: do not predict face when most of the face landmarks from
# pose are invisible.
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:input_video"
output_stream: "SIZE:image_size"
}
# Gets ROI for re-crop model from face-related pose landmarks.
node {
calculator: "FaceLandmarksFromPoseToRecropRoi"
input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "ROI:face_roi_from_pose"
}
# Detects faces within the face ROI calculated from pose landmarks. This is done
# to refine face ROI for further landmark detection as ROI calculated from
# pose landmarks may be inaccurate.
node {
calculator: "FaceDetectionShortRangeByRoiGpu"
input_stream: "IMAGE:input_video"
input_stream: "ROI:face_roi_from_pose"
output_stream: "DETECTIONS:face_detections"
}
# Calculates refined face ROI.
node {
calculator: "FaceDetectionFrontDetectionsToRoi"
input_stream: "DETECTIONS:face_detections"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "ROI:face_roi_from_detection"
}
# Gets face tracking rectangle (either face rectangle from the previous
# frame or face re-crop rectangle from the current frame) for face prediction.
node {
calculator: "FaceTracking"
input_stream: "LANDMARKS:face_landmarks"
input_stream: "FACE_RECROP_ROI:face_roi_from_detection"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "FACE_TRACKING_ROI:face_tracking_roi"
}
# Predicts face landmarks from the tracking rectangle.
node {
calculator: "FaceLandmarkGpu"
input_stream: "IMAGE:input_video"
input_stream: "ROI:face_tracking_roi"
input_side_packet: "WITH_ATTENTION:refine_landmarks"
output_stream: "LANDMARKS:face_landmarks"
}

51
mediapipe/modules/holistic_landmark/face_landmarks_from_pose_to_recrop_roi.pbtxt
View File

@ -1,51 +0,0 @@
# Converts face-related pose landmarks to re-crop ROI.
type: "FaceLandmarksFromPoseToRecropRoi"
# Face-related pose landmarks (There should be 11 of them).
# (NormalizedLandmarkList)
input_stream: "FACE_LANDMARKS_FROM_POSE:face_landmarks_from_pose"
# Image size (width & height). (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# ROI to be used for face detection. (NormalizedRect)
output_stream: "ROI:roi"
# Converts face-related pose landmarks to a detection that tightly encloses all
# landmarks.
node {
calculator: "LandmarksToDetectionCalculator"
input_stream: "NORM_LANDMARKS:face_landmarks_from_pose"
output_stream: "DETECTION:pose_face_detection"
}
# Converts face detection to a normalized face rectangle.
node {
calculator: "DetectionsToRectsCalculator"
input_stream: "DETECTION:pose_face_detection"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECT:pose_face_rect"
options: {
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
rotation_vector_start_keypoint_index: 5 # Right eye.
rotation_vector_end_keypoint_index: 2 # Left eye.
rotation_vector_target_angle_degrees: 0
}
}
}
# Expands face rectangle so that it becomes big enough for face detector to
# localize it accurately.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:pose_face_rect"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "roi"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 3.0
scale_y: 3.0
square_long: true
}
}
}

53
mediapipe/modules/holistic_landmark/face_landmarks_to_roi.pbtxt
View File

@ -1,53 +0,0 @@
# Converts face landmarks to ROI.
type: "FaceLandmarksToRoi"
# Face landmarks. (NormalizedLandmarkList)
input_stream: "LANDMARKS:face_landmarks"
# Image size (width & height). (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# ROI according to landmarks. (NormalizedRect)
output_stream: "ROI:roi"
# Converts face landmarks to a detection that tightly encloses all landmarks.
node {
calculator: "LandmarksToDetectionCalculator"
input_stream: "NORM_LANDMARKS:face_landmarks"
output_stream: "DETECTION:face_detection"
}
# Converts the face detection into a rectangle (normalized by image size)
# that encloses the face and is rotated such that the line connecting center of
# the wrist and MCP of the middle finger is aligned with the Y-axis of the
# rectangle.
node {
calculator: "DetectionsToRectsCalculator"
input_stream: "DETECTION:face_detection"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECT:face_landmarks_rect_tight"
options: {
[mediapipe.DetectionsToRectsCalculatorOptions.ext] {
rotation_vector_start_keypoint_index: 33 # Right side of left eye.
rotation_vector_end_keypoint_index: 263 # Left side of right eye.
rotation_vector_target_angle_degrees: 0
}
}
}
# Expands the face rectangle so that it's likely to contain the face even with
# some motion.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:face_landmarks_rect_tight"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "roi"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 1.5
scale_y: 1.5
# TODO: remove `square_long` where appropriat
square_long: true
}
}
}

61
mediapipe/modules/holistic_landmark/face_tracking.pbtxt
View File

@ -1,61 +0,0 @@
# Decides what ROI to use for face landmarks prediction: either previous frame
# landmarks ROI or the current frame face re-crop ROI.
type: "FaceTracking"
# Face landmarks from the current frame. They will be memorized for tracking on
# the next frame. (NormalizedLandmarkList)
input_stream: "LANDMARKS:face_landmarks"
# Face re-crop ROI from the current frame. (NormalizedRect)
input_stream: "FACE_RECROP_ROI:face_recrop_roi"
# Image size (width & height). (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# Face tracking ROI. Which is either face landmarks ROI from the previous frame
# if face is still tracked, or face re-crop ROI from the current frame
# otherwise. (NormalizedRect)
output_stream: "FACE_TRACKING_ROI:face_tracking_roi"
# Keeps track of face landmarks from the previous frame.
node {
calculator: "PreviousLoopbackCalculator"
input_stream: "MAIN:image_size"
input_stream: "LOOP:face_landmarks"
input_stream_info: {
tag_index: "LOOP"
back_edge: true
}
output_stream: "PREV_LOOP:prev_face_landmarks"
}
# Gets hand landarmsk rect.
node {
calculator: "FaceLandmarksToRoi"
input_stream: "LANDMARKS:prev_face_landmarks"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "ROI:prev_face_landmarks_rect"
}
# Checks that all requirements for tracking are satisfied and use face rectangle
# from the previous frame in that case. Otherwise - use face re-crop rectangle
# from the current frame.
node {
calculator: "RoiTrackingCalculator"
input_stream: "PREV_LANDMARKS:prev_face_landmarks"
input_stream: "PREV_LANDMARKS_RECT:prev_face_landmarks_rect"
input_stream: "RECROP_RECT:face_recrop_roi"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "TRACKING_RECT:face_tracking_roi"
options: {
[mediapipe.RoiTrackingCalculatorOptions.ext] {
rect_requirements: {
rotation_degrees: 15.0
translation: 0.1
scale: 0.3
}
landmarks_requirements: {
recrop_rect_margin: -0.2
}
}
}
}

78
mediapipe/modules/holistic_landmark/hand_landmarks_from_pose_cpu.pbtxt
View File

@ -1,78 +0,0 @@
# Predicts hand landmarks within a ROI derived from hand-related pose landmarks.
type: "HandLandmarksFromPoseCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:input_video"
# Hand-related pose landmarks in [wrist, pinky, index] order.
# (NormalizedLandmarkList)
input_stream: "HAND_LANDMARKS_FROM_POSE:hand_landmarks_from_pose"
# Hand landmarks. (NormalizedLandmarkList)
output_stream: "HAND_LANDMARKS:hand_landmarks"
# Debug outputs.
# Hand ROI derived from hand-related landmarks, which defines the search region
# for the hand re-crop model. (NormalizedRect)
output_stream: "HAND_ROI_FROM_POSE:hand_roi_from_pose"
# Refined hand crop rectangle predicted by hand re-crop model. (NormalizedRect)
output_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
# Rectangle used to predict hand landmarks. (NormalizedRect)
output_stream: "HAND_TRACKING_ROI:hand_tracking_roi"
# Gets hand visibility.
node {
calculator: "HandVisibilityFromHandLandmarksFromPose"
input_stream: "HAND_LANDMARKS_FROM_POSE:hand_landmarks_from_pose"
output_stream: "VISIBILITY:hand_visibility"
}
# Drops hand-related pose landmarks if pose wrist is not visible. It will
# prevent from predicting hand landmarks on the current frame.
node {
calculator: "GateCalculator"
input_stream: "hand_landmarks_from_pose"
input_stream: "ALLOW:hand_visibility"
output_stream: "ensured_hand_landmarks_from_pose"
}
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE:input_video"
output_stream: "SIZE:image_size"
}
# Gets ROI for re-crop model from hand-related pose landmarks.
node {
calculator: "HandLandmarksFromPoseToRecropRoi"
input_stream: "HAND_LANDMARKS_FROM_POSE:ensured_hand_landmarks_from_pose"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "ROI:hand_roi_from_pose"
}
# Predicts hand re-crop rectangle on the current frame.
node {
calculator: "HandRecropByRoiCpu",
input_stream: "IMAGE:input_video"
input_stream: "ROI:hand_roi_from_pose"
output_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
}
# Gets hand tracking rectangle (either hand rectangle from the previous
# frame or hand re-crop rectangle from the current frame) for hand prediction.
node {
calculator: "HandTracking"
input_stream: "LANDMARKS:hand_landmarks"
input_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "HAND_TRACKING_ROI:hand_tracking_roi"
}
# Predicts hand landmarks from the tracking rectangle.
node {
calculator: "HandLandmarkCpu"
input_stream: "IMAGE:input_video"
input_stream: "ROI:hand_tracking_roi"
output_stream: "LANDMARKS:hand_landmarks"
}

78
mediapipe/modules/holistic_landmark/hand_landmarks_from_pose_gpu.pbtxt
View File

@ -1,78 +0,0 @@
# Predicts hand landmarks within a ROI derived from hand-related pose landmarks.
type: "HandLandmarksFromPoseGpu"
# GPU image. (ImageFrame)
input_stream: "IMAGE:input_video"
# Hand-related pose landmarks in [wrist, pinky, index] order.
# (NormalizedLandmarkList)
input_stream: "HAND_LANDMARKS_FROM_POSE:hand_landmarks_from_pose"
# Hand landmarks. (NormalizedLandmarkList)
output_stream: "HAND_LANDMARKS:hand_landmarks"
# Debug outputs.
# Hand ROI derived from hand-related landmarks, which defines the search region
# for the hand re-crop model. (NormalizedRect)
output_stream: "HAND_ROI_FROM_POSE:hand_roi_from_pose"
# Refined hand crop rectangle predicted by hand re-crop model. (NormalizedRect)
output_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
# Rectangle used to predict hand landmarks. (NormalizedRect)
output_stream: "HAND_TRACKING_ROI:hand_tracking_roi"
# Gets hand visibility.
node {
calculator: "HandVisibilityFromHandLandmarksFromPose"
input_stream: "HAND_LANDMARKS_FROM_POSE:hand_landmarks_from_pose"
output_stream: "VISIBILITY:hand_visibility"
}
# Drops hand-related pose landmarks if pose wrist is not visible. It will
# prevent from predicting hand landmarks on the current frame.
node {
calculator: "GateCalculator"
input_stream: "hand_landmarks_from_pose"
input_stream: "ALLOW:hand_visibility"
output_stream: "ensured_hand_landmarks_from_pose"
}
# Extracts image size from the input images.
node {
calculator: "ImagePropertiesCalculator"
input_stream: "IMAGE_GPU:input_video"
output_stream: "SIZE:image_size"
}
# Gets ROI for re-crop model from hand-related pose landmarks.
node {
calculator: "HandLandmarksFromPoseToRecropRoi"
input_stream: "HAND_LANDMARKS_FROM_POSE:ensured_hand_landmarks_from_pose"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "ROI:hand_roi_from_pose"
}
# Predicts hand re-crop rectangle on the current frame.
node {
calculator: "HandRecropByRoiGpu",
input_stream: "IMAGE:input_video"
input_stream: "ROI:hand_roi_from_pose"
output_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
}
# Gets hand tracking rectangle (either hand rectangle from the previous
# frame or hand re-crop rectangle from the current frame) for hand prediction.
node {
calculator: "HandTracking"
input_stream: "LANDMARKS:hand_landmarks"
input_stream: "HAND_ROI_FROM_RECROP:hand_roi_from_recrop"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "HAND_TRACKING_ROI:hand_tracking_roi"
}
# Predicts hand landmarks from the tracking rectangle.
node {
calculator: "HandLandmarkGpu"
input_stream: "IMAGE:input_video"
input_stream: "ROI:hand_tracking_roi"
output_stream: "LANDMARKS:hand_landmarks"
}

45
mediapipe/modules/holistic_landmark/hand_landmarks_from_pose_to_recrop_roi.pbtxt
View File

@ -1,45 +0,0 @@
# Converts hand-related pose landmarks to hand re-crop ROI.
type: "HandLandmarksFromPoseToRecropRoi"
# Hand-related pose landmarks in [wrist, pinky, index] order.
# (NormalizedLandmarkList)
input_stream: "HAND_LANDMARKS_FROM_POSE:hand_landmarks_from_pose"
# Image size (width & height). (std::pair<int, int>)
input_stream: "IMAGE_SIZE:image_size"
# ROI to be used for re-crop prediction. (NormalizedRect)
output_stream: "ROI:roi"
# Converts hand-related pose landmarks to a detection that tightly encloses all
# of them.
node {
calculator: "LandmarksToDetectionCalculator"
input_stream: "NORM_LANDMARKS:hand_landmarks_from_pose"
output_stream: "DETECTION:hand_detection_from_pose"
}
# Converts hand detection to a normalized hand rectangle.
node {
calculator: "HandDetectionsFromPoseToRectsCalculator"
input_stream: "DETECTION:hand_detection_from_pose"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "NORM_RECT:hand_roi_from_pose"
}
# Expands the palm rectangle so that it becomes big enough for hand re-crop
# model to localize it accurately.
node {
calculator: "RectTransformationCalculator"
input_stream: "NORM_RECT:hand_roi_from_pose"
input_stream: "IMAGE_SIZE:image_size"
output_stream: "roi"
options: {
[mediapipe.RectTransformationCalculatorOptions.ext] {
scale_x: 2.7
scale_y: 2.7
shift_y: -0.1
square_long: true
}
}
}

76
mediapipe/modules/holistic_landmark/hand_landmarks_left_and_right_cpu.pbtxt
View File

@ -1,76 +0,0 @@
# Predicts left and right hand landmarks within corresponding ROIs derived from
# hand-related pose landmarks.
type: "HandLandmarksLeftAndRightCpu"
# CPU image. (ImageFrame)
input_stream: "IMAGE:input_video"
# Pose landmarks to derive initial hand location from. (NormalizedLandmarkList)
input_stream: "POSE_LANDMARKS:pose_landmarks"
# Left hand landmarks. (NormalizedLandmarkList)
output_stream: "LEFT_HAND_LANDMARKS:left_hand_landmarks"
# RIght hand landmarks. (NormalizedLandmarkList)
output_stream: "RIGHT_HAND_LANDMARKS:right_hand_landmarks"
# Debug outputs.
output_stream: "LEFT_HAND_ROI_FROM_POSE:left_hand_roi_from_pose"
output_stream: "LEFT_HAND_ROI_FROM_RECROP:left_hand_roi_from_recrop"
output_stream: "LEFT_HAND_TRACKING_ROI:left_hand_tracking_roi"
output_stream: "RIGHT_HAND_ROI_FROM_POSE:right_hand_roi_from_pose"
output_stream: "RIGHT_HAND_ROI_FROM_RECROP:right_hand_roi_from_recrop"
output_stream: "RIGHT_HAND_TRACKING_ROI:right_hand_tracking_roi"
# Extracts left-hand-related landmarks from the pose landmarks.
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "pose_landmarks"
output_stream: "left_hand_landmarks_from_pose"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 15 end: 16 }
ranges: { begin: 17 end: 18 }
ranges: { begin: 19 end: 20 }
combine_outputs: true
}
}
}
# Predicts left hand landmarks.
node {
calculator: "HandLandmarksFromPoseCpu"
input_stream: "IMAGE:input_video"
input_stream: "HAND_LANDMARKS_FROM_POSE:left_hand_landmarks_from_pose"
output_stream: "HAND_LANDMARKS:left_hand_landmarks"
# Debug outputs.
output_stream: "HAND_ROI_FROM_POSE:left_hand_roi_from_pose"
output_stream: "HAND_ROI_FROM_RECROP:left_hand_roi_from_recrop"
output_stream: "HAND_TRACKING_ROI:left_hand_tracking_roi"
}
# Extracts right-hand-related landmarks from the pose landmarks.
node {
calculator: "SplitNormalizedLandmarkListCalculator"
input_stream: "pose_landmarks"
output_stream: "right_hand_landmarks_from_pose"
options: {
[mediapipe.SplitVectorCalculatorOptions.ext] {
ranges: { begin: 16 end: 17 }
ranges: { begin: 18 end: 19 }
ranges: { begin: 20 end: 21 }
combine_outputs: true
}
}
}
# Extracts right-hand-related landmarks from the pose landmarks.
node {
calculator: "HandLandmarksFromPoseCpu"
input_stream: "IMAGE:input_video"
input_stream: "HAND_LANDMARKS_FROM_POSE:right_hand_landmarks_from_pose"
output_stream: "HAND_LANDMARKS:right_hand_landmarks"
# Debug outputs.
output_stream: "HAND_ROI_FROM_POSE:right_hand_roi_from_pose"
output_stream: "HAND_ROI_FROM_RECROP:right_hand_roi_from_recrop"
output_stream: "HAND_TRACKING_ROI:right_hand_tracking_roi"
}

Some files were not shown because too many files have changed in this diff Show More