From 12046fcf89536311491b5fe81450d7a2694b8168 Mon Sep 17 00:00:00 2001 From: liuyulvv Date: Fri, 12 Aug 2022 09:18:10 +0800 Subject: [PATCH] =?UTF-8?q?face=20detection=E5=92=8Clandmark=E6=94=AF?= =?UTF-8?q?=E6=8C=81onnxruntime=E7=9A=84cuda=E5=92=8Ctensorrt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../examples/desktop/face_detection/BUILD | 80 ++++++ mediapipe/graphs/face_detection/BUILD | 40 +++ ...ace_detection_desktop_live_onnx_cuda.pbtxt | 58 ++++ ...detection_desktop_live_onnx_tensorrt.pbtxt | 58 ++++ ...on_full_range_desktop_live_onnx_cuda.pbtxt | 58 ++++ ...ull_range_desktop_live_onnx_tensorrt.pbtxt | 58 ++++ mediapipe/modules/face_detection/BUILD | 95 ++++++- .../face_detection_full_range.onnx | Bin 2076774 -> 2077622 bytes .../face_detection_full_range_onnx_cuda.pbtxt | 37 +++ ...e_detection_full_range_onnx_tensorrt.pbtxt | 37 +++ .../face_detection_onnx_cuda.pbtxt | 155 +++++++++++ .../face_detection_onnx_tensorrt.pbtxt | 165 ++++++++++++ .../face_detection_short_range.onnx | Bin 418837 -> 418471 bytes ...tection_short_range_by_roi_onnx_cuda.pbtxt | 40 +++ ...ion_short_range_by_roi_onnx_tensorrt.pbtxt | 40 +++ ...face_detection_short_range_onnx_cuda.pbtxt | 40 +++ ..._detection_short_range_onnx_tensorrt.pbtxt | 40 +++ mediapipe/modules/face_landmark/BUILD | 81 ++++++ .../face_landmark_front_onnx_cuda.pbtxt | 247 ++++++++++++++++++ .../face_landmark_front_onnx_tensorrt.pbtxt | 247 ++++++++++++++++++ .../face_landmark_onnx_cuda.pbtxt | 166 ++++++++++++ .../face_landmark_onnx_tensorrt.pbtxt | 166 ++++++++++++ 22 files changed, 1907 insertions(+), 1 deletion(-) create mode 100644 mediapipe/graphs/face_detection/face_detection_desktop_live_onnx_cuda.pbtxt create mode 100644 mediapipe/graphs/face_detection/face_detection_desktop_live_onnx_tensorrt.pbtxt create mode 100644 mediapipe/graphs/face_detection/face_detection_full_range_desktop_live_onnx_cuda.pbtxt create mode 100644 mediapipe/graphs/face_detection/face_detection_full_range_desktop_live_onnx_tensorrt.pbtxt create mode 100644 mediapipe/modules/face_detection/face_detection_full_range_onnx_cuda.pbtxt create mode 100644 mediapipe/modules/face_detection/face_detection_full_range_onnx_tensorrt.pbtxt create mode 100644 mediapipe/modules/face_detection/face_detection_onnx_cuda.pbtxt create mode 100644 mediapipe/modules/face_detection/face_detection_onnx_tensorrt.pbtxt create mode 100644 mediapipe/modules/face_detection/face_detection_short_range_by_roi_onnx_cuda.pbtxt create mode 100644 mediapipe/modules/face_detection/face_detection_short_range_by_roi_onnx_tensorrt.pbtxt create mode 100644 mediapipe/modules/face_detection/face_detection_short_range_onnx_cuda.pbtxt create mode 100644 mediapipe/modules/face_detection/face_detection_short_range_onnx_tensorrt.pbtxt create mode 100644 mediapipe/modules/face_landmark/face_landmark_front_onnx_cuda.pbtxt create mode 100644 mediapipe/modules/face_landmark/face_landmark_front_onnx_tensorrt.pbtxt create mode 100644 mediapipe/modules/face_landmark/face_landmark_onnx_cuda.pbtxt create mode 100644 mediapipe/modules/face_landmark/face_landmark_onnx_tensorrt.pbtxt diff --git a/mediapipe/examples/desktop/face_detection/BUILD b/mediapipe/examples/desktop/face_detection/BUILD index 8cd75b44e..6d131ac68 100644 --- a/mediapipe/examples/desktop/face_detection/BUILD +++ b/mediapipe/examples/desktop/face_detection/BUILD @@ -24,6 +24,46 @@ cc_binary( ], ) +cc_binary( + name = "face_detection_full_range_cpu_fps", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main_fps", + "//mediapipe/graphs/face_detection:face_detection_full_range_desktop_live_deps", + ], +) + +cc_binary( + name = "face_detection_full_range_onnx_cuda", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main", + "//mediapipe/graphs/face_detection:face_detection_full_range_desktop_live_onnx_cuda_deps", + ], +) + +cc_binary( + name = "face_detection_full_range_onnx_cuda_fps", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main_fps", + "//mediapipe/graphs/face_detection:face_detection_full_range_desktop_live_onnx_cuda_deps", + ], +) + +cc_binary( + name = "face_detection_full_range_onnx_tensorrt", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main", + "//mediapipe/graphs/face_detection:face_detection_full_range_desktop_live_onnx_tensorrt_deps", + ], +) + +cc_binary( + name = "face_detection_full_range_onnx_tensorrt_fps", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main_fps", + "//mediapipe/graphs/face_detection:face_detection_full_range_desktop_live_onnx_tensorrt_deps", + ], +) + cc_binary( name = "face_detection_cpu", deps = [ @@ -32,6 +72,46 @@ cc_binary( ], ) +cc_binary( + name = "face_detection_cpu_fps", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main_fps", + "//mediapipe/graphs/face_detection:desktop_live_calculators", + ], +) + +cc_binary( + name = "face_detection_onnx_cuda", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main", + "//mediapipe/graphs/face_detection:desktop_live_onnx_cuda_calculators", + ], +) + +cc_binary( + name = "face_detection_onnx_cuda_fps", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main_fps", + "//mediapipe/graphs/face_detection:desktop_live_onnx_cuda_calculators", + ], +) + +cc_binary( + name = "face_detection_onnx_tensorrt", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main", + "//mediapipe/graphs/face_detection:desktop_live_onnx_tensorrt_calculators", + ], +) + +cc_binary( + name = "face_detection_onnx_tensorrt_fps", + deps = [ + "//mediapipe/examples/desktop:demo_run_graph_main_fps", + "//mediapipe/graphs/face_detection:desktop_live_onnx_tensorrt_calculators", + ], +) + # Linux only cc_binary( name = "face_detection_gpu", diff --git a/mediapipe/graphs/face_detection/BUILD b/mediapipe/graphs/face_detection/BUILD index 9e7cf2505..81eec6692 100644 --- a/mediapipe/graphs/face_detection/BUILD +++ b/mediapipe/graphs/face_detection/BUILD @@ -43,6 +43,26 @@ cc_library( ], ) +cc_library( + name = "desktop_live_onnx_cuda_calculators", + deps = [ + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/calculators/util:annotation_overlay_calculator", + "//mediapipe/calculators/util:detections_to_render_data_calculator", + "//mediapipe/modules/face_detection:face_detection_short_range_onnx_cuda", + ], +) + +cc_library( + name = "desktop_live_onnx_tensorrt_calculators", + deps = [ + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/calculators/util:annotation_overlay_calculator", + "//mediapipe/calculators/util:detections_to_render_data_calculator", + "//mediapipe/modules/face_detection:face_detection_short_range_onnx_tensorrt", + ], +) + cc_library( name = "desktop_live_gpu_calculators", deps = [ @@ -93,3 +113,23 @@ cc_library( "//mediapipe/modules/face_detection:face_detection_full_range_cpu", ], ) + +cc_library( + name = "face_detection_full_range_desktop_live_onnx_cuda_deps", + deps = [ + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/calculators/util:annotation_overlay_calculator", + "//mediapipe/calculators/util:detections_to_render_data_calculator", + "//mediapipe/modules/face_detection:face_detection_full_range_onnx_cuda", + ], +) + +cc_library( + name = "face_detection_full_range_desktop_live_onnx_tensorrt_deps", + deps = [ + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/calculators/util:annotation_overlay_calculator", + "//mediapipe/calculators/util:detections_to_render_data_calculator", + "//mediapipe/modules/face_detection:face_detection_full_range_onnx_tensorrt", + ], +) diff --git a/mediapipe/graphs/face_detection/face_detection_desktop_live_onnx_cuda.pbtxt b/mediapipe/graphs/face_detection/face_detection_desktop_live_onnx_cuda.pbtxt new file mode 100644 index 000000000..367327335 --- /dev/null +++ b/mediapipe/graphs/face_detection/face_detection_desktop_live_onnx_cuda.pbtxt @@ -0,0 +1,58 @@ +# MediaPipe graph that performs face mesh with onnxruntime cuda. + +# CPU buffer. (ImageFrame) +input_stream: "input_video" + +# Output image with rendered results. (ImageFrame) +output_stream: "output_video" +# Detected faces. (std::vector) +output_stream: "face_detections" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for downstream nodes +# (calculators and subgraphs) in the graph to finish their tasks before it +# passes through another image. All images that come in while waiting are +# dropped, limiting the number of in-flight images in most part of the graph to +# 1. This prevents the downstream nodes from queuing up incoming images and data +# excessively, which leads to increased latency and memory usage, unwanted in +# real-time mobile applications. It also eliminates unnecessarily computation, +# e.g., the output produced by a node may get dropped downstream if the +# subsequent nodes are still busy processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:output_video" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Subgraph that detects faces. +node { + calculator: "FaceDetectionShortRangeOnnxCUDA" + input_stream: "IMAGE:throttled_input_video" + output_stream: "DETECTIONS:face_detections" +} + +# Converts the detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionsToRenderDataCalculator" + input_stream: "DETECTIONS:face_detections" + output_stream: "RENDER_DATA:render_data" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { + thickness: 4.0 + color { r: 255 g: 0 b: 0 } + } + } +} + +# Draws annotations and overlays them on top of the input images. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "IMAGE:throttled_input_video" + input_stream: "render_data" + output_stream: "IMAGE:output_video" +} diff --git a/mediapipe/graphs/face_detection/face_detection_desktop_live_onnx_tensorrt.pbtxt b/mediapipe/graphs/face_detection/face_detection_desktop_live_onnx_tensorrt.pbtxt new file mode 100644 index 000000000..d3a7f097f --- /dev/null +++ b/mediapipe/graphs/face_detection/face_detection_desktop_live_onnx_tensorrt.pbtxt @@ -0,0 +1,58 @@ +# MediaPipe graph that performs face mesh with onnxruntime tensorrt. + +# CPU buffer. (ImageFrame) +input_stream: "input_video" + +# Output image with rendered results. (ImageFrame) +output_stream: "output_video" +# Detected faces. (std::vector) +output_stream: "face_detections" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for downstream nodes +# (calculators and subgraphs) in the graph to finish their tasks before it +# passes through another image. All images that come in while waiting are +# dropped, limiting the number of in-flight images in most part of the graph to +# 1. This prevents the downstream nodes from queuing up incoming images and data +# excessively, which leads to increased latency and memory usage, unwanted in +# real-time mobile applications. It also eliminates unnecessarily computation, +# e.g., the output produced by a node may get dropped downstream if the +# subsequent nodes are still busy processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:output_video" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Subgraph that detects faces. +node { + calculator: "FaceDetectionShortRangeOnnxTensorRT" + input_stream: "IMAGE:throttled_input_video" + output_stream: "DETECTIONS:face_detections" +} + +# Converts the detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionsToRenderDataCalculator" + input_stream: "DETECTIONS:face_detections" + output_stream: "RENDER_DATA:render_data" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { + thickness: 4.0 + color { r: 255 g: 0 b: 0 } + } + } +} + +# Draws annotations and overlays them on top of the input images. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "IMAGE:throttled_input_video" + input_stream: "render_data" + output_stream: "IMAGE:output_video" +} diff --git a/mediapipe/graphs/face_detection/face_detection_full_range_desktop_live_onnx_cuda.pbtxt b/mediapipe/graphs/face_detection/face_detection_full_range_desktop_live_onnx_cuda.pbtxt new file mode 100644 index 000000000..d33a772a3 --- /dev/null +++ b/mediapipe/graphs/face_detection/face_detection_full_range_desktop_live_onnx_cuda.pbtxt @@ -0,0 +1,58 @@ +# MediaPipe graph that performs face detection with onnxruntime on cuda. + +# Images on GPU coming into and out of the graph. +input_stream: "input_video" +output_stream: "output_video" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for +# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish +# generating the corresponding detections before it passes through another +# image. All images that come in while waiting are dropped, limiting the number +# of in-flight images between this calculator and +# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between +# from queuing up incoming images and data excessively, which leads to increased +# latency and memory usage, unwanted in real-time mobile applications. It also +# eliminates unnecessarily computation, e.g., a transformed image produced by +# ImageTransformationCalculator may get dropped downstream if the subsequent +# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy +# processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:detections" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Detects faces. +node { + calculator: "FaceDetectionFullRangeOnnxCUDA" + input_stream: "IMAGE:throttled_input_video" + output_stream: "DETECTIONS:detections" +} + +# Converts the detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionsToRenderDataCalculator" + input_stream: "DETECTIONS:detections" + output_stream: "RENDER_DATA:render_data" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { + thickness: 4.0 + color { r: 255 g: 0 b: 0 } + } + } +} + +# Draws annotations and overlays them on top of the input images. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "IMAGE:throttled_input_video" + input_stream: "render_data" + output_stream: "IMAGE:output_video" +} + diff --git a/mediapipe/graphs/face_detection/face_detection_full_range_desktop_live_onnx_tensorrt.pbtxt b/mediapipe/graphs/face_detection/face_detection_full_range_desktop_live_onnx_tensorrt.pbtxt new file mode 100644 index 000000000..4db446757 --- /dev/null +++ b/mediapipe/graphs/face_detection/face_detection_full_range_desktop_live_onnx_tensorrt.pbtxt @@ -0,0 +1,58 @@ +# MediaPipe graph that performs face detection with onnxruntime on tensorrt. + +# Images on GPU coming into and out of the graph. +input_stream: "input_video" +output_stream: "output_video" + +# Throttles the images flowing downstream for flow control. It passes through +# the very first incoming image unaltered, and waits for +# TfLiteTensorsToDetectionsCalculator downstream in the graph to finish +# generating the corresponding detections before it passes through another +# image. All images that come in while waiting are dropped, limiting the number +# of in-flight images between this calculator and +# TfLiteTensorsToDetectionsCalculator to 1. This prevents the nodes in between +# from queuing up incoming images and data excessively, which leads to increased +# latency and memory usage, unwanted in real-time mobile applications. It also +# eliminates unnecessarily computation, e.g., a transformed image produced by +# ImageTransformationCalculator may get dropped downstream if the subsequent +# TfLiteConverterCalculator or TfLiteInferenceCalculator is still busy +# processing previous inputs. +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:detections" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Detects faces. +node { + calculator: "FaceDetectionFullRangeOnnxTensorRT" + input_stream: "IMAGE:throttled_input_video" + output_stream: "DETECTIONS:detections" +} + +# Converts the detections to drawing primitives for annotation overlay. +node { + calculator: "DetectionsToRenderDataCalculator" + input_stream: "DETECTIONS:detections" + output_stream: "RENDER_DATA:render_data" + node_options: { + [type.googleapis.com/mediapipe.DetectionsToRenderDataCalculatorOptions] { + thickness: 4.0 + color { r: 255 g: 0 b: 0 } + } + } +} + +# Draws annotations and overlays them on top of the input images. +node { + calculator: "AnnotationOverlayCalculator" + input_stream: "IMAGE:throttled_input_video" + input_stream: "render_data" + output_stream: "IMAGE:output_video" +} + diff --git a/mediapipe/modules/face_detection/BUILD b/mediapipe/modules/face_detection/BUILD index 84c9388ea..d6815d5ac 100644 --- a/mediapipe/modules/face_detection/BUILD +++ b/mediapipe/modules/face_detection/BUILD @@ -17,7 +17,7 @@ load( "mediapipe_simple_subgraph", ) load("//mediapipe/framework/port:build_config.bzl", "mediapipe_proto_library") -load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test") +load("//mediapipe/framework:mediapipe_cc_test.bzl", "mediapipe_cc_test") #@unused licenses(["notice"]) @@ -35,6 +35,24 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "face_detection_short_range_by_roi_onnx_cuda", + graph = "face_detection_short_range_by_roi_onnx_cuda.pbtxt", + register_as = "FaceDetectionShortRangeByRoiOnnxCUDA", + deps = [ + ":face_detection_short_range_onnx_cuda", + ], +) + +mediapipe_simple_subgraph( + name = "face_detection_short_range_by_roi_onnx_tensorrt", + graph = "face_detection_short_range_by_roi_onnx_tensorrt.pbtxt", + register_as = "FaceDetectionShortRangeByRoiOnnxTensorRT", + deps = [ + ":face_detection_short_range_onnx_tensorrt", + ], +) + mediapipe_simple_subgraph( name = "face_detection_short_range_by_roi_gpu", graph = "face_detection_short_range_by_roi_gpu.pbtxt", @@ -74,6 +92,24 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "face_detection_short_range_onnx_cuda", + graph = "face_detection_short_range_onnx_cuda.pbtxt", + register_as = "FaceDetectionShortRangeOnnxCUDA", + deps = [ + ":face_detection_onnx_cuda", + ], +) + +mediapipe_simple_subgraph( + name = "face_detection_short_range_onnx_tensorrt", + graph = "face_detection_short_range_onnx_tensorrt.pbtxt", + register_as = "FaceDetectionShortRangeOnnxTensorRT", + deps = [ + ":face_detection_onnx_tensorrt", + ], +) + mediapipe_simple_subgraph( name = "face_detection_full_range", graph = "face_detection_full_range.pbtxt", @@ -83,6 +119,24 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "face_detection_full_range_onnx_cuda", + graph = "face_detection_full_range_onnx_cuda.pbtxt", + register_as = "FaceDetectionFullRangeOnnxCUDA", + deps = [ + ":face_detection_onnx_cuda", + ], +) + +mediapipe_simple_subgraph( + name = "face_detection_full_range_onnx_tensorrt", + graph = "face_detection_full_range_onnx_tensorrt.pbtxt", + register_as = "FaceDetectionFullRangeOnnxTensorRT", + deps = [ + ":face_detection_onnx_tensorrt", + ], +) + mediapipe_simple_subgraph( name = "face_detection_without_roi", graph = "face_detection_without_roi.pbtxt", @@ -110,6 +164,42 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "face_detection_onnx_cuda", + graph = "face_detection_onnx_cuda.pbtxt", + register_as = "FaceDetectionOnnxCUDA", + deps = [ + ":face_detection_cc_proto", + ":face_detection_options_lib", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/tensor:image_to_tensor_calculator", + "//mediapipe/calculators/tensor:inference_calculator_onnx_cuda", + "//mediapipe/calculators/tensor:tensors_to_detections_calculator", + "//mediapipe/calculators/tflite:ssd_anchors_calculator", + "//mediapipe/calculators/util:detection_projection_calculator", + "//mediapipe/calculators/util:non_max_suppression_calculator", + "//mediapipe/calculators/util:to_image_calculator", + ], +) + +mediapipe_simple_subgraph( + name = "face_detection_onnx_tensorrt", + graph = "face_detection_onnx_tensorrt.pbtxt", + register_as = "FaceDetectionOnnxTensorRT", + deps = [ + ":face_detection_cc_proto", + ":face_detection_options_lib", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/tensor:image_to_tensor_calculator", + "//mediapipe/calculators/tensor:inference_calculator_onnx_tensorrt", + "//mediapipe/calculators/tensor:tensors_to_detections_calculator", + "//mediapipe/calculators/tflite:ssd_anchors_calculator", + "//mediapipe/calculators/util:detection_projection_calculator", + "//mediapipe/calculators/util:non_max_suppression_calculator", + "//mediapipe/calculators/util:to_image_calculator", + ], +) + mediapipe_proto_library( name = "face_detection_proto", srcs = ["face_detection.proto"], @@ -168,8 +258,11 @@ mediapipe_simple_subgraph( exports_files( srcs = [ + "face_detection_full_range.onnx", "face_detection_full_range.tflite", + "face_detection_full_range_sparse.onnx", "face_detection_full_range_sparse.tflite", + "face_detection_short_range.onnx", "face_detection_short_range.tflite", ], ) diff --git a/mediapipe/modules/face_detection/face_detection_full_range.onnx b/mediapipe/modules/face_detection/face_detection_full_range.onnx index ef2d8df17aad0d52ad5cd98b5c2291e7af848e75..27c3a29b9a04801b8afc45287d2bd30124befe24 100644 GIT binary patch delta 20140 zcmdU14Qw1$e$UL#Zq{QvtT(aaIQSz@LcxJB^JaGDO_D3l2V7Ws1Og#k(qrwj9kAlq zAu%5ZEjGnDtJf9+1GfSaFiF}|k1Iu8l~H?D6-sCiwdJc))jL|66txINYJsXB(pA0w z-|X(Zd2jw3Q>jX7nrc+Ru4? zcw~C~{^9BIsmZR`cE7fJY~nusj}<+3P+t!BJ~~}GFf}n+nmRlUb{0tTAhinl-00Zh z>3bd+KRQ;b*%LcbGrnq?zcpRAtUik zJ=eBwM!#B*e$KaW4B#?huZF{$fOz~QeT9sKX;@~`AkQ}Qc^L|5V-<2xvT~o*=YOhp z4xCnWmk*Bu1s@!noPx1Cba)uX+3aEm z{R?i#jH_e63m6gsLufnXl7u0_Ffz)00NhWfSeOaWhI;T3T#YGoJAG_fNWKbL^uFQgRbEu zt#C~e0boY}*hv6Z>o*B*LLwit^Zv%A9sV^-yH!tTn_C5ZYyls*bBNEq!p-b_>!INX zONXbXCdMc4CZqn#OZUv&po<6fWrv4H85!cjWNdx4wSl`L?YlMxlI=unbs{{RBp#df zn{}BD5n-KtJBxxO5eKV>StPu|)A4fzY#sl2bNjf&F9+k7iFC#(VDxejddVGRSU$0< zbN?AEec)nct3)Xhpk0()E=Dfd$V-w7Y2R}( zXCpip&@2!$orL<%yO@YLZ&czH>@;{5RhKs*5uxD*4#!vHam1m_g6g3QATGTBN7 z$kMjXul*M0QMdsZB8HJ~s+Xun7vD@`a!9{X$bdAF-P)w=26bBUP&LEdswaQZdF%Q1 z^0m+QwyRy$Z~xO&YuS3Xx9x^}YrO6?y2RbuXxy)hhx3;Ssk&tP@8tIuenFQH3mqcF z$R}(herfrti61CZ1K4mAjFg_6U!KT2rqP8Rtd2lDK{v1Hyz4uPj-8BIU||u|vH5tL ze7w!%c+cq9@)0pb;7t*DxH@zL85k2Y%RmB|<9P{SdPhqp6#7ZHWLudhJl-C90(N#X zHh2Xjw^7^W$PkVU=!NYNk|JFQmG#f-2z?0$i&V390ngTjP`7NZX6wW`BF;p_8Egr0 z{xgYyNT~IY`mp3us|NE`4{$A9G6@E=-0^)DWx)o47-^nTB-nVa#UZvh#Ij5Ph%b+m z&k{kg{98IZ5>9AgTqB4Juo@ZZgoR-sTgIpc3a$YTSb(Qkp=ic}uFY{kG>okm#aKfY z4^ozYan-6mgrXLwC@7$ckUxXLHQk z>N7JXQ-%R%iTt~ISNhkjZf*cPVus@&1a#a2Ct!gCFIWCRKq2}SL~sfsICX_0gHu2m ziSPkqB#V9p4i861w)N7KL|SufWPqGP%rza3j9kY9GClfNL`vG6R-jY-C%akRW?w9A z`_FW@T_(ck_@8!ntdWwmBPVI%Z|Fgiw8(#^sS`=s_P^h=>YzLtNF?w)q22%_TU_dPJ3P^OM1r7t@f`}N&GVqX`n?*%LT#}4U zg`^@XG*m=|D6oPS@n2lu;lHpx%-66frX;X5E_s->SV2yhQ|nTAq z2pqDhjEFzJu>lb=6C4X+CoS+aK)4v;i~Hzu_l1Y8cCsB{`sqrn!pmf>mh#U z`iBP+H331gFExmY^318*1Yt)&#oWy`d2yFe7Y)lho$YopAL;WQ`(wim0C8 zN&$Mf8k3X+F~B1jG?AoyfGbMy6iiMfOl>GBvF0i2B{r8*gi9$RYX|_PUq_hja%S84 z4uAHV4i#h(d_nD+NdcSLc2Oj+sEt@tXskuR@;nxREf;h!!km~Iipu>&N2;+__kVq3 zBR*m6VkJMdVv(1NAtDP5nNL#Y5ycR@8bt#aLkRcz90ey7h#`WBp@glC35BG3*FeIq zYnnVCaUnzyA+$x`B?uvKkNONIacW^Q`NXCgA4ynm)}v+OERk@I=(1o63Na=y3dq5T zegp9p5ha32oJ7AD=O5CdXCbv7h4>!f#dQvm#RKp!UT_jo=MdL_k94e;6FnwtXw~_! zfK?)xj!Br(m|jSG)I<+b-f1pqh+Nl0r@Li(#j-FwwVAS4`jd8#xB?`NH z$e6dpc|)iJk%c4ZgA+VrafyYoB@dm?jA>+xD;5h;EIl%%XcVz51Jd8?ZkYby zD@uQ`Ee3pOB3?kSW~z5)<;VgT5$76BNh>YHC!r~RWEHNvrHsPx+dZWE-0OPl(gyrgPDayjCh!Y zC*^`2)PH4DM^3ha4U$U3Wj(#r=Ap^vrId}7QgI~)RcX8-xKiA$(Sr!762`LSwtI#T zkCjT6Q(%h=`CflI+}^ZA->nNL!SaZ3CCx))JC972N_N53msO8UKLGodL*S+p<0Iis zo$#j79^8~|!%gpin?{HITl-hcPV)re0eneidOA-IXxlT)}^PYhn@;^CgPa0~oV_SePPCIKicfg_T4ehKR$1!oL4b&-g6HO6c}A5mb7rJQPE>=Zah zY_3;qxhQx}d_)27TErs`Tq*A2nbyWi6?{nr@~Etbg+zoRa&WvfMG#N*94z-xE}87Q zWEw5ma$W~uWaViaoGb81q5SIa^fI!9-*vWxJL1F!v?74mKwa60r+erEUMJQAUYpvr z+H@eN`#aLP@LeT&bj9-27Ryt1gD?V!3C4!^FzN}$G2;owG0_CWPT~n>QQ2cVB%Wa$ zg2^5@DU>}fJYb5{iS4l5#50Uzf58j`RT$4Jd`rA>EW?CqkINn#d@Lz@U?F`Zd*F;s zESkJ~YZy)7E0jHuX~JVjs&TMo9Sw>yK7++4uoBXmc?&#Q#@AlL)Fz@h48(`1!yhf% zyg0Sx_zg_k8(aPcC9S}Dal)NljaBS65Yb~({G@gO@ zB6VW9h1aJxQlD0Z4~%*X4;W6~&+T66U$#4({e(^N0gWw|ob0Y4La&R09eZ2EgW^kF zehmeQoI5gh_mQ!qN2iXI4h+Mbg58xkl2{V3(!culz8(6Io;WqXQobEb6i`-ewWB}4 z8ek|G`{4*D1N ztoL8uvoTGmSKCY70ef!!xBYC77`5M;_xAX2?P=4fT{}>@n^n!xPYq>_>|pF-N`jZy zr!+%5wK_XgJDVE8)9)*Dt^Ankxxu0J>akXHLso@e%I|@dKs*$4owc6SzXwhz*}j^Q zuH_pi=JZp8V0yfLmjs(%OKE-jsXSQL60^)TgZUR!=*Zt`+2hr|W;}-$&uZaleYyw2 z@=x)zu&22kOe(!8&nX1Yv;!Q>cxFCF-m!vr_JHv%XO1gb7(MtU8Wo2G|M_`$dT066 zMK&sq5j=E28BGbpQgDKzuV_xonFTd_RsjI`^@Tz(^wQ<&_VH)pFF<;KJCHD^Bw6y?fq1? zB^+}VI25?*?<=Xz*=#ssX?W+d*at}WwGJS+vgrJ-p&WeLf?Y#-xK)fkmRtG9f1Ydg zpZElakrygKIrSTOAq;rVho7CC+^Cd=q^1Ab7qx zJy<^aKEsjN!Ebo#Tr1W;LgtU}UkaaWaD0DLdQd;rna#yORXsCs&Z+5^7^PCM;IFdx z=^d(mY8{*!$Jrpkb?>QKz7gkP7Ppvdsd2vkJ#YfLQRO`KXyxs(Ii>kDu!H$sG`KJ~ z7hHZ*I=wS}G6Ulto}PjAP{rBI{|mRjh~F&iNw*9jmXqM!Kh!cin})y>V&etN?}Mj) z+@l$({n=ct|J3rY-m^W%GD?FjbHFm~gqj@;Pf{tZ{PNe*O1bUJz+RZ#3cT~mo0;;i zx0G(!bt@+p(oOZfLgm-aYfbt;FgN39s?;s-dslrk^>DVc*0AIGH}44-%9Y2p6z?P)aw;uPXv`^75JJGydHdk!-i6iav+0xB?5tWd7kv7&Z)utwuHnH+ z@Xdcy(?+~&w&y?WpA(Ky{|EceX)z|+ZspHU%mMD4oVH!r^Q&{rqzsD_YRn|tn!h-- zD!4e*m7dj3wm?>T&sEx0YFE6+(iy~AVWG_YxxSU~=?u>Gt%7h6YshjcFFeZHt|x;G z%rz8e6^!Xk;7tQ3HHf4k5O5~=#7{5YFYIYe#~qfC%E#XTQ=>kos7Wh-aa2P)ZMySU z_AP_YWxA2h?KCHU}z*7RT$a#Pgs{I3kHEq`P{q=VI} zLGOTaE)%x~V%5V3wM_c-PR5lmx5M(W%7H)9*2N(=%*x-7Gsx?1mT^6|IICiiH%OxG z>4EP-POA#XCJQk%~%kk%j=HliM&wpkw zgwMM3q~Hlav~!)JML|H29gJ@vs&!+YgLBXlsRdz%NOJePt$ixDh8t7?$wqAZnxHLAtg?9ZrI zLez@l8+4EOzxVB@`IxD$QTgzNv=Y@6X_&$&4oQ8zto#{KdqEe|%D4UtD)6v|fw`by z(^`nzw1F3bj4pO6HAneL?O(38KL9E+bhl+7or~I%+0Vbk6O`(BmF@Q&!*Vt(Fqt3! zARUZ*;WMi+dc{1^fm-6w_j_2{iii#~@D;nH#d38DUSRXT@oXsE z1nmPe_`%9=gZ9Zl<=>Z0`HwDi!e3zuw$1e(W5qdZ3;q|?Iq-H|aKpo4bM-nP1=HOA zvvhaomIY2*#g5w zA*zDN`~1FC8g!^6ui@`31uviM)-()58*=n!=oI`0MGNy`-eIVT@)x?w`3tKvYnnpj zQxKMwZOGgmgr&2=F>MXgAK@(;oaO{S+SRLd_MZl=4lOmlUm^J6q;jlv(RQt1xuwo& zeQ&5?Ron1}LGGLC6-xi-fIEo-(Uh2-=y7mm4AGs8Y5S?%Q7f|P{+bvbWf#1<|g71C@ zveKH5TEo*C;$sHpm(@2j{fnMQg1z(Vxi4^{8E7WNn*P&lRwdWC(}oxP_uai(t{*0p zIxp21!wp700@1tZ5E_NxC*y2rKGMR58+O{Tf(y^9JJkNimWJ}W@rh>e#zkc$b;oXV z1m1riJ2E{s+IQf{)S=}cy(Cm{q9f^@(4`o}~fgG4q!fFv4WW1tg~K@x_U zK*A3#*A5trwQg;~zJ=COSdu8XTe^y=^(p^|W!SC4a-p;=b<1*5ww9G$*mVW1a#z`N z?tAavd(OSxQVZ22-TmEj@A=Mm&pr3P_nNQ0^3i{N>7zGi`t$uqZgUqG7VaPKt-NFA z9p1+0U;1eFbE$0afrUf&9LWw}xwvqzyZO@F4$L2!yL)3fo7;yU>~B^s{m{m8ehGfv zY71K=534f`~?9idT`N2ZaWVRM$wlizSP%5n+DQ!}cuQP#7H71}XVd9w)KG~pwWT9cQQ0FY5vXaSG z*L7vhBXxk2pw~gu(?docfsbuLc6HYPqq|mTuHnLI1{n<}L7p+)YjF2aNzy&8BgiH1 zG;#?NnB@Xh5bwtgXHjKCrczn&k`s!wz$r@z-^8LfjO!6!$HJD-5cR)^`@ z&mo5W1CHihM z{6^z)PH5G)^qZI8GuTryCuS^j;wtdlB4WM%LK~5YtEM`~g@&uThAk+{C6*MEKH(Cr2`-1kCaziGg${E0pH~=MqH)QkR-j8Rd4)cMJxMNkFOXkWQw;%} z`VG}ZFfv{+W(s_6anqk6ZenqxONQ4s=p}4R`Ak9E(o3yYmtOK>eFi&{Uh-bR%Z=dW z^>nHul_ZVY@&woozPob8WfryJAd?ju`;XLCGu0MMOiW%Jr%KD$ly0bOf;|(3t$)oD z6IRits;X5&H&#(P-Mo<6l^zR9ldC!1Xj{^)mFSXgUZc<8!Y}FOtA4y_X^9Eo)Jv>~ z$Yg?=$U2#@nl|0`J>oW6)7`Gx5|c${Xk7ZO)#}o3UaZexO(6Z|6#@9Y24wbLDl;Tk z$^_cOrtoJaQL2BdG?@lj;M?mtL0E7Tf|6pLt(bu0D<%q|X3b;6yt4o7*lY_WX*>WjNk8kmM%dREM~4IJDYKZuX5sjm|dQIq06+whoQ+jpT;5rI*^~L3+t{4}Muw z9lSBUyvV*BYRVHfKzZBN2<4U6M9PD=cpgP+Fr<_yFWDdP=mXF)HEXnHKx;Zjag5ZV zly*XixoIG{?1B|s7uZw6n5%6XBx7vX;8(Q53Er5Dk-|y9DGA6pApuyo<|hDJNT||P z5UuGX%<%jK`y}o&8kRm=HWDt|Nca`4Xo5F(pV<fe5sbU(@rEu%?lZ#&G}H zMSpKqGS*@& z9*Mt=vgt4KX!u*VuEp57MFzEK<64R?6*Up1XChbM4zD8)OBk0Duvg zYmxpeEY%jNLNh52BBGW_m&6(r?R7z~wsu549=%29;waqhD^dC*1q;3$F8#J_3Ov3k@byiBZ`l-FW~LY<1}GB13db&Py~ za5^?#ZF@bbR!pATNSc;Ui6M7bh9gFEI9ANcSb zzC})!Xm5iT-}k`zHP<7mI8bJpfk%-$9uev*YEG~ZTI)ku!#G6^wT7d*YRG5Iv#;Gw@H2B;WX9VHny?m zK?=4})5M}hk82U{+>z6yX1SKo2RZYWC|K*jc#*#w#+{!I5D5 z)rAF~apCihnYB|UhKw60+y)dqd6hlIHF1@2XJCJ8CR{9GXcJ&~`?UqU74L>?r=na7 z@9nv^PL8m}UPD(A-)`XKu^*j@I45HAEp@8L>r}j#OXeQi-C#7|id(-(46w2!^5{^f zdRR)D%Zx|!Elx+>b?a8xys#!_r}W!Wr~153#T&Ym-<)A6;F6pRLDO@txDhPMxp2bF zZ$b&4zkcfCq&~HLPjU~`{K4=IWBfWdZUrnv)yN^Un3XzFJoRM53VeE|M{l5Sk)dRR88tm|P3tIAw>i&+b3#ll=1o^3ze_JIb&SvJmfSA6E(44-+P`1{K@uLwT7JD;vp zYQf%{-?I!mN*+&&OQ-fz{lzN_KTKuH+IR8ENBd)+A3HaCRVJr>6<59O-<%%vc(Zs) zv|qo$Xcp&$_Vrgovvw*ot(lBBknOu3DIOd9Q8bd|)`Vo5weR_T?p)!%OcC12?-HZG z`CV?G(J--k@Ziw70=F8iE01g$d#>=Qp3JOnG+H*pb3f1Zrp7)Mja2p+*Anfs7fv1< z<1OQ&EKGlW!by$RAZm*3cp+=|e7kTyx0+P3?SJ{)m|-J(@4cUMqahnXzLj&QP-ZY=3v#*g2zVOkcQnWzNzxap!;Uoj6zEy~YgsZ;n4? z4k>0ZyyK0+u~hNXxy;eDKH`MKwa*uvJ}#8FqG_+39|WGLugQ^hc-aHXErydz0pXh) z^G=`9I97GIb0%+V91A;K`e@P8IF@1f8y^~vahKw2oZ**WAMQ0fj)@HaZGH$l&SZ}E zXmljE284f99daBV{N$#I@U6px4m9q|Je$%RCtSXCXwXSBJtjgfubVud3!{>xhEEL$lR2rIt;ES8{7PSLbI;fpQkgYcrWM3Ko%i^b<%u&)dA#!ouq|?LaK_!{b ztG_XEY|LmIDKad7w%BVnO$7Osr71%il~_07)pr2NEQwSSCLVpOu#YDLxYq%nSq;Db z%6M;T`6E#>AWyTlKXS+5vEXk%K9#Q4-SCB1bG-xUfy{LKt|xoO;iF3Ui&H(voET6( z`@+YafpIUBhJ3zkAHV2cn`!^_jq#PvX=R|+2>n{Ps z4M#ThZ(g3XsDy8AN*x<)|MP3(XrTPH*T>KG#qX*!;Z@%*sJD%9bE^bz?|iZcu+@6_ z=TG*mNjYa~nP-D5ZdnN*)xvwX_d6?tO}9W-)#};iEl$4u;@K3!{qUD(Q~T5U*s5N5 z?-xqP9KRD&Rri`=>b&Tmx8i@2_w7?(9Kc>G;Z0wfdT20s*Dcsf`Rqf7dh=Rn{F!j% zd1Q7TGEhvL-w5vuQ%)@kmtPNGeRj$jh$-=_XD|P?WW@KwS6|LO*QZl;kUmy5Dx*ryzj2kbA7@0Pk_6gclON_PK`Ex^Q9a##;~NS;puNqtxd%d^vdBa z_hZN+eYi8>?(a{5K7Q_GMq)19MmX`M(&nC9r!wck{O)khbj^9|S7mU9TkFH}HO_fQ z|9jK=C-RwcCYMPGr1{ua_+dBY43q$s0~|%E5;WX#dnrPhwp4u{es%j8qH>6B)o{7)l*+hu`_>xZ{*g6*D7pTyjewYtQh+ZeRlls z9YD>3rGzj!GIgvFe(u_oGhG7MwArORbqW9L;>q6W4#w0uR`?I!m~aHlVoV+Lg*)@u zCqxP3M0KH!|8E|2oYiNJLSY1p4P(QTVJe5)M~l5jSDk(~1;9NVCj(UuKk?L9@19ks zPo@Bs$T4BW(_(e~?YX`!(*P@*=CoM-XxCWpmQLnmu*zZoWG=>8>?wD{OH*JE{OyBY z!Rd$=IX)G!<3IWJc=(fFuk1Yv-~j}uIatQ6eEi@u1Mtd*2cKEeJ1ZIh++azC@c3i% z{qV|!kIfIn4QxHAU><+@!R7E84PSn+&^tS2oqBTc;Wy_8doid3IBGr&R|WO>#1(mX zEej{ESl)RUu8QvQ;a44a^@hW*_H}mDRbf7U)8j>Wtq5;=e7s8!D&WU|@Kg`H(%}!D za=P@OB7gjcZw$k0B>dqUOGN`~WL+hJ<0lq|!V?R_BC=3nq4KE=T*+kl_}t0K_T0&- zzE!~FEL4abszea3y=SoZ+0|#B=s_5lzN>;CKK@#&_vGp`xT}KVkXuwi4|l(n>z%jH zV6ET6)=9@%b>?&mjjdtkDl*z(`e##p+gG0vAY-amRFK2(+>sJEdb){K}7{GJohI(&YI~{ zxM=VgS2P0MN2Z+O^eN1yBivgpZ0kQ>az>`D>zVc}akze<=;S6IUkNUx@n0EW~o6eR-TL&*lPJ+bm z)Z`1<<gewRg z9UN}6J{6J-t{YGrTtpvHqVkq@jTaF+xl6k42GqLN=xf=C!K)0mTTkg2l>2nFX6jK$ zGFk+-I@wgi6+bIMVIxT>57Mq-Enq|`BY(TJ-l^*dSCJIpAo# zZ=bAgNEmMmYa_TmgufKPI-i#Gv(C-5rl6FpZpgKwMrn!85jU0{g(77sbc%J&QOuT6 zNZ(=Vyer;~gkG~;Ry6x)#ht9DO2_1Af@ADDqQc*PGly8Q+%tk+P;q>PZwt z)PVBeVI&b$@*h3^P1C^|%_8QxR{YD1O?-xtO$LRf3h#!L{KpSq6M<{~#DZ-EJJ*bu zG0=c42B;{?s)nT`b%iRDDrI{;lJHJe1G$FDB@EZu2uexnhEybpB)pY61bdvsq?CzL z35vfX9z`8LVB8Ty3B~77JKH0Kc86+&)=WZYrAu_yFmI>|}k5BvyQcf>ErE$jBR(Jdfk)!D7Xpmc_$lwC7n!3bxW0@9AVF z6;urp9~omXEI!2RAek`Nz;kz9u&^uFp5HQnsFTPYH+tsBT+S7QNJ&JH`mB)3k@!GR zKMzz@6mg`9Pqf(1D=Zb4CDh8Q*13HZk`Ci}L51$4Ae?5Iq8wu*HknsY%GTRZ5rNrOUaY@y~>b_yx#BYtG36NHV=K6T0D!5jWfW@e=*nMD}jd)TZ{ zSY7Fsvk?SF6B0dvCoN2+D+r~}jIu&1`o!ZZl$vSTCvybTFgux3SlYQ6Wrfr_?>0to zSSX!7Bw|&%a*U4HvK%QW9k3a8h13RIH~CkNvChpIsi3rTGi@rQ);T6&rgSw()JOc_ z1P!Nxu<_X?9H~w9DxP) +# NOTE: there will not be an output packet in the DETECTIONS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "DETECTIONS:detections" + +graph_options: { + [type.googleapis.com/mediapipe.FaceDetectionOptions] {} +} + +# Converts the input CPU or GPU image to the multi-backend image type (Image). +node: { + calculator: "ToImageCalculator" + input_stream: "IMAGE:image" + output_stream: "IMAGE:multi_backend_image" +} + +# Transforms the input image into a 128x128 tensor while keeping the aspect +# ratio (what is expected by the corresponding face detection model), resulting +# in potential letterboxing in the transformed image. +node: { + calculator: "ImageToTensorCalculator" + input_stream: "IMAGE:multi_backend_image" + input_stream: "NORM_RECT:roi" + output_stream: "TENSORS:input_tensors" + output_stream: "MATRIX:transform_matrix" + options: { + [mediapipe.ImageToTensorCalculatorOptions.ext] { + keep_aspect_ratio: true + output_tensor_float_range { + min: -1.0 + max: 1.0 + } + border_mode: BORDER_ZERO + } + } + option_value: "gpu_origin:options/gpu_origin" + option_value: "output_tensor_width:options/tensor_width" + option_value: "output_tensor_height:options/tensor_height" +} + +# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "InferenceCalculator" + input_stream: "TENSORS:input_tensors" + output_stream: "TENSORS:detection_tensors" + options: { + [mediapipe.InferenceCalculatorOptions.ext] { + delegate { cuda {} } + } + } + option_value: "model_path:options/model_path" +} + +# Detection tensors. (std::vector) +#input_stream: "TENSORS:detection_tensors" + +# A 4x4 row-major-order matrix that maps a point represented in the detection +# tensors to a desired coordinate system, e.g., in the original input image +# before scaling/cropping. (std::array) +#input_stream: "MATRIX:transform_matrix" + +# Detected faces. (std::vector) +# NOTE: there will not be an output packet in the DETECTIONS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +#output_stream: "DETECTIONS:detections" + +# Generates a single side packet containing a vector of SSD anchors based on +# the specification in the options. +node { + calculator: "SsdAnchorsCalculator" + output_side_packet: "anchors" + options: { + [mediapipe.SsdAnchorsCalculatorOptions.ext] { + num_layers: 1 + min_scale: 0.1484375 + max_scale: 0.75 + anchor_offset_x: 0.5 + anchor_offset_y: 0.5 + aspect_ratios: 1.0 + fixed_anchor_size: true + } + } + option_value: "input_size_width:tensor_width" + option_value: "input_size_height:tensor_height" + option_value: "num_layers:num_layers" + option_value: "strides:strides" + option_value: "interpolated_scale_aspect_ratio:interpolated_scale_aspect_ratio" +} + +# Decodes the detection tensors generated by the TensorFlow Lite model, based on +# the SSD anchors and the specification in the options, into a vector of +# detections. Each detection describes a detected object. +node { + calculator: "TensorsToDetectionsCalculator" + input_stream: "TENSORS:detection_tensors" + input_side_packet: "ANCHORS:anchors" + output_stream: "DETECTIONS:unfiltered_detections" + options: { + [mediapipe.TensorsToDetectionsCalculatorOptions.ext] { + num_classes: 1 + num_coords: 16 + box_coord_offset: 0 + keypoint_coord_offset: 4 + num_keypoints: 6 + num_values_per_keypoint: 2 + sigmoid_score: true + score_clipping_thresh: 100.0 + reverse_output_order: true + } + } + option_value: "num_boxes:num_boxes" + option_value: "x_scale:x_scale" + option_value: "y_scale:y_scale" + option_value: "h_scale:h_scale" + option_value: "w_scale:w_scale" + option_value: "min_score_thresh:min_score_thresh" +} + +# Performs non-max suppression to remove excessive detections. +node { + calculator: "NonMaxSuppressionCalculator" + input_stream: "unfiltered_detections" + output_stream: "filtered_detections" + options: { + [mediapipe.NonMaxSuppressionCalculatorOptions.ext] { + min_suppression_threshold: 0.3 + overlap_type: INTERSECTION_OVER_UNION + algorithm: WEIGHTED + } + } +} + +# Projects the detections from input tensor to the corresponding locations on +# the original image (input to the graph). +node { + calculator: "DetectionProjectionCalculator" + input_stream: "DETECTIONS:filtered_detections" + input_stream: "PROJECTION_MATRIX:transform_matrix" + output_stream: "DETECTIONS:detections" +} diff --git a/mediapipe/modules/face_detection/face_detection_onnx_tensorrt.pbtxt b/mediapipe/modules/face_detection/face_detection_onnx_tensorrt.pbtxt new file mode 100644 index 000000000..321736b5f --- /dev/null +++ b/mediapipe/modules/face_detection/face_detection_onnx_tensorrt.pbtxt @@ -0,0 +1,165 @@ +# MediaPipe graph to detect faces. +# +# EXAMPLE: +# node { +# calculator: "FaceDetectionFrontCpu" +# input_stream: "IMAGE:image" +# input_stream: "ROI:roi" +# output_stream: "DETECTIONS:face_detections" +# } + +type: "FaceDetectionOnnxTensorRT" + +# The input image, either ImageFrame, GpuBuffer, or (multi-backend) Image. +input_stream: "IMAGE:image" + +# ROI (region of interest) within the given image where faces should be +# detected. (NormalizedRect) +input_stream: "ROI:roi" + +# Detected faces. (std::vector) +# NOTE: there will not be an output packet in the DETECTIONS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "DETECTIONS:detections" + +graph_options: { + [type.googleapis.com/mediapipe.FaceDetectionOptions] {} +} + +# Converts the input CPU or GPU image to the multi-backend image type (Image). +node: { + calculator: "ToImageCalculator" + input_stream: "IMAGE:image" + output_stream: "IMAGE:multi_backend_image" +} + +# Transforms the input image into a 128x128 tensor while keeping the aspect +# ratio (what is expected by the corresponding face detection model), resulting +# in potential letterboxing in the transformed image. +node: { + calculator: "ImageToTensorCalculator" + input_stream: "IMAGE:multi_backend_image" + input_stream: "NORM_RECT:roi" + output_stream: "TENSORS:input_tensors" + output_stream: "MATRIX:transform_matrix" + options: { + [mediapipe.ImageToTensorCalculatorOptions.ext] { + keep_aspect_ratio: true + output_tensor_float_range { + min: -1.0 + max: 1.0 + } + border_mode: BORDER_ZERO + } + } + option_value: "gpu_origin:options/gpu_origin" + option_value: "output_tensor_width:options/tensor_width" + option_value: "output_tensor_height:options/tensor_height" +} + +# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "InferenceCalculator" + input_stream: "TENSORS:input_tensors" + output_stream: "TENSORS:detection_tensors" + options: { + [mediapipe.InferenceCalculatorOptions.ext] { + delegate { tensorrt {} } + } + } + option_value: "model_path:options/model_path" +} + +# Detection tensors. (std::vector) +#input_stream: "TENSORS:detection_tensors" + +# A 4x4 row-major-order matrix that maps a point represented in the detection +# tensors to a desired coordinate system, e.g., in the original input image +# before scaling/cropping. (std::array) +#input_stream: "MATRIX:transform_matrix" + +# Detected faces. (std::vector) +# NOTE: there will not be an output packet in the DETECTIONS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +#output_stream: "DETECTIONS:detections" + +# Generates a single side packet containing a vector of SSD anchors based on +# the specification in the options. +node { + calculator: "SsdAnchorsCalculator" + output_side_packet: "anchors" + options: { + [mediapipe.SsdAnchorsCalculatorOptions.ext] { + num_layers: 1 + min_scale: 0.1484375 + max_scale: 0.75 + anchor_offset_x: 0.5 + anchor_offset_y: 0.5 + aspect_ratios: 1.0 + fixed_anchor_size: true + } + } + option_value: "input_size_width:tensor_width" + option_value: "input_size_height:tensor_height" + option_value: "num_layers:num_layers" + option_value: "strides:strides" + option_value: "interpolated_scale_aspect_ratio:interpolated_scale_aspect_ratio" +} + +# Decodes the detection tensors generated by the TensorFlow Lite model, based on +# the SSD anchors and the specification in the options, into a vector of +# detections. Each detection describes a detected object. +node { + calculator: "TensorsToDetectionsCalculator" + input_stream: "TENSORS:detection_tensors" + input_side_packet: "ANCHORS:anchors" + output_stream: "DETECTIONS:unfiltered_detections" + options: { + [mediapipe.TensorsToDetectionsCalculatorOptions.ext] { + num_classes: 1 + num_coords: 16 + box_coord_offset: 0 + keypoint_coord_offset: 4 + num_keypoints: 6 + num_values_per_keypoint: 2 + sigmoid_score: true + score_clipping_thresh: 100.0 + reverse_output_order: true + } + } + option_value: "num_boxes:num_boxes" + option_value: "x_scale:x_scale" + option_value: "y_scale:y_scale" + option_value: "h_scale:h_scale" + option_value: "w_scale:w_scale" + option_value: "min_score_thresh:min_score_thresh" +} + +# Performs non-max suppression to remove excessive detections. +node { + calculator: "NonMaxSuppressionCalculator" + input_stream: "unfiltered_detections" + output_stream: "filtered_detections" + options: { + [mediapipe.NonMaxSuppressionCalculatorOptions.ext] { + min_suppression_threshold: 0.3 + overlap_type: INTERSECTION_OVER_UNION + algorithm: WEIGHTED + } + } +} + +# Projects the detections from input tensor to the corresponding locations on +# the original image (input to the graph). +node { + calculator: "DetectionProjectionCalculator" + input_stream: "DETECTIONS:filtered_detections" + input_stream: "PROJECTION_MATRIX:transform_matrix" + output_stream: "DETECTIONS:detections" +} diff --git a/mediapipe/modules/face_detection/face_detection_short_range.onnx b/mediapipe/modules/face_detection/face_detection_short_range.onnx index 638600236a3dba071ad265a40aa9bd45a752f143..18edac978f588b043f784033b4cb5d4bc775852f 100644 GIT binary patch delta 9392 zcmbVSYm6J!753O;d-m=l&g{dxCXda=5OBlBGxyGmM%hgW?Z$|ZK!B8o9JBV4tv2>{ z*MvxjZbX-=4@FIS8zE3y(%rVHDx@MA$yA~cP-v^tKUIHGQE7vURH>EPQvS3>ZO_cu zGoHD3v!eTuXU^Q2d(QWL_uO;u_~NfduDmlcDM@m&KBLyE)iar1Wn6*3QN>cUnaA(= z$F&hOf#S2(`6ugg&vdPNS}hlg#JnS)LEhN~eYE#w0r{8dWdGnK))mwbZ5Hdrnc7^r zSevgGi&!`0gnMJA^*JWq36IUCP;#-pFk7xHPDUrAmxUxyOz)`sMT6V zrRn9sQzW04Q{~Eh{p3@#igj}+P5`D&&7t1e_ucy?cFD^y<@}R7>O=R?j1!X;~g=bWmR-nAULn7++QjK zD`zUz8aRJyz63xk8QVyO(V->ilXG3}@qvWafwLBv*MQT`My&4ys~jW1Df@7ad9igE z+cqXj57OMKfH*g>-C>>X7y7qi-G@>2U{sk7vPl&ip?Q$4^!yp4BzyLVPV}P&%A>4ed6AlA}#{dh7 z%fQ#sIt=i9P)yb_jQwP$c?@VQ14|(WNPq#$k{MZJmI1sj1K49gb5j}#r8GgI?{;r} zq?OCUSCzmx(E=<%Fb+#k1{9>pn*zBFa^>AU4=vq|!sZR*By607o8Q8z3=#)1f+=Ro zsa*)MIKt3^!f-}ri>M9q+gEzmQ3=l@&Elv;^;M#KD$!Y$?BL}$;&H3XgY-n-U^L@= z&^0j;143<6aG2=<0-+z?Fwp*tJ(?OBaNVt78}ImD2>;Spnb#8?{>V{S~0L0=i*5@UG8SS~RppBS@~7^Y2t7%M;w_|GGT zlTiugltY*pJYp;kF)UIpF@GN%e)NaD^$gbsjuGpc+|^z*eSK_xr0w zKSrLBiPnX8*YNPrj|9ZI!(JK^VsNC6P0L4A@ex&+KC-7!As9`5V@NHw{IJ~e<9-%q z%ZFKkwZ=%A)<(8P_rh9ZB)t?wSIGo$1#cZ`H$!-Hb~&)cF%r#?yB8`?EL0X3YYUKU zI!_!Vgwx)J$G>g{AdjNXCbHadGweEO8te?E!Z!NauI>F(h-pjr_YWMzsw#Iu|0hfH z64kOyl=m7z&yR>AoU$L10h)-Vsgj@j+7zsY5& zJW0cr46E~&+%_l`)tiPN;|rYY6mK3T#FxA3%e}P}_c+dUXDxILaX|4ks>(yo_6D=R zK|ix)+tOaN&efoh)u52U4@byh&^eVOQF%s^cQMa(ZsHXcl?OT-w4Fz8T?b-bZwfD? zVC+MTeTdn0dUh|mHz;Y1;U#z)Vz-}@lj^i;@NFsl(CO8k%#E@?L*Ow^WoIq0I6*XC0r6j5W3;& zlvB_Vof}h2r$Lz#AHdCC>8G$U-3URB3z)ZTB(PlZrghCnkh8t8HH;wf5!7UQeMg$E z?Qmu)I1B-&1%Yd?_rucT3?#yPb3oeC3j(Kcfn!6t5pWzAFt=nN9YvV#1IMmUS?Os$ za5_lur#l_#!G|Y3INlT-ygJ>S9T^ME_neh2G3_GfW~>sq$Deo=Sgd!Rg{m5u%1ub~ zfhKZ)=MBZ8npa^m-^>{8*?iy-Vq4qO`5l4j9I{}0X2~yIP)eK<3SXdkmq7#T|Ca1%XvhmOXk3sGQ8CSdQ-$K}mX=)Ac-&q;|yfr=>TmI*H{0$3!&YerDOe)Jf;!w57p<)M%P z;#Tqy_W;jshn=26+6q^;75EnYTJArs8qkz81>#r<{prwRPsRQ>kP{( zOw!WsTj+DUZx>CYb@Kl1agpHG3Az0ynm@WN23sJFqenkX(*L}$6;4QF+lyisExl%6 zQ5$bAE%$b>UXDs9;8&0Yy8p^Rl$gz(7lg!lIx!2a45M*kb|Bup`o16?myWbs!3C}P z(5`r*+OaHy-@LvM62? zz;kdd0Xv?B9}5oD^SjfO{omA+3&9s7fZvDD#^UG2OEKw$+bzM3yuK`mAM{Fjw=1GH zet+g#!fOLvEWSM;O}V{e+_+fHB9{=X zHjZB@$GTU)X6xX%f1_$Nl~pmmEIvCT9d~b3wMO||xyKcdsy3c|ZMk;>OEC~A^Qs%9 zUlTUQ*HtrLxXSC(a}mN#!Lwy)7=Bw>BnGegn{)TTU}Xq}(xUmo<53tO?Oqeu zPz!7SAFS?}+$82&6vp5i7PLAsEw=FhgH9=46x$?%L2IP<9TdD2l)ySEoL?`-zIg6Z zRGM_b&~f4Hr*X6!Zf<8EtgG~&>N0xU1e{`0T#CDp>q_$-Be8P}U^obw)Wa}}_2!qq zDm;f+2qCq-`H>Cb>n)KdV3V$J`O|o2(>Kro8`0@aWZFweF-)|^MFL67OJkUjhVa>C z0j*?VW zB4_`+FS2%FBG=BhaGOrgDKDdyW0DEE+|v@)ny-k0nCtKbQrL`Td)J~ShE=oi(m1qg zD;F#}ZRD?^m3bKRrvEfZV&{G%njH^QN#mW|vS6)~$)KcTGg+#Y*>Y`sqL4^`Z7d7q zKGojDt1Q$j<?m8qfV2Lyh#Dqji>0{&A4zU|- zD>iAU5F3yS6p4Zv=z^*i6BSXRs;__sfeM6D(Z`RLKhTOw4T!2p^jAv@s8Z8&=5e1h z87)SgSYcmrj`Tb8}}q)$G-!t8*itz1gAf($$%{ zg*%pvHP_G2oiWGC<>E-Dv%$Yu8BOnhPSM$idW-vT$BCI!OVg)EGJ4}AHk~SGcicIW zaXTuPvqM?GBj=(whRv*%=5}jE?Opl%=4(gl^g*|04{bC~OfJownOK^cpM$nKz3Pn} zoIZ7j{*@!+4oEIEYEtD4;f}~CpfsDGdM%tg)WTtiK(e_`Rw%Ee0g&I_JxTh z-~oDWgWJ}{`s7VJ^R@_0+`)MR zD5K@}iL>Q}`T0{|x;ZtHvCgW8hi}xy+oIQPf{Q4gby@9b6_hqdJf-uj(r6x|G+5^= zjnffkM8bMAo)R>MMnvPhtZ`1zczBr6czBo&!3ERc9?a!>pEY!B3g8_@cJe!9pJ8WCJgT=x0RO*+4N;&Ot_)Cv9TAe3 zb*Wfq4777O5xyL^E=1t!cn$(%g1pPQLD z4$IMlt%*h_+e~_K?SOa;r@#A_KgjTuV9vZgyd>JuV4(43G{l*uWG# zgk|W=%|ZovUF~eg%}@cip8oP0fZ`$_YXz47Vle+jKN)b#D>i2Mhqit);BvLg|H|6b zEkEEkpwL`5umuz?*^DeeNf4mKhJGnD^ko5XpeqUh#9H^X2QIKEpf3O#0V(CJc4vkC zL%W+(;1UMmm@Ns+mV%ir`H^k1k&TTRW}$5jX5UVx4nbkXZ0(^rWj0HhZJG4yyo^XK zxJY8V6dK#Ox#wdWN4g@l7$;IzDTZbhl#9?!;=Kg%o)9Z7OG!EDWqgmj$48KWFG#A+-w%}Ml(}uVdF;jx~41}J5q;Np@w4{u(~jHD$ym&>tbe@ zkcW$q@tn=jO`QE%u=UEqV3_UzxIXw6n#@_T!!Fu;hx4 z>M85y&g4xnCs6pQ4zaGS)ZIoz2 z9m#UQ6sv?Lp;|SwI3%GdY`nofpy@ClUO1E=$k+=ow32ohdN}Pe1eX#+SUhkH`L(FX zkXwubLnV$OY|k=OyBrN*2+I+|5L1v6hJ=dL%ra>mF2Vxo8kr%c@^O0crg~1!uhvR$ z*CCkiPAUj^&L;pYKsXMenhcCGw-f~qO^!os&vID17!BYM%LBq8Q;rf2g?iL9*(ftZ zqijs(kf}tRUOWy1D#OmC!(JYDZiLGK9M2_g);TWy8d8i$x0HkBj8A9Sp5>A$<^uL_ z0bJttpK!?(c7#izvNKIK9?j5r44AP@rQ!4fGq%#7bk>}^`{|kk_u9n7hCAzb`#X+u z3lZ+AiRsMdf9>R{iN(d46El+&OY^7UeG2Hnb}89BcGb+^v3`D4PeMfDN>nk61!K+W>BW;13)5vMAjw5<{XzG>{sO;+$+AA0 z%q}o-dipkKz8K&i8|yhQ)&~sTdaT*tLL$gT%%i^XCcKf&=5fvSUG(mti_JXBx^`Q8 z#VjxmIxPDR=BR@&3J!7$D^okS@gf#^4KeJAhG8)uJD%AeCG1jLfs}XIKdFZ_(u+Q|xV@mQ9MJqB7y;nC?`#ez;j(PTr?z z>zQ`m7PC6or!Kai(NfGKujZ(}eMy~M_k>HHn8EEs@2Sg*^whJ|p1L`*tR%&@uC4Z| zV?cXqKz!<1YroSAkENbHd+mR{kK=wC(T;+}#468Z`~Lf4Q_8wOMYM7DsaM?EFZZcq zO52j-EO)|L{pm(^GInphma;DI>Wsnxq08Qa%{@j4Yx$zh*#V0QoyJ}>B9Qm%T2L+ zJG7%Npis1*xgk2$=$P3H6HGQU;h8;f-q1$9Rznv)$;TS1PuotfdHZ!+`8djOQOMb! zy&CbkD42HsF6Dd^7?8CG4n@I&2OG5kudj?f^Ht>^YGtDq0se!s?mn$9-L|LhwcgzW zc`Xj0!1Z-ss_wO?cGs=`{+D&K zzH$+F_0FxF_bYE^i@qrpMx>wf&X1#!-^^HAtK#4VXt+oRuC7?*LXumb4eoIfE zUyW$T;IB)Ezki`?`-O{Nj6afo`X~%|R0PrW!9ICwEVjLS)qwHD5$Q__`=bY9v4-^1 zh8CAb%u)sHmPG|%@2MKhQDMw1!+yISdn3{l>_C>`zxC0W`bZd#QQ+(WH5|uiFm_Q1 z$1y7W?#?LU9zcR6KZp()yYj1uy4-XAFAAW>MSm%K7rNey&!wMN0JB#_(XE*Moi9h! zTYJtwq5x`C#4(DEG{@Av8(t}bUQtceR7`j7-jTfAvme@`Zd_mOJ09<=e*Jj7zk0** z_{PfYW_J(jeHUV+^rWU3h91!r_=)OK?VRHLGrP&(tUBL)7)gCC023ZC1^>8peQY#K~Y&kpL+aj8s( z19jjWYr!_8^$K=OX>+~`;BBQHdPD?$5D?acCjx6M0a-y)Wpe|N1jtw`i`++G0c^=6 z^feMZcn9b370E=}Bkznn87PvYz7~-B;+$S5^$I(w2Z(V2&d3x|xTjRm3ksfp+Qf6N z;(i-e#!@N_3e!hA@eTlEYDxqJtxQM;p$_~Oj=EqgTqLE=IYj1-r5yql791$cx>5+Q zE0j%wGZe`gR}07(=LX$gR@WkNH$Yd}X9`DW;EYywhC+B}Qse%sgc)n;8Sl93r&(9}#+$Fe2u|NcW1{aDX z{<+6n{NP6e+}q-R!C01Qqj=Lj4U@&95YHbkJvyg0VWUg=QC@ejUk!g|uV&7NDxMSp zWE~?7c(UXT0PUKR6<{vy5S)i`%h_z%`fgW?%mGCT9LR1_C~svgBx5DQuHspxJ)%Z% z=<_-3lUGRE8*g7NBx4^=sY(s|92$d8Si;%}gvtw}Y;Xm?!G^bc(sZIwYB3&-2y=J{!Sy}VdaEKhMQBxA6gQfY^d!PY(fYd=Q}$O{ndj}Gg+{pUrBlL8((tHNhjNh>PMcbh~ZlYUxAAB`@neEh`D9LobEE z@5k72F_;6!O|aw2YDAzL>^3n=$MxThi|#t(BRDdfW?qPkqs9Qic`FBPI$%`)y|g&0 ZV;$hrvs)RKF#!tR;<$HmNg!BhYM diff --git a/mediapipe/modules/face_detection/face_detection_short_range_by_roi_onnx_cuda.pbtxt b/mediapipe/modules/face_detection/face_detection_short_range_by_roi_onnx_cuda.pbtxt new file mode 100644 index 000000000..10dd4774f --- /dev/null +++ b/mediapipe/modules/face_detection/face_detection_short_range_by_roi_onnx_cuda.pbtxt @@ -0,0 +1,40 @@ +type: "FaceDetectionShortRangeByRoiOnnxCUDA" + +input_stream: "IMAGE:image" + +input_stream: "ROI:roi" + +output_stream: "DETECTIONS:detections" + +graph_options: { + [type.googleapis.com/mediapipe.FaceDetectionOptions] {} +} + +node { + calculator: "FaceDetectionOnnxCUDA" + input_stream: "IMAGE:image" + input_stream: "ROI:roi" + output_stream: "DETECTIONS:detections" + node_options: { + [type.googleapis.com/mediapipe.FaceDetectionOptions] { + model_path: "mediapipe/modules/face_detection/face_detection_short_range.onnx" + tensor_width: 128 + tensor_height: 128 + + num_layers: 4 + strides: 8 + strides: 16 + strides: 16 + strides: 16 + interpolated_scale_aspect_ratio: 1.0 + + num_boxes: 896 + x_scale: 128.0 + y_scale: 128.0 + h_scale: 128.0 + w_scale: 128.0 + min_score_thresh: 0.5 + } + } + option_value: "OPTIONS:options" +} diff --git a/mediapipe/modules/face_detection/face_detection_short_range_by_roi_onnx_tensorrt.pbtxt b/mediapipe/modules/face_detection/face_detection_short_range_by_roi_onnx_tensorrt.pbtxt new file mode 100644 index 000000000..9d431912e --- /dev/null +++ b/mediapipe/modules/face_detection/face_detection_short_range_by_roi_onnx_tensorrt.pbtxt @@ -0,0 +1,40 @@ +type: "FaceDetectionShortRangeByRoiOnnxTensorRT" + +input_stream: "IMAGE:image" + +input_stream: "ROI:roi" + +output_stream: "DETECTIONS:detections" + +graph_options: { + [type.googleapis.com/mediapipe.FaceDetectionOptions] {} +} + +node { + calculator: "FaceDetectionOnnxTensorRT" + input_stream: "IMAGE:image" + input_stream: "ROI:roi" + output_stream: "DETECTIONS:detections" + node_options: { + [type.googleapis.com/mediapipe.FaceDetectionOptions] { + model_path: "mediapipe/modules/face_detection/face_detection_short_range.onnx" + tensor_width: 128 + tensor_height: 128 + + num_layers: 4 + strides: 8 + strides: 16 + strides: 16 + strides: 16 + interpolated_scale_aspect_ratio: 1.0 + + num_boxes: 896 + x_scale: 128.0 + y_scale: 128.0 + h_scale: 128.0 + w_scale: 128.0 + min_score_thresh: 0.5 + } + } + option_value: "OPTIONS:options" +} diff --git a/mediapipe/modules/face_detection/face_detection_short_range_onnx_cuda.pbtxt b/mediapipe/modules/face_detection/face_detection_short_range_onnx_cuda.pbtxt new file mode 100644 index 000000000..9d79fb6ac --- /dev/null +++ b/mediapipe/modules/face_detection/face_detection_short_range_onnx_cuda.pbtxt @@ -0,0 +1,40 @@ +type: "FaceDetectionShortRangeOnnxCUDA" + +input_stream: "IMAGE:image" + +input_stream: "ROI:roi" + +output_stream: "DETECTIONS:detections" + +graph_options: { + [type.googleapis.com/mediapipe.FaceDetectionOptions] {} +} + +node { + calculator: "FaceDetectionOnnxCUDA" + input_stream: "IMAGE:image" + input_stream: "ROI:roi" + output_stream: "DETECTIONS:detections" + node_options: { + [type.googleapis.com/mediapipe.FaceDetectionOptions] { + model_path: "mediapipe/modules/face_detection/face_detection_short_range.onnx" + tensor_width: 128 + tensor_height: 128 + + num_layers: 4 + strides: 8 + strides: 16 + strides: 16 + strides: 16 + interpolated_scale_aspect_ratio: 1.0 + + num_boxes: 896 + x_scale: 128.0 + y_scale: 128.0 + h_scale: 128.0 + w_scale: 128.0 + min_score_thresh: 0.5 + } + } + option_value: "OPTIONS:options" +} diff --git a/mediapipe/modules/face_detection/face_detection_short_range_onnx_tensorrt.pbtxt b/mediapipe/modules/face_detection/face_detection_short_range_onnx_tensorrt.pbtxt new file mode 100644 index 000000000..c54dff27d --- /dev/null +++ b/mediapipe/modules/face_detection/face_detection_short_range_onnx_tensorrt.pbtxt @@ -0,0 +1,40 @@ +type: "FaceDetectionShortRangeOnnxTensorRT" + +input_stream: "IMAGE:image" + +input_stream: "ROI:roi" + +output_stream: "DETECTIONS:detections" + +graph_options: { + [type.googleapis.com/mediapipe.FaceDetectionOptions] {} +} + +node { + calculator: "FaceDetectionOnnxTensorRT" + input_stream: "IMAGE:image" + input_stream: "ROI:roi" + output_stream: "DETECTIONS:detections" + node_options: { + [type.googleapis.com/mediapipe.FaceDetectionOptions] { + model_path: "mediapipe/modules/face_detection/face_detection_short_range.onnx" + tensor_width: 128 + tensor_height: 128 + + num_layers: 4 + strides: 8 + strides: 16 + strides: 16 + strides: 16 + interpolated_scale_aspect_ratio: 1.0 + + num_boxes: 896 + x_scale: 128.0 + y_scale: 128.0 + h_scale: 128.0 + w_scale: 128.0 + min_score_thresh: 0.5 + } + } + option_value: "OPTIONS:options" +} diff --git a/mediapipe/modules/face_landmark/BUILD b/mediapipe/modules/face_landmark/BUILD index f155e46d5..331319fcf 100644 --- a/mediapipe/modules/face_landmark/BUILD +++ b/mediapipe/modules/face_landmark/BUILD @@ -42,6 +42,45 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "face_landmark_onnx_cuda", + graph = "face_landmark_onnx_cuda.pbtxt", + register_as = "FaceLandmarkOnnxCUDA", + deps = [ + ":tensors_to_face_landmarks", + ":tensors_to_face_landmarks_with_attention", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:split_vector_calculator", + "//mediapipe/calculators/tensor:image_to_tensor_calculator", + "//mediapipe/calculators/tensor:inference_calculator_onnx_cuda", + "//mediapipe/calculators/tensor:tensors_to_floats_calculator", + "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator", + "//mediapipe/calculators/util:landmark_projection_calculator", + "//mediapipe/calculators/util:thresholding_calculator", + "//mediapipe/framework/tool:switch_container", + ], +) + +mediapipe_simple_subgraph( + name = "face_landmark_onnx_tensorrt", + graph = "face_landmark_onnx_tensorrt.pbtxt", + register_as = "FaceLandmarkOnnxTensorRT", + deps = [ + ":tensors_to_face_landmarks", + ":tensors_to_face_landmarks_with_attention", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:split_vector_calculator", + "//mediapipe/calculators/tensor:image_to_tensor_calculator", + "//mediapipe/calculators/tensor:inference_calculator", + "//mediapipe/calculators/tensor:inference_calculator_onnx_tensorrt", + "//mediapipe/calculators/tensor:tensors_to_floats_calculator", + "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator", + "//mediapipe/calculators/util:landmark_projection_calculator", + "//mediapipe/calculators/util:thresholding_calculator", + "//mediapipe/framework/tool:switch_container", + ], +) + mediapipe_simple_subgraph( name = "face_landmark_gpu", graph = "face_landmark_gpu.pbtxt", @@ -84,6 +123,48 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "face_landmark_front_onnx_cuda", + graph = "face_landmark_front_onnx_cuda.pbtxt", + register_as = "FaceLandmarkFrontOnnxCUDA", + deps = [ + ":face_detection_front_detection_to_roi", + ":face_landmark_landmarks_to_roi", + ":face_landmark_onnx_cuda", + "//mediapipe/calculators/core:begin_loop_calculator", + "//mediapipe/calculators/core:clip_vector_size_calculator", + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:end_loop_calculator", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:previous_loopback_calculator", + "//mediapipe/calculators/image:image_properties_calculator", + "//mediapipe/calculators/util:association_norm_rect_calculator", + "//mediapipe/calculators/util:collection_has_min_size_calculator", + "//mediapipe/modules/face_detection:face_detection_short_range_onnx_cuda", + ], +) + +mediapipe_simple_subgraph( + name = "face_landmark_front_onnx_tensorrt", + graph = "face_landmark_front_onnx_tensorrt.pbtxt", + register_as = "FaceLandmarkFrontOnnxTensorRT", + deps = [ + ":face_detection_front_detection_to_roi", + ":face_landmark_landmarks_to_roi", + ":face_landmark_onnx_tensorrt", + "//mediapipe/calculators/core:begin_loop_calculator", + "//mediapipe/calculators/core:clip_vector_size_calculator", + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:end_loop_calculator", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:previous_loopback_calculator", + "//mediapipe/calculators/image:image_properties_calculator", + "//mediapipe/calculators/util:association_norm_rect_calculator", + "//mediapipe/calculators/util:collection_has_min_size_calculator", + "//mediapipe/modules/face_detection:face_detection_short_range_onnx_tensorrt", + ], +) + mediapipe_simple_subgraph( name = "face_landmark_front_gpu", graph = "face_landmark_front_gpu.pbtxt", diff --git a/mediapipe/modules/face_landmark/face_landmark_front_onnx_cuda.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_onnx_cuda.pbtxt new file mode 100644 index 000000000..fa1283b14 --- /dev/null +++ b/mediapipe/modules/face_landmark/face_landmark_front_onnx_cuda.pbtxt @@ -0,0 +1,247 @@ +# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is +# executed with onnxruntime on cuda.) This graph tries to skip face detection as much as possible +# by using previously detected/predicted landmarks for new images. +# +# It is required that "face_detection_short_range.onnxruntime" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.onnxruntime" +# path during execution. +# +# It is required that "face_landmark.onnxruntime" is available at +# "mediapipe/modules/face_landmark/face_landmark.onnxruntime" +# path during execution if `with_attention` is not set or set to `false`. +# +# It is required that "face_landmark_with_attention.onnxruntime" is available at +# "mediapipe/modules/face_landmark/face_landmark_with_attention.onnxruntime" +# path during execution if `with_attention` is set to `true`. +# +# EXAMPLE: +# node { +# calculator: "FaceLandmarkFrontOnnxCUDA" +# input_stream: "IMAGE:image" +# input_side_packet: "NUM_FACES:num_faces" +# input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks" +# input_side_packet: "WITH_ATTENTION:with_attention" +# output_stream: "LANDMARKS:multi_face_landmarks" +# } + +type: "FaceLandmarkFrontOnnxCUDA" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:image" + +# Max number of faces to detect/track. (int) +input_side_packet: "NUM_FACES:num_faces" + +# Whether landmarks on the previous image should be used to help localize +# landmarks on the current image. (bool) +input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks" + +# Whether to run face mesh model with attention on lips and eyes. (bool) +# Attention provides more accuracy on lips and eye regions as well as iris +# landmarks. +input_side_packet: "WITH_ATTENTION:with_attention" + +# Collection of detected/predicted faces, each represented as a list of 468 face +# landmarks. (std::vector) +# NOTE: there will not be an output packet in the LANDMARKS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:multi_face_landmarks" + +# Extra outputs (for debugging, for instance). +# Detected faces. (std::vector) +output_stream: "DETECTIONS:face_detections" +# Regions of interest calculated based on landmarks. +# (std::vector) +output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" +# Regions of interest calculated based on face detections. +# (std::vector) +output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" + +# When the optional input side packet "use_prev_landmarks" is either absent or +# set to true, uses the landmarks on the previous image to help localize +# landmarks on the current image. +node { + calculator: "GateCalculator" + input_side_packet: "ALLOW:use_prev_landmarks" + input_stream: "prev_face_rects_from_landmarks" + output_stream: "gated_prev_face_rects_from_landmarks" + options: { + [mediapipe.GateCalculatorOptions.ext] { + allow: true + } + } +} + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided num_faces. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:gated_prev_face_rects_from_landmarks" + input_side_packet: "num_faces" + output_stream: "prev_has_enough_faces" +} + +# Drops the incoming image if enough faces have already been identified from the +# previous image. Otherwise, passes the incoming image through to trigger a new +# round of face detection. +node { + calculator: "GateCalculator" + input_stream: "image" + input_stream: "DISALLOW:prev_has_enough_faces" + output_stream: "gated_image" + options: { + [mediapipe.GateCalculatorOptions.ext] { + empty_packets_as_allow: true + } + } +} + +# Detects faces. +node { + calculator: "FaceDetectionShortRangeOnnxCUDA" + input_stream: "IMAGE:gated_image" + output_stream: "DETECTIONS:all_face_detections" +} + +# Makes sure there are no more detections than the provided num_faces. +node { + calculator: "ClipDetectionVectorSizeCalculator" + input_stream: "all_face_detections" + output_stream: "face_detections" + input_side_packet: "num_faces" +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:gated_image" + output_stream: "SIZE:gated_image_size" +} + +# Outputs each element of face_detections at a fake timestamp for the rest of +# the graph to process. Clones the image size packet for each face_detection at +# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp +# for downstream calculators to inform them that all elements in the vector have +# been processed. +node { + calculator: "BeginLoopDetectionCalculator" + input_stream: "ITERABLE:face_detections" + input_stream: "CLONE:gated_image_size" + output_stream: "ITEM:face_detection" + output_stream: "CLONE:detections_loop_image_size" + output_stream: "BATCH_END:detections_loop_end_timestamp" +} + +# Calculates region of interest based on face detections, so that can be used +# to detect landmarks. +node { + calculator: "FaceDetectionFrontDetectionToRoi" + input_stream: "DETECTION:face_detection" + input_stream: "IMAGE_SIZE:detections_loop_image_size" + output_stream: "ROI:face_rect_from_detection" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_detection" + input_stream: "BATCH_END:detections_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_detections" +} + +# Performs association between NormalizedRect vector elements from previous +# image and rects based on face detections from the current image. This +# calculator ensures that the output face_rects vector doesn't contain +# overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "face_rects_from_detections" + input_stream: "gated_prev_face_rects_from_landmarks" + output_stream: "face_rects" + options: { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.5 + } + } +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:image" + output_stream: "SIZE:image_size" +} + +# Outputs each element of face_rects at a fake timestamp for the rest of the +# graph to process. Clones image and image size packets for each +# single_face_rect at the fake timestamp. At the end of the loop, outputs the +# BATCH_END timestamp for downstream calculators to inform them that all +# elements in the vector have been processed. +node { + calculator: "BeginLoopNormalizedRectCalculator" + input_stream: "ITERABLE:face_rects" + input_stream: "CLONE:0:image" + input_stream: "CLONE:1:image_size" + output_stream: "ITEM:face_rect" + output_stream: "CLONE:0:landmarks_loop_image" + output_stream: "CLONE:1:landmarks_loop_image_size" + output_stream: "BATCH_END:landmarks_loop_end_timestamp" +} + +# Detects face landmarks within specified region of interest of the image. +node { + calculator: "FaceLandmarkOnnxCUDA" + input_stream: "IMAGE:landmarks_loop_image" + input_stream: "ROI:face_rect" + input_side_packet: "WITH_ATTENTION:with_attention" + output_stream: "LANDMARKS:face_landmarks" +} + +# Calculates region of interest based on face landmarks, so that can be reused +# for subsequent image. +node { + calculator: "FaceLandmarkLandmarksToRoi" + input_stream: "LANDMARKS:face_landmarks" + input_stream: "IMAGE_SIZE:landmarks_loop_image_size" + output_stream: "ROI:face_rect_from_landmarks" +} + +# Collects a set of landmarks for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedLandmarkListVectorCalculator" + input_stream: "ITEM:face_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:multi_face_landmarks" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_landmarks" +} + +# Caches face rects calculated from landmarks, and upon the arrival of the next +# input image, sends out the cached rects with timestamps replaced by that of +# the input image, essentially generating a packet that carries the previous +# face rects. Note that upon the arrival of the very first input image, a +# timestamp bound update occurs to jump start the feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:image" + input_stream: "LOOP:face_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_face_rects_from_landmarks" +} diff --git a/mediapipe/modules/face_landmark/face_landmark_front_onnx_tensorrt.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_onnx_tensorrt.pbtxt new file mode 100644 index 000000000..fca5f7105 --- /dev/null +++ b/mediapipe/modules/face_landmark/face_landmark_front_onnx_tensorrt.pbtxt @@ -0,0 +1,247 @@ +# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is +# executed with onnxruntime on tensorrt.) This graph tries to skip face detection as much as possible +# by using previously detected/predicted landmarks for new images. +# +# It is required that "face_detection_short_range.onnxruntime" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.onnxruntime" +# path during execution. +# +# It is required that "face_landmark.onnxruntime" is available at +# "mediapipe/modules/face_landmark/face_landmark.onnxruntime" +# path during execution if `with_attention` is not set or set to `false`. +# +# It is required that "face_landmark_with_attention.onnxruntime" is available at +# "mediapipe/modules/face_landmark/face_landmark_with_attention.onnxruntime" +# path during execution if `with_attention` is set to `true`. +# +# EXAMPLE: +# node { +# calculator: "FaceLandmarkFrontTensorRT" +# input_stream: "IMAGE:image" +# input_side_packet: "NUM_FACES:num_faces" +# input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks" +# input_side_packet: "WITH_ATTENTION:with_attention" +# output_stream: "LANDMARKS:multi_face_landmarks" +# } + +type: "FaceLandmarkFrontTensorRT" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:image" + +# Max number of faces to detect/track. (int) +input_side_packet: "NUM_FACES:num_faces" + +# Whether landmarks on the previous image should be used to help localize +# landmarks on the current image. (bool) +input_side_packet: "USE_PREV_LANDMARKS:use_prev_landmarks" + +# Whether to run face mesh model with attention on lips and eyes. (bool) +# Attention provides more accuracy on lips and eye regions as well as iris +# landmarks. +input_side_packet: "WITH_ATTENTION:with_attention" + +# Collection of detected/predicted faces, each represented as a list of 468 face +# landmarks. (std::vector) +# NOTE: there will not be an output packet in the LANDMARKS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:multi_face_landmarks" + +# Extra outputs (for debugging, for instance). +# Detected faces. (std::vector) +output_stream: "DETECTIONS:face_detections" +# Regions of interest calculated based on landmarks. +# (std::vector) +output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" +# Regions of interest calculated based on face detections. +# (std::vector) +output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" + +# When the optional input side packet "use_prev_landmarks" is either absent or +# set to true, uses the landmarks on the previous image to help localize +# landmarks on the current image. +node { + calculator: "GateCalculator" + input_side_packet: "ALLOW:use_prev_landmarks" + input_stream: "prev_face_rects_from_landmarks" + output_stream: "gated_prev_face_rects_from_landmarks" + options: { + [mediapipe.GateCalculatorOptions.ext] { + allow: true + } + } +} + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided num_faces. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:gated_prev_face_rects_from_landmarks" + input_side_packet: "num_faces" + output_stream: "prev_has_enough_faces" +} + +# Drops the incoming image if enough faces have already been identified from the +# previous image. Otherwise, passes the incoming image through to trigger a new +# round of face detection. +node { + calculator: "GateCalculator" + input_stream: "image" + input_stream: "DISALLOW:prev_has_enough_faces" + output_stream: "gated_image" + options: { + [mediapipe.GateCalculatorOptions.ext] { + empty_packets_as_allow: true + } + } +} + +# Detects faces. +node { + calculator: "FaceDetectionShortRangeOnnxTensorRT" + input_stream: "IMAGE:gated_image" + output_stream: "DETECTIONS:all_face_detections" +} + +# Makes sure there are no more detections than the provided num_faces. +node { + calculator: "ClipDetectionVectorSizeCalculator" + input_stream: "all_face_detections" + output_stream: "face_detections" + input_side_packet: "num_faces" +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:gated_image" + output_stream: "SIZE:gated_image_size" +} + +# Outputs each element of face_detections at a fake timestamp for the rest of +# the graph to process. Clones the image size packet for each face_detection at +# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp +# for downstream calculators to inform them that all elements in the vector have +# been processed. +node { + calculator: "BeginLoopDetectionCalculator" + input_stream: "ITERABLE:face_detections" + input_stream: "CLONE:gated_image_size" + output_stream: "ITEM:face_detection" + output_stream: "CLONE:detections_loop_image_size" + output_stream: "BATCH_END:detections_loop_end_timestamp" +} + +# Calculates region of interest based on face detections, so that can be used +# to detect landmarks. +node { + calculator: "FaceDetectionFrontDetectionToRoi" + input_stream: "DETECTION:face_detection" + input_stream: "IMAGE_SIZE:detections_loop_image_size" + output_stream: "ROI:face_rect_from_detection" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_detection" + input_stream: "BATCH_END:detections_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_detections" +} + +# Performs association between NormalizedRect vector elements from previous +# image and rects based on face detections from the current image. This +# calculator ensures that the output face_rects vector doesn't contain +# overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "face_rects_from_detections" + input_stream: "gated_prev_face_rects_from_landmarks" + output_stream: "face_rects" + options: { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.5 + } + } +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:image" + output_stream: "SIZE:image_size" +} + +# Outputs each element of face_rects at a fake timestamp for the rest of the +# graph to process. Clones image and image size packets for each +# single_face_rect at the fake timestamp. At the end of the loop, outputs the +# BATCH_END timestamp for downstream calculators to inform them that all +# elements in the vector have been processed. +node { + calculator: "BeginLoopNormalizedRectCalculator" + input_stream: "ITERABLE:face_rects" + input_stream: "CLONE:0:image" + input_stream: "CLONE:1:image_size" + output_stream: "ITEM:face_rect" + output_stream: "CLONE:0:landmarks_loop_image" + output_stream: "CLONE:1:landmarks_loop_image_size" + output_stream: "BATCH_END:landmarks_loop_end_timestamp" +} + +# Detects face landmarks within specified region of interest of the image. +node { + calculator: "FaceLandmarkOnnxTensorRT" + input_stream: "IMAGE:landmarks_loop_image" + input_stream: "ROI:face_rect" + input_side_packet: "WITH_ATTENTION:with_attention" + output_stream: "LANDMARKS:face_landmarks" +} + +# Calculates region of interest based on face landmarks, so that can be reused +# for subsequent image. +node { + calculator: "FaceLandmarkLandmarksToRoi" + input_stream: "LANDMARKS:face_landmarks" + input_stream: "IMAGE_SIZE:landmarks_loop_image_size" + output_stream: "ROI:face_rect_from_landmarks" +} + +# Collects a set of landmarks for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedLandmarkListVectorCalculator" + input_stream: "ITEM:face_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:multi_face_landmarks" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_landmarks" +} + +# Caches face rects calculated from landmarks, and upon the arrival of the next +# input image, sends out the cached rects with timestamps replaced by that of +# the input image, essentially generating a packet that carries the previous +# face rects. Note that upon the arrival of the very first input image, a +# timestamp bound update occurs to jump start the feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:image" + input_stream: "LOOP:face_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_face_rects_from_landmarks" +} diff --git a/mediapipe/modules/face_landmark/face_landmark_onnx_cuda.pbtxt b/mediapipe/modules/face_landmark/face_landmark_onnx_cuda.pbtxt new file mode 100644 index 000000000..a006ea15f --- /dev/null +++ b/mediapipe/modules/face_landmark/face_landmark_onnx_cuda.pbtxt @@ -0,0 +1,166 @@ +# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is +# executed with onnxruntime on cuda.) +# +# It is required that "face_landmark.onnx" is available at +# "mediapipe/modules/face_landmark/face_landmark.onnx" +# path during execution if `with_attention` is not set or set to `false`. +# +# It is required that "face_landmark_with_attention.onnx" is available at +# "mediapipe/modules/face_landmark/face_landmark_with_attention.onnx" +# path during execution if `with_attention` is set to `true`. +# +# EXAMPLE: +# node { +# calculator: "FaceLandmarkOnnxCUDA" +# input_stream: "IMAGE:image" +# input_stream: "ROI:face_roi" +# input_side_packet: "WITH_ATTENTION:with_attention" +# output_stream: "LANDMARKS:face_landmarks" +# } + +type: "FaceLandmarkOnnxCUDA" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:image" +# ROI (region of interest) within the given image where a face is located. +# (NormalizedRect) +input_stream: "ROI:roi" +# Whether to run face mesh model with attention on lips and eyes. (bool) +# Attention provides more accuracy on lips and eye regions as well as iris +# landmarks. +input_side_packet: "WITH_ATTENTION:with_attention" + +# 468 or 478 facial landmarks within the given ROI. (NormalizedLandmarkList) +# +# Number of landmarks depends on the WITH_ATTENTION flag. If it's `true` - then +# there will be 478 landmarks with refined lips, eyes and irises (10 extra +# landmarks are for irises), otherwise 468 non-refined landmarks are returned. +# +# NOTE: if a face is not present within the given ROI, for this particular +# timestamp there will not be an output packet in the LANDMARKS stream. However, +# the MediaPipe framework will internally inform the downstream calculators of +# the absence of this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:face_landmarks" + +# Transforms the input image into a 192x192 tensor. +node: { + calculator: "ImageToTensorCalculator" + input_stream: "IMAGE:image" + input_stream: "NORM_RECT:roi" + output_stream: "TENSORS:input_tensors" + options: { + [mediapipe.ImageToTensorCalculatorOptions.ext] { + output_tensor_width: 192 + output_tensor_height: 192 + output_tensor_float_range { + min: 0.0 + max: 1.0 + } + } + } +} + +node { + calculator: "InferenceCalculator" + input_stream: "TENSORS:input_tensors" + output_stream: "TENSORS:output_tensors" + options: { + [mediapipe.InferenceCalculatorOptions.ext] { + delegate { cuda {} } + model_path: "mediapipe/modules/face_landmark/face_landmark.onnx" + } + } +} + +# Splits a vector of tensors into landmark tensors and face flag tensor. +node { + calculator: "SwitchContainer" + input_side_packet: "ENABLE:with_attention" + input_stream: "output_tensors" + output_stream: "landmark_tensors" + output_stream: "face_flag_tensor" + options: { + [mediapipe.SwitchContainerOptions.ext] { + contained_node: { + calculator: "SplitTensorVectorCalculator" + options: { + [mediapipe.SplitVectorCalculatorOptions.ext] { + ranges: { begin: 0 end: 1 } + ranges: { begin: 1 end: 2 } + } + } + } + contained_node: { + calculator: "SplitTensorVectorCalculator" + options: { + [mediapipe.SplitVectorCalculatorOptions.ext] { + ranges: { begin: 0 end: 6 } + ranges: { begin: 6 end: 7 } + } + } + } + } + } +} + +# Converts the face-flag tensor into a float that represents the confidence +# score of face presence. +node { + calculator: "TensorsToFloatsCalculator" + input_stream: "TENSORS:face_flag_tensor" + output_stream: "FLOAT:face_presence_score" + options { + [mediapipe.TensorsToFloatsCalculatorOptions.ext] { + activation: SIGMOID + } + } +} + +# Applies a threshold to the confidence score to determine whether a face is +# present. +node { + calculator: "ThresholdingCalculator" + input_stream: "FLOAT:face_presence_score" + output_stream: "FLAG:face_presence" + options: { + [mediapipe.ThresholdingCalculatorOptions.ext] { + threshold: 0.5 + } + } +} + +# Drop landmarks tensors if face is not present. +node { + calculator: "GateCalculator" + input_stream: "landmark_tensors" + input_stream: "ALLOW:face_presence" + output_stream: "ensured_landmark_tensors" +} + +# Decodes the landmark tensors into a vector of landmarks, where the landmark +# coordinates are normalized by the size of the input image to the model. +node { + calculator: "SwitchContainer" + input_side_packet: "ENABLE:with_attention" + input_stream: "TENSORS:ensured_landmark_tensors" + output_stream: "LANDMARKS:landmarks" + options: { + [mediapipe.SwitchContainerOptions.ext] { + contained_node: { + calculator: "TensorsToFaceLandmarks" + } + contained_node: { + calculator: "TensorsToFaceLandmarksWithAttention" + } + } + } +} + +# Projects the landmarks from the cropped face image to the corresponding +# locations on the full image before cropping (input to the graph). +node { + calculator: "LandmarkProjectionCalculator" + input_stream: "NORM_LANDMARKS:landmarks" + input_stream: "NORM_RECT:roi" + output_stream: "NORM_LANDMARKS:face_landmarks" +} diff --git a/mediapipe/modules/face_landmark/face_landmark_onnx_tensorrt.pbtxt b/mediapipe/modules/face_landmark/face_landmark_onnx_tensorrt.pbtxt new file mode 100644 index 000000000..37af474c8 --- /dev/null +++ b/mediapipe/modules/face_landmark/face_landmark_onnx_tensorrt.pbtxt @@ -0,0 +1,166 @@ +# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is +# executed with onnxruntime on TensorRT.) +# +# It is required that "face_landmark.onnx" is available at +# "mediapipe/modules/face_landmark/face_landmark.onnx" +# path during execution if `with_attention` is not set or set to `false`. +# +# It is required that "face_landmark_with_attention.onnx" is available at +# "mediapipe/modules/face_landmark/face_landmark_with_attention.onnx" +# path during execution if `with_attention` is set to `true`. +# +# EXAMPLE: +# node { +# calculator: "FaceLandmarkOnnxTensorrt" +# input_stream: "IMAGE:image" +# input_stream: "ROI:face_roi" +# input_side_packet: "WITH_ATTENTION:with_attention" +# output_stream: "LANDMARKS:face_landmarks" +# } + +type: "FaceLandmarkOnnxTensorrt" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:image" +# ROI (region of interest) within the given image where a face is located. +# (NormalizedRect) +input_stream: "ROI:roi" +# Whether to run face mesh model with attention on lips and eyes. (bool) +# Attention provides more accuracy on lips and eye regions as well as iris +# landmarks. +input_side_packet: "WITH_ATTENTION:with_attention" + +# 468 or 478 facial landmarks within the given ROI. (NormalizedLandmarkList) +# +# Number of landmarks depends on the WITH_ATTENTION flag. If it's `true` - then +# there will be 478 landmarks with refined lips, eyes and irises (10 extra +# landmarks are for irises), otherwise 468 non-refined landmarks are returned. +# +# NOTE: if a face is not present within the given ROI, for this particular +# timestamp there will not be an output packet in the LANDMARKS stream. However, +# the MediaPipe framework will internally inform the downstream calculators of +# the absence of this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:face_landmarks" + +# Transforms the input image into a 192x192 tensor. +node: { + calculator: "ImageToTensorCalculator" + input_stream: "IMAGE:image" + input_stream: "NORM_RECT:roi" + output_stream: "TENSORS:input_tensors" + options: { + [mediapipe.ImageToTensorCalculatorOptions.ext] { + output_tensor_width: 192 + output_tensor_height: 192 + output_tensor_float_range { + min: 0.0 + max: 1.0 + } + } + } +} + +node { + calculator: "InferenceCalculator" + input_stream: "TENSORS:input_tensors" + output_stream: "TENSORS:output_tensors" + options: { + [mediapipe.InferenceCalculatorOptions.ext] { + delegate { tensorrt {} } + model_path: "mediapipe/modules/face_landmark/face_landmark.onnx" + } + } +} + +# Splits a vector of tensors into landmark tensors and face flag tensor. +node { + calculator: "SwitchContainer" + input_side_packet: "ENABLE:with_attention" + input_stream: "output_tensors" + output_stream: "landmark_tensors" + output_stream: "face_flag_tensor" + options: { + [mediapipe.SwitchContainerOptions.ext] { + contained_node: { + calculator: "SplitTensorVectorCalculator" + options: { + [mediapipe.SplitVectorCalculatorOptions.ext] { + ranges: { begin: 0 end: 1 } + ranges: { begin: 1 end: 2 } + } + } + } + contained_node: { + calculator: "SplitTensorVectorCalculator" + options: { + [mediapipe.SplitVectorCalculatorOptions.ext] { + ranges: { begin: 0 end: 6 } + ranges: { begin: 6 end: 7 } + } + } + } + } + } +} + +# Converts the face-flag tensor into a float that represents the confidence +# score of face presence. +node { + calculator: "TensorsToFloatsCalculator" + input_stream: "TENSORS:face_flag_tensor" + output_stream: "FLOAT:face_presence_score" + options { + [mediapipe.TensorsToFloatsCalculatorOptions.ext] { + activation: SIGMOID + } + } +} + +# Applies a threshold to the confidence score to determine whether a face is +# present. +node { + calculator: "ThresholdingCalculator" + input_stream: "FLOAT:face_presence_score" + output_stream: "FLAG:face_presence" + options: { + [mediapipe.ThresholdingCalculatorOptions.ext] { + threshold: 0.5 + } + } +} + +# Drop landmarks tensors if face is not present. +node { + calculator: "GateCalculator" + input_stream: "landmark_tensors" + input_stream: "ALLOW:face_presence" + output_stream: "ensured_landmark_tensors" +} + +# Decodes the landmark tensors into a vector of landmarks, where the landmark +# coordinates are normalized by the size of the input image to the model. +node { + calculator: "SwitchContainer" + input_side_packet: "ENABLE:with_attention" + input_stream: "TENSORS:ensured_landmark_tensors" + output_stream: "LANDMARKS:landmarks" + options: { + [mediapipe.SwitchContainerOptions.ext] { + contained_node: { + calculator: "TensorsToFaceLandmarks" + } + contained_node: { + calculator: "TensorsToFaceLandmarksWithAttention" + } + } + } +} + +# Projects the landmarks from the cropped face image to the corresponding +# locations on the full image before cropping (input to the graph). +node { + calculator: "LandmarkProjectionCalculator" + input_stream: "NORM_LANDMARKS:landmarks" + input_stream: "NORM_RECT:roi" + output_stream: "NORM_LANDMARKS:face_landmarks" +}