From fd7f357c18712896c3725b40f19b991aa15b255f Mon Sep 17 00:00:00 2001 From: dmaletskiy Date: Thu, 1 Jul 2021 12:51:14 +0300 Subject: [PATCH] feat: Added face mesh with face counter example Change List: - added face counter with "clock" (trigger that allow to thrack all input events) - face counter can be used for checking whether face was detected (can be used as flag to get face landmarks from ouput stream) --- mediapipe/calculators/util/BUILD | 14 + .../util/counting_vector_size_calculator.cc | 26 ++ .../util/counting_vector_size_calculator.h | 79 ++++++ mediapipe/modules/face_landmark/BUILD | 22 ++ ...landmark_front_cpu_with_face_counter.pbtxt | 249 ++++++++++++++++++ 5 files changed, 390 insertions(+) create mode 100644 mediapipe/calculators/util/counting_vector_size_calculator.cc create mode 100644 mediapipe/calculators/util/counting_vector_size_calculator.h create mode 100644 mediapipe/modules/face_landmark/face_landmark_front_cpu_with_face_counter.pbtxt diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD index e759ff990..869b4387e 100644 --- a/mediapipe/calculators/util/BUILD +++ b/mediapipe/calculators/util/BUILD @@ -18,6 +18,20 @@ licenses(["notice"]) package(default_visibility = ["//visibility:public"]) +cc_library( + name = "counting_vector_size_calculator", + srcs = ["counting_vector_size_calculator.cc"], + hdrs = ["counting_vector_size_calculator.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:landmark_cc_proto", + ], + alwayslink = 1, +) + cc_library( name = "alignment_points_to_rects_calculator", srcs = ["alignment_points_to_rects_calculator.cc"], diff --git a/mediapipe/calculators/util/counting_vector_size_calculator.cc b/mediapipe/calculators/util/counting_vector_size_calculator.cc new file mode 100644 index 000000000..c2203686f --- /dev/null +++ b/mediapipe/calculators/util/counting_vector_size_calculator.cc @@ -0,0 +1,26 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/util/counting_vector_size_calculator.h" + +#include "mediapipe/framework/formats/landmark.pb.h" + +namespace mediapipe { + +typedef CountingVectorSizeCalculator< + std::vector<::mediapipe::NormalizedLandmarkList>> + CountingNormalizedLandmarkListVectorSizeCalculator; + +REGISTER_CALCULATOR(CountingNormalizedLandmarkListVectorSizeCalculator); +} // namespace mediapipe diff --git a/mediapipe/calculators/util/counting_vector_size_calculator.h b/mediapipe/calculators/util/counting_vector_size_calculator.h new file mode 100644 index 000000000..4921d3c27 --- /dev/null +++ b/mediapipe/calculators/util/counting_vector_size_calculator.h @@ -0,0 +1,79 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H +#define MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/landmark.pb.h" + +namespace mediapipe { + +// A calculator that counts the size of the input vector. It was created to +// aid in polling packets in the output stream synchronously. If there is +// a clock stream, it will output a value of 0 even if the input vector stream +// is empty. If not, it will output some value only if there is an input vector. +// The clock stream must have the same time stamp as the vector stream, and +// it must be the stream where packets are transmitted while the graph is +// running. (e.g. Any input stream of graph) +// +// It is designed to be used like: +// +// Example config: +// node { +// calculator: "CountingWithVectorSizeCalculator" +// input_stream: "CLOCK:triger_signal" +// input_stream: "VECTOR:input_vector" +// output_stream: "COUNT:vector_count" +// } +// +// node { +// calculator: "CountingWithVectorSizeCalculator" +// input_stream: "VECTOR:input_vector" +// output_stream: "COUNT:vector_count" +// } + +template +class CountingVectorSizeCalculator : public CalculatorBase { +public: + static ::mediapipe::Status GetContract(CalculatorContract *cc) { + if (cc->Inputs().HasTag("CLOCK")) { + cc->Inputs().Tag("CLOCK").SetAny(); + } + + RET_CHECK(cc->Inputs().HasTag("VECTOR")); + cc->Inputs().Tag("VECTOR").Set(); + RET_CHECK(cc->Outputs().HasTag("COUNT")); + cc->Outputs().Tag("COUNT").Set(); + + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext *cc) { + std::unique_ptr face_count; + if (!cc->Inputs().Tag("VECTOR").IsEmpty()) { + const auto &landmarks = cc->Inputs().Tag("VECTOR").Get(); + face_count = absl::make_unique(landmarks.size()); + } else { + face_count = absl::make_unique(0); + } + cc->Outputs().Tag("COUNT").Add(face_count.release(), cc->InputTimestamp()); + + return ::mediapipe::OkStatus(); + }; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H diff --git a/mediapipe/modules/face_landmark/BUILD b/mediapipe/modules/face_landmark/BUILD index 77560022e..30720c1b0 100644 --- a/mediapipe/modules/face_landmark/BUILD +++ b/mediapipe/modules/face_landmark/BUILD @@ -74,6 +74,28 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "face_landmark_front_cpu_with_face_counter", + graph = "face_landmark_front_cpu_with_face_counter.pbtxt", + register_as = "FaceLandmarkFrontCpuWithFaceCounter", + deps = [ + ":face_detection_front_detection_to_roi", + ":face_landmark_cpu", + ":face_landmark_landmarks_to_roi", + "//mediapipe/calculators/core:begin_loop_calculator", + "//mediapipe/calculators/core:clip_vector_size_calculator", + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:end_loop_calculator", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:previous_loopback_calculator", + "//mediapipe/calculators/image:image_properties_calculator", + "//mediapipe/calculators/util:association_norm_rect_calculator", + "//mediapipe/calculators/util:collection_has_min_size_calculator", + "//mediapipe/calculators/util:counting_vector_size_calculator", + "//mediapipe/modules/face_detection:face_detection_short_range_cpu", + ], +) + mediapipe_simple_subgraph( name = "face_landmark_front_gpu", graph = "face_landmark_front_gpu.pbtxt", diff --git a/mediapipe/modules/face_landmark/face_landmark_front_cpu_with_face_counter.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_cpu_with_face_counter.pbtxt new file mode 100644 index 000000000..5389a8293 --- /dev/null +++ b/mediapipe/modules/face_landmark/face_landmark_front_cpu_with_face_counter.pbtxt @@ -0,0 +1,249 @@ +# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is +# executed on CPU.) This graph tries to skip face detection as much as possible +# by using previously detected/predicted landmarks for new images. +# +# It is required that "face_detection_short_range.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.tflite" +# path during execution. +# +# It is required that "face_landmark.tflite" is available at +# "mediapipe/modules/face_landmark/face_landmark.tflite" +# path during execution. +# +# EXAMPLE: +# node { +# calculator: "FaceLandmarkFrontCpu" +# input_stream: "IMAGE:image" +# input_side_packet: "NUM_FACES:num_faces" +# output_stream: "LANDMARKS:multi_face_landmarks" +# } + +type: "FaceLandmarkFrontCpu" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:image" + +# Max number of faces to detect/track. (int) +input_side_packet: "NUM_FACES:num_faces" + +# Collection of detected/predicted faces, each represented as a list of 468 face +# landmarks. (std::vector) +# NOTE: there will not be an output packet in the LANDMARKS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:multi_face_landmarks" + +# Extra outputs (for debugging, for instance). +# Detected faces. (std::vector) +output_stream: "DETECTIONS:face_detections" +# Regions of interest calculated based on landmarks. +# (std::vector) +output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" +# Regions of interest calculated based on face detections. +# (std::vector) +output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" + +# (int) +output_stream: "FACE_COUNT_FROM_LANDMARKS:face_count" + + +# Defines whether landmarks on the previous image should be used to help +# localize landmarks on the current image. +node { + name: "ConstantSidePacketCalculator" + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:use_prev_landmarks" + options: { + [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { + packet { bool_value: true } + } + } +} +node { + calculator: "GateCalculator" + input_side_packet: "ALLOW:use_prev_landmarks" + input_stream: "prev_face_rects_from_landmarks" + output_stream: "gated_prev_face_rects_from_landmarks" +} + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided num_faces. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:gated_prev_face_rects_from_landmarks" + input_side_packet: "num_faces" + output_stream: "prev_has_enough_faces" +} + +# Drops the incoming image if enough faces have already been identified from the +# previous image. Otherwise, passes the incoming image through to trigger a new +# round of face detection. +node { + calculator: "GateCalculator" + input_stream: "image" + input_stream: "DISALLOW:prev_has_enough_faces" + output_stream: "gated_image" + options: { + [mediapipe.GateCalculatorOptions.ext] { + empty_packets_as_allow: true + } + } +} + +# Detects faces. +node { + calculator: "FaceDetectionShortRangeCpu" + input_stream: "IMAGE:gated_image" + output_stream: "DETECTIONS:all_face_detections" +} + +# Makes sure there are no more detections than the provided num_faces. +node { + calculator: "ClipDetectionVectorSizeCalculator" + input_stream: "all_face_detections" + output_stream: "face_detections" + input_side_packet: "num_faces" +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:gated_image" + output_stream: "SIZE:gated_image_size" +} + +# Outputs each element of face_detections at a fake timestamp for the rest of +# the graph to process. Clones the image size packet for each face_detection at +# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp +# for downstream calculators to inform them that all elements in the vector have +# been processed. +node { + calculator: "BeginLoopDetectionCalculator" + input_stream: "ITERABLE:face_detections" + input_stream: "CLONE:gated_image_size" + output_stream: "ITEM:face_detection" + output_stream: "CLONE:detections_loop_image_size" + output_stream: "BATCH_END:detections_loop_end_timestamp" +} + +# Calculates region of interest based on face detections, so that can be used +# to detect landmarks. +node { + calculator: "FaceDetectionFrontDetectionToRoi" + input_stream: "DETECTION:face_detection" + input_stream: "IMAGE_SIZE:detections_loop_image_size" + output_stream: "ROI:face_rect_from_detection" +} + +# Counting a multi_faceLandmarks vector size. The image stream is only used to +# make the calculator work even when there is no input vector. +node { + calculator: "CountingNormalizedLandmarkListVectorSizeCalculator" + input_stream: "CLOCK:image" + input_stream: "VECTOR:multi_face_landmarks" + output_stream: "COUNT:face_count" +} + + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_detection" + input_stream: "BATCH_END:detections_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_detections" +} + +# Performs association between NormalizedRect vector elements from previous +# image and rects based on face detections from the current image. This +# calculator ensures that the output face_rects vector doesn't contain +# overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "face_rects_from_detections" + input_stream: "gated_prev_face_rects_from_landmarks" + output_stream: "face_rects" + options: { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.5 + } + } +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:image" + output_stream: "SIZE:image_size" +} + +# Outputs each element of face_rects at a fake timestamp for the rest of the +# graph to process. Clones image and image size packets for each +# single_face_rect at the fake timestamp. At the end of the loop, outputs the +# BATCH_END timestamp for downstream calculators to inform them that all +# elements in the vector have been processed. +node { + calculator: "BeginLoopNormalizedRectCalculator" + input_stream: "ITERABLE:face_rects" + input_stream: "CLONE:0:image" + input_stream: "CLONE:1:image_size" + output_stream: "ITEM:face_rect" + output_stream: "CLONE:0:landmarks_loop_image" + output_stream: "CLONE:1:landmarks_loop_image_size" + output_stream: "BATCH_END:landmarks_loop_end_timestamp" +} + +# Detects face landmarks within specified region of interest of the image. +node { + calculator: "FaceLandmarkCpu" + input_stream: "IMAGE:landmarks_loop_image" + input_stream: "ROI:face_rect" + output_stream: "LANDMARKS:face_landmarks" +} + +# Calculates region of interest based on face landmarks, so that can be reused +# for subsequent image. +node { + calculator: "FaceLandmarkLandmarksToRoi" + input_stream: "LANDMARKS:face_landmarks" + input_stream: "IMAGE_SIZE:landmarks_loop_image_size" + output_stream: "ROI:face_rect_from_landmarks" +} + +# Collects a set of landmarks for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedLandmarkListVectorCalculator" + input_stream: "ITEM:face_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:multi_face_landmarks" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_landmarks" +} + +# Caches face rects calculated from landmarks, and upon the arrival of the next +# input image, sends out the cached rects with timestamps replaced by that of +# the input image, essentially generating a packet that carries the previous +# face rects. Note that upon the arrival of the very first input image, a +# timestamp bound update occurs to jump start the feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:image" + input_stream: "LOOP:face_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_face_rects_from_landmarks" +}