feat: Added face mesh with face counter example

Change List:
- added face counter with "clock" (trigger that allow to thrack all input events)
- face counter can be used for checking whether face was detected (can be used as flag to get face landmarks from ouput stream)

mediapipe/calculators/util/BUILD

@@ -18,6 +18,20 @@ licenses(["notice"])
 
 package(default_visibility = ["//visibility:public"])
 
+cc_library(
+    name = "counting_vector_size_calculator",
+    srcs = ["counting_vector_size_calculator.cc"],
+    hdrs = ["counting_vector_size_calculator.h"],
+    visibility = [
+        "//visibility:public",
+    ],
+    deps = [
+        "//mediapipe/framework:calculator_framework",
+        "//mediapipe/framework/formats:landmark_cc_proto",
+    ],
+    alwayslink = 1,
+)
+
 cc_library(
     name = "alignment_points_to_rects_calculator",
     srcs = ["alignment_points_to_rects_calculator.cc"],

mediapipe/calculators/util/counting_vector_size_calculator.cc

@@ -0,0 +1,26 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/calculators/util/counting_vector_size_calculator.h"
+
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+namespace mediapipe {
+
+typedef CountingVectorSizeCalculator<
+    std::vector<::mediapipe::NormalizedLandmarkList>>
+    CountingNormalizedLandmarkListVectorSizeCalculator;
+
+REGISTER_CALCULATOR(CountingNormalizedLandmarkListVectorSizeCalculator);
+} // namespace mediapipe

mediapipe/calculators/util/counting_vector_size_calculator.h

@@ -0,0 +1,79 @@
+// Copyright 2020 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H
+#define MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H
+
+#include "mediapipe/framework/calculator_framework.h"
+#include "mediapipe/framework/formats/landmark.pb.h"
+
+namespace mediapipe {
+
+// A calculator that counts the size of the input vector. It was created to
+// aid in polling packets in the output stream synchronously. If there is
+// a clock stream, it will output a value of 0 even if the input vector stream
+// is empty. If not, it will output some value only if there is an input vector.
+// The clock stream must have the same time stamp as the vector stream, and
+// it must be the stream where packets are transmitted while the graph is
+// running. (e.g. Any input stream of graph)
+//
+// It is designed to be used like:
+//
+// Example config:
+// node {
+//   calculator: "CountingWithVectorSizeCalculator"
+//   input_stream: "CLOCK:triger_signal"
+//   input_stream: "VECTOR:input_vector"
+//   output_stream: "COUNT:vector_count"
+// }
+//
+// node {
+//   calculator: "CountingWithVectorSizeCalculator"
+//   input_stream: "VECTOR:input_vector"
+//   output_stream: "COUNT:vector_count"
+// }
+
+template <typename VectorT>
+class CountingVectorSizeCalculator : public CalculatorBase {
+public:
+  static ::mediapipe::Status GetContract(CalculatorContract *cc) {
+    if (cc->Inputs().HasTag("CLOCK")) {
+      cc->Inputs().Tag("CLOCK").SetAny();
+    }
+
+    RET_CHECK(cc->Inputs().HasTag("VECTOR"));
+    cc->Inputs().Tag("VECTOR").Set<VectorT>();
+    RET_CHECK(cc->Outputs().HasTag("COUNT"));
+    cc->Outputs().Tag("COUNT").Set<int>();
+
+    return ::mediapipe::OkStatus();
+  }
+
+  ::mediapipe::Status Process(CalculatorContext *cc) {
+    std::unique_ptr<int> face_count;
+    if (!cc->Inputs().Tag("VECTOR").IsEmpty()) {
+      const auto &landmarks = cc->Inputs().Tag("VECTOR").Get<VectorT>();
+      face_count = absl::make_unique<int>(landmarks.size());
+    } else {
+      face_count = absl::make_unique<int>(0);
+    }
+    cc->Outputs().Tag("COUNT").Add(face_count.release(), cc->InputTimestamp());
+
+    return ::mediapipe::OkStatus();
+  };
+};
+
+} // namespace mediapipe
+
+#endif // MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H

mediapipe/modules/face_landmark/BUILD

@@ -74,6 +74,28 @@ mediapipe_simple_subgraph(
     ],
 )
 
+mediapipe_simple_subgraph(
+    name = "face_landmark_front_cpu_with_face_counter",
+    graph = "face_landmark_front_cpu_with_face_counter.pbtxt",
+    register_as = "FaceLandmarkFrontCpuWithFaceCounter",
+    deps = [
+        ":face_detection_front_detection_to_roi",
+        ":face_landmark_cpu",
+        ":face_landmark_landmarks_to_roi",
+        "//mediapipe/calculators/core:begin_loop_calculator",
+        "//mediapipe/calculators/core:clip_vector_size_calculator",
+        "//mediapipe/calculators/core:constant_side_packet_calculator",
+        "//mediapipe/calculators/core:end_loop_calculator",
+        "//mediapipe/calculators/core:gate_calculator",
+        "//mediapipe/calculators/core:previous_loopback_calculator",
+        "//mediapipe/calculators/image:image_properties_calculator",
+        "//mediapipe/calculators/util:association_norm_rect_calculator",
+        "//mediapipe/calculators/util:collection_has_min_size_calculator",
+        "//mediapipe/calculators/util:counting_vector_size_calculator",
+        "//mediapipe/modules/face_detection:face_detection_short_range_cpu",
+    ],
+)
+
 mediapipe_simple_subgraph(
     name = "face_landmark_front_gpu",
     graph = "face_landmark_front_gpu.pbtxt",

mediapipe/modules/face_landmark/face_landmark_front_cpu_with_face_counter.pbtxt

@@ -0,0 +1,249 @@
+# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is
+# executed on CPU.) This graph tries to skip face detection as much as possible
+# by using previously detected/predicted landmarks for new images.
+#
+# It is required that "face_detection_short_range.tflite" is available at
+# "mediapipe/modules/face_detection/face_detection_short_range.tflite"
+# path during execution.
+#
+# It is required that "face_landmark.tflite" is available at
+# "mediapipe/modules/face_landmark/face_landmark.tflite"
+# path during execution.
+#
+# EXAMPLE:
+#   node {
+#     calculator: "FaceLandmarkFrontCpu"
+#     input_stream: "IMAGE:image"
+#     input_side_packet: "NUM_FACES:num_faces"
+#     output_stream: "LANDMARKS:multi_face_landmarks"
+#   }
+
+type: "FaceLandmarkFrontCpu"
+
+# CPU image. (ImageFrame)
+input_stream: "IMAGE:image"
+
+# Max number of faces to detect/track. (int)
+input_side_packet: "NUM_FACES:num_faces"
+
+# Collection of detected/predicted faces, each represented as a list of 468 face
+# landmarks. (std::vector<NormalizedLandmarkList>)
+# NOTE: there will not be an output packet in the LANDMARKS stream for this
+# particular timestamp if none of faces detected. However, the MediaPipe
+# framework will internally inform the downstream calculators of the absence of
+# this packet so that they don't wait for it unnecessarily.
+output_stream: "LANDMARKS:multi_face_landmarks"
+
+# Extra outputs (for debugging, for instance).
+# Detected faces. (std::vector<Detection>)
+output_stream: "DETECTIONS:face_detections"
+# Regions of interest calculated based on landmarks.
+# (std::vector<NormalizedRect>)
+output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks"
+# Regions of interest calculated based on face detections.
+# (std::vector<NormalizedRect>)
+output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections"
+
+# (int)
+output_stream: "FACE_COUNT_FROM_LANDMARKS:face_count"
+
+
+# Defines whether landmarks on the previous image should be used to help
+# localize landmarks on the current image.
+node {
+  name: "ConstantSidePacketCalculator"
+  calculator: "ConstantSidePacketCalculator"
+  output_side_packet: "PACKET:use_prev_landmarks"
+  options: {
+    [mediapipe.ConstantSidePacketCalculatorOptions.ext]: {
+      packet { bool_value: true }
+    }
+  }
+}
+node {
+  calculator: "GateCalculator"
+  input_side_packet: "ALLOW:use_prev_landmarks"
+  input_stream: "prev_face_rects_from_landmarks"
+  output_stream: "gated_prev_face_rects_from_landmarks"
+}
+
+# Determines if an input vector of NormalizedRect has a size greater than or
+# equal to the provided num_faces.
+node {
+  calculator: "NormalizedRectVectorHasMinSizeCalculator"
+  input_stream: "ITERABLE:gated_prev_face_rects_from_landmarks"
+  input_side_packet: "num_faces"
+  output_stream: "prev_has_enough_faces"
+}
+
+# Drops the incoming image if enough faces have already been identified from the
+# previous image. Otherwise, passes the incoming image through to trigger a new
+# round of face detection.
+node {
+  calculator: "GateCalculator"
+  input_stream: "image"
+  input_stream: "DISALLOW:prev_has_enough_faces"
+  output_stream: "gated_image"
+  options: {
+    [mediapipe.GateCalculatorOptions.ext] {
+      empty_packets_as_allow: true
+    }
+  }
+}
+
+# Detects faces.
+node {
+  calculator: "FaceDetectionShortRangeCpu"
+  input_stream: "IMAGE:gated_image"
+  output_stream: "DETECTIONS:all_face_detections"
+}
+
+# Makes sure there are no more detections than the provided num_faces.
+node {
+  calculator: "ClipDetectionVectorSizeCalculator"
+  input_stream: "all_face_detections"
+  output_stream: "face_detections"
+  input_side_packet: "num_faces"
+}
+
+# Calculate size of the image.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:gated_image"
+  output_stream: "SIZE:gated_image_size"
+}
+
+# Outputs each element of face_detections at a fake timestamp for the rest of
+# the graph to process. Clones the image size packet for each face_detection at
+# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp
+# for downstream calculators to inform them that all elements in the vector have
+# been processed.
+node {
+  calculator: "BeginLoopDetectionCalculator"
+  input_stream: "ITERABLE:face_detections"
+  input_stream: "CLONE:gated_image_size"
+  output_stream: "ITEM:face_detection"
+  output_stream: "CLONE:detections_loop_image_size"
+  output_stream: "BATCH_END:detections_loop_end_timestamp"
+}
+
+# Calculates region of interest based on face detections, so that can be used
+# to detect landmarks.
+node {
+  calculator: "FaceDetectionFrontDetectionToRoi"
+  input_stream: "DETECTION:face_detection"
+  input_stream: "IMAGE_SIZE:detections_loop_image_size"
+  output_stream: "ROI:face_rect_from_detection"
+}
+
+# Counting a multi_faceLandmarks vector size. The image stream is only used to 
+# make the calculator work even when there is no input vector.
+node {
+  calculator: "CountingNormalizedLandmarkListVectorSizeCalculator"
+  input_stream: "CLOCK:image"
+  input_stream: "VECTOR:multi_face_landmarks"
+  output_stream: "COUNT:face_count"
+}
+
+
+# Collects a NormalizedRect for each face into a vector. Upon receiving the
+# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
+# timestamp.
+node {
+  calculator: "EndLoopNormalizedRectCalculator"
+  input_stream: "ITEM:face_rect_from_detection"
+  input_stream: "BATCH_END:detections_loop_end_timestamp"
+  output_stream: "ITERABLE:face_rects_from_detections"
+}
+
+# Performs association between NormalizedRect vector elements from previous
+# image and rects based on face detections from the current image. This
+# calculator ensures that the output face_rects vector doesn't contain
+# overlapping regions based on the specified min_similarity_threshold.
+node {
+  calculator: "AssociationNormRectCalculator"
+  input_stream: "face_rects_from_detections"
+  input_stream: "gated_prev_face_rects_from_landmarks"
+  output_stream: "face_rects"
+  options: {
+    [mediapipe.AssociationCalculatorOptions.ext] {
+      min_similarity_threshold: 0.5
+    }
+  }
+}
+
+# Calculate size of the image.
+node {
+  calculator: "ImagePropertiesCalculator"
+  input_stream: "IMAGE:image"
+  output_stream: "SIZE:image_size"
+}
+
+# Outputs each element of face_rects at a fake timestamp for the rest of the
+# graph to process. Clones image and image size packets for each
+# single_face_rect at the fake timestamp. At the end of the loop, outputs the
+# BATCH_END timestamp for downstream calculators to inform them that all
+# elements in the vector have been processed.
+node {
+  calculator: "BeginLoopNormalizedRectCalculator"
+  input_stream: "ITERABLE:face_rects"
+  input_stream: "CLONE:0:image"
+  input_stream: "CLONE:1:image_size"
+  output_stream: "ITEM:face_rect"
+  output_stream: "CLONE:0:landmarks_loop_image"
+  output_stream: "CLONE:1:landmarks_loop_image_size"
+  output_stream: "BATCH_END:landmarks_loop_end_timestamp"
+}
+
+# Detects face landmarks within specified region of interest of the image.
+node {
+  calculator: "FaceLandmarkCpu"
+  input_stream: "IMAGE:landmarks_loop_image"
+  input_stream: "ROI:face_rect"
+  output_stream: "LANDMARKS:face_landmarks"
+}
+
+# Calculates region of interest based on face landmarks, so that can be reused
+# for subsequent image.
+node {
+  calculator: "FaceLandmarkLandmarksToRoi"
+  input_stream: "LANDMARKS:face_landmarks"
+  input_stream: "IMAGE_SIZE:landmarks_loop_image_size"
+  output_stream: "ROI:face_rect_from_landmarks"
+}
+
+# Collects a set of landmarks for each face into a vector. Upon receiving the
+# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END
+# timestamp.
+node {
+  calculator: "EndLoopNormalizedLandmarkListVectorCalculator"
+  input_stream: "ITEM:face_landmarks"
+  input_stream: "BATCH_END:landmarks_loop_end_timestamp"
+  output_stream: "ITERABLE:multi_face_landmarks"
+}
+
+# Collects a NormalizedRect for each face into a vector. Upon receiving the
+# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END
+# timestamp.
+node {
+  calculator: "EndLoopNormalizedRectCalculator"
+  input_stream: "ITEM:face_rect_from_landmarks"
+  input_stream: "BATCH_END:landmarks_loop_end_timestamp"
+  output_stream: "ITERABLE:face_rects_from_landmarks"
+}
+
+# Caches face rects calculated from landmarks, and upon the arrival of the next
+# input image, sends out the cached rects with timestamps replaced by that of
+# the input image, essentially generating a packet that carries the previous
+# face rects. Note that upon the arrival of the very first input image, a
+# timestamp bound update occurs to jump start the feedback loop.
+node {
+  calculator: "PreviousLoopbackCalculator"
+  input_stream: "MAIN:image"
+  input_stream: "LOOP:face_rects_from_landmarks"
+  input_stream_info: {
+    tag_index: "LOOP"
+    back_edge: true
+  }
+  output_stream: "PREV_LOOP:prev_face_rects_from_landmarks"
+}