From fd7f357c18712896c3725b40f19b991aa15b255f Mon Sep 17 00:00:00 2001 From: dmaletskiy Date: Thu, 1 Jul 2021 12:51:14 +0300 Subject: [PATCH 1/8] feat: Added face mesh with face counter example Change List: - added face counter with "clock" (trigger that allow to thrack all input events) - face counter can be used for checking whether face was detected (can be used as flag to get face landmarks from ouput stream) --- mediapipe/calculators/util/BUILD | 14 + .../util/counting_vector_size_calculator.cc | 26 ++ .../util/counting_vector_size_calculator.h | 79 ++++++ mediapipe/modules/face_landmark/BUILD | 22 ++ ...landmark_front_cpu_with_face_counter.pbtxt | 249 ++++++++++++++++++ 5 files changed, 390 insertions(+) create mode 100644 mediapipe/calculators/util/counting_vector_size_calculator.cc create mode 100644 mediapipe/calculators/util/counting_vector_size_calculator.h create mode 100644 mediapipe/modules/face_landmark/face_landmark_front_cpu_with_face_counter.pbtxt diff --git a/mediapipe/calculators/util/BUILD b/mediapipe/calculators/util/BUILD index e759ff990..869b4387e 100644 --- a/mediapipe/calculators/util/BUILD +++ b/mediapipe/calculators/util/BUILD @@ -18,6 +18,20 @@ licenses(["notice"]) package(default_visibility = ["//visibility:public"]) +cc_library( + name = "counting_vector_size_calculator", + srcs = ["counting_vector_size_calculator.cc"], + hdrs = ["counting_vector_size_calculator.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:landmark_cc_proto", + ], + alwayslink = 1, +) + cc_library( name = "alignment_points_to_rects_calculator", srcs = ["alignment_points_to_rects_calculator.cc"], diff --git a/mediapipe/calculators/util/counting_vector_size_calculator.cc b/mediapipe/calculators/util/counting_vector_size_calculator.cc new file mode 100644 index 000000000..c2203686f --- /dev/null +++ b/mediapipe/calculators/util/counting_vector_size_calculator.cc @@ -0,0 +1,26 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mediapipe/calculators/util/counting_vector_size_calculator.h" + +#include "mediapipe/framework/formats/landmark.pb.h" + +namespace mediapipe { + +typedef CountingVectorSizeCalculator< + std::vector<::mediapipe::NormalizedLandmarkList>> + CountingNormalizedLandmarkListVectorSizeCalculator; + +REGISTER_CALCULATOR(CountingNormalizedLandmarkListVectorSizeCalculator); +} // namespace mediapipe diff --git a/mediapipe/calculators/util/counting_vector_size_calculator.h b/mediapipe/calculators/util/counting_vector_size_calculator.h new file mode 100644 index 000000000..4921d3c27 --- /dev/null +++ b/mediapipe/calculators/util/counting_vector_size_calculator.h @@ -0,0 +1,79 @@ +// Copyright 2020 The MediaPipe Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H +#define MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H + +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/formats/landmark.pb.h" + +namespace mediapipe { + +// A calculator that counts the size of the input vector. It was created to +// aid in polling packets in the output stream synchronously. If there is +// a clock stream, it will output a value of 0 even if the input vector stream +// is empty. If not, it will output some value only if there is an input vector. +// The clock stream must have the same time stamp as the vector stream, and +// it must be the stream where packets are transmitted while the graph is +// running. (e.g. Any input stream of graph) +// +// It is designed to be used like: +// +// Example config: +// node { +// calculator: "CountingWithVectorSizeCalculator" +// input_stream: "CLOCK:triger_signal" +// input_stream: "VECTOR:input_vector" +// output_stream: "COUNT:vector_count" +// } +// +// node { +// calculator: "CountingWithVectorSizeCalculator" +// input_stream: "VECTOR:input_vector" +// output_stream: "COUNT:vector_count" +// } + +template +class CountingVectorSizeCalculator : public CalculatorBase { +public: + static ::mediapipe::Status GetContract(CalculatorContract *cc) { + if (cc->Inputs().HasTag("CLOCK")) { + cc->Inputs().Tag("CLOCK").SetAny(); + } + + RET_CHECK(cc->Inputs().HasTag("VECTOR")); + cc->Inputs().Tag("VECTOR").Set(); + RET_CHECK(cc->Outputs().HasTag("COUNT")); + cc->Outputs().Tag("COUNT").Set(); + + return ::mediapipe::OkStatus(); + } + + ::mediapipe::Status Process(CalculatorContext *cc) { + std::unique_ptr face_count; + if (!cc->Inputs().Tag("VECTOR").IsEmpty()) { + const auto &landmarks = cc->Inputs().Tag("VECTOR").Get(); + face_count = absl::make_unique(landmarks.size()); + } else { + face_count = absl::make_unique(0); + } + cc->Outputs().Tag("COUNT").Add(face_count.release(), cc->InputTimestamp()); + + return ::mediapipe::OkStatus(); + }; +}; + +} // namespace mediapipe + +#endif // MEDIAPIPE_CALCULATORS_UTIL_COUNTING_VECTOR_SIZE_CALCULATOR_H diff --git a/mediapipe/modules/face_landmark/BUILD b/mediapipe/modules/face_landmark/BUILD index 77560022e..30720c1b0 100644 --- a/mediapipe/modules/face_landmark/BUILD +++ b/mediapipe/modules/face_landmark/BUILD @@ -74,6 +74,28 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "face_landmark_front_cpu_with_face_counter", + graph = "face_landmark_front_cpu_with_face_counter.pbtxt", + register_as = "FaceLandmarkFrontCpuWithFaceCounter", + deps = [ + ":face_detection_front_detection_to_roi", + ":face_landmark_cpu", + ":face_landmark_landmarks_to_roi", + "//mediapipe/calculators/core:begin_loop_calculator", + "//mediapipe/calculators/core:clip_vector_size_calculator", + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:end_loop_calculator", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:previous_loopback_calculator", + "//mediapipe/calculators/image:image_properties_calculator", + "//mediapipe/calculators/util:association_norm_rect_calculator", + "//mediapipe/calculators/util:collection_has_min_size_calculator", + "//mediapipe/calculators/util:counting_vector_size_calculator", + "//mediapipe/modules/face_detection:face_detection_short_range_cpu", + ], +) + mediapipe_simple_subgraph( name = "face_landmark_front_gpu", graph = "face_landmark_front_gpu.pbtxt", diff --git a/mediapipe/modules/face_landmark/face_landmark_front_cpu_with_face_counter.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_cpu_with_face_counter.pbtxt new file mode 100644 index 000000000..5389a8293 --- /dev/null +++ b/mediapipe/modules/face_landmark/face_landmark_front_cpu_with_face_counter.pbtxt @@ -0,0 +1,249 @@ +# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is +# executed on CPU.) This graph tries to skip face detection as much as possible +# by using previously detected/predicted landmarks for new images. +# +# It is required that "face_detection_short_range.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.tflite" +# path during execution. +# +# It is required that "face_landmark.tflite" is available at +# "mediapipe/modules/face_landmark/face_landmark.tflite" +# path during execution. +# +# EXAMPLE: +# node { +# calculator: "FaceLandmarkFrontCpu" +# input_stream: "IMAGE:image" +# input_side_packet: "NUM_FACES:num_faces" +# output_stream: "LANDMARKS:multi_face_landmarks" +# } + +type: "FaceLandmarkFrontCpu" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:image" + +# Max number of faces to detect/track. (int) +input_side_packet: "NUM_FACES:num_faces" + +# Collection of detected/predicted faces, each represented as a list of 468 face +# landmarks. (std::vector) +# NOTE: there will not be an output packet in the LANDMARKS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:multi_face_landmarks" + +# Extra outputs (for debugging, for instance). +# Detected faces. (std::vector) +output_stream: "DETECTIONS:face_detections" +# Regions of interest calculated based on landmarks. +# (std::vector) +output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" +# Regions of interest calculated based on face detections. +# (std::vector) +output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" + +# (int) +output_stream: "FACE_COUNT_FROM_LANDMARKS:face_count" + + +# Defines whether landmarks on the previous image should be used to help +# localize landmarks on the current image. +node { + name: "ConstantSidePacketCalculator" + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:use_prev_landmarks" + options: { + [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { + packet { bool_value: true } + } + } +} +node { + calculator: "GateCalculator" + input_side_packet: "ALLOW:use_prev_landmarks" + input_stream: "prev_face_rects_from_landmarks" + output_stream: "gated_prev_face_rects_from_landmarks" +} + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided num_faces. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:gated_prev_face_rects_from_landmarks" + input_side_packet: "num_faces" + output_stream: "prev_has_enough_faces" +} + +# Drops the incoming image if enough faces have already been identified from the +# previous image. Otherwise, passes the incoming image through to trigger a new +# round of face detection. +node { + calculator: "GateCalculator" + input_stream: "image" + input_stream: "DISALLOW:prev_has_enough_faces" + output_stream: "gated_image" + options: { + [mediapipe.GateCalculatorOptions.ext] { + empty_packets_as_allow: true + } + } +} + +# Detects faces. +node { + calculator: "FaceDetectionShortRangeCpu" + input_stream: "IMAGE:gated_image" + output_stream: "DETECTIONS:all_face_detections" +} + +# Makes sure there are no more detections than the provided num_faces. +node { + calculator: "ClipDetectionVectorSizeCalculator" + input_stream: "all_face_detections" + output_stream: "face_detections" + input_side_packet: "num_faces" +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:gated_image" + output_stream: "SIZE:gated_image_size" +} + +# Outputs each element of face_detections at a fake timestamp for the rest of +# the graph to process. Clones the image size packet for each face_detection at +# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp +# for downstream calculators to inform them that all elements in the vector have +# been processed. +node { + calculator: "BeginLoopDetectionCalculator" + input_stream: "ITERABLE:face_detections" + input_stream: "CLONE:gated_image_size" + output_stream: "ITEM:face_detection" + output_stream: "CLONE:detections_loop_image_size" + output_stream: "BATCH_END:detections_loop_end_timestamp" +} + +# Calculates region of interest based on face detections, so that can be used +# to detect landmarks. +node { + calculator: "FaceDetectionFrontDetectionToRoi" + input_stream: "DETECTION:face_detection" + input_stream: "IMAGE_SIZE:detections_loop_image_size" + output_stream: "ROI:face_rect_from_detection" +} + +# Counting a multi_faceLandmarks vector size. The image stream is only used to +# make the calculator work even when there is no input vector. +node { + calculator: "CountingNormalizedLandmarkListVectorSizeCalculator" + input_stream: "CLOCK:image" + input_stream: "VECTOR:multi_face_landmarks" + output_stream: "COUNT:face_count" +} + + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_detection" + input_stream: "BATCH_END:detections_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_detections" +} + +# Performs association between NormalizedRect vector elements from previous +# image and rects based on face detections from the current image. This +# calculator ensures that the output face_rects vector doesn't contain +# overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "face_rects_from_detections" + input_stream: "gated_prev_face_rects_from_landmarks" + output_stream: "face_rects" + options: { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.5 + } + } +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:image" + output_stream: "SIZE:image_size" +} + +# Outputs each element of face_rects at a fake timestamp for the rest of the +# graph to process. Clones image and image size packets for each +# single_face_rect at the fake timestamp. At the end of the loop, outputs the +# BATCH_END timestamp for downstream calculators to inform them that all +# elements in the vector have been processed. +node { + calculator: "BeginLoopNormalizedRectCalculator" + input_stream: "ITERABLE:face_rects" + input_stream: "CLONE:0:image" + input_stream: "CLONE:1:image_size" + output_stream: "ITEM:face_rect" + output_stream: "CLONE:0:landmarks_loop_image" + output_stream: "CLONE:1:landmarks_loop_image_size" + output_stream: "BATCH_END:landmarks_loop_end_timestamp" +} + +# Detects face landmarks within specified region of interest of the image. +node { + calculator: "FaceLandmarkCpu" + input_stream: "IMAGE:landmarks_loop_image" + input_stream: "ROI:face_rect" + output_stream: "LANDMARKS:face_landmarks" +} + +# Calculates region of interest based on face landmarks, so that can be reused +# for subsequent image. +node { + calculator: "FaceLandmarkLandmarksToRoi" + input_stream: "LANDMARKS:face_landmarks" + input_stream: "IMAGE_SIZE:landmarks_loop_image_size" + output_stream: "ROI:face_rect_from_landmarks" +} + +# Collects a set of landmarks for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedLandmarkListVectorCalculator" + input_stream: "ITEM:face_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:multi_face_landmarks" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_landmarks" +} + +# Caches face rects calculated from landmarks, and upon the arrival of the next +# input image, sends out the cached rects with timestamps replaced by that of +# the input image, essentially generating a packet that carries the previous +# face rects. Note that upon the arrival of the very first input image, a +# timestamp bound update occurs to jump start the feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:image" + input_stream: "LOOP:face_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_face_rects_from_landmarks" +} From 872386a6bb13dd18829c831068bb1a4597b65f00 Mon Sep 17 00:00:00 2001 From: dmaletskiy Date: Thu, 1 Jul 2021 12:55:40 +0300 Subject: [PATCH 2/8] feat: Added test dll face mesh example Change List: - added `'face_mesh_dll/face_mesh_lib` that will be builded as windows dynamic library - added 'face_mesh_dll/face_mesh_cpu.cpp` as simple test of `'face_mesh_dll/face_mesh_lib` - currently, face_mesh_lib only prints in console `face_count` and `first face landmark` --- .../examples/desktop/face_mesh_dll/BUILD | 65 +++++++ .../desktop/face_mesh_dll/face_mesh_cpu.cpp | 56 ++++++ .../desktop/face_mesh_dll/face_mesh_lib.cpp | 179 ++++++++++++++++++ .../desktop/face_mesh_dll/face_mesh_lib.h | 64 +++++++ .../face_mesh_dll/windows_dll_library.bzl | 62 ++++++ 5 files changed, 426 insertions(+) create mode 100644 mediapipe/examples/desktop/face_mesh_dll/BUILD create mode 100644 mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp create mode 100644 mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp create mode 100644 mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h create mode 100644 mediapipe/examples/desktop/face_mesh_dll/windows_dll_library.bzl diff --git a/mediapipe/examples/desktop/face_mesh_dll/BUILD b/mediapipe/examples/desktop/face_mesh_dll/BUILD new file mode 100644 index 000000000..ff5709093 --- /dev/null +++ b/mediapipe/examples/desktop/face_mesh_dll/BUILD @@ -0,0 +1,65 @@ +# Copyright 2019 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("windows_dll_library.bzl", "windows_dll_library") + +licenses(["notice"]) + +filegroup( + name = "srcs", + srcs = glob(["**"]), + visibility = ["//examples:__pkg__"], +) + +package(default_visibility = ["//mediapipe/examples:__subpackages__"]) + +# Define the shared library +windows_dll_library( + name = "face_mesh_lib", + srcs = ["face_mesh_lib.cpp"], + hdrs = ["face_mesh_lib.h"], + # Define COMPILING_DLL to export symbols during compiling the DLL. + copts = ["-DCOMPILING_DLL"], + deps = [ + "//mediapipe/framework:calculator_framework", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/port:file_helpers", + "//mediapipe/framework/port:opencv_highgui", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/framework/port:opencv_video", + "//mediapipe/framework/port:parse_text_proto", + "//mediapipe/framework/port:status", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/flags:parse", + + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:flow_limiter_calculator", + "//mediapipe/modules/face_landmark:face_landmark_front_cpu_with_face_counter", + + + ] +) + +# **Implicitly link to face_mesh_lib.dll** + +## Link to face_mesh_lib.dll through its import library. +cc_binary( + name = "face_mesh_cpu", + srcs = ["face_mesh_cpu.cpp"], + deps = [ + ":face_mesh_lib", + ], +) \ No newline at end of file diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp new file mode 100644 index 000000000..e44375864 --- /dev/null +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp @@ -0,0 +1,56 @@ +#include "face_mesh_lib.h" + +int main(int argc, char **argv) { + google::InitGoogleLogging(argv[0]); + absl::ParseCommandLine(argc, argv); + + + cv::VideoCapture capture; + capture.open(0); + if (!capture.isOpened()) { + return -1; + } + + constexpr char kWindowName[] = "MediaPipe"; + + cv::namedWindow(kWindowName, /*flags=WINDOW_AUTOSIZE*/ 1); +#if (CV_MAJOR_VERSION >= 3) && (CV_MINOR_VERSION >= 2) + capture.set(cv::CAP_PROP_FRAME_WIDTH, 640); + capture.set(cv::CAP_PROP_FRAME_HEIGHT, 480); + capture.set(cv::CAP_PROP_FPS, 30); +#endif + + LOG(INFO) << "VideoCapture initialized."; + + FaceMeshDetector *faceMeshDetector = FaceMeshDetector_Construct(); + + LOG(INFO) << "FaceMeshDetector constructed."; + + LOG(INFO) << "Start grabbing and processing frames."; + bool grab_frames = true; + + while (grab_frames) { + // Capture opencv camera. + cv::Mat camera_frame_raw; + capture >> camera_frame_raw; + if (camera_frame_raw.empty()) { + LOG(INFO) << "Ignore empty frames from camera."; + continue; + } + cv::Mat camera_frame; + cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); + cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); + + FaceMeshDetector_ProcessFrame(faceMeshDetector, camera_frame); + + const int pressed_key = cv::waitKey(5); + if (pressed_key >= 0 && pressed_key != 255) + grab_frames = false; + + cv::imshow(kWindowName, camera_frame_raw); + } + + LOG(INFO) << "Shutting down."; + + FaceMeshDetector_Destruct(faceMeshDetector); +} \ No newline at end of file diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp new file mode 100644 index 000000000..a918e3719 --- /dev/null +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp @@ -0,0 +1,179 @@ +#include + +#include "face_mesh_lib.h" + +#define DEBUG + +FaceMeshDetector::FaceMeshDetector() { + const auto status = InitFaceMeshDetector(); + if (!status.ok()) { + LOG(INFO) << "Failed constructing FaceMeshDetector."; + } +} + +absl::Status FaceMeshDetector::InitFaceMeshDetector() { + LOG(INFO) << "Get calculator graph config contents: " << graphConfig; + + mediapipe::CalculatorGraphConfig config = + mediapipe::ParseTextProtoOrDie( + graphConfig); + + LOG(INFO) << "Initialize the calculator graph."; + + MP_RETURN_IF_ERROR(graph.Initialize(config)); + + LOG(INFO) << "Start running the calculator graph."; + + ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller landmarks_poller, + graph.AddOutputStreamPoller(kOutputStream_landmarks)); + ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller face_count_poller, + graph.AddOutputStreamPoller(kOutputStream_faceCount)); + + landmarks_poller_ptr = std::make_unique( + std::move(landmarks_poller)); + face_count_poller_ptr = std::make_unique( + std::move(face_count_poller)); + + MP_RETURN_IF_ERROR(graph.StartRun({})); + + return absl::Status(); +} + +absl::Status FaceMeshDetector::ProcessFrameWithStatus(cv::Mat &camera_frame) { + // Wrap Mat into an ImageFrame. + auto input_frame = absl::make_unique( + mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, + mediapipe::ImageFrame::kDefaultAlignmentBoundary); + cv::Mat input_frame_mat = mediapipe::formats::MatView(input_frame.get()); + camera_frame.copyTo(input_frame_mat); + + // Send image packet into the graph. + + size_t frame_timestamp_us = + (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6; + MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( + kInputStream, mediapipe::Adopt(input_frame.release()) + .At(mediapipe::Timestamp(frame_timestamp_us)))); + LOG(INFO) << "Pushed new frame."; + +#ifdef DEBUG + LOG(INFO) << "Pushed new frame."; +#endif + mediapipe::Packet face_count_packet; + if (!face_count_poller_ptr || + !face_count_poller_ptr->Next(&face_count_packet)) { + LOG(INFO) << "Failed during getting next face_count_packet."; + + return absl::Status(); + } + auto &face_count = face_count_packet.Get(); + +#ifdef DEBUG + LOG(INFO) << "Got face_count: " << face_count; +#endif + + if (!face_count) { + return absl::Status(); + } + + mediapipe::Packet face_landmarks_packet; + if (!landmarks_poller_ptr || + !landmarks_poller_ptr->Next(&face_landmarks_packet)) { + LOG(INFO) << "Failed during getting next landmarks_packet."; + + return absl::Status(); + } + + auto &output_landmarks_vector = + face_landmarks_packet + .Get<::std::vector<::mediapipe::NormalizedLandmarkList>>(); + + auto &output_landmarks = output_landmarks_vector[0]; + +#ifdef DEBUG + LOG(INFO) << "Got landmarks_packet: " << output_landmarks.landmark_size(); +#endif + + auto &landmark = output_landmarks.landmark(0); +#ifdef DEBUG + LOG(INFO) << "First landmark: x - " << landmark.x() << ", y - " + << landmark.y() << ", z - " << landmark.z(); +#endif + + return absl::Status(); +} + +std::vector * +FaceMeshDetector::ProcessFrame(cv::Mat &camera_frame) { + ProcessFrameWithStatus(camera_frame); + + return new std::vector(); +} + +extern "C" { +DLLEXPORT FaceMeshDetector *FaceMeshDetector_Construct() { + return new FaceMeshDetector(); +} + +DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector) { + delete detector; +} + +DLLEXPORT void *FaceMeshDetector_ProcessFrame(FaceMeshDetector *detector, + cv::Mat &camera_frame) { + return reinterpret_cast(detector->ProcessFrame(camera_frame)); +} +} + +const char FaceMeshDetector::kInputStream[] = "input_video"; +const char FaceMeshDetector::kOutputStream_landmarks[] = "multi_face_landmarks"; +const char FaceMeshDetector::kOutputStream_faceCount[] = "face_count"; + +const std::string FaceMeshDetector::graphConfig = R"pb( +# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU. + +# Input image. (ImageFrame) +input_stream: "input_video" + +# Collection of detected/processed faces, each represented as a list of +# landmarks. (std::vector) +output_stream: "multi_face_landmarks" + +# Detected faces count. (int) +output_stream: "face_count" + +node { + calculator: "FlowLimiterCalculator" + input_stream: "input_video" + input_stream: "FINISHED:multi_face_landmarks" + input_stream_info: { + tag_index: "FINISHED" + back_edge: true + } + output_stream: "throttled_input_video" +} + +# Defines side packets for further use in the graph. +node { + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:num_faces" + node_options: { + [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { + packet { int_value: 1 } + } + } +} + +# Subgraph that detects faces and corresponding landmarks. +node { + calculator: "FaceLandmarkFrontCpuWithFaceCounter" + input_stream: "IMAGE:throttled_input_video" + input_side_packet: "NUM_FACES:num_faces" + output_stream: "LANDMARKS:multi_face_landmarks" + output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" + output_stream: "DETECTIONS:face_detections" + output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" + output_stream: "FACE_COUNT_FROM_LANDMARKS:face_count" +} + +)pb"; diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h new file mode 100644 index 000000000..7c62b01e4 --- /dev/null +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h @@ -0,0 +1,64 @@ +#ifndef FACE_MESH_LIBRARY_H +#define FACE_MESH_LIBRARY_H + +#ifdef COMPILING_DLL +#define DLLEXPORT __declspec(dllexport) +#else +#define DLLEXPORT __declspec(dllimport) +#endif + +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "mediapipe/framework/calculator_framework.h" +#include "mediapipe/framework/calculator_graph.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/framework/formats/image_frame_opencv.h" +#include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/port/file_helpers.h" +#include "mediapipe/framework/port/opencv_highgui_inc.h" +#include "mediapipe/framework/port/opencv_imgproc_inc.h" +#include "mediapipe/framework/port/opencv_video_inc.h" +#include "mediapipe/framework/port/parse_text_proto.h" +#include "mediapipe/framework/port/status.h" + +class FaceMeshDetector { +public: + FaceMeshDetector(); + ~FaceMeshDetector() = default; + std::vector *ProcessFrame(cv::Mat &camera_frame); + +private: + absl::Status InitFaceMeshDetector(); + absl::Status ProcessFrameWithStatus(cv::Mat &camera_frame); + + static const char kInputStream[]; + static const char kOutputStream_landmarks[]; + static const char kOutputStream_faceCount[]; + + static const std::string graphConfig; + + mediapipe::CalculatorGraph graph; + + std::unique_ptr landmarks_poller_ptr; + std::unique_ptr face_count_poller_ptr; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +DLLEXPORT FaceMeshDetector *FaceMeshDetector_Construct(); + +DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector); + +DLLEXPORT void *FaceMeshDetector_ProcessFrame(FaceMeshDetector *detector, + cv::Mat &camera_frame); + +#ifdef __cplusplus +}; +#endif +#endif \ No newline at end of file diff --git a/mediapipe/examples/desktop/face_mesh_dll/windows_dll_library.bzl b/mediapipe/examples/desktop/face_mesh_dll/windows_dll_library.bzl new file mode 100644 index 000000000..69c243d60 --- /dev/null +++ b/mediapipe/examples/desktop/face_mesh_dll/windows_dll_library.bzl @@ -0,0 +1,62 @@ +""" +This is a simple windows_dll_library rule for builing a DLL Windows +that can be depended on by other cc rules. +Example useage: + windows_dll_library( + name = "hellolib", + srcs = [ + "hello-library.cpp", + ], + hdrs = ["hello-library.h"], + # Define COMPILING_DLL to export symbols during compiling the DLL. + copts = ["/DCOMPILING_DLL"], + ) +""" + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_import", "cc_library") + +def windows_dll_library( + name, + srcs = [], + deps = [], + hdrs = [], + visibility = None, + **kwargs): + """A simple windows_dll_library rule for builing a DLL Windows.""" + dll_name = name + ".dll" + import_lib_name = name + "_import_lib" + import_target_name = name + "_dll_import" + + # Build the shared library + cc_binary( + name = dll_name, + srcs = srcs + hdrs, + deps = deps, + linkshared = 1, + **kwargs + ) + + # Get the import library for the dll + native.filegroup( + name = import_lib_name, + srcs = [":" + dll_name], + output_group = "interface_library", + ) + + # Because we cannot directly depend on cc_binary from other cc rules in deps attribute, + # we use cc_import as a bridge to depend on the dll. + cc_import( + name = import_target_name, + interface_library = ":" + import_lib_name, + shared_library = ":" + dll_name, + ) + + # Create a new cc_library to also include the headers needed for the shared library + cc_library( + name = name, + hdrs = hdrs, + visibility = visibility, + deps = deps + [ + ":" + import_target_name, + ], + ) \ No newline at end of file From de5fc2a532af0a951b93203e0ecadeff76e2aaf5 Mon Sep 17 00:00:00 2001 From: dmaletskiy Date: Thu, 1 Jul 2021 13:38:45 +0300 Subject: [PATCH 3/8] feat: added ouputing of multi-face_landmarks through dll to c++ Change List: - now FaceMeshDetector::ProcessFrame2D returns pointer to vector with all facelandmars - added converting of `mediapipe::NormalizedLandmarkList` to `std::vector` to make more independent of mediapipe (to use in custom projects) --- .../desktop/face_mesh_dll/face_mesh_cpu.cpp | 17 ++++++- .../desktop/face_mesh_dll/face_mesh_lib.cpp | 47 ++++++++++++------- .../desktop/face_mesh_dll/face_mesh_lib.h | 15 +++--- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp index e44375864..5956dade4 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp @@ -4,7 +4,6 @@ int main(int argc, char **argv) { google::InitGoogleLogging(argv[0]); absl::ParseCommandLine(argc, argv); - cv::VideoCapture capture; capture.open(0); if (!capture.isOpened()) { @@ -41,7 +40,21 @@ int main(int argc, char **argv) { cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); - FaceMeshDetector_ProcessFrame(faceMeshDetector, camera_frame); + std::unique_ptr>> multi_face_landmarks( + reinterpret_cast> *>( + FaceMeshDetector_ProcessFrame2D(faceMeshDetector, camera_frame))); + + const auto multi_face_landmarks_num = multi_face_landmarks->size(); + + LOG(INFO) << "Got multi_face_landmarks_num: " << multi_face_landmarks_num; + + if (multi_face_landmarks_num) { + auto &face_landmarks = multi_face_landmarks->operator[](0); + auto &landmark = face_landmarks[0]; + + LOG(INFO) << "First landmark: x - " << landmark.x << ", y - " + << landmark.y; + } const int pressed_key = cv::waitKey(5); if (pressed_key >= 0 && pressed_key != 255) diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp index a918e3719..dd8cb06eb 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp @@ -2,8 +2,6 @@ #include "face_mesh_lib.h" -#define DEBUG - FaceMeshDetector::FaceMeshDetector() { const auto status = InitFaceMeshDetector(); if (!status.ok()) { @@ -39,7 +37,10 @@ absl::Status FaceMeshDetector::InitFaceMeshDetector() { return absl::Status(); } -absl::Status FaceMeshDetector::ProcessFrameWithStatus(cv::Mat &camera_frame) { +absl::Status FaceMeshDetector::ProcessFrameWithStatus( + cv::Mat &camera_frame, + std::unique_ptr>> + &multi_face_landmarks) { // Wrap Mat into an ImageFrame. auto input_frame = absl::make_unique( mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, @@ -88,26 +89,38 @@ absl::Status FaceMeshDetector::ProcessFrameWithStatus(cv::Mat &camera_frame) { face_landmarks_packet .Get<::std::vector<::mediapipe::NormalizedLandmarkList>>(); - auto &output_landmarks = output_landmarks_vector[0]; + multi_face_landmarks->reserve(output_landmarks_vector.size()); + + for (const auto &normalizedLandmarkList : output_landmarks_vector) { + multi_face_landmarks->emplace_back(); + + auto &face_landmarks = multi_face_landmarks->back(); + + const auto landmarks_num = normalizedLandmarkList.landmark_size(); #ifdef DEBUG - LOG(INFO) << "Got landmarks_packet: " << output_landmarks.landmark_size(); + LOG(INFO) << "Got landmarks_num: " << landmarks_num; #endif - auto &landmark = output_landmarks.landmark(0); -#ifdef DEBUG - LOG(INFO) << "First landmark: x - " << landmark.x() << ", y - " - << landmark.y() << ", z - " << landmark.z(); -#endif + face_landmarks.reserve(landmarks_num); + + for (int i = 0; i < landmarks_num; ++i) { + auto &landmark = normalizedLandmarkList.landmark(i); + + face_landmarks.emplace_back(landmark.x(), landmark.y()); + } + } return absl::Status(); } -std::vector * -FaceMeshDetector::ProcessFrame(cv::Mat &camera_frame) { - ProcessFrameWithStatus(camera_frame); +std::vector> * +FaceMeshDetector::ProcessFrame2D(cv::Mat &camera_frame) { + auto landmarks = std::make_unique>>(); - return new std::vector(); + ProcessFrameWithStatus(camera_frame, landmarks); + + return landmarks.release(); } extern "C" { @@ -119,9 +132,9 @@ DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector) { delete detector; } -DLLEXPORT void *FaceMeshDetector_ProcessFrame(FaceMeshDetector *detector, - cv::Mat &camera_frame) { - return reinterpret_cast(detector->ProcessFrame(camera_frame)); +DLLEXPORT void *FaceMeshDetector_ProcessFrame2D(FaceMeshDetector *detector, + cv::Mat &camera_frame) { + return reinterpret_cast(detector->ProcessFrame2D(camera_frame)); } } diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h index 7c62b01e4..9788d6803 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h @@ -8,8 +8,8 @@ #endif #include -#include #include +#include #include "absl/flags/flag.h" #include "absl/flags/parse.h" @@ -29,11 +29,14 @@ class FaceMeshDetector { public: FaceMeshDetector(); ~FaceMeshDetector() = default; - std::vector *ProcessFrame(cv::Mat &camera_frame); + std::vector> *ProcessFrame2D(cv::Mat &camera_frame); private: absl::Status InitFaceMeshDetector(); - absl::Status ProcessFrameWithStatus(cv::Mat &camera_frame); + absl::Status + ProcessFrameWithStatus(cv::Mat &camera_frame, + std::unique_ptr>> + &multi_face_landmarks); static const char kInputStream[]; static const char kOutputStream_landmarks[]; @@ -42,7 +45,7 @@ private: static const std::string graphConfig; mediapipe::CalculatorGraph graph; - + std::unique_ptr landmarks_poller_ptr; std::unique_ptr face_count_poller_ptr; }; @@ -55,8 +58,8 @@ DLLEXPORT FaceMeshDetector *FaceMeshDetector_Construct(); DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector); -DLLEXPORT void *FaceMeshDetector_ProcessFrame(FaceMeshDetector *detector, - cv::Mat &camera_frame); +DLLEXPORT void *FaceMeshDetector_ProcessFrame2D(FaceMeshDetector *detector, + cv::Mat &camera_frame); #ifdef __cplusplus }; From c48c01ca75ba95d596e65a46a7c69a4c2e928290 Mon Sep 17 00:00:00 2001 From: dmaletskiy Date: Thu, 1 Jul 2021 16:00:21 +0300 Subject: [PATCH 4/8] feat: updated face mesh lib class name --- .../desktop/face_mesh_dll/face_mesh_cpu.cpp | 2 +- .../desktop/face_mesh_dll/face_mesh_lib.cpp | 30 ++++++++++--------- .../desktop/face_mesh_dll/face_mesh_lib.h | 18 +++++------ 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp index 5956dade4..210d19c07 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp @@ -21,7 +21,7 @@ int main(int argc, char **argv) { LOG(INFO) << "VideoCapture initialized."; - FaceMeshDetector *faceMeshDetector = FaceMeshDetector_Construct(); + MPFaceMeshDetector *faceMeshDetector = FaceMeshDetector_Construct(); LOG(INFO) << "FaceMeshDetector constructed."; diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp index dd8cb06eb..ec15f9ee8 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp @@ -2,14 +2,14 @@ #include "face_mesh_lib.h" -FaceMeshDetector::FaceMeshDetector() { +MPFaceMeshDetector::MPFaceMeshDetector() { const auto status = InitFaceMeshDetector(); if (!status.ok()) { LOG(INFO) << "Failed constructing FaceMeshDetector."; } } -absl::Status FaceMeshDetector::InitFaceMeshDetector() { +absl::Status MPFaceMeshDetector::InitFaceMeshDetector() { LOG(INFO) << "Get calculator graph config contents: " << graphConfig; mediapipe::CalculatorGraphConfig config = @@ -37,8 +37,8 @@ absl::Status FaceMeshDetector::InitFaceMeshDetector() { return absl::Status(); } -absl::Status FaceMeshDetector::ProcessFrameWithStatus( - cv::Mat &camera_frame, +absl::Status MPFaceMeshDetector::ProcessFrameWithStatus( + const cv::Mat &camera_frame, std::unique_ptr>> &multi_face_landmarks) { // Wrap Mat into an ImageFrame. @@ -115,7 +115,7 @@ absl::Status FaceMeshDetector::ProcessFrameWithStatus( } std::vector> * -FaceMeshDetector::ProcessFrame2D(cv::Mat &camera_frame) { +MPFaceMeshDetector::ProcessFrame2D(const cv::Mat &camera_frame) { auto landmarks = std::make_unique>>(); ProcessFrameWithStatus(camera_frame, landmarks); @@ -124,25 +124,27 @@ FaceMeshDetector::ProcessFrame2D(cv::Mat &camera_frame) { } extern "C" { -DLLEXPORT FaceMeshDetector *FaceMeshDetector_Construct() { - return new FaceMeshDetector(); +DLLEXPORT MPFaceMeshDetector *FaceMeshDetector_Construct() { + return new MPFaceMeshDetector(); } -DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector) { +DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector) { delete detector; } -DLLEXPORT void *FaceMeshDetector_ProcessFrame2D(FaceMeshDetector *detector, - cv::Mat &camera_frame) { +DLLEXPORT void * +FaceMeshDetector_ProcessFrame2D(MPFaceMeshDetector *detector, + const cv::Mat &camera_frame) { return reinterpret_cast(detector->ProcessFrame2D(camera_frame)); } } -const char FaceMeshDetector::kInputStream[] = "input_video"; -const char FaceMeshDetector::kOutputStream_landmarks[] = "multi_face_landmarks"; -const char FaceMeshDetector::kOutputStream_faceCount[] = "face_count"; +const char MPFaceMeshDetector::kInputStream[] = "input_video"; +const char MPFaceMeshDetector::kOutputStream_landmarks[] = + "multi_face_landmarks"; +const char MPFaceMeshDetector::kOutputStream_faceCount[] = "face_count"; -const std::string FaceMeshDetector::graphConfig = R"pb( +const std::string MPFaceMeshDetector::graphConfig = R"pb( # MediaPipe graph that performs face mesh with TensorFlow Lite on CPU. # Input image. (ImageFrame) diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h index 9788d6803..eef5cc8c9 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h @@ -25,16 +25,16 @@ #include "mediapipe/framework/port/parse_text_proto.h" #include "mediapipe/framework/port/status.h" -class FaceMeshDetector { +class MPFaceMeshDetector { public: - FaceMeshDetector(); - ~FaceMeshDetector() = default; - std::vector> *ProcessFrame2D(cv::Mat &camera_frame); + MPFaceMeshDetector(); + ~MPFaceMeshDetector() = default; + std::vector> *ProcessFrame2D(const cv::Mat &camera_frame); private: absl::Status InitFaceMeshDetector(); absl::Status - ProcessFrameWithStatus(cv::Mat &camera_frame, + ProcessFrameWithStatus(const cv::Mat &camera_frame, std::unique_ptr>> &multi_face_landmarks); @@ -54,12 +54,12 @@ private: extern "C" { #endif -DLLEXPORT FaceMeshDetector *FaceMeshDetector_Construct(); +DLLEXPORT MPFaceMeshDetector *FaceMeshDetector_Construct(); -DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector); +DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector); -DLLEXPORT void *FaceMeshDetector_ProcessFrame2D(FaceMeshDetector *detector, - cv::Mat &camera_frame); +DLLEXPORT void *FaceMeshDetector_ProcessFrame2D(MPFaceMeshDetector *detector, + const cv::Mat &camera_frame); #ifdef __cplusplus }; From 26b367dc69a7c2b88ec1c89d26cbbc4a4ec7ac41 Mon Sep 17 00:00:00 2001 From: dmaletskiy Date: Fri, 2 Jul 2021 11:58:01 +0300 Subject: [PATCH 5/8] fix: resolved code review comments Change List: - removed some debug lines --- .../examples/desktop/face_mesh_dll/face_mesh_lib.cpp | 11 ----------- .../examples/desktop/face_mesh_dll/face_mesh_lib.h | 1 - 2 files changed, 12 deletions(-) diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp index ec15f9ee8..54ac3185a 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp @@ -57,9 +57,6 @@ absl::Status MPFaceMeshDetector::ProcessFrameWithStatus( .At(mediapipe::Timestamp(frame_timestamp_us)))); LOG(INFO) << "Pushed new frame."; -#ifdef DEBUG - LOG(INFO) << "Pushed new frame."; -#endif mediapipe::Packet face_count_packet; if (!face_count_poller_ptr || !face_count_poller_ptr->Next(&face_count_packet)) { @@ -69,10 +66,6 @@ absl::Status MPFaceMeshDetector::ProcessFrameWithStatus( } auto &face_count = face_count_packet.Get(); -#ifdef DEBUG - LOG(INFO) << "Got face_count: " << face_count; -#endif - if (!face_count) { return absl::Status(); } @@ -98,10 +91,6 @@ absl::Status MPFaceMeshDetector::ProcessFrameWithStatus( const auto landmarks_num = normalizedLandmarkList.landmark_size(); -#ifdef DEBUG - LOG(INFO) << "Got landmarks_num: " << landmarks_num; -#endif - face_landmarks.reserve(landmarks_num); for (int i = 0; i < landmarks_num; ++i) { diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h index eef5cc8c9..d6fe713e4 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h @@ -28,7 +28,6 @@ class MPFaceMeshDetector { public: MPFaceMeshDetector(); - ~MPFaceMeshDetector() = default; std::vector> *ProcessFrame2D(const cv::Mat &camera_frame); private: From b7dd4cfe723899dc572e22016e4f4d48636d223b Mon Sep 17 00:00:00 2001 From: dmaletskiy Date: Mon, 12 Jul 2021 17:52:15 +0300 Subject: [PATCH 6/8] feat: Added face mesh DLL example with side models Change List: - added graphs for running face mesh dll example with face_detections and face_landmarks models paths saved in side pockets (these pathed can be configured in `MPFaceMeshDetector` constructor - added possibility to set maximum nuber of faces to detect (by default 1) --- .../examples/desktop/face_mesh_dll/BUILD | 6 +- .../desktop/face_mesh_dll/face_mesh_cpu.cpp | 39 ++- .../desktop/face_mesh_dll/face_mesh_lib.cpp | 198 ++++++++++---- .../desktop/face_mesh_dll/face_mesh_lib.h | 38 ++- mediapipe/modules/face_detection/BUILD | 12 + ...detection_short_range_side_model_cpu.pbtxt | 86 ++++++ mediapipe/modules/face_landmark/BUILD | 38 +++ ...ont_side_model_cpu_with_face_counter.pbtxt | 256 ++++++++++++++++++ .../face_landmark_side_model_cpu.pbtxt | 143 ++++++++++ 9 files changed, 741 insertions(+), 75 deletions(-) create mode 100644 mediapipe/modules/face_detection/face_detection_short_range_side_model_cpu.pbtxt create mode 100644 mediapipe/modules/face_landmark/face_landmark_front_side_model_cpu_with_face_counter.pbtxt create mode 100644 mediapipe/modules/face_landmark/face_landmark_side_model_cpu.pbtxt diff --git a/mediapipe/examples/desktop/face_mesh_dll/BUILD b/mediapipe/examples/desktop/face_mesh_dll/BUILD index ff5709093..3a20d0f43 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/BUILD +++ b/mediapipe/examples/desktop/face_mesh_dll/BUILD @@ -47,9 +47,9 @@ windows_dll_library( "//mediapipe/calculators/core:constant_side_packet_calculator", "//mediapipe/calculators/core:flow_limiter_calculator", - "//mediapipe/modules/face_landmark:face_landmark_front_cpu_with_face_counter", - - + "//mediapipe/calculators/tflite:tflite_model_calculator", + "//mediapipe/calculators/util:local_file_contents_calculator", + "//mediapipe/modules/face_landmark:face_landmark_front_side_model_cpu_with_face_counter", ] ) diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp index 210d19c07..90462477a 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp @@ -21,7 +21,22 @@ int main(int argc, char **argv) { LOG(INFO) << "VideoCapture initialized."; - MPFaceMeshDetector *faceMeshDetector = FaceMeshDetector_Construct(); + // Maximum number of faces that can be detected + constexpr int maxNumFaces = 1; + constexpr char face_detection_model_path[] = + "mediapipe/modules/face_detection/face_detection_short_range.tflite"; + constexpr char face_landmark_model_path[] = + "mediapipe/modules/face_landmark/face_landmark.tflite"; + + MPFaceMeshDetector *faceMeshDetector = FaceMeshDetector_Construct( + maxNumFaces, face_detection_model_path, face_landmark_model_path); + + // allocate memory for face landmarks + auto multiFaceLandmarks = new cv::Point2f *[maxNumFaces]; + constexpr auto mediapipeFaceLandmarksNum = 468; + for (int i = 0; i < maxNumFaces; ++i) { + multiFaceLandmarks[i] = new cv::Point2f[mediapipeFaceLandmarksNum]; + } LOG(INFO) << "FaceMeshDetector constructed."; @@ -36,26 +51,26 @@ int main(int argc, char **argv) { LOG(INFO) << "Ignore empty frames from camera."; continue; } + cv::Mat camera_frame; cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); - std::unique_ptr>> multi_face_landmarks( - reinterpret_cast> *>( - FaceMeshDetector_ProcessFrame2D(faceMeshDetector, camera_frame))); + int faceCount = + FaceMeshDetector_GetFaceCount(faceMeshDetector, camera_frame); - const auto multi_face_landmarks_num = multi_face_landmarks->size(); + LOG(INFO) << "Detected faces num: " << faceCount; - LOG(INFO) << "Got multi_face_landmarks_num: " << multi_face_landmarks_num; + if (faceCount > 0) { - if (multi_face_landmarks_num) { - auto &face_landmarks = multi_face_landmarks->operator[](0); + FaceMeshDetector_GetFaceLandmarks(faceMeshDetector, multiFaceLandmarks); + + auto &face_landmarks = multiFaceLandmarks[0]; auto &landmark = face_landmarks[0]; LOG(INFO) << "First landmark: x - " << landmark.x << ", y - " << landmark.y; } - const int pressed_key = cv::waitKey(5); if (pressed_key >= 0 && pressed_key != 255) grab_frames = false; @@ -65,5 +80,11 @@ int main(int argc, char **argv) { LOG(INFO) << "Shutting down."; + // deallocate memory for face landmarks + for (int i = 0; i < maxNumFaces; ++i) { + delete[] multiFaceLandmarks[i]; + } + delete[] multiFaceLandmarks; + FaceMeshDetector_Destruct(faceMeshDetector); } \ No newline at end of file diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp index 54ac3185a..5bba0efea 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp @@ -2,20 +2,51 @@ #include "face_mesh_lib.h" -MPFaceMeshDetector::MPFaceMeshDetector() { - const auto status = InitFaceMeshDetector(); +#define DEBUG + +MPFaceMeshDetector::MPFaceMeshDetector(int numFaces, + const char *face_detection_model_path, + const char *face_landmark_model_path) { + const auto status = InitFaceMeshDetector(numFaces, face_detection_model_path, + face_landmark_model_path); if (!status.ok()) { LOG(INFO) << "Failed constructing FaceMeshDetector."; + LOG(INFO) << status.message(); } } -absl::Status MPFaceMeshDetector::InitFaceMeshDetector() { - LOG(INFO) << "Get calculator graph config contents: " << graphConfig; +absl::Status +MPFaceMeshDetector::InitFaceMeshDetector(int numFaces, + const char *face_detection_model_path, + const char *face_landmark_model_path) { + if (numFaces <= 0) { + numFaces = 1; + } + + if (face_detection_model_path == nullptr) { + face_detection_model_path = + "mediapipe/modules/face_detection/face_detection_short_range.tflite"; + } + + if (face_landmark_model_path == nullptr) { + face_landmark_model_path = + "mediapipe/modules/face_landmark/face_landmark.tflite"; + } + + auto preparedGraphConfig = absl::StrReplaceAll( + graphConfig, {{"$numFaces", std::to_string(numFaces)}}); + preparedGraphConfig = absl::StrReplaceAll( + preparedGraphConfig, + {{"$faceDetectionModelPath", face_detection_model_path}}); + preparedGraphConfig = absl::StrReplaceAll( + preparedGraphConfig, + {{"$faceLandmarkModelPath", face_landmark_model_path}}); + + LOG(INFO) << "Get calculator graph config contents: " << preparedGraphConfig; mediapipe::CalculatorGraphConfig config = mediapipe::ParseTextProtoOrDie( - graphConfig); - + preparedGraphConfig); LOG(INFO) << "Initialize the calculator graph."; MP_RETURN_IF_ERROR(graph.Initialize(config)); @@ -34,13 +65,13 @@ absl::Status MPFaceMeshDetector::InitFaceMeshDetector() { MP_RETURN_IF_ERROR(graph.StartRun({})); - return absl::Status(); + LOG(INFO) << "MPFaceMeshDetector constructed successfully."; + + return absl::OkStatus(); } -absl::Status MPFaceMeshDetector::ProcessFrameWithStatus( - const cv::Mat &camera_frame, - std::unique_ptr>> - &multi_face_landmarks) { +absl::Status +MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) { // Wrap Mat into an ImageFrame. auto input_frame = absl::make_unique( mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, @@ -49,82 +80,99 @@ absl::Status MPFaceMeshDetector::ProcessFrameWithStatus( camera_frame.copyTo(input_frame_mat); // Send image packet into the graph. - - size_t frame_timestamp_us = - (double)cv::getTickCount() / (double)cv::getTickFrequency() * 1e6; + size_t frame_timestamp_us = static_cast(cv::getTickCount()) / + static_cast(cv::getTickFrequency()) * 1e6; MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( kInputStream, mediapipe::Adopt(input_frame.release()) .At(mediapipe::Timestamp(frame_timestamp_us)))); - LOG(INFO) << "Pushed new frame."; - mediapipe::Packet face_count_packet; if (!face_count_poller_ptr || !face_count_poller_ptr->Next(&face_count_packet)) { - LOG(INFO) << "Failed during getting next face_count_packet."; - - return absl::Status(); + return absl::CancelledError( + "Failed during getting next face_count_packet."); } + auto &face_count = face_count_packet.Get(); - if (!face_count) { - return absl::Status(); + faceCount = face_count; + + return absl::OkStatus(); +} + +int MPFaceMeshDetector::GetFaceCount(const cv::Mat &camera_frame) { + const auto status = GetFaceCountWithStatus(camera_frame); + if (!status.ok()) { + LOG(INFO) << "Failed GetFaceCount."; + LOG(INFO) << status.message(); + } + + return faceCount; +} + +absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus( + cv::Point2f **multi_face_landmarks) { + + if (faceCount <= 0) { + return absl::CancelledError( + "Failed during gettinglandmarks, because faceCount is <= 0."); } mediapipe::Packet face_landmarks_packet; if (!landmarks_poller_ptr || !landmarks_poller_ptr->Next(&face_landmarks_packet)) { - LOG(INFO) << "Failed during getting next landmarks_packet."; - - return absl::Status(); + return absl::CancelledError("Failed during getting next landmarks_packet."); } auto &output_landmarks_vector = face_landmarks_packet .Get<::std::vector<::mediapipe::NormalizedLandmarkList>>(); - multi_face_landmarks->reserve(output_landmarks_vector.size()); - - for (const auto &normalizedLandmarkList : output_landmarks_vector) { - multi_face_landmarks->emplace_back(); - - auto &face_landmarks = multi_face_landmarks->back(); - + for (int i = 0; i < faceCount; ++i) { + const auto &normalizedLandmarkList = output_landmarks_vector[i]; const auto landmarks_num = normalizedLandmarkList.landmark_size(); + auto &face_landmarks = multi_face_landmarks[i]; - face_landmarks.reserve(landmarks_num); - - for (int i = 0; i < landmarks_num; ++i) { - auto &landmark = normalizedLandmarkList.landmark(i); - - face_landmarks.emplace_back(landmark.x(), landmark.y()); + for (int j = 0; j < landmarks_num; ++j) { + const auto &landmark = normalizedLandmarkList.landmark(j); + face_landmarks[j].x = landmark.x(); + face_landmarks[j].y = landmark.y(); } } - return absl::Status(); + faceCount = -1; + + return absl::OkStatus(); } -std::vector> * -MPFaceMeshDetector::ProcessFrame2D(const cv::Mat &camera_frame) { - auto landmarks = std::make_unique>>(); - - ProcessFrameWithStatus(camera_frame, landmarks); - - return landmarks.release(); +void MPFaceMeshDetector::GetFaceLandmarks(cv::Point2f **multi_face_landmarks) { + const auto status = GetFaceLandmarksWithStatus(multi_face_landmarks); + if (!status.ok()) { + LOG(INFO) << "Failed GetFaceLandmarks."; + LOG(INFO) << status.message(); + } } extern "C" { -DLLEXPORT MPFaceMeshDetector *FaceMeshDetector_Construct() { - return new MPFaceMeshDetector(); +DLLEXPORT MPFaceMeshDetector * +FaceMeshDetector_Construct(int numFaces, const char *face_detection_model_path, + const char *face_landmark_model_path) { + return new MPFaceMeshDetector(numFaces, face_detection_model_path, + face_landmark_model_path); } DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector) { delete detector; } -DLLEXPORT void * -FaceMeshDetector_ProcessFrame2D(MPFaceMeshDetector *detector, - const cv::Mat &camera_frame) { - return reinterpret_cast(detector->ProcessFrame2D(camera_frame)); +DLLEXPORT int FaceMeshDetector_GetFaceCount(MPFaceMeshDetector *detector, + const cv::Mat &camera_frame) { + return detector->GetFaceCount(camera_frame); +} + +DLLEXPORT void +FaceMeshDetector_GetFaceLandmarks(MPFaceMeshDetector *detector, + cv::Point2f **multi_face_landmarks) { + detector->GetFaceLandmarks(multi_face_landmarks); } } @@ -163,16 +211,60 @@ node { output_side_packet: "PACKET:num_faces" node_options: { [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { - packet { int_value: 1 } + packet { int_value: $numFaces } } } } +# Defines side packets for further use in the graph. +node { + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:face_detection_model_path" + options: { + [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { + packet { string_value: "$faceDetectionModelPath" } + } + } +} + +# Defines side packets for further use in the graph. +node { + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:face_landmark_model_path" + node_options: { + [type.googleapis.com/mediapipe.ConstantSidePacketCalculatorOptions]: { + packet { string_value: "$faceLandmarkModelPath" } + } + } +} + +node { + calculator: "LocalFileContentsCalculator" + input_side_packet: "FILE_PATH:0:face_detection_model_path" + input_side_packet: "FILE_PATH:1:face_landmark_model_path" + output_side_packet: "CONTENTS:0:face_detection_model_blob" + output_side_packet: "CONTENTS:1:face_landmark_model_blob" +} + +node { + calculator: "TfLiteModelCalculator" + input_side_packet: "MODEL_BLOB:face_detection_model_blob" + output_side_packet: "MODEL:face_detection_model" +} +node { + calculator: "TfLiteModelCalculator" + input_side_packet: "MODEL_BLOB:face_landmark_model_blob" + output_side_packet: "MODEL:face_landmark_model" +} + + # Subgraph that detects faces and corresponding landmarks. node { - calculator: "FaceLandmarkFrontCpuWithFaceCounter" + calculator: "FaceLandmarkFrontSideModelCpuWithFaceCounter" input_stream: "IMAGE:throttled_input_video" input_side_packet: "NUM_FACES:num_faces" + input_side_packet: "MODEL:0:face_detection_model" + input_side_packet: "MODEL:1:face_landmark_model" output_stream: "LANDMARKS:multi_face_landmarks" output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" output_stream: "DETECTIONS:face_detections" diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h index d6fe713e4..88c3ed680 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h @@ -13,11 +13,13 @@ #include "absl/flags/flag.h" #include "absl/flags/parse.h" +#include "absl/strings/str_replace.h" #include "mediapipe/framework/calculator_framework.h" #include "mediapipe/framework/calculator_graph.h" #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/output_stream_poller.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/opencv_highgui_inc.h" #include "mediapipe/framework/port/opencv_imgproc_inc.h" @@ -27,15 +29,20 @@ class MPFaceMeshDetector { public: - MPFaceMeshDetector(); - std::vector> *ProcessFrame2D(const cv::Mat &camera_frame); + MPFaceMeshDetector(int numFaces, const char *face_detection_model_path, + const char *face_landmark_model_path); + int GetFaceCount(const cv::Mat &camera_frame); + void GetFaceLandmarks(cv::Point2f **multi_face_landmarks); private: - absl::Status InitFaceMeshDetector(); - absl::Status - ProcessFrameWithStatus(const cv::Mat &camera_frame, - std::unique_ptr>> - &multi_face_landmarks); + absl::Status InitFaceMeshDetector(int numFaces, + const char *face_detection_model_path, + const char *face_landmark_model_path); + absl::Status ProcessFrameWithStatus( + const cv::Mat &camera_frame, + std::vector> &multi_face_landmarks); + absl::Status GetFaceCountWithStatus(const cv::Mat &camera_frame); + absl::Status GetFaceLandmarksWithStatus(cv::Point2f **multi_face_landmarks); static const char kInputStream[]; static const char kOutputStream_landmarks[]; @@ -47,18 +54,29 @@ private: std::unique_ptr landmarks_poller_ptr; std::unique_ptr face_count_poller_ptr; + + int faceCount = -1; }; #ifdef __cplusplus extern "C" { #endif -DLLEXPORT MPFaceMeshDetector *FaceMeshDetector_Construct(); +DLLEXPORT MPFaceMeshDetector *FaceMeshDetector_Construct( + int numFaces = 1, + const char *face_detection_model_path = + "mediapipe/modules/face_detection/face_detection_short_range.tflite", + const char *face_landmark_model_path = + "mediapipe/modules/face_landmark/face_landmark.tflite"); + DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector); -DLLEXPORT void *FaceMeshDetector_ProcessFrame2D(MPFaceMeshDetector *detector, - const cv::Mat &camera_frame); +DLLEXPORT int FaceMeshDetector_GetFaceCount(MPFaceMeshDetector *detector, + const cv::Mat &camera_frame); +DLLEXPORT void +FaceMeshDetector_GetFaceLandmarks(MPFaceMeshDetector *detector, + cv::Point2f **multi_face_landmarks); #ifdef __cplusplus }; diff --git a/mediapipe/modules/face_detection/BUILD b/mediapipe/modules/face_detection/BUILD index 839418c77..4a0b41544 100644 --- a/mediapipe/modules/face_detection/BUILD +++ b/mediapipe/modules/face_detection/BUILD @@ -57,6 +57,18 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "face_detection_short_range_side_model_cpu", + graph = "face_detection_short_range_side_model_cpu.pbtxt", + register_as = "FaceDetectionShortRangeSideModelCpu", + deps = [ + ":face_detection_short_range_common", + "//mediapipe/calculators/tensor:image_to_tensor_calculator", + "//mediapipe/calculators/tensor:inference_calculator", + "//mediapipe/calculators/util:to_image_calculator", + ], +) + mediapipe_simple_subgraph( name = "face_detection_short_range_gpu", graph = "face_detection_short_range_gpu.pbtxt", diff --git a/mediapipe/modules/face_detection/face_detection_short_range_side_model_cpu.pbtxt b/mediapipe/modules/face_detection/face_detection_short_range_side_model_cpu.pbtxt new file mode 100644 index 000000000..57639bab2 --- /dev/null +++ b/mediapipe/modules/face_detection/face_detection_short_range_side_model_cpu.pbtxt @@ -0,0 +1,86 @@ +# MediaPipe graph to detect faces. (CPU input, and inference is executed on +# CPU.) +# +# It is required that "face_detection_short_range.tflite" is available at +# "mediapipe/modules/face_detection/face_detection_short_range.tflite" +# path during execution. +# +# EXAMPLE: +# node { +# calculator: "FaceDetectionShortRangeCpu" +# input_stream: "IMAGE:image" +# input_side_packet: "MODEL:face_detection_model" +# output_stream: "DETECTIONS:face_detections" +# } + +type: "FaceDetectionShortRangeCpu" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:image" + +# TfLite model to detect faces. +# (std::unique_ptr>) +# NOTE: mediapipe/modules/face_detection/face_detection_short_range.tflite +# model only, can be passed here, otherwise - results are undefined. +input_side_packet: "MODEL:face_detection_model" + +# Detected faces. (std::vector) +# NOTE: there will not be an output packet in the DETECTIONS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "DETECTIONS:detections" + +# Converts the input CPU image (ImageFrame) to the multi-backend image type +# (Image). +node: { + calculator: "ToImageCalculator" + input_stream: "IMAGE_CPU:image" + output_stream: "IMAGE:multi_backend_image" +} + +# Transforms the input image into a 128x128 tensor while keeping the aspect +# ratio (what is expected by the corresponding face detection model), resulting +# in potential letterboxing in the transformed image. +node: { + calculator: "ImageToTensorCalculator" + input_stream: "IMAGE:multi_backend_image" + output_stream: "TENSORS:input_tensors" + output_stream: "MATRIX:transform_matrix" + options: { + [mediapipe.ImageToTensorCalculatorOptions.ext] { + output_tensor_width: 128 + output_tensor_height: 128 + keep_aspect_ratio: true + output_tensor_float_range { + min: -1.0 + max: 1.0 + } + border_mode: BORDER_ZERO + } + } +} + +# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "InferenceCalculator" + input_stream: "TENSORS:input_tensors" + output_stream: "TENSORS:detection_tensors" + input_side_packet: "MODEL:face_detection_model" + options { + [mediapipe.InferenceCalculatorOptions.ext] { + delegate { tflite {} } + } + } +} + +# Performs tensor post processing to generate face detections. +node { + calculator: "FaceDetectionShortRangeCommon" + input_stream: "TENSORS:detection_tensors" + input_stream: "MATRIX:transform_matrix" + output_stream: "DETECTIONS:detections" +} diff --git a/mediapipe/modules/face_landmark/BUILD b/mediapipe/modules/face_landmark/BUILD index 30720c1b0..6e642d7fc 100644 --- a/mediapipe/modules/face_landmark/BUILD +++ b/mediapipe/modules/face_landmark/BUILD @@ -37,6 +37,22 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "face_landmark_side_model_cpu", + graph = "face_landmark_side_model_cpu.pbtxt", + register_as = "FaceLandmarkSideModelCpu", + deps = [ + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:split_vector_calculator", + "//mediapipe/calculators/tensor:image_to_tensor_calculator", + "//mediapipe/calculators/tensor:inference_calculator", + "//mediapipe/calculators/tensor:tensors_to_floats_calculator", + "//mediapipe/calculators/tensor:tensors_to_landmarks_calculator", + "//mediapipe/calculators/util:landmark_projection_calculator", + "//mediapipe/calculators/util:thresholding_calculator", + ], +) + mediapipe_simple_subgraph( name = "face_landmark_gpu", graph = "face_landmark_gpu.pbtxt", @@ -96,6 +112,28 @@ mediapipe_simple_subgraph( ], ) +mediapipe_simple_subgraph( + name = "face_landmark_front_side_model_cpu_with_face_counter", + graph = "face_landmark_front_side_model_cpu_with_face_counter.pbtxt", + register_as = "FaceLandmarkFrontSideModelCpuWithFaceCounter", + deps = [ + ":face_detection_front_detection_to_roi", + ":face_landmark_side_model_cpu", + ":face_landmark_landmarks_to_roi", + "//mediapipe/calculators/core:begin_loop_calculator", + "//mediapipe/calculators/core:clip_vector_size_calculator", + "//mediapipe/calculators/core:constant_side_packet_calculator", + "//mediapipe/calculators/core:end_loop_calculator", + "//mediapipe/calculators/core:gate_calculator", + "//mediapipe/calculators/core:previous_loopback_calculator", + "//mediapipe/calculators/image:image_properties_calculator", + "//mediapipe/calculators/util:association_norm_rect_calculator", + "//mediapipe/calculators/util:collection_has_min_size_calculator", + "//mediapipe/calculators/util:counting_vector_size_calculator", + "//mediapipe/modules/face_detection:face_detection_short_range_side_model_cpu", + ], +) + mediapipe_simple_subgraph( name = "face_landmark_front_gpu", graph = "face_landmark_front_gpu.pbtxt", diff --git a/mediapipe/modules/face_landmark/face_landmark_front_side_model_cpu_with_face_counter.pbtxt b/mediapipe/modules/face_landmark/face_landmark_front_side_model_cpu_with_face_counter.pbtxt new file mode 100644 index 000000000..dc83f17b7 --- /dev/null +++ b/mediapipe/modules/face_landmark/face_landmark_front_side_model_cpu_with_face_counter.pbtxt @@ -0,0 +1,256 @@ +# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is +# executed on CPU.) This graph tries to skip face detection as much as possible +# by using previously detected/predicted landmarks for new images. +# +# EXAMPLE: +# node { +# calculator: "FaceLandmarkFrontSideModelCpu" +# input_stream: "IMAGE:image" +# input_side_packet: "NUM_FACES:num_faces" +# input_side_packet: "MODEL:0:face_detection_model" +# input_side_packet: "MODEL:1:face_landmark_model" +# output_stream: "LANDMARKS:multi_face_landmarks" +# } + +type: "FaceLandmarkFrontSideModelCpu" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:image" + +# Max number of faces to detect/track. (int) +input_side_packet: "NUM_FACES:num_faces" +# TfLite model to detect faces. +# (std::unique_ptr>) +# NOTE: mediapipe/modules/face_detection/face_detection_short_range.tflite +# model only, can be passed here, otherwise - results are undefined. +input_side_packet: "MODEL:0:face_detection_model" +# TfLite model to detect face landmarks. +# (std::unique_ptr>) +# NOTE: mediapipe/modules/face_landmark/face_landmark.tflite model +# only, can be passed here, otherwise - results are undefined. +input_side_packet: "MODEL:1:face_landmark_model" + +# Collection of detected/predicted faces, each represented as a list of 468 face +# landmarks. (std::vector) +# NOTE: there will not be an output packet in the LANDMARKS stream for this +# particular timestamp if none of faces detected. However, the MediaPipe +# framework will internally inform the downstream calculators of the absence of +# this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:multi_face_landmarks" + +# Extra outputs (for debugging, for instance). +# Detected faces. (std::vector) +output_stream: "DETECTIONS:face_detections" +# Regions of interest calculated based on landmarks. +# (std::vector) +output_stream: "ROIS_FROM_LANDMARKS:face_rects_from_landmarks" +# Regions of interest calculated based on face detections. +# (std::vector) +output_stream: "ROIS_FROM_DETECTIONS:face_rects_from_detections" + +# (int) +output_stream: "FACE_COUNT_FROM_LANDMARKS:face_count" + + +# Defines whether landmarks on the previous image should be used to help +# localize landmarks on the current image. +node { + name: "ConstantSidePacketCalculator" + calculator: "ConstantSidePacketCalculator" + output_side_packet: "PACKET:use_prev_landmarks" + options: { + [mediapipe.ConstantSidePacketCalculatorOptions.ext]: { + packet { bool_value: true } + } + } +} +node { + calculator: "GateCalculator" + input_side_packet: "ALLOW:use_prev_landmarks" + input_stream: "prev_face_rects_from_landmarks" + output_stream: "gated_prev_face_rects_from_landmarks" +} + +# Determines if an input vector of NormalizedRect has a size greater than or +# equal to the provided num_faces. +node { + calculator: "NormalizedRectVectorHasMinSizeCalculator" + input_stream: "ITERABLE:prev_face_rects_from_landmarks" + input_side_packet: "num_faces" + output_stream: "prev_has_enough_faces" +} + +# Drops the incoming image if FaceLandmarkCpu was able to identify face presence +# in the previous image. Otherwise, passes the incoming image through to trigger +# a new round of face detection in FaceDetectionShortRangeCpu. +node { + calculator: "GateCalculator" + input_stream: "image" + input_stream: "DISALLOW:prev_has_enough_faces" + output_stream: "gated_image" + options: { + [mediapipe.GateCalculatorOptions.ext] { + empty_packets_as_allow: true + } + } +} + +# Detects faces. +node { + calculator: "FaceDetectionShortRangeSideModelCpu" + input_stream: "IMAGE:gated_image" + input_side_packet: "MODEL:face_detection_model" + output_stream: "DETECTIONS:all_face_detections" +} + +# Makes sure there are no more detections than the provided num_faces. +node { + calculator: "ClipDetectionVectorSizeCalculator" + input_stream: "all_face_detections" + output_stream: "face_detections" + input_side_packet: "num_faces" +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:gated_image" + output_stream: "SIZE:gated_image_size" +} + +# Outputs each element of face_detections at a fake timestamp for the rest of +# the graph to process. Clones the image size packet for each face_detection at +# the fake timestamp. At the end of the loop, outputs the BATCH_END timestamp +# for downstream calculators to inform them that all elements in the vector have +# been processed. +node { + calculator: "BeginLoopDetectionCalculator" + input_stream: "ITERABLE:face_detections" + input_stream: "CLONE:gated_image_size" + output_stream: "ITEM:face_detection" + output_stream: "CLONE:detections_loop_image_size" + output_stream: "BATCH_END:detections_loop_end_timestamp" +} + +# Calculates region of interest based on face detections, so that can be used +# to detect landmarks. +node { + calculator: "FaceDetectionFrontDetectionToRoi" + input_stream: "DETECTION:face_detection" + input_stream: "IMAGE_SIZE:detections_loop_image_size" + output_stream: "ROI:face_rect_from_detection" +} + +# Counting a multi_faceLandmarks vector size. The image stream is only used to +# make the calculator work even when there is no input vector. +node { + calculator: "CountingNormalizedLandmarkListVectorSizeCalculator" + input_stream: "CLOCK:image" + input_stream: "VECTOR:multi_face_landmarks" + output_stream: "COUNT:face_count" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_detection" + input_stream: "BATCH_END:detections_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_detections" +} + +# Performs association between NormalizedRect vector elements from previous +# image and rects based on face detections from the current image. This +# calculator ensures that the output face_rects vector doesn't contain +# overlapping regions based on the specified min_similarity_threshold. +node { + calculator: "AssociationNormRectCalculator" + input_stream: "face_rects_from_detections" + input_stream: "prev_face_rects_from_landmarks" + output_stream: "face_rects" + options: { + [mediapipe.AssociationCalculatorOptions.ext] { + min_similarity_threshold: 0.5 + } + } +} + +# Calculate size of the image. +node { + calculator: "ImagePropertiesCalculator" + input_stream: "IMAGE:image" + output_stream: "SIZE:image_size" +} + +# Outputs each element of face_rects at a fake timestamp for the rest of the +# graph to process. Clones image and image size packets for each +# single_face_rect at the fake timestamp. At the end of the loop, outputs the +# BATCH_END timestamp for downstream calculators to inform them that all +# elements in the vector have been processed. +node { + calculator: "BeginLoopNormalizedRectCalculator" + input_stream: "ITERABLE:face_rects" + input_stream: "CLONE:0:image" + input_stream: "CLONE:1:image_size" + output_stream: "ITEM:face_rect" + output_stream: "CLONE:0:landmarks_loop_image" + output_stream: "CLONE:1:landmarks_loop_image_size" + output_stream: "BATCH_END:landmarks_loop_end_timestamp" +} + +# Detects face landmarks within specified region of interest of the image. +node { + calculator: "FaceLandmarkSideModelCpu" + input_stream: "IMAGE:landmarks_loop_image" + input_stream: "ROI:face_rect" + input_side_packet: "MODEL:face_landmark_model" + output_stream: "LANDMARKS:face_landmarks" +} + +# Calculates region of interest based on face landmarks, so that can be reused +# for subsequent image. +node { + calculator: "FaceLandmarkLandmarksToRoi" + input_stream: "LANDMARKS:face_landmarks" + input_stream: "IMAGE_SIZE:landmarks_loop_image_size" + output_stream: "ROI:face_rect_from_landmarks" +} + +# Collects a set of landmarks for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of landmarks at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedLandmarkListVectorCalculator" + input_stream: "ITEM:face_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:multi_face_landmarks" +} + +# Collects a NormalizedRect for each face into a vector. Upon receiving the +# BATCH_END timestamp, outputs the vector of NormalizedRect at the BATCH_END +# timestamp. +node { + calculator: "EndLoopNormalizedRectCalculator" + input_stream: "ITEM:face_rect_from_landmarks" + input_stream: "BATCH_END:landmarks_loop_end_timestamp" + output_stream: "ITERABLE:face_rects_from_landmarks" +} + +# Caches face rects calculated from landmarks, and upon the arrival of the next +# input image, sends out the cached rects with timestamps replaced by that of +# the input image, essentially generating a packet that carries the previous +# face rects. Note that upon the arrival of the very first input image, a +# timestamp bound update occurs to jump start the feedback loop. +node { + calculator: "PreviousLoopbackCalculator" + input_stream: "MAIN:image" + input_stream: "LOOP:face_rects_from_landmarks" + input_stream_info: { + tag_index: "LOOP" + back_edge: true + } + output_stream: "PREV_LOOP:prev_face_rects_from_landmarks" +} diff --git a/mediapipe/modules/face_landmark/face_landmark_side_model_cpu.pbtxt b/mediapipe/modules/face_landmark/face_landmark_side_model_cpu.pbtxt new file mode 100644 index 000000000..d8537fd82 --- /dev/null +++ b/mediapipe/modules/face_landmark/face_landmark_side_model_cpu.pbtxt @@ -0,0 +1,143 @@ +# MediaPipe graph to detect/predict face landmarks. (CPU input, and inference is +# executed on CPU.) +# +# It is required that "face_landmark.tflite" is available at +# "mediapipe/modules/face_landmark/face_landmark.tflite" +# path during execution. +# +# EXAMPLE: +# node { +# calculator: "FaceLandmarkCpu" +# input_stream: "IMAGE:image" +# input_stream: "ROI:face_roi" +# input_side_packet: "MODEL:face_landmark_model" +# output_stream: "LANDMARKS:face_landmarks" +# } + +type: "FaceLandmarkCpu" + +# CPU image. (ImageFrame) +input_stream: "IMAGE:image" +# ROI (region of interest) within the given image where a face is located. +# (NormalizedRect) +input_stream: "ROI:roi" + +# TfLite model to detect face landmarks. +# (std::unique_ptr>) +# NOTE: mediapipe/modules/face_landmark/face_landmark.tflite model +# only, can be passed here, otherwise - results are undefined. +input_side_packet: "MODEL:face_landmark_model" + + +# 468 face landmarks within the given ROI. (NormalizedLandmarkList) +# NOTE: if a face is not present within the given ROI, for this particular +# timestamp there will not be an output packet in the LANDMARKS stream. However, +# the MediaPipe framework will internally inform the downstream calculators of +# the absence of this packet so that they don't wait for it unnecessarily. +output_stream: "LANDMARKS:face_landmarks" + +# Transforms the input image into a 192x192 tensor. +node: { + calculator: "ImageToTensorCalculator" + input_stream: "IMAGE:image" + input_stream: "NORM_RECT:roi" + output_stream: "TENSORS:input_tensors" + options: { + [mediapipe.ImageToTensorCalculatorOptions.ext] { + output_tensor_width: 192 + output_tensor_height: 192 + output_tensor_float_range { + min: 0.0 + max: 1.0 + } + } + } +} + +# Runs a TensorFlow Lite model on CPU that takes an image tensor and outputs a +# vector of tensors representing, for instance, detection boxes/keypoints and +# scores. +node { + calculator: "InferenceCalculator" + input_stream: "TENSORS:input_tensors" + output_stream: "TENSORS:output_tensors" + input_side_packet: "MODEL:face_landmark_model" + options { + [mediapipe.InferenceCalculatorOptions.ext] { + delegate { tflite {} } + } + } +} + +# Splits a vector of tensors into multiple vectors. +node { + calculator: "SplitTensorVectorCalculator" + input_stream: "output_tensors" + output_stream: "landmark_tensors" + output_stream: "face_flag_tensor" + options: { + [mediapipe.SplitVectorCalculatorOptions.ext] { + ranges: { begin: 0 end: 1 } + ranges: { begin: 1 end: 2 } + } + } +} + +# Converts the face-flag tensor into a float that represents the confidence +# score of face presence. +node { + calculator: "TensorsToFloatsCalculator" + input_stream: "TENSORS:face_flag_tensor" + output_stream: "FLOAT:face_presence_score" + options { + [mediapipe.TensorsToFloatsCalculatorOptions.ext] { + activation: SIGMOID + } + } +} + +# Applies a threshold to the confidence score to determine whether a face is +# present. +node { + calculator: "ThresholdingCalculator" + input_stream: "FLOAT:face_presence_score" + output_stream: "FLAG:face_presence" + options: { + [mediapipe.ThresholdingCalculatorOptions.ext] { + threshold: 0.5 + } + } +} + +# Drop landmarks tensors if face is not present. +node { + calculator: "GateCalculator" + input_stream: "landmark_tensors" + input_stream: "ALLOW:face_presence" + output_stream: "ensured_landmark_tensors" +} + +# Decodes the landmark tensors into a vector of landmarks, where the landmark +# coordinates are normalized by the size of the input image to the model. +node { + calculator: "TensorsToLandmarksCalculator" + input_stream: "TENSORS:ensured_landmark_tensors" + output_stream: "NORM_LANDMARKS:landmarks" + options: { + [mediapipe.TensorsToLandmarksCalculatorOptions.ext] { + num_landmarks: 468 + input_image_width: 192 + input_image_height: 192 + } + } +} + +# Projects the landmarks from the cropped face image to the corresponding +# locations on the full image before cropping (input to the graph). +node { + calculator: "LandmarkProjectionCalculator" + input_stream: "NORM_LANDMARKS:landmarks" + input_stream: "NORM_RECT:roi" + output_stream: "NORM_LANDMARKS:face_landmarks" +} From 6d89ef3e9efb3a44358466b30f5750bbc224dbc6 Mon Sep 17 00:00:00 2001 From: dmaletskiy Date: Wed, 14 Jul 2021 12:19:34 +0300 Subject: [PATCH 7/8] feat: Modified face_mesh_lib functionality Change List: - Combined `MPFaceMeshDetector::GetFaceCount` and `MPFaceMeshDetector::GetFaceLandmarks` into `MPFaceMeshDetector::ProcessFrame2D` - Added `MPFaceMeshDetectorLandmarksNum` that exports number of face landmarks detected by MediaPipe --- .../desktop/face_mesh_dll/face_mesh_cpu.cpp | 24 +++-- .../desktop/face_mesh_dll/face_mesh_lib.cpp | 88 ++++++++----------- .../desktop/face_mesh_dll/face_mesh_lib.h | 45 +++++----- 3 files changed, 68 insertions(+), 89 deletions(-) diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp index 90462477a..0d9011dd0 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp @@ -28,15 +28,15 @@ int main(int argc, char **argv) { constexpr char face_landmark_model_path[] = "mediapipe/modules/face_landmark/face_landmark.tflite"; - MPFaceMeshDetector *faceMeshDetector = FaceMeshDetector_Construct( + MPFaceMeshDetector *faceMeshDetector = MPFaceMeshDetectorConstruct( maxNumFaces, face_detection_model_path, face_landmark_model_path); - // allocate memory for face landmarks + // Allocate memory for face landmarks. auto multiFaceLandmarks = new cv::Point2f *[maxNumFaces]; - constexpr auto mediapipeFaceLandmarksNum = 468; for (int i = 0; i < maxNumFaces; ++i) { - multiFaceLandmarks[i] = new cv::Point2f[mediapipeFaceLandmarksNum]; + multiFaceLandmarks[i] = new cv::Point2f[MPFaceMeshDetectorLandmarksNum]; } + const auto faceCount = std::make_unique(); LOG(INFO) << "FaceMeshDetector constructed."; @@ -56,21 +56,19 @@ int main(int argc, char **argv) { cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); - int faceCount = - FaceMeshDetector_GetFaceCount(faceMeshDetector, camera_frame); + MPFaceMeshDetectorProcessFrame2D(faceMeshDetector, camera_frame, + faceCount.get(), multiFaceLandmarks); - LOG(INFO) << "Detected faces num: " << faceCount; - - if (faceCount > 0) { - - FaceMeshDetector_GetFaceLandmarks(faceMeshDetector, multiFaceLandmarks); + LOG(INFO) << "Detected faces num: " << *faceCount; + if (*faceCount > 0) { auto &face_landmarks = multiFaceLandmarks[0]; auto &landmark = face_landmarks[0]; LOG(INFO) << "First landmark: x - " << landmark.x << ", y - " << landmark.y; } + const int pressed_key = cv::waitKey(5); if (pressed_key >= 0 && pressed_key != 255) grab_frames = false; @@ -80,11 +78,11 @@ int main(int argc, char **argv) { LOG(INFO) << "Shutting down."; - // deallocate memory for face landmarks + // Deallocate memory for face landmarks. for (int i = 0; i < maxNumFaces; ++i) { delete[] multiFaceLandmarks[i]; } delete[] multiFaceLandmarks; - FaceMeshDetector_Destruct(faceMeshDetector); + MPFaceMeshDetectorDestruct(faceMeshDetector); } \ No newline at end of file diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp index 5bba0efea..167a38f93 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp @@ -1,9 +1,5 @@ -#include - #include "face_mesh_lib.h" -#define DEBUG - MPFaceMeshDetector::MPFaceMeshDetector(int numFaces, const char *face_detection_model_path, const char *face_landmark_model_path) { @@ -19,9 +15,7 @@ absl::Status MPFaceMeshDetector::InitFaceMeshDetector(int numFaces, const char *face_detection_model_path, const char *face_landmark_model_path) { - if (numFaces <= 0) { - numFaces = 1; - } + numFaces = std::max(numFaces, 1); if (face_detection_model_path == nullptr) { face_detection_model_path = @@ -33,6 +27,7 @@ MPFaceMeshDetector::InitFaceMeshDetector(int numFaces, "mediapipe/modules/face_landmark/face_landmark.tflite"; } + // Prepare graph config. auto preparedGraphConfig = absl::StrReplaceAll( graphConfig, {{"$numFaces", std::to_string(numFaces)}}); preparedGraphConfig = absl::StrReplaceAll( @@ -70,8 +65,11 @@ MPFaceMeshDetector::InitFaceMeshDetector(int numFaces, return absl::OkStatus(); } -absl::Status -MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) { +absl::Status MPFaceMeshDetector::ProcessFrame2DWithStatus( + const cv::Mat &camera_frame, int *numFaces, + cv::Point2f **multi_face_landmarks) { + *numFaces = 0; + // Wrap Mat into an ImageFrame. auto input_frame = absl::make_unique( mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, @@ -85,6 +83,8 @@ MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) { MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( kInputStream, mediapipe::Adopt(input_frame.release()) .At(mediapipe::Timestamp(frame_timestamp_us)))); + + // Get face count. mediapipe::Packet face_count_packet; if (!face_count_poller_ptr || !face_count_poller_ptr->Next(&face_count_packet)) { @@ -94,29 +94,11 @@ MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) { auto &face_count = face_count_packet.Get(); - faceCount = face_count; - - return absl::OkStatus(); -} - -int MPFaceMeshDetector::GetFaceCount(const cv::Mat &camera_frame) { - const auto status = GetFaceCountWithStatus(camera_frame); - if (!status.ok()) { - LOG(INFO) << "Failed GetFaceCount."; - LOG(INFO) << status.message(); - } - - return faceCount; -} - -absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus( - cv::Point2f **multi_face_landmarks) { - - if (faceCount <= 0) { - return absl::CancelledError( - "Failed during gettinglandmarks, because faceCount is <= 0."); + if (face_count <= 0) { + return absl::OkStatus(); } + // Get face landmarks. mediapipe::Packet face_landmarks_packet; if (!landmarks_poller_ptr || !landmarks_poller_ptr->Next(&face_landmarks_packet)) { @@ -127,9 +109,15 @@ absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus( face_landmarks_packet .Get<::std::vector<::mediapipe::NormalizedLandmarkList>>(); - for (int i = 0; i < faceCount; ++i) { + // Convert landmarks to cv::Point2f**. + for (int i = 0; i < face_count; ++i) { const auto &normalizedLandmarkList = output_landmarks_vector[i]; const auto landmarks_num = normalizedLandmarkList.landmark_size(); + + if (landmarks_num != kLandmarksNum) { + return absl::CancelledError("Detected unexpected landmarks number."); + } + auto &face_landmarks = multi_face_landmarks[i]; for (int j = 0; j < landmarks_num; ++j) { @@ -139,47 +127,43 @@ absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus( } } - faceCount = -1; + *numFaces = face_count; return absl::OkStatus(); } -void MPFaceMeshDetector::GetFaceLandmarks(cv::Point2f **multi_face_landmarks) { - const auto status = GetFaceLandmarksWithStatus(multi_face_landmarks); +void MPFaceMeshDetector::ProcessFrame2D(const cv::Mat &camera_frame, + int *numFaces, + cv::Point2f **multi_face_landmarks) { + const auto status = + ProcessFrame2DWithStatus(camera_frame, numFaces, multi_face_landmarks); if (!status.ok()) { - LOG(INFO) << "Failed GetFaceLandmarks."; + LOG(INFO) << "Failed ProcessFrame2D."; LOG(INFO) << status.message(); } } extern "C" { DLLEXPORT MPFaceMeshDetector * -FaceMeshDetector_Construct(int numFaces, const char *face_detection_model_path, - const char *face_landmark_model_path) { +MPFaceMeshDetectorConstruct(int numFaces, const char *face_detection_model_path, + const char *face_landmark_model_path) { return new MPFaceMeshDetector(numFaces, face_detection_model_path, face_landmark_model_path); } -DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector) { +DLLEXPORT void MPFaceMeshDetectorDestruct(MPFaceMeshDetector *detector) { delete detector; } -DLLEXPORT int FaceMeshDetector_GetFaceCount(MPFaceMeshDetector *detector, - const cv::Mat &camera_frame) { - return detector->GetFaceCount(camera_frame); -} - DLLEXPORT void -FaceMeshDetector_GetFaceLandmarks(MPFaceMeshDetector *detector, - cv::Point2f **multi_face_landmarks) { - detector->GetFaceLandmarks(multi_face_landmarks); -} +MPFaceMeshDetectorProcessFrame2D(MPFaceMeshDetector *detector, + const cv::Mat &camera_frame, int *numFaces, + cv::Point2f **multi_face_landmarks) { + detector->ProcessFrame2D(camera_frame, numFaces, multi_face_landmarks); } -const char MPFaceMeshDetector::kInputStream[] = "input_video"; -const char MPFaceMeshDetector::kOutputStream_landmarks[] = - "multi_face_landmarks"; -const char MPFaceMeshDetector::kOutputStream_faceCount[] = "face_count"; +DLLEXPORT const int MPFaceMeshDetectorLandmarksNum = MPFaceMeshDetector::kLandmarksNum; +} const std::string MPFaceMeshDetector::graphConfig = R"pb( # MediaPipe graph that performs face mesh with TensorFlow Lite on CPU. @@ -197,7 +181,7 @@ output_stream: "face_count" node { calculator: "FlowLimiterCalculator" input_stream: "input_video" - input_stream: "FINISHED:multi_face_landmarks" + input_stream: "FINISHED:face_count" input_stream_info: { tag_index: "FINISHED" back_edge: true diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h index 88c3ed680..4b40417bc 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h @@ -10,6 +10,7 @@ #include #include #include +#include #include "absl/flags/flag.h" #include "absl/flags/parse.h" @@ -29,24 +30,25 @@ class MPFaceMeshDetector { public: + static constexpr auto kLandmarksNum = 468; + MPFaceMeshDetector(int numFaces, const char *face_detection_model_path, const char *face_landmark_model_path); - int GetFaceCount(const cv::Mat &camera_frame); - void GetFaceLandmarks(cv::Point2f **multi_face_landmarks); + + void ProcessFrame2D(const cv::Mat &camera_frame, int *numFaces, + cv::Point2f **multi_face_landmarks); private: absl::Status InitFaceMeshDetector(int numFaces, const char *face_detection_model_path, const char *face_landmark_model_path); - absl::Status ProcessFrameWithStatus( - const cv::Mat &camera_frame, - std::vector> &multi_face_landmarks); - absl::Status GetFaceCountWithStatus(const cv::Mat &camera_frame); - absl::Status GetFaceLandmarksWithStatus(cv::Point2f **multi_face_landmarks); + absl::Status ProcessFrame2DWithStatus(const cv::Mat &camera_frame, + int *numFaces, + cv::Point2f **multi_face_landmarks); - static const char kInputStream[]; - static const char kOutputStream_landmarks[]; - static const char kOutputStream_faceCount[]; + static constexpr auto kInputStream = "input_video"; + static constexpr auto kOutputStream_landmarks = "multi_face_landmarks"; + static constexpr auto kOutputStream_faceCount = "face_count"; static const std::string graphConfig; @@ -54,29 +56,24 @@ private: std::unique_ptr landmarks_poller_ptr; std::unique_ptr face_count_poller_ptr; - - int faceCount = -1; }; #ifdef __cplusplus extern "C" { #endif -DLLEXPORT MPFaceMeshDetector *FaceMeshDetector_Construct( - int numFaces = 1, - const char *face_detection_model_path = - "mediapipe/modules/face_detection/face_detection_short_range.tflite", - const char *face_landmark_model_path = - "mediapipe/modules/face_landmark/face_landmark.tflite"); +DLLEXPORT MPFaceMeshDetector * +MPFaceMeshDetectorConstruct(int numFaces, const char *face_detection_model_path, + const char *face_landmark_model_path); +DLLEXPORT void MPFaceMeshDetectorDestruct(MPFaceMeshDetector *detector); -DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector); - -DLLEXPORT int FaceMeshDetector_GetFaceCount(MPFaceMeshDetector *detector, - const cv::Mat &camera_frame); DLLEXPORT void -FaceMeshDetector_GetFaceLandmarks(MPFaceMeshDetector *detector, - cv::Point2f **multi_face_landmarks); +MPFaceMeshDetectorProcessFrame2D(MPFaceMeshDetector *detector, + const cv::Mat &camera_frame, int *numFaces, + cv::Point2f **multi_face_landmarks); + +DLLEXPORT extern const int MPFaceMeshDetectorLandmarksNum; #ifdef __cplusplus }; From 393ad8ffccea5020c7fec4429ee170a717acd903 Mon Sep 17 00:00:00 2001 From: dmaletskiy Date: Thu, 12 Aug 2021 17:40:07 +0300 Subject: [PATCH 8/8] Added possibility to get detected face bounding boxes --- .../examples/desktop/face_mesh_dll/BUILD | 1 + .../desktop/face_mesh_dll/face_mesh_cpu.cpp | 20 +- .../desktop/face_mesh_dll/face_mesh_lib.cpp | 189 +++++++++++++++--- .../desktop/face_mesh_dll/face_mesh_lib.h | 44 +++- 4 files changed, 213 insertions(+), 41 deletions(-) diff --git a/mediapipe/examples/desktop/face_mesh_dll/BUILD b/mediapipe/examples/desktop/face_mesh_dll/BUILD index 3a20d0f43..8b6029790 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/BUILD +++ b/mediapipe/examples/desktop/face_mesh_dll/BUILD @@ -36,6 +36,7 @@ windows_dll_library( "//mediapipe/framework/formats:image_frame", "//mediapipe/framework/formats:image_frame_opencv", "//mediapipe/framework/formats:landmark_cc_proto", + "//mediapipe/framework/formats:rect_cc_proto", "//mediapipe/framework/port:file_helpers", "//mediapipe/framework/port:opencv_highgui", "//mediapipe/framework/port:opencv_imgproc", diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp index 0d9011dd0..83762a1a1 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp @@ -36,7 +36,8 @@ int main(int argc, char **argv) { for (int i = 0; i < maxNumFaces; ++i) { multiFaceLandmarks[i] = new cv::Point2f[MPFaceMeshDetectorLandmarksNum]; } - const auto faceCount = std::make_unique(); + + std::vector multiFaceBoundingBoxes(maxNumFaces); LOG(INFO) << "FaceMeshDetector constructed."; @@ -54,14 +55,21 @@ int main(int argc, char **argv) { cv::Mat camera_frame; cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); - cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); - MPFaceMeshDetectorProcessFrame2D(faceMeshDetector, camera_frame, - faceCount.get(), multiFaceLandmarks); + int faceCount = 0; - LOG(INFO) << "Detected faces num: " << *faceCount; + MPFaceMeshDetectorDetectFaces(faceMeshDetector, camera_frame, + multiFaceBoundingBoxes.data(), &faceCount); - if (*faceCount > 0) { + if (faceCount > 0) { + auto &face_bounding_box = multiFaceBoundingBoxes[0]; + + cv::rectangle(camera_frame_raw, face_bounding_box, cv::Scalar(0, 255, 0), + 3); + + int landmarksNum = 0; + MPFaceMeshDetectorDetect2DLandmarks(faceMeshDetector, multiFaceLandmarks, + &landmarksNum); auto &face_landmarks = multiFaceLandmarks[0]; auto &landmark = face_landmarks[0]; diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp index 167a38f93..b3082e58c 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp @@ -52,11 +52,17 @@ MPFaceMeshDetector::InitFaceMeshDetector(int numFaces, graph.AddOutputStreamPoller(kOutputStream_landmarks)); ASSIGN_OR_RETURN(mediapipe::OutputStreamPoller face_count_poller, graph.AddOutputStreamPoller(kOutputStream_faceCount)); + ASSIGN_OR_RETURN( + mediapipe::OutputStreamPoller face_rects_from_landmarks_poller, + graph.AddOutputStreamPoller(kOutputStream_face_rects_from_landmarks)); landmarks_poller_ptr = std::make_unique( std::move(landmarks_poller)); face_count_poller_ptr = std::make_unique( std::move(face_count_poller)); + face_rects_from_landmarks_poller_ptr = + std::make_unique( + std::move(face_rects_from_landmarks_poller)); MP_RETURN_IF_ERROR(graph.StartRun({})); @@ -65,10 +71,19 @@ MPFaceMeshDetector::InitFaceMeshDetector(int numFaces, return absl::OkStatus(); } -absl::Status MPFaceMeshDetector::ProcessFrame2DWithStatus( - const cv::Mat &camera_frame, int *numFaces, - cv::Point2f **multi_face_landmarks) { +absl::Status +MPFaceMeshDetector::DetectFacesWithStatus(const cv::Mat &camera_frame, + cv::Rect *multi_face_bounding_boxes, + int *numFaces) { + if (!numFaces || !multi_face_bounding_boxes) { + return absl::InvalidArgumentError( + "MPFaceMeshDetector::DetectFacesWithStatus requires notnull pointer to " + "save results data."); + } + + // Reset face counts. *numFaces = 0; + face_count = 0; // Wrap Mat into an ImageFrame. auto input_frame = absl::make_unique( @@ -92,26 +107,89 @@ absl::Status MPFaceMeshDetector::ProcessFrame2DWithStatus( "Failed during getting next face_count_packet."); } - auto &face_count = face_count_packet.Get(); + auto &face_count_val = face_count_packet.Get(); - if (face_count <= 0) { + if (face_count_val <= 0) { return absl::OkStatus(); } + // Get face bounding boxes. + mediapipe::Packet face_rects_from_landmarks_packet; + if (!face_rects_from_landmarks_poller_ptr || + !face_rects_from_landmarks_poller_ptr->Next( + &face_rects_from_landmarks_packet)) { + return absl::CancelledError( + "Failed during getting next face_rects_from_landmarks_packet."); + } + + auto &face_bounding_boxes = + face_rects_from_landmarks_packet + .Get<::std::vector<::mediapipe::NormalizedRect>>(); + + image_width = camera_frame.cols; + image_height = camera_frame.rows; + const auto image_width_f = static_cast(image_width); + const auto image_height_f = static_cast(image_height); + + // Convert vector (center based Rects) to cv::Rect* + // (leftTop based Rects). + for (int i = 0; i < face_count_val; ++i) { + const auto &normalized_bounding_box = face_bounding_boxes[i]; + auto &bounding_box = multi_face_bounding_boxes[i]; + + const auto width = + static_cast(normalized_bounding_box.width() * image_width_f); + const auto height = + static_cast(normalized_bounding_box.height() * image_height_f); + + bounding_box.x = + static_cast(normalized_bounding_box.x_center() * image_width_f) - + (width >> 1); + bounding_box.y = + static_cast(normalized_bounding_box.y_center() * image_height_f) - + (height >> 1); + bounding_box.width = width; + bounding_box.height = height; + } + // Get face landmarks. - mediapipe::Packet face_landmarks_packet; if (!landmarks_poller_ptr || !landmarks_poller_ptr->Next(&face_landmarks_packet)) { return absl::CancelledError("Failed during getting next landmarks_packet."); } - auto &output_landmarks_vector = + *numFaces = face_count_val; + face_count = face_count_val; + + return absl::OkStatus(); +} + +void MPFaceMeshDetector::DetectFaces(const cv::Mat &camera_frame, + cv::Rect *multi_face_bounding_boxes, + int *numFaces) { + const auto status = + DetectFacesWithStatus(camera_frame, multi_face_bounding_boxes, numFaces); + if (!status.ok()) { + LOG(INFO) << "MPFaceMeshDetector::DetectFaces failed: " << status.message(); + } +} +absl::Status MPFaceMeshDetector::DetectLandmarksWithStatus( + cv::Point2f **multi_face_landmarks) { + + if (face_landmarks_packet.IsEmpty()) { + return absl::CancelledError("Face landmarks packet is empty."); + } + + auto &face_landmarks = face_landmarks_packet .Get<::std::vector<::mediapipe::NormalizedLandmarkList>>(); + const auto image_width_f = static_cast(image_width); + const auto image_height_f = static_cast(image_height); + // Convert landmarks to cv::Point2f**. for (int i = 0; i < face_count; ++i) { - const auto &normalizedLandmarkList = output_landmarks_vector[i]; + const auto &normalizedLandmarkList = face_landmarks[i]; const auto landmarks_num = normalizedLandmarkList.landmark_size(); if (landmarks_num != kLandmarksNum) { @@ -122,25 +200,70 @@ absl::Status MPFaceMeshDetector::ProcessFrame2DWithStatus( for (int j = 0; j < landmarks_num; ++j) { const auto &landmark = normalizedLandmarkList.landmark(j); - face_landmarks[j].x = landmark.x(); - face_landmarks[j].y = landmark.y(); + face_landmarks[j].x = landmark.x() * image_width_f; + face_landmarks[j].y = landmark.y() * image_height_f; } } - *numFaces = face_count; - return absl::OkStatus(); } -void MPFaceMeshDetector::ProcessFrame2D(const cv::Mat &camera_frame, - int *numFaces, - cv::Point2f **multi_face_landmarks) { - const auto status = - ProcessFrame2DWithStatus(camera_frame, numFaces, multi_face_landmarks); - if (!status.ok()) { - LOG(INFO) << "Failed ProcessFrame2D."; - LOG(INFO) << status.message(); +absl::Status MPFaceMeshDetector::DetectLandmarksWithStatus( + cv::Point3f **multi_face_landmarks) { + + if (face_landmarks_packet.IsEmpty()) { + return absl::CancelledError("Face landmarks packet is empty."); } + + auto &face_landmarks = + face_landmarks_packet + .Get<::std::vector<::mediapipe::NormalizedLandmarkList>>(); + + const auto image_width_f = static_cast(image_width); + const auto image_height_f = static_cast(image_height); + + // Convert landmarks to cv::Point3f**. + for (int i = 0; i < face_count; ++i) { + const auto &normalized_landmark_list = face_landmarks[i]; + const auto landmarks_num = normalized_landmark_list.landmark_size(); + + if (landmarks_num != kLandmarksNum) { + return absl::CancelledError("Detected unexpected landmarks number."); + } + + auto &face_landmarks = multi_face_landmarks[i]; + + for (int j = 0; j < landmarks_num; ++j) { + const auto &landmark = normalized_landmark_list.landmark(j); + face_landmarks[j].x = landmark.x() * image_width_f; + face_landmarks[j].y = landmark.y() * image_height_f; + face_landmarks[j].z = landmark.z(); + } + } + + return absl::OkStatus(); +} + +void MPFaceMeshDetector::DetectLandmarks(cv::Point2f **multi_face_landmarks, + int *numFaces) { + *numFaces = 0; + const auto status = DetectLandmarksWithStatus(multi_face_landmarks); + if (!status.ok()) { + LOG(INFO) << "MPFaceMeshDetector::DetectLandmarks failed: " + << status.message(); + } + *numFaces = face_count; +} + +void MPFaceMeshDetector::DetectLandmarks(cv::Point3f **multi_face_landmarks, + int *numFaces) { + *numFaces = 0; + const auto status = DetectLandmarksWithStatus(multi_face_landmarks); + if (!status.ok()) { + LOG(INFO) << "MPFaceMeshDetector::DetectLandmarks failed: " + << status.message(); + } + *numFaces = face_count; } extern "C" { @@ -155,14 +278,26 @@ DLLEXPORT void MPFaceMeshDetectorDestruct(MPFaceMeshDetector *detector) { delete detector; } +DLLEXPORT void MPFaceMeshDetectorDetectFaces( + MPFaceMeshDetector *detector, const cv::Mat &camera_frame, + cv::Rect *multi_face_bounding_boxes, int *numFaces) { + detector->DetectFaces(camera_frame, multi_face_bounding_boxes, numFaces); +} DLLEXPORT void -MPFaceMeshDetectorProcessFrame2D(MPFaceMeshDetector *detector, - const cv::Mat &camera_frame, int *numFaces, - cv::Point2f **multi_face_landmarks) { - detector->ProcessFrame2D(camera_frame, numFaces, multi_face_landmarks); +MPFaceMeshDetectorDetect2DLandmarks(MPFaceMeshDetector *detector, + cv::Point2f **multi_face_landmarks, + int *numFaces) { + detector->DetectLandmarks(multi_face_landmarks, numFaces); +} +DLLEXPORT void +MPFaceMeshDetectorDetect3DLandmarks(MPFaceMeshDetector *detector, + cv::Point3f **multi_face_landmarks, + int *numFaces) { + detector->DetectLandmarks(multi_face_landmarks, numFaces); } -DLLEXPORT const int MPFaceMeshDetectorLandmarksNum = MPFaceMeshDetector::kLandmarksNum; +DLLEXPORT const int MPFaceMeshDetectorLandmarksNum = + MPFaceMeshDetector::kLandmarksNum; } const std::string MPFaceMeshDetector::graphConfig = R"pb( @@ -178,6 +313,10 @@ output_stream: "multi_face_landmarks" # Detected faces count. (int) output_stream: "face_count" +# Regions of interest calculated based on landmarks. +# (std::vector) +output_stream: "face_rects_from_landmarks" + node { calculator: "FlowLimiterCalculator" input_stream: "input_video" diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h index 4b40417bc..6705b42e6 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h @@ -20,6 +20,7 @@ #include "mediapipe/framework/formats/image_frame.h" #include "mediapipe/framework/formats/image_frame_opencv.h" #include "mediapipe/framework/formats/landmark.pb.h" +#include "mediapipe/framework/formats/rect.pb.h" #include "mediapipe/framework/output_stream_poller.h" #include "mediapipe/framework/port/file_helpers.h" #include "mediapipe/framework/port/opencv_highgui_inc.h" @@ -30,25 +31,33 @@ class MPFaceMeshDetector { public: - static constexpr auto kLandmarksNum = 468; - MPFaceMeshDetector(int numFaces, const char *face_detection_model_path, const char *face_landmark_model_path); - void ProcessFrame2D(const cv::Mat &camera_frame, int *numFaces, - cv::Point2f **multi_face_landmarks); + void DetectFaces(const cv::Mat &camera_frame, + cv::Rect *multi_face_bounding_boxes, int *numFaces); + + void DetectLandmarks(cv::Point2f **multi_face_landmarks, int *numFaces); + void DetectLandmarks(cv::Point3f **multi_face_landmarks, int *numFaces); + + static constexpr auto kLandmarksNum = 468; private: absl::Status InitFaceMeshDetector(int numFaces, const char *face_detection_model_path, const char *face_landmark_model_path); - absl::Status ProcessFrame2DWithStatus(const cv::Mat &camera_frame, - int *numFaces, - cv::Point2f **multi_face_landmarks); + absl::Status DetectFacesWithStatus(const cv::Mat &camera_frame, + cv::Rect *multi_face_bounding_boxes, + int *numFaces); + + absl::Status DetectLandmarksWithStatus(cv::Point2f **multi_face_landmarks); + absl::Status DetectLandmarksWithStatus(cv::Point3f **multi_face_landmarks); static constexpr auto kInputStream = "input_video"; static constexpr auto kOutputStream_landmarks = "multi_face_landmarks"; static constexpr auto kOutputStream_faceCount = "face_count"; + static constexpr auto kOutputStream_face_rects_from_landmarks = + "face_rects_from_landmarks"; static const std::string graphConfig; @@ -56,6 +65,13 @@ private: std::unique_ptr landmarks_poller_ptr; std::unique_ptr face_count_poller_ptr; + std::unique_ptr + face_rects_from_landmarks_poller_ptr; + + int face_count; + int image_width; + int image_height; + mediapipe::Packet face_landmarks_packet; }; #ifdef __cplusplus @@ -68,10 +84,18 @@ MPFaceMeshDetectorConstruct(int numFaces, const char *face_detection_model_path, DLLEXPORT void MPFaceMeshDetectorDestruct(MPFaceMeshDetector *detector); +DLLEXPORT void MPFaceMeshDetectorDetectFaces( + MPFaceMeshDetector *detector, const cv::Mat &camera_frame, + cv::Rect *multi_face_bounding_boxes, int *numFaces); + DLLEXPORT void -MPFaceMeshDetectorProcessFrame2D(MPFaceMeshDetector *detector, - const cv::Mat &camera_frame, int *numFaces, - cv::Point2f **multi_face_landmarks); +MPFaceMeshDetectorDetect2DLandmarks(MPFaceMeshDetector *detector, + cv::Point2f **multi_face_landmarks, + int *numFaces); +DLLEXPORT void +MPFaceMeshDetectorDetect3DLandmarks(MPFaceMeshDetector *detector, + cv::Point3f **multi_face_landmarks, + int *numFaces); DLLEXPORT extern const int MPFaceMeshDetectorLandmarksNum;