From de5fc2a532af0a951b93203e0ecadeff76e2aaf5 Mon Sep 17 00:00:00 2001 From: dmaletskiy Date: Thu, 1 Jul 2021 13:38:45 +0300 Subject: [PATCH] feat: added ouputing of multi-face_landmarks through dll to c++ Change List: - now FaceMeshDetector::ProcessFrame2D returns pointer to vector with all facelandmars - added converting of `mediapipe::NormalizedLandmarkList` to `std::vector` to make more independent of mediapipe (to use in custom projects) --- .../desktop/face_mesh_dll/face_mesh_cpu.cpp | 17 ++++++- .../desktop/face_mesh_dll/face_mesh_lib.cpp | 47 ++++++++++++------- .../desktop/face_mesh_dll/face_mesh_lib.h | 15 +++--- 3 files changed, 54 insertions(+), 25 deletions(-) diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp index e44375864..5956dade4 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp @@ -4,7 +4,6 @@ int main(int argc, char **argv) { google::InitGoogleLogging(argv[0]); absl::ParseCommandLine(argc, argv); - cv::VideoCapture capture; capture.open(0); if (!capture.isOpened()) { @@ -41,7 +40,21 @@ int main(int argc, char **argv) { cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); - FaceMeshDetector_ProcessFrame(faceMeshDetector, camera_frame); + std::unique_ptr>> multi_face_landmarks( + reinterpret_cast> *>( + FaceMeshDetector_ProcessFrame2D(faceMeshDetector, camera_frame))); + + const auto multi_face_landmarks_num = multi_face_landmarks->size(); + + LOG(INFO) << "Got multi_face_landmarks_num: " << multi_face_landmarks_num; + + if (multi_face_landmarks_num) { + auto &face_landmarks = multi_face_landmarks->operator[](0); + auto &landmark = face_landmarks[0]; + + LOG(INFO) << "First landmark: x - " << landmark.x << ", y - " + << landmark.y; + } const int pressed_key = cv::waitKey(5); if (pressed_key >= 0 && pressed_key != 255) diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp index a918e3719..dd8cb06eb 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp @@ -2,8 +2,6 @@ #include "face_mesh_lib.h" -#define DEBUG - FaceMeshDetector::FaceMeshDetector() { const auto status = InitFaceMeshDetector(); if (!status.ok()) { @@ -39,7 +37,10 @@ absl::Status FaceMeshDetector::InitFaceMeshDetector() { return absl::Status(); } -absl::Status FaceMeshDetector::ProcessFrameWithStatus(cv::Mat &camera_frame) { +absl::Status FaceMeshDetector::ProcessFrameWithStatus( + cv::Mat &camera_frame, + std::unique_ptr>> + &multi_face_landmarks) { // Wrap Mat into an ImageFrame. auto input_frame = absl::make_unique( mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, @@ -88,26 +89,38 @@ absl::Status FaceMeshDetector::ProcessFrameWithStatus(cv::Mat &camera_frame) { face_landmarks_packet .Get<::std::vector<::mediapipe::NormalizedLandmarkList>>(); - auto &output_landmarks = output_landmarks_vector[0]; + multi_face_landmarks->reserve(output_landmarks_vector.size()); + + for (const auto &normalizedLandmarkList : output_landmarks_vector) { + multi_face_landmarks->emplace_back(); + + auto &face_landmarks = multi_face_landmarks->back(); + + const auto landmarks_num = normalizedLandmarkList.landmark_size(); #ifdef DEBUG - LOG(INFO) << "Got landmarks_packet: " << output_landmarks.landmark_size(); + LOG(INFO) << "Got landmarks_num: " << landmarks_num; #endif - auto &landmark = output_landmarks.landmark(0); -#ifdef DEBUG - LOG(INFO) << "First landmark: x - " << landmark.x() << ", y - " - << landmark.y() << ", z - " << landmark.z(); -#endif + face_landmarks.reserve(landmarks_num); + + for (int i = 0; i < landmarks_num; ++i) { + auto &landmark = normalizedLandmarkList.landmark(i); + + face_landmarks.emplace_back(landmark.x(), landmark.y()); + } + } return absl::Status(); } -std::vector * -FaceMeshDetector::ProcessFrame(cv::Mat &camera_frame) { - ProcessFrameWithStatus(camera_frame); +std::vector> * +FaceMeshDetector::ProcessFrame2D(cv::Mat &camera_frame) { + auto landmarks = std::make_unique>>(); - return new std::vector(); + ProcessFrameWithStatus(camera_frame, landmarks); + + return landmarks.release(); } extern "C" { @@ -119,9 +132,9 @@ DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector) { delete detector; } -DLLEXPORT void *FaceMeshDetector_ProcessFrame(FaceMeshDetector *detector, - cv::Mat &camera_frame) { - return reinterpret_cast(detector->ProcessFrame(camera_frame)); +DLLEXPORT void *FaceMeshDetector_ProcessFrame2D(FaceMeshDetector *detector, + cv::Mat &camera_frame) { + return reinterpret_cast(detector->ProcessFrame2D(camera_frame)); } } diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h index 7c62b01e4..9788d6803 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h @@ -8,8 +8,8 @@ #endif #include -#include #include +#include #include "absl/flags/flag.h" #include "absl/flags/parse.h" @@ -29,11 +29,14 @@ class FaceMeshDetector { public: FaceMeshDetector(); ~FaceMeshDetector() = default; - std::vector *ProcessFrame(cv::Mat &camera_frame); + std::vector> *ProcessFrame2D(cv::Mat &camera_frame); private: absl::Status InitFaceMeshDetector(); - absl::Status ProcessFrameWithStatus(cv::Mat &camera_frame); + absl::Status + ProcessFrameWithStatus(cv::Mat &camera_frame, + std::unique_ptr>> + &multi_face_landmarks); static const char kInputStream[]; static const char kOutputStream_landmarks[]; @@ -42,7 +45,7 @@ private: static const std::string graphConfig; mediapipe::CalculatorGraph graph; - + std::unique_ptr landmarks_poller_ptr; std::unique_ptr face_count_poller_ptr; }; @@ -55,8 +58,8 @@ DLLEXPORT FaceMeshDetector *FaceMeshDetector_Construct(); DLLEXPORT void FaceMeshDetector_Destruct(FaceMeshDetector *detector); -DLLEXPORT void *FaceMeshDetector_ProcessFrame(FaceMeshDetector *detector, - cv::Mat &camera_frame); +DLLEXPORT void *FaceMeshDetector_ProcessFrame2D(FaceMeshDetector *detector, + cv::Mat &camera_frame); #ifdef __cplusplus };