diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp index 90462477a..0d9011dd0 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp @@ -28,15 +28,15 @@ int main(int argc, char **argv) { constexpr char face_landmark_model_path[] = "mediapipe/modules/face_landmark/face_landmark.tflite"; - MPFaceMeshDetector *faceMeshDetector = FaceMeshDetector_Construct( + MPFaceMeshDetector *faceMeshDetector = MPFaceMeshDetectorConstruct( maxNumFaces, face_detection_model_path, face_landmark_model_path); - // allocate memory for face landmarks + // Allocate memory for face landmarks. auto multiFaceLandmarks = new cv::Point2f *[maxNumFaces]; - constexpr auto mediapipeFaceLandmarksNum = 468; for (int i = 0; i < maxNumFaces; ++i) { - multiFaceLandmarks[i] = new cv::Point2f[mediapipeFaceLandmarksNum]; + multiFaceLandmarks[i] = new cv::Point2f[MPFaceMeshDetectorLandmarksNum]; } + const auto faceCount = std::make_unique(); LOG(INFO) << "FaceMeshDetector constructed."; @@ -56,21 +56,19 @@ int main(int argc, char **argv) { cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB); cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1); - int faceCount = - FaceMeshDetector_GetFaceCount(faceMeshDetector, camera_frame); + MPFaceMeshDetectorProcessFrame2D(faceMeshDetector, camera_frame, + faceCount.get(), multiFaceLandmarks); - LOG(INFO) << "Detected faces num: " << faceCount; - - if (faceCount > 0) { - - FaceMeshDetector_GetFaceLandmarks(faceMeshDetector, multiFaceLandmarks); + LOG(INFO) << "Detected faces num: " << *faceCount; + if (*faceCount > 0) { auto &face_landmarks = multiFaceLandmarks[0]; auto &landmark = face_landmarks[0]; LOG(INFO) << "First landmark: x - " << landmark.x << ", y - " << landmark.y; } + const int pressed_key = cv::waitKey(5); if (pressed_key >= 0 && pressed_key != 255) grab_frames = false; @@ -80,11 +78,11 @@ int main(int argc, char **argv) { LOG(INFO) << "Shutting down."; - // deallocate memory for face landmarks + // Deallocate memory for face landmarks. for (int i = 0; i < maxNumFaces; ++i) { delete[] multiFaceLandmarks[i]; } delete[] multiFaceLandmarks; - FaceMeshDetector_Destruct(faceMeshDetector); + MPFaceMeshDetectorDestruct(faceMeshDetector); } \ No newline at end of file diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp index 5bba0efea..167a38f93 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp @@ -1,9 +1,5 @@ -#include - #include "face_mesh_lib.h" -#define DEBUG - MPFaceMeshDetector::MPFaceMeshDetector(int numFaces, const char *face_detection_model_path, const char *face_landmark_model_path) { @@ -19,9 +15,7 @@ absl::Status MPFaceMeshDetector::InitFaceMeshDetector(int numFaces, const char *face_detection_model_path, const char *face_landmark_model_path) { - if (numFaces <= 0) { - numFaces = 1; - } + numFaces = std::max(numFaces, 1); if (face_detection_model_path == nullptr) { face_detection_model_path = @@ -33,6 +27,7 @@ MPFaceMeshDetector::InitFaceMeshDetector(int numFaces, "mediapipe/modules/face_landmark/face_landmark.tflite"; } + // Prepare graph config. auto preparedGraphConfig = absl::StrReplaceAll( graphConfig, {{"$numFaces", std::to_string(numFaces)}}); preparedGraphConfig = absl::StrReplaceAll( @@ -70,8 +65,11 @@ MPFaceMeshDetector::InitFaceMeshDetector(int numFaces, return absl::OkStatus(); } -absl::Status -MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) { +absl::Status MPFaceMeshDetector::ProcessFrame2DWithStatus( + const cv::Mat &camera_frame, int *numFaces, + cv::Point2f **multi_face_landmarks) { + *numFaces = 0; + // Wrap Mat into an ImageFrame. auto input_frame = absl::make_unique( mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows, @@ -85,6 +83,8 @@ MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) { MP_RETURN_IF_ERROR(graph.AddPacketToInputStream( kInputStream, mediapipe::Adopt(input_frame.release()) .At(mediapipe::Timestamp(frame_timestamp_us)))); + + // Get face count. mediapipe::Packet face_count_packet; if (!face_count_poller_ptr || !face_count_poller_ptr->Next(&face_count_packet)) { @@ -94,29 +94,11 @@ MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) { auto &face_count = face_count_packet.Get(); - faceCount = face_count; - - return absl::OkStatus(); -} - -int MPFaceMeshDetector::GetFaceCount(const cv::Mat &camera_frame) { - const auto status = GetFaceCountWithStatus(camera_frame); - if (!status.ok()) { - LOG(INFO) << "Failed GetFaceCount."; - LOG(INFO) << status.message(); - } - - return faceCount; -} - -absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus( - cv::Point2f **multi_face_landmarks) { - - if (faceCount <= 0) { - return absl::CancelledError( - "Failed during gettinglandmarks, because faceCount is <= 0."); + if (face_count <= 0) { + return absl::OkStatus(); } + // Get face landmarks. mediapipe::Packet face_landmarks_packet; if (!landmarks_poller_ptr || !landmarks_poller_ptr->Next(&face_landmarks_packet)) { @@ -127,9 +109,15 @@ absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus( face_landmarks_packet .Get<::std::vector<::mediapipe::NormalizedLandmarkList>>(); - for (int i = 0; i < faceCount; ++i) { + // Convert landmarks to cv::Point2f**. + for (int i = 0; i < face_count; ++i) { const auto &normalizedLandmarkList = output_landmarks_vector[i]; const auto landmarks_num = normalizedLandmarkList.landmark_size(); + + if (landmarks_num != kLandmarksNum) { + return absl::CancelledError("Detected unexpected landmarks number."); + } + auto &face_landmarks = multi_face_landmarks[i]; for (int j = 0; j < landmarks_num; ++j) { @@ -139,47 +127,43 @@ absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus( } } - faceCount = -1; + *numFaces = face_count; return absl::OkStatus(); } -void MPFaceMeshDetector::GetFaceLandmarks(cv::Point2f **multi_face_landmarks) { - const auto status = GetFaceLandmarksWithStatus(multi_face_landmarks); +void MPFaceMeshDetector::ProcessFrame2D(const cv::Mat &camera_frame, + int *numFaces, + cv::Point2f **multi_face_landmarks) { + const auto status = + ProcessFrame2DWithStatus(camera_frame, numFaces, multi_face_landmarks); if (!status.ok()) { - LOG(INFO) << "Failed GetFaceLandmarks."; + LOG(INFO) << "Failed ProcessFrame2D."; LOG(INFO) << status.message(); } } extern "C" { DLLEXPORT MPFaceMeshDetector * -FaceMeshDetector_Construct(int numFaces, const char *face_detection_model_path, - const char *face_landmark_model_path) { +MPFaceMeshDetectorConstruct(int numFaces, const char *face_detection_model_path, + const char *face_landmark_model_path) { return new MPFaceMeshDetector(numFaces, face_detection_model_path, face_landmark_model_path); } -DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector) { +DLLEXPORT void MPFaceMeshDetectorDestruct(MPFaceMeshDetector *detector) { delete detector; } -DLLEXPORT int FaceMeshDetector_GetFaceCount(MPFaceMeshDetector *detector, - const cv::Mat &camera_frame) { - return detector->GetFaceCount(camera_frame); -} - DLLEXPORT void -FaceMeshDetector_GetFaceLandmarks(MPFaceMeshDetector *detector, - cv::Point2f **multi_face_landmarks) { - detector->GetFaceLandmarks(multi_face_landmarks); -} +MPFaceMeshDetectorProcessFrame2D(MPFaceMeshDetector *detector, + const cv::Mat &camera_frame, int *numFaces, + cv::Point2f **multi_face_landmarks) { + detector->ProcessFrame2D(camera_frame, numFaces, multi_face_landmarks); } -const char MPFaceMeshDetector::kInputStream[] = "input_video"; -const char MPFaceMeshDetector::kOutputStream_landmarks[] = - "multi_face_landmarks"; -const char MPFaceMeshDetector::kOutputStream_faceCount[] = "face_count"; +DLLEXPORT const int MPFaceMeshDetectorLandmarksNum = MPFaceMeshDetector::kLandmarksNum; +} const std::string MPFaceMeshDetector::graphConfig = R"pb( # MediaPipe graph that performs face mesh with TensorFlow Lite on CPU. @@ -197,7 +181,7 @@ output_stream: "face_count" node { calculator: "FlowLimiterCalculator" input_stream: "input_video" - input_stream: "FINISHED:multi_face_landmarks" + input_stream: "FINISHED:face_count" input_stream_info: { tag_index: "FINISHED" back_edge: true diff --git a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h index 88c3ed680..4b40417bc 100644 --- a/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h +++ b/mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h @@ -10,6 +10,7 @@ #include #include #include +#include #include "absl/flags/flag.h" #include "absl/flags/parse.h" @@ -29,24 +30,25 @@ class MPFaceMeshDetector { public: + static constexpr auto kLandmarksNum = 468; + MPFaceMeshDetector(int numFaces, const char *face_detection_model_path, const char *face_landmark_model_path); - int GetFaceCount(const cv::Mat &camera_frame); - void GetFaceLandmarks(cv::Point2f **multi_face_landmarks); + + void ProcessFrame2D(const cv::Mat &camera_frame, int *numFaces, + cv::Point2f **multi_face_landmarks); private: absl::Status InitFaceMeshDetector(int numFaces, const char *face_detection_model_path, const char *face_landmark_model_path); - absl::Status ProcessFrameWithStatus( - const cv::Mat &camera_frame, - std::vector> &multi_face_landmarks); - absl::Status GetFaceCountWithStatus(const cv::Mat &camera_frame); - absl::Status GetFaceLandmarksWithStatus(cv::Point2f **multi_face_landmarks); + absl::Status ProcessFrame2DWithStatus(const cv::Mat &camera_frame, + int *numFaces, + cv::Point2f **multi_face_landmarks); - static const char kInputStream[]; - static const char kOutputStream_landmarks[]; - static const char kOutputStream_faceCount[]; + static constexpr auto kInputStream = "input_video"; + static constexpr auto kOutputStream_landmarks = "multi_face_landmarks"; + static constexpr auto kOutputStream_faceCount = "face_count"; static const std::string graphConfig; @@ -54,29 +56,24 @@ private: std::unique_ptr landmarks_poller_ptr; std::unique_ptr face_count_poller_ptr; - - int faceCount = -1; }; #ifdef __cplusplus extern "C" { #endif -DLLEXPORT MPFaceMeshDetector *FaceMeshDetector_Construct( - int numFaces = 1, - const char *face_detection_model_path = - "mediapipe/modules/face_detection/face_detection_short_range.tflite", - const char *face_landmark_model_path = - "mediapipe/modules/face_landmark/face_landmark.tflite"); +DLLEXPORT MPFaceMeshDetector * +MPFaceMeshDetectorConstruct(int numFaces, const char *face_detection_model_path, + const char *face_landmark_model_path); +DLLEXPORT void MPFaceMeshDetectorDestruct(MPFaceMeshDetector *detector); -DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector); - -DLLEXPORT int FaceMeshDetector_GetFaceCount(MPFaceMeshDetector *detector, - const cv::Mat &camera_frame); DLLEXPORT void -FaceMeshDetector_GetFaceLandmarks(MPFaceMeshDetector *detector, - cv::Point2f **multi_face_landmarks); +MPFaceMeshDetectorProcessFrame2D(MPFaceMeshDetector *detector, + const cv::Mat &camera_frame, int *numFaces, + cv::Point2f **multi_face_landmarks); + +DLLEXPORT extern const int MPFaceMeshDetectorLandmarksNum; #ifdef __cplusplus };