cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Modified face_mesh_lib functionality

Browse Source 
Change List:
- Combined `MPFaceMeshDetector::GetFaceCount` and `MPFaceMeshDetector::GetFaceLandmarks` into `MPFaceMeshDetector::ProcessFrame2D`
- Added `MPFaceMeshDetectorLandmarksNum` that exports number of face landmarks detected by MediaPipe
This commit is contained in:
dmaletskiy 2021-07-14 12:19:34 +03:00
parent b7dd4cfe72
commit 6d89ef3e9e
3 changed files with 68 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
mediapipe/examples/desktop/face_mesh_dll/face_mesh_cpu.cpp
View File

@ -28,15 +28,15 @@ int main(int argc, char **argv) {
constexpr char face_landmark_model_path[] =
"mediapipe/modules/face_landmark/face_landmark.tflite";
MPFaceMeshDetector *faceMeshDetector = FaceMeshDetector_Construct(
MPFaceMeshDetector *faceMeshDetector = MPFaceMeshDetectorConstruct(
maxNumFaces, face_detection_model_path, face_landmark_model_path);
// allocate memory for face landmarks
// Allocate memory for face landmarks.
auto multiFaceLandmarks = new cv::Point2f *[maxNumFaces];
constexpr auto mediapipeFaceLandmarksNum = 468;
for (int i = 0; i < maxNumFaces; ++i) {
multiFaceLandmarks[i] = new cv::Point2f[mediapipeFaceLandmarksNum];
multiFaceLandmarks[i] = new cv::Point2f[MPFaceMeshDetectorLandmarksNum];
}
const auto faceCount = std::make_unique<int>();
LOG(INFO) << "FaceMeshDetector constructed.";
@ -56,21 +56,19 @@ int main(int argc, char **argv) {
cv::cvtColor(camera_frame_raw, camera_frame, cv::COLOR_BGR2RGB);
cv::flip(camera_frame, camera_frame, /*flipcode=HORIZONTAL*/ 1);
int faceCount =
FaceMeshDetector_GetFaceCount(faceMeshDetector, camera_frame);
MPFaceMeshDetectorProcessFrame2D(faceMeshDetector, camera_frame,
faceCount.get(), multiFaceLandmarks);
LOG(INFO) << "Detected faces num: " << faceCount;
if (faceCount > 0) {
FaceMeshDetector_GetFaceLandmarks(faceMeshDetector, multiFaceLandmarks);
LOG(INFO) << "Detected faces num: " << *faceCount;
if (*faceCount > 0) {
auto &face_landmarks = multiFaceLandmarks[0];
auto &landmark = face_landmarks[0];
LOG(INFO) << "First landmark: x - " << landmark.x << ", y - "
<< landmark.y;
}
const int pressed_key = cv::waitKey(5);
if (pressed_key >= 0 && pressed_key != 255)
grab_frames = false;
@ -80,11 +78,11 @@ int main(int argc, char **argv) {
LOG(INFO) << "Shutting down.";
// deallocate memory for face landmarks
// Deallocate memory for face landmarks.
for (int i = 0; i < maxNumFaces; ++i) {
delete[] multiFaceLandmarks[i];
}
delete[] multiFaceLandmarks;
FaceMeshDetector_Destruct(faceMeshDetector);
MPFaceMeshDetectorDestruct(faceMeshDetector);
}

82
mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.cpp
View File

@ -1,9 +1,5 @@
#include <windows.h>
#include "face_mesh_lib.h"
#define DEBUG
MPFaceMeshDetector::MPFaceMeshDetector(int numFaces,
const char *face_detection_model_path,
const char *face_landmark_model_path) {
@ -19,9 +15,7 @@ absl::Status
MPFaceMeshDetector::InitFaceMeshDetector(int numFaces,
const char *face_detection_model_path,
const char *face_landmark_model_path) {
if (numFaces <= 0) {
numFaces = 1;
}
numFaces = std::max(numFaces, 1);
if (face_detection_model_path == nullptr) {
face_detection_model_path =
@ -33,6 +27,7 @@ MPFaceMeshDetector::InitFaceMeshDetector(int numFaces,
"mediapipe/modules/face_landmark/face_landmark.tflite";
}
// Prepare graph config.
auto preparedGraphConfig = absl::StrReplaceAll(
graphConfig, {{"$numFaces", std::to_string(numFaces)}});
preparedGraphConfig = absl::StrReplaceAll(
@ -70,8 +65,11 @@ MPFaceMeshDetector::InitFaceMeshDetector(int numFaces,
return absl::OkStatus();
}
absl::Status
MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) {
absl::Status MPFaceMeshDetector::ProcessFrame2DWithStatus(
const cv::Mat &camera_frame, int *numFaces,
cv::Point2f **multi_face_landmarks) {
*numFaces = 0;
// Wrap Mat into an ImageFrame.
auto input_frame = absl::make_unique<mediapipe::ImageFrame>(
mediapipe::ImageFormat::SRGB, camera_frame.cols, camera_frame.rows,
@ -85,6 +83,8 @@ MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) {
MP_RETURN_IF_ERROR(graph.AddPacketToInputStream(
kInputStream, mediapipe::Adopt(input_frame.release())
.At(mediapipe::Timestamp(frame_timestamp_us))));
// Get face count.
mediapipe::Packet face_count_packet;
if (!face_count_poller_ptr ||
!face_count_poller_ptr->Next(&face_count_packet)) {
@ -94,29 +94,11 @@ MPFaceMeshDetector::GetFaceCountWithStatus(const cv::Mat &camera_frame) {
auto &face_count = face_count_packet.Get<int>();
faceCount = face_count;
if (face_count <= 0) {
return absl::OkStatus();
}
int MPFaceMeshDetector::GetFaceCount(const cv::Mat &camera_frame) {
const auto status = GetFaceCountWithStatus(camera_frame);
if (!status.ok()) {
LOG(INFO) << "Failed GetFaceCount.";
LOG(INFO) << status.message();
}
return faceCount;
}
absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus(
cv::Point2f **multi_face_landmarks) {
if (faceCount <= 0) {
return absl::CancelledError(
"Failed during gettinglandmarks, because faceCount is <= 0.");
}
// Get face landmarks.
mediapipe::Packet face_landmarks_packet;
if (!landmarks_poller_ptr ||
!landmarks_poller_ptr->Next(&face_landmarks_packet)) {
@ -127,9 +109,15 @@ absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus(
face_landmarks_packet
.Get<::std::vector<::mediapipe::NormalizedLandmarkList>>();
for (int i = 0; i < faceCount; ++i) {
// Convert landmarks to cv::Point2f**.
for (int i = 0; i < face_count; ++i) {
const auto &normalizedLandmarkList = output_landmarks_vector[i];
const auto landmarks_num = normalizedLandmarkList.landmark_size();
if (landmarks_num != kLandmarksNum) {
return absl::CancelledError("Detected unexpected landmarks number.");
}
auto &face_landmarks = multi_face_landmarks[i];
for (int j = 0; j < landmarks_num; ++j) {
@ -139,47 +127,43 @@ absl::Status MPFaceMeshDetector::GetFaceLandmarksWithStatus(
}
}
faceCount = -1;
*numFaces = face_count;
return absl::OkStatus();
}
void MPFaceMeshDetector::GetFaceLandmarks(cv::Point2f **multi_face_landmarks) {
const auto status = GetFaceLandmarksWithStatus(multi_face_landmarks);
void MPFaceMeshDetector::ProcessFrame2D(const cv::Mat &camera_frame,
int *numFaces,
cv::Point2f **multi_face_landmarks) {
const auto status =
ProcessFrame2DWithStatus(camera_frame, numFaces, multi_face_landmarks);
if (!status.ok()) {
LOG(INFO) << "Failed GetFaceLandmarks.";
LOG(INFO) << "Failed ProcessFrame2D.";
LOG(INFO) << status.message();
}
}
extern "C" {
DLLEXPORT MPFaceMeshDetector *
FaceMeshDetector_Construct(int numFaces, const char *face_detection_model_path,
MPFaceMeshDetectorConstruct(int numFaces, const char *face_detection_model_path,
const char *face_landmark_model_path) {
return new MPFaceMeshDetector(numFaces, face_detection_model_path,
face_landmark_model_path);
}
DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector) {
DLLEXPORT void MPFaceMeshDetectorDestruct(MPFaceMeshDetector *detector) {
delete detector;
}
DLLEXPORT int FaceMeshDetector_GetFaceCount(MPFaceMeshDetector *detector,
const cv::Mat &camera_frame) {
return detector->GetFaceCount(camera_frame);
}
DLLEXPORT void
FaceMeshDetector_GetFaceLandmarks(MPFaceMeshDetector *detector,
MPFaceMeshDetectorProcessFrame2D(MPFaceMeshDetector *detector,
const cv::Mat &camera_frame, int *numFaces,
cv::Point2f **multi_face_landmarks) {
detector->GetFaceLandmarks(multi_face_landmarks);
}
detector->ProcessFrame2D(camera_frame, numFaces, multi_face_landmarks);
}
const char MPFaceMeshDetector::kInputStream[] = "input_video";
const char MPFaceMeshDetector::kOutputStream_landmarks[] =
"multi_face_landmarks";
const char MPFaceMeshDetector::kOutputStream_faceCount[] = "face_count";
DLLEXPORT const int MPFaceMeshDetectorLandmarksNum = MPFaceMeshDetector::kLandmarksNum;
}
const std::string MPFaceMeshDetector::graphConfig = R"pb(
# MediaPipe graph that performs face mesh with TensorFlow Lite on CPU.
@ -197,7 +181,7 @@ output_stream: "face_count"
node {
calculator: "FlowLimiterCalculator"
input_stream: "input_video"
input_stream: "FINISHED:multi_face_landmarks"
input_stream: "FINISHED:face_count"
input_stream_info: {
tag_index: "FINISHED"
back_edge: true

43
mediapipe/examples/desktop/face_mesh_dll/face_mesh_lib.h
View File

@ -10,6 +10,7 @@
#include <cstdlib>
#include <memory>
#include <string>
#include <windows.h>
#include "absl/flags/flag.h"
#include "absl/flags/parse.h"
@ -29,24 +30,25 @@
class MPFaceMeshDetector {
public:
static constexpr auto kLandmarksNum = 468;
MPFaceMeshDetector(int numFaces, const char *face_detection_model_path,
const char *face_landmark_model_path);
int GetFaceCount(const cv::Mat &camera_frame);
void GetFaceLandmarks(cv::Point2f **multi_face_landmarks);
void ProcessFrame2D(const cv::Mat &camera_frame, int *numFaces,
cv::Point2f **multi_face_landmarks);
private:
absl::Status InitFaceMeshDetector(int numFaces,
const char *face_detection_model_path,
const char *face_landmark_model_path);
absl::Status ProcessFrameWithStatus(
const cv::Mat &camera_frame,
std::vector<std::vector<cv::Point2f>> &multi_face_landmarks);
absl::Status GetFaceCountWithStatus(const cv::Mat &camera_frame);
absl::Status GetFaceLandmarksWithStatus(cv::Point2f **multi_face_landmarks);
absl::Status ProcessFrame2DWithStatus(const cv::Mat &camera_frame,
int *numFaces,
cv::Point2f **multi_face_landmarks);
static const char kInputStream[];
static const char kOutputStream_landmarks[];
static const char kOutputStream_faceCount[];
static constexpr auto kInputStream = "input_video";
static constexpr auto kOutputStream_landmarks = "multi_face_landmarks";
static constexpr auto kOutputStream_faceCount = "face_count";
static const std::string graphConfig;
@ -54,30 +56,25 @@ private:
std::unique_ptr<mediapipe::OutputStreamPoller> landmarks_poller_ptr;
std::unique_ptr<mediapipe::OutputStreamPoller> face_count_poller_ptr;
int faceCount = -1;
};
#ifdef __cplusplus
extern "C" {
#endif
DLLEXPORT MPFaceMeshDetector *FaceMeshDetector_Construct(
int numFaces = 1,
const char *face_detection_model_path =
"mediapipe/modules/face_detection/face_detection_short_range.tflite",
const char *face_landmark_model_path =
"mediapipe/modules/face_landmark/face_landmark.tflite");
DLLEXPORT MPFaceMeshDetector *
MPFaceMeshDetectorConstruct(int numFaces, const char *face_detection_model_path,
const char *face_landmark_model_path);
DLLEXPORT void MPFaceMeshDetectorDestruct(MPFaceMeshDetector *detector);
DLLEXPORT void FaceMeshDetector_Destruct(MPFaceMeshDetector *detector);
DLLEXPORT int FaceMeshDetector_GetFaceCount(MPFaceMeshDetector *detector,
const cv::Mat &camera_frame);
DLLEXPORT void
FaceMeshDetector_GetFaceLandmarks(MPFaceMeshDetector *detector,
MPFaceMeshDetectorProcessFrame2D(MPFaceMeshDetector *detector,
const cv::Mat &camera_frame, int *numFaces,
cv::Point2f **multi_face_landmarks);
DLLEXPORT extern const int MPFaceMeshDetectorLandmarksNum;
#ifdef __cplusplus
};
#endif