Merge pull request #5021 from kinaryml:c-face-landmarker-api

PiperOrigin-RevId: 591062403
This commit is contained in:
Copybara-Service 2023-12-14 15:07:10 -08:00
commit d6b8c2257b
16 changed files with 1449 additions and 11 deletions

View File

@ -43,6 +43,33 @@ cc_test(
],
)
cc_library(
name = "matrix",
hdrs = ["matrix.h"],
)
cc_library(
name = "matrix_converter",
srcs = ["matrix_converter.cc"],
hdrs = ["matrix_converter.h"],
deps = [
":matrix",
"@eigen_archive//:eigen3",
],
)
cc_test(
name = "matrix_converter_test",
srcs = ["matrix_converter_test.cc"],
deps = [
":matrix",
":matrix_converter",
"//mediapipe/framework/port:gtest",
"@com_google_googletest//:gtest_main",
"@eigen_archive//:eigen3",
],
)
cc_library(
name = "landmark",
hdrs = ["landmark.h"],

View File

@ -0,0 +1,41 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_H_
#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_H_
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
// Data are stored in column-major order by default.
struct Matrix {
// The number of rows in the matrix.
uint32_t rows;
// The number of columns in the matrix.
uint32_t cols;
// The matrix data stored in a column-first layout.
float* data;
};
#ifdef __cplusplus
} // extern C
#endif
#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_H_

View File

@ -0,0 +1,43 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "mediapipe/tasks/c/components/containers/matrix_converter.h"
#include <cstring>
#include "Eigen/Core"
#include "mediapipe/tasks/c/components/containers/matrix.h"
namespace mediapipe::tasks::c::components::containers {
void CppConvertToMatrix(const Eigen::MatrixXf& in, ::Matrix* out) {
out->rows = in.rows();
out->cols = in.cols();
out->data = new float[out->rows * out->cols];
// Copies data from an Eigen matrix (default column-major as used by
// MediaPipe) to a C-style matrix, preserving the sequence of elements as per
// the Eigen matrix's internal storage (column-major order by default).
memcpy(out->data, in.data(), sizeof(float) * out->rows * out->cols);
}
void CppCloseMatrix(::Matrix* m) {
if (m->data) {
delete[] m->data;
m->data = nullptr;
}
}
} // namespace mediapipe::tasks::c::components::containers

View File

@ -0,0 +1,30 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_CONVERTER_H_
#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_CONVERTER_H_
#include "Eigen/Core"
#include "mediapipe/tasks/c/components/containers/matrix.h"
namespace mediapipe::tasks::c::components::containers {
void CppConvertToMatrix(const Eigen::MatrixXf& in, ::Matrix* out);
void CppCloseMatrix(::Matrix* m);
} // namespace mediapipe::tasks::c::components::containers
#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_CONVERTER_H_

View File

@ -0,0 +1,49 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "mediapipe/tasks/c/components/containers/matrix_converter.h"
#include "Eigen/Core"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/tasks/c/components/containers/matrix.h"
namespace mediapipe::tasks::c::components::containers {
TEST(MatrixConversionTest, ConvertsEigenMatrixToCMatrixAndFreesMemory) {
// Initialize an Eigen::MatrixXf
Eigen::MatrixXf cpp_matrix(2, 2);
cpp_matrix << 1.0f, 2.0f, 3.0f, 4.0f;
// Convert this Eigen matrix to C-style Matrix
::Matrix c_matrix;
CppConvertToMatrix(cpp_matrix, &c_matrix);
// Verify the conversion
EXPECT_EQ(c_matrix.rows, 2);
EXPECT_EQ(c_matrix.cols, 2);
ASSERT_NE(c_matrix.data, nullptr);
EXPECT_FLOAT_EQ(c_matrix.data[0], 1.0f);
EXPECT_FLOAT_EQ(c_matrix.data[1], 3.0f);
EXPECT_FLOAT_EQ(c_matrix.data[2], 2.0f);
EXPECT_FLOAT_EQ(c_matrix.data[3], 4.0f);
// Close the C-style Matrix
CppCloseMatrix(&c_matrix);
// Verify that memory is freed
EXPECT_EQ(c_matrix.data, nullptr);
}
} // namespace mediapipe::tasks::c::components::containers

View File

@ -0,0 +1,149 @@
# Copyright 2023 The MediaPipe Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
cc_library(
name = "face_landmarker_result",
hdrs = ["face_landmarker_result.h"],
visibility = ["//visibility:public"],
deps = [
"//mediapipe/tasks/c/components/containers:category",
"//mediapipe/tasks/c/components/containers:landmark",
"//mediapipe/tasks/c/components/containers:matrix",
],
)
cc_library(
name = "face_landmarker_result_converter",
srcs = ["face_landmarker_result_converter.cc"],
hdrs = ["face_landmarker_result_converter.h"],
deps = [
":face_landmarker_result",
"//mediapipe/tasks/c/components/containers:category",
"//mediapipe/tasks/c/components/containers:category_converter",
"//mediapipe/tasks/c/components/containers:landmark",
"//mediapipe/tasks/c/components/containers:landmark_converter",
"//mediapipe/tasks/c/components/containers:matrix",
"//mediapipe/tasks/c/components/containers:matrix_converter",
"//mediapipe/tasks/cc/components/containers:category",
"//mediapipe/tasks/cc/components/containers:landmark",
"//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_result",
],
)
cc_test(
name = "face_landmarker_result_converter_test",
srcs = ["face_landmarker_result_converter_test.cc"],
linkstatic = 1,
deps = [
":face_landmarker_result",
":face_landmarker_result_converter",
"//mediapipe/framework/formats:matrix",
"//mediapipe/framework/port:gtest",
"//mediapipe/tasks/c/components/containers:landmark",
"//mediapipe/tasks/cc/components/containers:category",
"//mediapipe/tasks/cc/components/containers:classification_result",
"//mediapipe/tasks/cc/components/containers:landmark",
"//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_result",
"@com_google_googletest//:gtest_main",
"@eigen_archive//:eigen3",
],
)
cc_library(
name = "face_landmarker_lib",
srcs = ["face_landmarker.cc"],
hdrs = ["face_landmarker.h"],
visibility = ["//visibility:public"],
deps = [
":face_landmarker_result",
":face_landmarker_result_converter",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/tasks/c/core:base_options",
"//mediapipe/tasks/c/core:base_options_converter",
"//mediapipe/tasks/c/vision/core:common",
"//mediapipe/tasks/cc/vision/core:running_mode",
"//mediapipe/tasks/cc/vision/face_landmarker",
"//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_result",
"//mediapipe/tasks/cc/vision/utils:image_utils",
"@com_google_absl//absl/log:absl_log",
"@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor",
],
alwayslink = 1,
)
cc_test(
name = "face_landmarker_test",
srcs = ["face_landmarker_test.cc"],
data = [
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/tasks/testdata/vision:test_images",
"//mediapipe/tasks/testdata/vision:test_models",
],
linkstatic = 1,
deps = [
":face_landmarker_lib",
":face_landmarker_result",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/port:gtest",
"//mediapipe/tasks/c/components/containers:landmark",
"//mediapipe/tasks/c/vision/core:common",
"//mediapipe/tasks/cc/vision/utils:image_utils",
"@com_google_absl//absl/flags:flag",
"@com_google_absl//absl/strings",
"@com_google_googletest//:gtest_main",
],
)
# bazel build -c opt --linkopt -s --strip always --define MEDIAPIPE_DISABLE_GPU=1 \
# //mediapipe/tasks/c/vision/face_landmarker:libface_landmarker.so
cc_binary(
name = "libface_landmarker.so",
linkopts = [
"-Wl,-soname=libface_landmarker.so",
"-fvisibility=hidden",
],
linkshared = True,
tags = [
"manual",
"nobuilder",
"notap",
],
deps = [":face_landmarker_lib"],
)
# bazel build --config darwin_arm64 -c opt --strip always --define MEDIAPIPE_DISABLE_GPU=1 \
# //mediapipe/tasks/c/vision/face_landmarker:libface_landmarker.dylib
cc_binary(
name = "libface_landmarker.dylib",
linkopts = [
"-Wl,-install_name,libface_landmarker.dylib",
"-fvisibility=hidden",
],
linkshared = True,
tags = [
"manual",
"nobuilder",
"notap",
],
deps = [":face_landmarker_lib"],
)

View File

@ -0,0 +1,287 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker.h"
#include <cstdint>
#include <cstdlib>
#include <memory>
#include <utility>
#include "absl/log/absl_log.h"
#include "absl/status/status.h"
#include "absl/status/statusor.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/tasks/c/core/base_options_converter.h"
#include "mediapipe/tasks/c/vision/core/common.h"
#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h"
#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.h"
#include "mediapipe/tasks/cc/vision/core/running_mode.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h"
#include "mediapipe/tasks/cc/vision/utils/image_utils.h"
namespace mediapipe::tasks::c::vision::face_landmarker {
namespace {
using ::mediapipe::tasks::c::components::containers::
CppCloseFaceLandmarkerResult;
using ::mediapipe::tasks::c::components::containers::
CppConvertToFaceLandmarkerResult;
using ::mediapipe::tasks::c::core::CppConvertToBaseOptions;
using ::mediapipe::tasks::vision::CreateImageFromBuffer;
using ::mediapipe::tasks::vision::core::RunningMode;
using ::mediapipe::tasks::vision::face_landmarker::FaceLandmarker;
typedef ::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult
CppFaceLandmarkerResult;
int CppProcessError(absl::Status status, char** error_msg) {
if (error_msg) {
*error_msg = strdup(status.ToString().c_str());
}
return status.raw_code();
}
} // namespace
void CppConvertToFaceLandmarkerOptions(
const FaceLandmarkerOptions& in,
mediapipe::tasks::vision::face_landmarker::FaceLandmarkerOptions* out) {
out->num_faces = in.num_faces;
out->min_face_detection_confidence = in.min_face_detection_confidence;
out->min_face_presence_confidence = in.min_face_presence_confidence;
out->min_tracking_confidence = in.min_tracking_confidence;
out->output_face_blendshapes = in.output_face_blendshapes;
out->output_facial_transformation_matrixes =
in.output_facial_transformation_matrixes;
}
FaceLandmarker* CppFaceLandmarkerCreate(const FaceLandmarkerOptions& options,
char** error_msg) {
auto cpp_options = std::make_unique<
::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerOptions>();
CppConvertToBaseOptions(options.base_options, &cpp_options->base_options);
CppConvertToFaceLandmarkerOptions(options, cpp_options.get());
cpp_options->running_mode = static_cast<RunningMode>(options.running_mode);
// Enable callback for processing live stream data when the running mode is
// set to RunningMode::LIVE_STREAM.
if (cpp_options->running_mode == RunningMode::LIVE_STREAM) {
if (options.result_callback == nullptr) {
const absl::Status status = absl::InvalidArgumentError(
"Provided null pointer to callback function.");
ABSL_LOG(ERROR) << "Failed to create FaceLandmarker: " << status;
CppProcessError(status, error_msg);
return nullptr;
}
FaceLandmarkerOptions::result_callback_fn result_callback =
options.result_callback;
cpp_options->result_callback =
[result_callback](absl::StatusOr<CppFaceLandmarkerResult> cpp_result,
const Image& image, int64_t timestamp) {
char* error_msg = nullptr;
if (!cpp_result.ok()) {
ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status();
CppProcessError(cpp_result.status(), &error_msg);
result_callback({}, MpImage(), timestamp, error_msg);
free(error_msg);
return;
}
// Result is valid for the lifetime of the callback function.
FaceLandmarkerResult result;
CppConvertToFaceLandmarkerResult(*cpp_result, &result);
const auto& image_frame = image.GetImageFrameSharedPtr();
const MpImage mp_image = {
.type = MpImage::IMAGE_FRAME,
.image_frame = {
.format = static_cast<::ImageFormat>(image_frame->Format()),
.image_buffer = image_frame->PixelData(),
.width = image_frame->Width(),
.height = image_frame->Height()}};
result_callback(&result, mp_image, timestamp,
/* error_msg= */ nullptr);
CppCloseFaceLandmarkerResult(&result);
};
}
auto landmarker = FaceLandmarker::Create(std::move(cpp_options));
if (!landmarker.ok()) {
ABSL_LOG(ERROR) << "Failed to create FaceLandmarker: "
<< landmarker.status();
CppProcessError(landmarker.status(), error_msg);
return nullptr;
}
return landmarker->release();
}
int CppFaceLandmarkerDetect(void* landmarker, const MpImage& image,
FaceLandmarkerResult* result, char** error_msg) {
if (image.type == MpImage::GPU_BUFFER) {
const absl::Status status =
absl::InvalidArgumentError("GPU Buffer not supported yet.");
ABSL_LOG(ERROR) << "Detection failed: " << status.message();
return CppProcessError(status, error_msg);
}
const auto img = CreateImageFromBuffer(
static_cast<ImageFormat::Format>(image.image_frame.format),
image.image_frame.image_buffer, image.image_frame.width,
image.image_frame.height);
if (!img.ok()) {
ABSL_LOG(ERROR) << "Failed to create Image: " << img.status();
return CppProcessError(img.status(), error_msg);
}
auto cpp_landmarker = static_cast<FaceLandmarker*>(landmarker);
auto cpp_result = cpp_landmarker->Detect(*img);
if (!cpp_result.ok()) {
ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status();
return CppProcessError(cpp_result.status(), error_msg);
}
CppConvertToFaceLandmarkerResult(*cpp_result, result);
return 0;
}
int CppFaceLandmarkerDetectForVideo(void* landmarker, const MpImage& image,
int64_t timestamp_ms,
FaceLandmarkerResult* result,
char** error_msg) {
if (image.type == MpImage::GPU_BUFFER) {
absl::Status status =
absl::InvalidArgumentError("GPU Buffer not supported yet");
ABSL_LOG(ERROR) << "Detection failed: " << status.message();
return CppProcessError(status, error_msg);
}
const auto img = CreateImageFromBuffer(
static_cast<ImageFormat::Format>(image.image_frame.format),
image.image_frame.image_buffer, image.image_frame.width,
image.image_frame.height);
if (!img.ok()) {
ABSL_LOG(ERROR) << "Failed to create Image: " << img.status();
return CppProcessError(img.status(), error_msg);
}
auto cpp_landmarker = static_cast<FaceLandmarker*>(landmarker);
auto cpp_result = cpp_landmarker->DetectForVideo(*img, timestamp_ms);
if (!cpp_result.ok()) {
ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status();
return CppProcessError(cpp_result.status(), error_msg);
}
CppConvertToFaceLandmarkerResult(*cpp_result, result);
return 0;
}
int CppFaceLandmarkerDetectAsync(void* landmarker, const MpImage& image,
int64_t timestamp_ms, char** error_msg) {
if (image.type == MpImage::GPU_BUFFER) {
absl::Status status =
absl::InvalidArgumentError("GPU Buffer not supported yet");
ABSL_LOG(ERROR) << "Detection failed: " << status.message();
return CppProcessError(status, error_msg);
}
const auto img = CreateImageFromBuffer(
static_cast<ImageFormat::Format>(image.image_frame.format),
image.image_frame.image_buffer, image.image_frame.width,
image.image_frame.height);
if (!img.ok()) {
ABSL_LOG(ERROR) << "Failed to create Image: " << img.status();
return CppProcessError(img.status(), error_msg);
}
auto cpp_landmarker = static_cast<FaceLandmarker*>(landmarker);
auto cpp_result = cpp_landmarker->DetectAsync(*img, timestamp_ms);
if (!cpp_result.ok()) {
ABSL_LOG(ERROR) << "Data preparation for the landmark detection failed: "
<< cpp_result;
return CppProcessError(cpp_result, error_msg);
}
return 0;
}
void CppFaceLandmarkerCloseResult(FaceLandmarkerResult* result) {
CppCloseFaceLandmarkerResult(result);
}
int CppFaceLandmarkerClose(void* landmarker, char** error_msg) {
auto cpp_landmarker = static_cast<FaceLandmarker*>(landmarker);
auto result = cpp_landmarker->Close();
if (!result.ok()) {
ABSL_LOG(ERROR) << "Failed to close FaceLandmarker: " << result;
return CppProcessError(result, error_msg);
}
delete cpp_landmarker;
return 0;
}
} // namespace mediapipe::tasks::c::vision::face_landmarker
extern "C" {
void* face_landmarker_create(struct FaceLandmarkerOptions* options,
char** error_msg) {
return mediapipe::tasks::c::vision::face_landmarker::CppFaceLandmarkerCreate(
*options, error_msg);
}
int face_landmarker_detect_image(void* landmarker, const MpImage& image,
FaceLandmarkerResult* result,
char** error_msg) {
return mediapipe::tasks::c::vision::face_landmarker::CppFaceLandmarkerDetect(
landmarker, image, result, error_msg);
}
int face_landmarker_detect_for_video(void* landmarker, const MpImage& image,
int64_t timestamp_ms,
FaceLandmarkerResult* result,
char** error_msg) {
return mediapipe::tasks::c::vision::face_landmarker::
CppFaceLandmarkerDetectForVideo(landmarker, image, timestamp_ms, result,
error_msg);
}
int face_landmarker_detect_async(void* landmarker, const MpImage& image,
int64_t timestamp_ms, char** error_msg) {
return mediapipe::tasks::c::vision::face_landmarker::
CppFaceLandmarkerDetectAsync(landmarker, image, timestamp_ms, error_msg);
}
void face_landmarker_close_result(FaceLandmarkerResult* result) {
mediapipe::tasks::c::vision::face_landmarker::CppFaceLandmarkerCloseResult(
result);
}
int face_landmarker_close(void* landmarker, char** error_ms) {
return mediapipe::tasks::c::vision::face_landmarker::CppFaceLandmarkerClose(
landmarker, error_ms);
}
} // extern "C"

View File

@ -0,0 +1,156 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_FACE_LANDMARKER_H_
#define MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_FACE_LANDMARKER_H_
#include "mediapipe/tasks/c/core/base_options.h"
#include "mediapipe/tasks/c/vision/core/common.h"
#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h"
#ifndef MP_EXPORT
#define MP_EXPORT __attribute__((visibility("default")))
#endif // MP_EXPORT
#ifdef __cplusplus
extern "C" {
#endif
// The options for configuring a MediaPipe face landmarker task.
struct FaceLandmarkerOptions {
// Base options for configuring MediaPipe Tasks, such as specifying the model
// file with metadata, accelerator options, op resolver, etc.
struct BaseOptions base_options;
// The running mode of the task. Default to the image mode.
// FaceLandmarker has three running modes:
// 1) The image mode for recognizing face landmarks on single image inputs.
// 2) The video mode for recognizing face landmarks on the decoded frames of a
// video.
// 3) The live stream mode for recognizing face landmarks on the live stream
// of input data, such as from camera. In this mode, the "result_callback"
// below must be specified to receive the detection results asynchronously.
RunningMode running_mode;
// The maximum number of faces can be detected by the FaceLandmarker.
int num_faces = 1;
// The minimum confidence score for the face detection to be considered
// successful.
float min_face_detection_confidence = 0.5;
// The minimum confidence score of face presence score in the face landmark
// detection.
float min_face_presence_confidence = 0.5;
// The minimum confidence score for the face tracking to be considered
// successful.
float min_tracking_confidence = 0.5;
// Whether FaceLandmarker outputs face blendshapes classification. Face
// blendshapes are used for rendering the 3D face model.
bool output_face_blendshapes = false;
// Whether FaceLandmarker outputs facial transformation_matrix. Facial
// transformation matrix is used to transform the face landmarks in canonical
// face to the detected face, so that users can apply face effects on the
// detected landmarks.
bool output_facial_transformation_matrixes = false;
// The user-defined result callback for processing live stream data.
// The result callback should only be specified when the running mode is set
// to RunningMode::LIVE_STREAM. Arguments of the callback function include:
// the pointer to recognition result, the image that result was obtained
// on, the timestamp relevant to recognition results and pointer to error
// message in case of any failure. The validity of the passed arguments is
// true for the lifetime of the callback function.
//
// A caller is responsible for closing face landmarker result.
typedef void (*result_callback_fn)(const FaceLandmarkerResult* result,
const MpImage& image, int64_t timestamp_ms,
char* error_msg);
result_callback_fn result_callback;
};
// Creates an FaceLandmarker from the provided `options`.
// Returns a pointer to the face landmarker on success.
// If an error occurs, returns `nullptr` and sets the error parameter to an
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT void* face_landmarker_create(struct FaceLandmarkerOptions* options,
char** error_msg);
// Performs face landmark detection on the input `image`. Returns `0` on
// success. If an error occurs, returns an error code and sets the error
// parameter to an an error message (if `error_msg` is not `nullptr`). You must
// free the memory allocated for the error message.
MP_EXPORT int face_landmarker_detect_image(void* landmarker,
const MpImage& image,
FaceLandmarkerResult* result,
char** error_msg);
// Performs face landmark detection on the provided video frame.
// Only use this method when the FaceLandmarker is created with the video
// running mode.
// The image can be of any size with format RGB or RGBA. It's required to
// provide the video frame's timestamp (in milliseconds). The input timestamps
// must be monotonically increasing.
// If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT int face_landmarker_detect_for_video(void* landmarker,
const MpImage& image,
int64_t timestamp_ms,
FaceLandmarkerResult* result,
char** error_msg);
// Sends live image data to face landmark detection, and the results will be
// available via the `result_callback` provided in the FaceLandmarkerOptions.
// Only use this method when the FaceLandmarker is created with the live
// stream running mode.
// The image can be of any size with format RGB or RGBA. It's required to
// provide a timestamp (in milliseconds) to indicate when the input image is
// sent to the face landmarker. The input timestamps must be monotonically
// increasing.
// The `result_callback` provides:
// - The recognition results as an FaceLandmarkerResult object.
// - The const reference to the corresponding input image that the face
// landmarker runs on. Note that the const reference to the image will no
// longer be valid when the callback returns. To access the image data
// outside of the callback, callers need to make a copy of the image.
// - The input timestamp in milliseconds.
// If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT int face_landmarker_detect_async(void* landmarker,
const MpImage& image,
int64_t timestamp_ms,
char** error_msg);
// Frees the memory allocated inside a FaceLandmarkerResult result.
// Does not free the result pointer itself.
MP_EXPORT void face_landmarker_close_result(FaceLandmarkerResult* result);
// Frees face landmarker.
// If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT int face_landmarker_close(void* landmarker, char** error_msg);
#ifdef __cplusplus
} // extern C
#endif
#endif // MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_FACE_LANDMARKER_H_

View File

@ -0,0 +1,59 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_RESULT_FACE_LANDMARKER_RESULT_H_
#define MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_RESULT_FACE_LANDMARKER_RESULT_H_
#include <cstdint>
#include "mediapipe/tasks/c/components/containers/category.h"
#include "mediapipe/tasks/c/components/containers/landmark.h"
#include "mediapipe/tasks/c/components/containers/matrix.h"
#ifndef MP_EXPORT
#define MP_EXPORT __attribute__((visibility("default")))
#endif // MP_EXPORT
#ifdef __cplusplus
extern "C" {
#endif
// The hand landmarker result from HandLandmarker, where each vector
// element represents a single hand detected in the image.
struct FaceLandmarkerResult {
// Detected face landmarks in normalized image coordinates.
struct NormalizedLandmarks* face_landmarks;
// The number of elements in the face_landmarks array.
uint32_t face_landmarks_count;
// Optional face blendshapes results.
struct Categories* face_blendshapes;
// The number of elements in the face_blendshapes array.
uint32_t face_blendshapes_count;
// Optional facial transformation matrixes.
struct Matrix* facial_transformation_matrixes;
// The number of elements in the facial_transformation_matrixes array.
uint32_t facial_transformation_matrixes_count;
};
#ifdef __cplusplus
} // extern C
#endif
#endif // MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_RESULT_FACE_LANDMARKER_RESULT_H_

View File

@ -0,0 +1,117 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.h"
#include <cstdint>
#include <vector>
#include "mediapipe/tasks/c/components/containers/category.h"
#include "mediapipe/tasks/c/components/containers/category_converter.h"
#include "mediapipe/tasks/c/components/containers/landmark.h"
#include "mediapipe/tasks/c/components/containers/landmark_converter.h"
#include "mediapipe/tasks/c/components/containers/matrix.h"
#include "mediapipe/tasks/c/components/containers/matrix_converter.h"
#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h"
#include "mediapipe/tasks/cc/components/containers/category.h"
#include "mediapipe/tasks/cc/components/containers/landmark.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h"
namespace mediapipe::tasks::c::components::containers {
using CppCategory = ::mediapipe::tasks::components::containers::Category;
using CppNormalizedLandmark =
::mediapipe::tasks::components::containers::NormalizedLandmark;
void CppConvertToFaceLandmarkerResult(
const ::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult& in,
FaceLandmarkerResult* out) {
out->face_landmarks_count = in.face_landmarks.size();
out->face_landmarks = new NormalizedLandmarks[out->face_landmarks_count];
for (uint32_t i = 0; i < out->face_landmarks_count; ++i) {
std::vector<CppNormalizedLandmark> cpp_normalized_landmarks;
for (uint32_t j = 0; j < in.face_landmarks[i].landmarks.size(); ++j) {
const auto& cpp_landmark = in.face_landmarks[i].landmarks[j];
cpp_normalized_landmarks.push_back(cpp_landmark);
}
CppConvertToNormalizedLandmarks(cpp_normalized_landmarks,
&out->face_landmarks[i]);
}
if (in.face_blendshapes.has_value()) {
out->face_blendshapes_count = in.face_blendshapes->size();
out->face_blendshapes = new Categories[out->face_blendshapes_count];
for (uint32_t i = 0; i < out->face_blendshapes_count; ++i) {
uint32_t categories_count =
in.face_blendshapes.value()[i].categories.size();
out->face_blendshapes[i].categories_count = categories_count;
out->face_blendshapes[i].categories = new Category[categories_count];
for (uint32_t j = 0; j < categories_count; ++j) {
const auto& cpp_category = in.face_blendshapes.value()[i].categories[j];
CppConvertToCategory(cpp_category,
&out->face_blendshapes[i].categories[j]);
}
}
} else {
out->face_blendshapes_count = 0;
out->face_blendshapes = nullptr;
}
if (in.facial_transformation_matrixes.has_value()) {
out->facial_transformation_matrixes_count =
in.facial_transformation_matrixes.value().size();
out->facial_transformation_matrixes =
new ::Matrix[out->facial_transformation_matrixes_count];
for (uint32_t i = 0; i < out->facial_transformation_matrixes_count; ++i) {
CppConvertToMatrix(in.facial_transformation_matrixes.value()[i],
&out->facial_transformation_matrixes[i]);
}
} else {
out->facial_transformation_matrixes_count = 0;
out->facial_transformation_matrixes = nullptr;
}
}
void CppCloseFaceLandmarkerResult(FaceLandmarkerResult* result) {
for (uint32_t i = 0; i < result->face_blendshapes_count; ++i) {
for (uint32_t j = 0; j < result->face_blendshapes[i].categories_count;
++j) {
CppCloseCategory(&result->face_blendshapes[i].categories[j]);
}
delete[] result->face_blendshapes[i].categories;
}
delete[] result->face_blendshapes;
for (uint32_t i = 0; i < result->face_landmarks_count; ++i) {
CppCloseNormalizedLandmarks(&result->face_landmarks[i]);
}
delete[] result->face_landmarks;
for (uint32_t i = 0; i < result->facial_transformation_matrixes_count; ++i) {
CppCloseMatrix(&result->facial_transformation_matrixes[i]);
}
delete[] result->facial_transformation_matrixes;
result->face_blendshapes_count = 0;
result->face_landmarks_count = 0;
result->facial_transformation_matrixes_count = 0;
result->face_blendshapes = nullptr;
result->face_landmarks = nullptr;
result->facial_transformation_matrixes = nullptr;
}
} // namespace mediapipe::tasks::c::components::containers

View File

@ -0,0 +1,32 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_FACE_LANDMARKER_RESULT_CONVERTER_H_
#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_FACE_LANDMARKER_RESULT_CONVERTER_H_
#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h"
namespace mediapipe::tasks::c::components::containers {
void CppConvertToFaceLandmarkerResult(
const mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult& in,
FaceLandmarkerResult* out);
void CppCloseFaceLandmarkerResult(FaceLandmarkerResult* result);
} // namespace mediapipe::tasks::c::components::containers
#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_FACE_LANDMARKER_RESULT_CONVERTER_H_

View File

@ -0,0 +1,154 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.h"
#include <cstddef>
#include <cstdint>
#include <string>
#include <vector>
#include "Eigen/Core"
#include "mediapipe/framework/formats/matrix.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/tasks/c/components/containers/landmark.h"
#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h"
#include "mediapipe/tasks/cc/components/containers/category.h"
#include "mediapipe/tasks/cc/components/containers/classification_result.h"
#include "mediapipe/tasks/cc/components/containers/landmark.h"
#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h"
namespace mediapipe::tasks::c::components::containers {
void InitFaceLandmarkerResult(
::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult*
cpp_result) {
// Initialize face_landmarks
mediapipe::tasks::components::containers::NormalizedLandmark
cpp_normalized_landmark = {/* x= */ 0.1f, /* y= */ 0.2f, /* z= */ 0.3f};
mediapipe::tasks::components::containers::NormalizedLandmarks
cpp_normalized_landmarks;
cpp_normalized_landmarks.landmarks.push_back(cpp_normalized_landmark);
cpp_result->face_landmarks.push_back(cpp_normalized_landmarks);
// Initialize face_blendshapes
mediapipe::tasks::components::containers::Category cpp_category = {
/* index= */ 1,
/* score= */ 0.8f,
/* category_name= */ "blendshape_label_1",
/* display_name= */ "blendshape_display_name_1"};
mediapipe::tasks::components::containers::Classifications
classifications_for_blendshapes;
classifications_for_blendshapes.categories.push_back(cpp_category);
cpp_result->face_blendshapes =
std::vector<mediapipe::tasks::components::containers::Classifications>{
classifications_for_blendshapes};
cpp_result->face_blendshapes->push_back(classifications_for_blendshapes);
// Initialize facial_transformation_matrixes
Eigen::MatrixXf cpp_matrix(2, 2);
cpp_matrix << 1.0f, 2.0f, 3.0f, 4.0f;
cpp_result->facial_transformation_matrixes = std::vector<Matrix>{cpp_matrix};
}
TEST(FaceLandmarkerResultConverterTest, ConvertsCustomResult) {
// Initialize a C++ FaceLandmarkerResult
::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult cpp_result;
InitFaceLandmarkerResult(&cpp_result);
FaceLandmarkerResult c_result;
CppConvertToFaceLandmarkerResult(cpp_result, &c_result);
// Verify conversion of face_landmarks
EXPECT_EQ(c_result.face_landmarks_count, cpp_result.face_landmarks.size());
for (uint32_t i = 0; i < c_result.face_landmarks_count; ++i) {
EXPECT_EQ(c_result.face_landmarks[i].landmarks_count,
cpp_result.face_landmarks[i].landmarks.size());
for (uint32_t j = 0; j < c_result.face_landmarks[i].landmarks_count; ++j) {
const auto& cpp_landmark = cpp_result.face_landmarks[i].landmarks[j];
EXPECT_FLOAT_EQ(c_result.face_landmarks[i].landmarks[j].x,
cpp_landmark.x);
EXPECT_FLOAT_EQ(c_result.face_landmarks[i].landmarks[j].y,
cpp_landmark.y);
EXPECT_FLOAT_EQ(c_result.face_landmarks[i].landmarks[j].z,
cpp_landmark.z);
}
}
// Verify conversion of face_blendshapes
EXPECT_EQ(c_result.face_blendshapes_count,
cpp_result.face_blendshapes.value().size());
for (uint32_t i = 0; i < c_result.face_blendshapes_count; ++i) {
const auto& cpp_face_blendshapes = cpp_result.face_blendshapes.value();
EXPECT_EQ(c_result.face_blendshapes[i].categories_count,
cpp_face_blendshapes[i].categories.size());
for (uint32_t j = 0; j < c_result.face_blendshapes[i].categories_count;
++j) {
const auto& cpp_category = cpp_face_blendshapes[i].categories[j];
EXPECT_EQ(c_result.face_blendshapes[i].categories[j].index,
cpp_category.index);
EXPECT_FLOAT_EQ(c_result.face_blendshapes[i].categories[j].score,
cpp_category.score);
EXPECT_EQ(
std::string(c_result.face_blendshapes[i].categories[j].category_name),
cpp_category.category_name);
}
}
// Verify conversion of facial_transformation_matrixes
EXPECT_EQ(c_result.facial_transformation_matrixes_count,
cpp_result.facial_transformation_matrixes.value().size());
for (uint32_t i = 0; i < c_result.facial_transformation_matrixes_count; ++i) {
const auto& cpp_facial_transformation_matrixes =
cpp_result.facial_transformation_matrixes.value();
// Assuming Matrix struct contains data array and dimensions
const auto& cpp_matrix = cpp_facial_transformation_matrixes[i];
EXPECT_EQ(c_result.facial_transformation_matrixes[i].rows,
cpp_matrix.rows());
EXPECT_EQ(c_result.facial_transformation_matrixes[i].cols,
cpp_matrix.cols());
// Check each element of the matrix
for (int32_t row = 0; row < cpp_matrix.rows(); ++row) {
for (int32_t col = 0; col < cpp_matrix.cols(); ++col) {
size_t index = col * cpp_matrix.rows() + row; // Column-major index
EXPECT_FLOAT_EQ(c_result.facial_transformation_matrixes[i].data[index],
cpp_matrix(row, col));
}
}
}
CppCloseFaceLandmarkerResult(&c_result);
}
TEST(FaceLandmarkerResultConverterTest, FreesMemory) {
::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult cpp_result;
InitFaceLandmarkerResult(&cpp_result);
FaceLandmarkerResult c_result;
CppConvertToFaceLandmarkerResult(cpp_result, &c_result);
EXPECT_NE(c_result.face_blendshapes, nullptr);
EXPECT_NE(c_result.face_landmarks, nullptr);
EXPECT_NE(c_result.facial_transformation_matrixes, nullptr);
CppCloseFaceLandmarkerResult(&c_result);
EXPECT_EQ(c_result.face_blendshapes, nullptr);
EXPECT_EQ(c_result.face_landmarks, nullptr);
EXPECT_EQ(c_result.facial_transformation_matrixes, nullptr);
}
} // namespace mediapipe::tasks::c::components::containers

View File

@ -0,0 +1,292 @@
/* Copyright 2023 The MediaPipe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/
#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker.h"
#include <cstdint>
#include <cstdio>
#include <cstdlib>
#include <string>
#include "absl/flags/flag.h"
#include "absl/strings/string_view.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/formats/image.h"
#include "mediapipe/framework/port/gmock.h"
#include "mediapipe/framework/port/gtest.h"
#include "mediapipe/tasks/c/components/containers/landmark.h"
#include "mediapipe/tasks/c/vision/core/common.h"
#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h"
#include "mediapipe/tasks/cc/vision/utils/image_utils.h"
namespace {
using ::mediapipe::file::JoinPath;
using ::mediapipe::tasks::vision::DecodeImageFromFile;
using testing::HasSubstr;
constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/";
constexpr char kModelName[] = "face_landmarker_v2_with_blendshapes.task";
constexpr char kImageFile[] = "portrait.jpg";
constexpr float kLandmarksPrecision = 0.03;
constexpr float kBlendshapesPrecision = 0.12;
constexpr float kFacialTransformationMatrixPrecision = 0.05;
constexpr int kIterations = 100;
std::string GetFullPath(absl::string_view file_name) {
return JoinPath("./", kTestDataDirectory, file_name);
}
void AssertHandLandmarkerResult(const FaceLandmarkerResult* result,
const float blendshapes_precision,
const float landmark_precision,
const float matrix_precison) {
// Expects to have the same number of faces detected.
EXPECT_EQ(result->face_blendshapes_count, 1);
// Actual blendshapes matches expected blendshapes.
EXPECT_EQ(
std::string{result->face_blendshapes[0].categories[0].category_name},
"_neutral");
EXPECT_NEAR(result->face_blendshapes[0].categories[0].score, 0.0f,
blendshapes_precision);
// Actual landmarks match expected landmarks.
EXPECT_NEAR(result->face_landmarks[0].landmarks[0].x, 0.4977f,
landmark_precision);
EXPECT_NEAR(result->face_landmarks[0].landmarks[0].y, 0.2485f,
landmark_precision);
EXPECT_NEAR(result->face_landmarks[0].landmarks[0].z, -0.0305f,
landmark_precision);
// Expects to have at least one facial transformation matrix.
EXPECT_GE(result->facial_transformation_matrixes_count, 1);
// Actual matrix matches expected matrix.
// Assuming the expected matrix is 2x2 for demonstration.
const float expected_matrix[4] = {0.9991f, 0.0166f, -0.0374f, 0.0f};
for (int i = 0; i < 4; ++i) {
printf(">> %f <<", result->facial_transformation_matrixes[0].data[i]);
EXPECT_NEAR(result->facial_transformation_matrixes[0].data[i],
expected_matrix[i], matrix_precison);
}
}
TEST(FaceLandmarkerTest, ImageModeTest) {
const auto image = DecodeImageFromFile(GetFullPath(kImageFile));
ASSERT_TRUE(image.ok());
const std::string model_path = GetFullPath(kModelName);
FaceLandmarkerOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* running_mode= */ RunningMode::IMAGE,
/* num_faces= */ 1,
/* min_face_detection_confidence= */ 0.5,
/* min_face_presence_confidence= */ 0.5,
/* min_tracking_confidence= */ 0.5,
/* output_face_blendshapes = */ true,
/* output_facial_transformation_matrixes = */ true,
};
void* landmarker = face_landmarker_create(&options, /* error_msg */ nullptr);
EXPECT_NE(landmarker, nullptr);
const auto& image_frame = image->GetImageFrameSharedPtr();
const MpImage mp_image = {
.type = MpImage::IMAGE_FRAME,
.image_frame = {.format = static_cast<ImageFormat>(image_frame->Format()),
.image_buffer = image_frame->PixelData(),
.width = image_frame->Width(),
.height = image_frame->Height()}};
FaceLandmarkerResult result;
face_landmarker_detect_image(landmarker, mp_image, &result,
/* error_msg */ nullptr);
AssertHandLandmarkerResult(&result, kBlendshapesPrecision,
kLandmarksPrecision,
kFacialTransformationMatrixPrecision);
face_landmarker_close_result(&result);
face_landmarker_close(landmarker, /* error_msg */ nullptr);
}
TEST(FaceLandmarkerTest, VideoModeTest) {
const auto image = DecodeImageFromFile(GetFullPath(kImageFile));
ASSERT_TRUE(image.ok());
const std::string model_path = GetFullPath(kModelName);
FaceLandmarkerOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* running_mode= */ RunningMode::VIDEO,
/* num_faces= */ 1,
/* min_face_detection_confidence= */ 0.5,
/* min_face_presence_confidence= */ 0.5,
/* min_tracking_confidence= */ 0.5,
/* output_face_blendshapes = */ true,
/* output_facial_transformation_matrixes = */ true,
};
void* landmarker = face_landmarker_create(&options,
/* error_msg */ nullptr);
EXPECT_NE(landmarker, nullptr);
const auto& image_frame = image->GetImageFrameSharedPtr();
const MpImage mp_image = {
.type = MpImage::IMAGE_FRAME,
.image_frame = {.format = static_cast<ImageFormat>(image_frame->Format()),
.image_buffer = image_frame->PixelData(),
.width = image_frame->Width(),
.height = image_frame->Height()}};
for (int i = 0; i < kIterations; ++i) {
FaceLandmarkerResult result;
face_landmarker_detect_for_video(landmarker, mp_image, i, &result,
/* error_msg */ nullptr);
AssertHandLandmarkerResult(&result, kBlendshapesPrecision,
kLandmarksPrecision,
kFacialTransformationMatrixPrecision);
face_landmarker_close_result(&result);
}
face_landmarker_close(landmarker, /* error_msg */ nullptr);
}
// A structure to support LiveStreamModeTest below. This structure holds a
// static method `Fn` for a callback function of C API. A `static` qualifier
// allows to take an address of the method to follow API style. Another static
// struct member is `last_timestamp` that is used to verify that current
// timestamp is greater than the previous one.
struct LiveStreamModeCallback {
static int64_t last_timestamp;
static void Fn(const FaceLandmarkerResult* landmarker_result,
const MpImage& image, int64_t timestamp, char* error_msg) {
ASSERT_NE(landmarker_result, nullptr);
ASSERT_EQ(error_msg, nullptr);
AssertHandLandmarkerResult(landmarker_result, kBlendshapesPrecision,
kLandmarksPrecision,
kFacialTransformationMatrixPrecision);
EXPECT_GT(image.image_frame.width, 0);
EXPECT_GT(image.image_frame.height, 0);
EXPECT_GT(timestamp, last_timestamp);
++last_timestamp;
}
};
int64_t LiveStreamModeCallback::last_timestamp = -1;
TEST(FaceLandmarkerTest, LiveStreamModeTest) {
const auto image = DecodeImageFromFile(GetFullPath(kImageFile));
ASSERT_TRUE(image.ok());
const std::string model_path = GetFullPath(kModelName);
FaceLandmarkerOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* running_mode= */ RunningMode::LIVE_STREAM,
/* num_faces= */ 1,
/* min_face_detection_confidence= */ 0.5,
/* min_face_presence_confidence= */ 0.5,
/* min_tracking_confidence= */ 0.5,
/* output_face_blendshapes = */ true,
/* output_facial_transformation_matrixes = */ true,
/* result_callback= */ LiveStreamModeCallback::Fn,
};
void* landmarker = face_landmarker_create(&options, /* error_msg */
nullptr);
EXPECT_NE(landmarker, nullptr);
const auto& image_frame = image->GetImageFrameSharedPtr();
const MpImage mp_image = {
.type = MpImage::IMAGE_FRAME,
.image_frame = {.format = static_cast<ImageFormat>(image_frame->Format()),
.image_buffer = image_frame->PixelData(),
.width = image_frame->Width(),
.height = image_frame->Height()}};
for (int i = 0; i < kIterations; ++i) {
EXPECT_GE(face_landmarker_detect_async(landmarker, mp_image, i,
/* error_msg */ nullptr),
0);
}
face_landmarker_close(landmarker, /* error_msg */ nullptr);
// Due to the flow limiter, the total of outputs might be smaller than the
// number of iterations.
EXPECT_LE(LiveStreamModeCallback::last_timestamp, kIterations);
EXPECT_GT(LiveStreamModeCallback::last_timestamp, 0);
}
TEST(FaceLandmarkerTest, InvalidArgumentHandling) {
// It is an error to set neither the asset buffer nor the path.
FaceLandmarkerOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ nullptr},
/* running_mode= */ RunningMode::IMAGE,
/* num_faces= */ 1,
/* min_face_detection_confidence= */ 0.5,
/* min_face_presence_confidence= */ 0.5,
/* min_tracking_confidence= */ 0.5,
/* output_face_blendshapes = */ true,
/* output_facial_transformation_matrixes = */ true,
};
char* error_msg;
void* landmarker = face_landmarker_create(&options, &error_msg);
EXPECT_EQ(landmarker, nullptr);
EXPECT_THAT(
error_msg,
HasSubstr("INVALID_ARGUMENT: BLENDSHAPES Tag and blendshapes model must "
"be both set. Get BLENDSHAPES is set: true, blendshapes model "
"is set: false [MediaPipeTasksStatus='601']"));
free(error_msg);
}
TEST(FaceLandmarkerTest, FailedRecognitionHandling) {
const std::string model_path = GetFullPath(kModelName);
FaceLandmarkerOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* running_mode= */ RunningMode::IMAGE,
/* num_faces= */ 1,
/* min_face_detection_confidence= */ 0.5,
/* min_face_presence_confidence= */ 0.5,
/* min_tracking_confidence= */ 0.5,
/* output_face_blendshapes = */ true,
/* output_facial_transformation_matrixes = */ true,
};
void* landmarker = face_landmarker_create(&options, /* error_msg */
nullptr);
EXPECT_NE(landmarker, nullptr);
const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}};
FaceLandmarkerResult result;
char* error_msg;
face_landmarker_detect_image(landmarker, mp_image, &result, &error_msg);
EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet"));
free(error_msg);
face_landmarker_close(landmarker, /* error_msg */ nullptr);
}
} // namespace

View File

@ -68,7 +68,7 @@ struct HandLandmarkerOptions {
// true for the lifetime of the callback function.
//
// A caller is responsible for closing hand landmarker result.
typedef void (*result_callback_fn)(HandLandmarkerResult* result,
typedef void (*result_callback_fn)(const HandLandmarkerResult* result,
const MpImage& image, int64_t timestamp_ms,
char* error_msg);
result_callback_fn result_callback;

View File

@ -47,7 +47,7 @@ std::string GetFullPath(absl::string_view file_name) {
return JoinPath("./", kTestDataDirectory, file_name);
}
void MatchesHandLandmarkerResult(HandLandmarkerResult* result,
void AssertHandLandmarkerResult(const HandLandmarkerResult* result,
const float score_precision,
const float landmark_precision) {
// Expects to have the same number of hands detected.
@ -104,7 +104,7 @@ TEST(HandLandmarkerTest, ImageModeTest) {
HandLandmarkerResult result;
hand_landmarker_detect_image(landmarker, mp_image, &result,
/* error_msg */ nullptr);
MatchesHandLandmarkerResult(&result, kScorePrecision, kLandmarkPrecision);
AssertHandLandmarkerResult(&result, kScorePrecision, kLandmarkPrecision);
hand_landmarker_close_result(&result);
hand_landmarker_close(landmarker, /* error_msg */ nullptr);
}
@ -141,7 +141,7 @@ TEST(HandLandmarkerTest, VideoModeTest) {
hand_landmarker_detect_for_video(landmarker, mp_image, i, &result,
/* error_msg */ nullptr);
MatchesHandLandmarkerResult(&result, kScorePrecision, kLandmarkPrecision);
AssertHandLandmarkerResult(&result, kScorePrecision, kLandmarkPrecision);
hand_landmarker_close_result(&result);
}
hand_landmarker_close(landmarker, /* error_msg */ nullptr);
@ -154,11 +154,11 @@ TEST(HandLandmarkerTest, VideoModeTest) {
// timestamp is greater than the previous one.
struct LiveStreamModeCallback {
static int64_t last_timestamp;
static void Fn(HandLandmarkerResult* landmarker_result, const MpImage& image,
int64_t timestamp, char* error_msg) {
static void Fn(const HandLandmarkerResult* landmarker_result,
const MpImage& image, int64_t timestamp, char* error_msg) {
ASSERT_NE(landmarker_result, nullptr);
ASSERT_EQ(error_msg, nullptr);
MatchesHandLandmarkerResult(landmarker_result, kScorePrecision,
AssertHandLandmarkerResult(landmarker_result, kScorePrecision,
kLandmarkPrecision);
EXPECT_GT(image.image_frame.width, 0);
EXPECT_GT(image.image_frame.height, 0);
@ -183,7 +183,7 @@ TEST(HandLandmarkerTest, LiveStreamModeTest) {
/* min_hand_detection_confidence= */ 0.5,
/* min_hand_presence_confidence= */ 0.5,
/* min_tracking_confidence= */ 0.5,
/* result_callback= */ LiveStreamModeCallback::Fn,
/* result_callback_fn= */ LiveStreamModeCallback::Fn,
};
void* landmarker = hand_landmarker_create(&options, /* error_msg */ nullptr);

View File

@ -48,6 +48,7 @@ mediapipe_files(srcs = [
"face_landmark.tflite",
"face_landmarker.task",
"face_landmarker_v2.task",
"face_landmarker_v2_with_blendshapes.task",
"face_stylizer_color_ink.task",
"fist.jpg",
"fist.png",
@ -185,6 +186,7 @@ filegroup(
"face_detection_short_range.tflite",
"face_landmarker.task",
"face_landmarker_v2.task",
"face_landmarker_v2_with_blendshapes.task",
"face_stylizer_color_ink.task",
"gesture_recognizer.task",
"hair_segmentation.tflite",