From fec4dff0d66f1700ba31c948808b7b129a822a26 Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 11 Dec 2023 04:58:15 -0800 Subject: [PATCH 1/6] Added Face Landmarker C Tasks API and tests --- mediapipe/tasks/c/components/containers/BUILD | 26 ++ .../tasks/c/components/containers/matrix.h | 41 +++ .../components/containers/matrix_converter.cc | 45 +++ .../components/containers/matrix_converter.h | 30 ++ .../containers/matrix_converter_test.cc | 52 ++++ .../tasks/c/vision/face_landmarker/BUILD | 148 +++++++++ .../vision/face_landmarker/face_landmarker.cc | 287 +++++++++++++++++ .../vision/face_landmarker/face_landmarker.h | 156 ++++++++++ .../face_landmarker/face_landmarker_result.h | 59 ++++ .../face_landmarker_result_converter.cc | 116 +++++++ .../face_landmarker_result_converter.h | 32 ++ .../face_landmarker_result_converter_test.cc | 157 ++++++++++ .../face_landmarker/face_landmarker_test.cc | 291 ++++++++++++++++++ mediapipe/tasks/testdata/vision/BUILD | 2 + 14 files changed, 1442 insertions(+) create mode 100644 mediapipe/tasks/c/components/containers/matrix.h create mode 100644 mediapipe/tasks/c/components/containers/matrix_converter.cc create mode 100644 mediapipe/tasks/c/components/containers/matrix_converter.h create mode 100644 mediapipe/tasks/c/components/containers/matrix_converter_test.cc create mode 100644 mediapipe/tasks/c/vision/face_landmarker/BUILD create mode 100644 mediapipe/tasks/c/vision/face_landmarker/face_landmarker.cc create mode 100644 mediapipe/tasks/c/vision/face_landmarker/face_landmarker.h create mode 100644 mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h create mode 100644 mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.cc create mode 100644 mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.h create mode 100644 mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter_test.cc create mode 100644 mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc diff --git a/mediapipe/tasks/c/components/containers/BUILD b/mediapipe/tasks/c/components/containers/BUILD index 3c4b557b3..510bdcd81 100644 --- a/mediapipe/tasks/c/components/containers/BUILD +++ b/mediapipe/tasks/c/components/containers/BUILD @@ -43,6 +43,32 @@ cc_test( ], ) +cc_library( + name = "matrix", + hdrs = ["matrix.h"], +) + +cc_library( + name = "matrix_converter", + srcs = ["matrix_converter.cc"], + hdrs = ["matrix_converter.h"], + deps = [ + ":matrix", + "@eigen_archive//:eigen3", + ], +) + +cc_test( + name = "matrix_converter_test", + srcs = ["matrix_converter_test.cc"], + deps = [ + ":matrix", + ":matrix_converter", + "//mediapipe/framework/port:gtest", + "@com_google_googletest//:gtest_main", + ], +) + cc_library( name = "landmark", hdrs = ["landmark.h"], diff --git a/mediapipe/tasks/c/components/containers/matrix.h b/mediapipe/tasks/c/components/containers/matrix.h new file mode 100644 index 000000000..71ec7474c --- /dev/null +++ b/mediapipe/tasks/c/components/containers/matrix.h @@ -0,0 +1,41 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_H_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// Data are stored in column-major order by default. +struct Matrix { + // The number of rows in the matrix. + long rows; + + // The number of rows in the matrix. + long cols; + + // The matrix data. + float* data; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_H_ diff --git a/mediapipe/tasks/c/components/containers/matrix_converter.cc b/mediapipe/tasks/c/components/containers/matrix_converter.cc new file mode 100644 index 000000000..6d823a424 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/matrix_converter.cc @@ -0,0 +1,45 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/matrix_converter.h" + +#include + +#include "Eigen/Core" +#include "mediapipe/tasks/c/components/containers/matrix.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToMatrix(const Eigen::MatrixXf& in, ::Matrix* out) { + out->rows = in.rows(); + out->cols = in.cols(); + out->data = new float[out->rows * out->cols]; + + // Copy data from Eigen matrix to C matrix in column-major order + for (int col = 0; col < out->cols; ++col) { + for (int row = 0; row < out->rows; ++row) { + out->data[col * out->rows + row] = in(row, col); + } + } +} + +void CppCloseMatrix(::Matrix* in) { + if (in->data) { + delete[] in->data; + in->data = nullptr; + } +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/matrix_converter.h b/mediapipe/tasks/c/components/containers/matrix_converter.h new file mode 100644 index 000000000..cc92f5c98 --- /dev/null +++ b/mediapipe/tasks/c/components/containers/matrix_converter.h @@ -0,0 +1,30 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_CONVERTER_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_CONVERTER_H_ + +#include "Eigen/Core" +#include "mediapipe/tasks/c/components/containers/matrix.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToMatrix(const Eigen::MatrixXf& in, ::Matrix* out); + +void CppCloseMatrix(::Matrix* data); + +} // namespace mediapipe::tasks::c::components::containers + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_MATRIX_CONVERTER_H_ diff --git a/mediapipe/tasks/c/components/containers/matrix_converter_test.cc b/mediapipe/tasks/c/components/containers/matrix_converter_test.cc new file mode 100644 index 000000000..01248203e --- /dev/null +++ b/mediapipe/tasks/c/components/containers/matrix_converter_test.cc @@ -0,0 +1,52 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/components/containers/matrix_converter.h" + +#include +#include +#include + +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/components/containers/matrix.h" + +namespace mediapipe::tasks::c::components::containers { + +TEST(MatrixConversionTest, ConvertsEigenMatrixToCMatrixAndFreesMemory) { + // Initialize an Eigen::MatrixXf + Eigen::MatrixXf cpp_matrix(2, 2); + cpp_matrix << 1.0f, 2.0f, 3.0f, 4.0f; + + // Convert this Eigen matrix to C-style Matrix + ::Matrix c_matrix; + CppConvertToMatrix(cpp_matrix, &c_matrix); + + // Verify the conversion + EXPECT_EQ(c_matrix.rows, 2); + EXPECT_EQ(c_matrix.cols, 2); + ASSERT_NE(c_matrix.data, nullptr); + EXPECT_FLOAT_EQ(c_matrix.data[0], 1.0f); + EXPECT_FLOAT_EQ(c_matrix.data[1], 2.0f); + EXPECT_FLOAT_EQ(c_matrix.data[2], 3.0f); + EXPECT_FLOAT_EQ(c_matrix.data[3], 4.0f); + + // Close the C-style Matrix + CppCloseMatrix(&c_matrix); + + // Verify that memory is freed + EXPECT_EQ(c_matrix.data, nullptr); +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/vision/face_landmarker/BUILD b/mediapipe/tasks/c/vision/face_landmarker/BUILD new file mode 100644 index 000000000..083bfa592 --- /dev/null +++ b/mediapipe/tasks/c/vision/face_landmarker/BUILD @@ -0,0 +1,148 @@ +# Copyright 2023 The MediaPipe Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//mediapipe/tasks:internal"]) + +licenses(["notice"]) + +cc_library( + name = "face_landmarker_result", + hdrs = ["face_landmarker_result.h"], + visibility = ["//visibility:public"], + deps = [ + "//mediapipe/tasks/c/components/containers:category", + "//mediapipe/tasks/c/components/containers:landmark", + "//mediapipe/tasks/c/components/containers:matrix", + ], +) + +cc_library( + name = "face_landmarker_result_converter", + srcs = ["face_landmarker_result_converter.cc"], + hdrs = ["face_landmarker_result_converter.h"], + deps = [ + ":face_landmarker_result", + "//mediapipe/tasks/c/components/containers:category", + "//mediapipe/tasks/c/components/containers:category_converter", + "//mediapipe/tasks/c/components/containers:landmark", + "//mediapipe/tasks/c/components/containers:landmark_converter", + "//mediapipe/tasks/c/components/containers:matrix", + "//mediapipe/tasks/c/components/containers:matrix_converter", + "//mediapipe/tasks/cc/components/containers:category", + "//mediapipe/tasks/cc/components/containers:landmark", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_result", + ], +) + +cc_test( + name = "face_landmarker_result_converter_test", + srcs = ["face_landmarker_result_converter_test.cc"], + linkstatic = 1, + deps = [ + ":face_landmarker_result", + ":face_landmarker_result_converter", + "//mediapipe/framework/port:gtest", + "//mediapipe/framework/formats:matrix", + "//mediapipe/tasks/c/components/containers:landmark", + "//mediapipe/tasks/cc/components/containers:category", + "//mediapipe/tasks/cc/components/containers:classification_result", + "//mediapipe/tasks/cc/components/containers:landmark", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_result", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "face_landmarker_lib", + srcs = ["face_landmarker.cc"], + hdrs = ["face_landmarker.h"], + visibility = ["//visibility:public"], + deps = [ + ":face_landmarker_result", + ":face_landmarker_result_converter", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/formats:image_frame", + "//mediapipe/tasks/c/core:base_options", + "//mediapipe/tasks/c/core:base_options_converter", + "//mediapipe/tasks/c/vision/core:common", + "//mediapipe/tasks/cc/vision/core:running_mode", + "//mediapipe/tasks/cc/vision/face_landmarker", + "//mediapipe/tasks/cc/vision/face_landmarker:face_landmarker_result", + "//mediapipe/tasks/cc/vision/utils:image_utils", + "@com_google_absl//absl/log:absl_log", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + ], + alwayslink = 1, +) + +cc_test( + name = "face_landmarker_test", + srcs = ["face_landmarker_test.cc"], + data = [ + "//mediapipe/framework/formats:image_frame_opencv", + "//mediapipe/framework/port:opencv_core", + "//mediapipe/framework/port:opencv_imgproc", + "//mediapipe/tasks/testdata/vision:test_images", + "//mediapipe/tasks/testdata/vision:test_models", + ], + linkstatic = 1, + deps = [ + ":face_landmarker_lib", + ":face_landmarker_result", + "//mediapipe/framework/deps:file_path", + "//mediapipe/framework/formats:image", + "//mediapipe/framework/port:gtest", + "//mediapipe/tasks/c/components/containers:landmark", + "//mediapipe/tasks/c/vision/core:common", + "//mediapipe/tasks/cc/vision/utils:image_utils", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +# bazel build -c opt --linkopt -s --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision/face_landmarker:libface_landmarker.so +cc_binary( + name = "libface_landmarker.so", + linkopts = [ + "-Wl,-soname=libface_landmarker.so", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [":face_landmarker_lib"], +) + +# bazel build --config darwin_arm64 -c opt --strip always --define MEDIAPIPE_DISABLE_GPU=1 \ +# //mediapipe/tasks/c/vision/face_landmarker:libface_landmarker.dylib +cc_binary( + name = "libface_landmarker.dylib", + linkopts = [ + "-Wl,-install_name,libface_landmarker.dylib", + "-fvisibility=hidden", + ], + linkshared = True, + tags = [ + "manual", + "nobuilder", + "notap", + ], + deps = [":face_landmarker_lib"], +) diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker.cc b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker.cc new file mode 100644 index 000000000..47f26a120 --- /dev/null +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker.cc @@ -0,0 +1,287 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker.h" + +#include +#include +#include +#include + +#include "absl/log/absl_log.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/formats/image_frame.h" +#include "mediapipe/tasks/c/core/base_options_converter.h" +#include "mediapipe/tasks/c/vision/core/common.h" +#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h" +#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.h" +#include "mediapipe/tasks/cc/vision/core/running_mode.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace mediapipe::tasks::c::vision::face_landmarker { + +namespace { + +using ::mediapipe::tasks::c::components::containers:: + CppCloseFaceLandmarkerResult; +using ::mediapipe::tasks::c::components::containers:: + CppConvertToFaceLandmarkerResult; +using ::mediapipe::tasks::c::core::CppConvertToBaseOptions; +using ::mediapipe::tasks::vision::CreateImageFromBuffer; +using ::mediapipe::tasks::vision::core::RunningMode; +using ::mediapipe::tasks::vision::face_landmarker::FaceLandmarker; +typedef ::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult + CppFaceLandmarkerResult; + +int CppProcessError(absl::Status status, char** error_msg) { + if (error_msg) { + *error_msg = strdup(status.ToString().c_str()); + } + return status.raw_code(); +} + +} // namespace + +void CppConvertToFaceLandmarkerOptions( + const FaceLandmarkerOptions& in, + mediapipe::tasks::vision::face_landmarker::FaceLandmarkerOptions* out) { + out->num_faces = in.num_faces; + out->min_face_detection_confidence = in.min_face_detection_confidence; + out->min_face_presence_confidence = in.min_face_presence_confidence; + out->min_tracking_confidence = in.min_tracking_confidence; + out->output_face_blendshapes = in.output_face_blendshapes; + out->output_facial_transformation_matrixes = + in.output_facial_transformation_matrixes; +} + +FaceLandmarker* CppFaceLandmarkerCreate(const FaceLandmarkerOptions& options, + char** error_msg) { + auto cpp_options = std::make_unique< + ::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerOptions>(); + + CppConvertToBaseOptions(options.base_options, &cpp_options->base_options); + CppConvertToFaceLandmarkerOptions(options, cpp_options.get()); + cpp_options->running_mode = static_cast(options.running_mode); + + // Enable callback for processing live stream data when the running mode is + // set to RunningMode::LIVE_STREAM. + if (cpp_options->running_mode == RunningMode::LIVE_STREAM) { + if (options.result_callback == nullptr) { + const absl::Status status = absl::InvalidArgumentError( + "Provided null pointer to callback function."); + ABSL_LOG(ERROR) << "Failed to create FaceLandmarker: " << status; + CppProcessError(status, error_msg); + return nullptr; + } + + FaceLandmarkerOptions::result_callback_fn result_callback = + options.result_callback; + cpp_options->result_callback = + [result_callback](absl::StatusOr cpp_result, + const Image& image, int64_t timestamp) { + char* error_msg = nullptr; + + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Recognition failed: " << cpp_result.status(); + CppProcessError(cpp_result.status(), &error_msg); + result_callback(nullptr, MpImage(), timestamp, error_msg); + free(error_msg); + return; + } + + // Result is valid for the lifetime of the callback function. + FaceLandmarkerResult result; + CppConvertToFaceLandmarkerResult(*cpp_result, &result); + + const auto& image_frame = image.GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = { + .format = static_cast<::ImageFormat>(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + result_callback(&result, mp_image, timestamp, + /* error_msg= */ nullptr); + + CppCloseFaceLandmarkerResult(&result); + }; + } + + auto landmarker = FaceLandmarker::Create(std::move(cpp_options)); + if (!landmarker.ok()) { + ABSL_LOG(ERROR) << "Failed to create FaceLandmarker: " + << landmarker.status(); + CppProcessError(landmarker.status(), error_msg); + return nullptr; + } + return landmarker->release(); +} + +int CppFaceLandmarkerDetect(void* landmarker, const MpImage& image, + FaceLandmarkerResult* result, char** error_msg) { + if (image.type == MpImage::GPU_BUFFER) { + const absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet."); + + ABSL_LOG(ERROR) << "Recognition failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image.image_frame.format), + image.image_frame.image_buffer, image.image_frame.width, + image.image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_landmarker = static_cast(landmarker); + auto cpp_result = cpp_landmarker->Detect(*img); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Recognition failed: " << cpp_result.status(); + return CppProcessError(cpp_result.status(), error_msg); + } + CppConvertToFaceLandmarkerResult(*cpp_result, result); + return 0; +} + +int CppFaceLandmarkerDetectForVideo(void* landmarker, const MpImage& image, + int64_t timestamp_ms, + FaceLandmarkerResult* result, + char** error_msg) { + if (image.type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Recognition failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image.image_frame.format), + image.image_frame.image_buffer, image.image_frame.width, + image.image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_landmarker = static_cast(landmarker); + auto cpp_result = cpp_landmarker->DetectForVideo(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Recognition failed: " << cpp_result.status(); + return CppProcessError(cpp_result.status(), error_msg); + } + CppConvertToFaceLandmarkerResult(*cpp_result, result); + return 0; +} + +int CppFaceLandmarkerDetectAsync(void* landmarker, const MpImage& image, + int64_t timestamp_ms, char** error_msg) { + if (image.type == MpImage::GPU_BUFFER) { + absl::Status status = + absl::InvalidArgumentError("GPU Buffer not supported yet"); + + ABSL_LOG(ERROR) << "Recognition failed: " << status.message(); + return CppProcessError(status, error_msg); + } + + const auto img = CreateImageFromBuffer( + static_cast(image.image_frame.format), + image.image_frame.image_buffer, image.image_frame.width, + image.image_frame.height); + + if (!img.ok()) { + ABSL_LOG(ERROR) << "Failed to create Image: " << img.status(); + return CppProcessError(img.status(), error_msg); + } + + auto cpp_landmarker = static_cast(landmarker); + auto cpp_result = cpp_landmarker->DetectAsync(*img, timestamp_ms); + if (!cpp_result.ok()) { + ABSL_LOG(ERROR) << "Data preparation for the landmark detection failed: " + << cpp_result; + return CppProcessError(cpp_result, error_msg); + } + return 0; +} + +void CppFaceLandmarkerCloseResult(FaceLandmarkerResult* result) { + CppCloseFaceLandmarkerResult(result); +} + +int CppFaceLandmarkerClose(void* landmarker, char** error_msg) { + auto cpp_landmarker = static_cast(landmarker); + auto result = cpp_landmarker->Close(); + if (!result.ok()) { + ABSL_LOG(ERROR) << "Failed to close FaceLandmarker: " << result; + return CppProcessError(result, error_msg); + } + delete cpp_landmarker; + return 0; +} + +} // namespace mediapipe::tasks::c::vision::face_landmarker + +extern "C" { + +void* face_landmarker_create(struct FaceLandmarkerOptions* options, + char** error_msg) { + return mediapipe::tasks::c::vision::face_landmarker::CppFaceLandmarkerCreate( + *options, error_msg); +} + +int face_landmarker_detect_image(void* landmarker, const MpImage& image, + FaceLandmarkerResult* result, + char** error_msg) { + return mediapipe::tasks::c::vision::face_landmarker::CppFaceLandmarkerDetect( + landmarker, image, result, error_msg); +} + +int face_landmarker_detect_for_video(void* landmarker, const MpImage& image, + int64_t timestamp_ms, + FaceLandmarkerResult* result, + char** error_msg) { + return mediapipe::tasks::c::vision::face_landmarker:: + CppFaceLandmarkerDetectForVideo(landmarker, image, timestamp_ms, result, + error_msg); +} + +int face_landmarker_detect_async(void* landmarker, const MpImage& image, + int64_t timestamp_ms, char** error_msg) { + return mediapipe::tasks::c::vision::face_landmarker:: + CppFaceLandmarkerDetectAsync(landmarker, image, timestamp_ms, error_msg); +} + +void face_landmarker_close_result(FaceLandmarkerResult* result) { + mediapipe::tasks::c::vision::face_landmarker::CppFaceLandmarkerCloseResult( + result); +} + +int face_landmarker_close(void* landmarker, char** error_ms) { + return mediapipe::tasks::c::vision::face_landmarker::CppFaceLandmarkerClose( + landmarker, error_ms); +} + +} // extern "C" diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker.h b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker.h new file mode 100644 index 000000000..6256e9ce7 --- /dev/null +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker.h @@ -0,0 +1,156 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_FACE_LANDMARKER_H_ +#define MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_FACE_LANDMARKER_H_ + +#include "mediapipe/tasks/c/core/base_options.h" +#include "mediapipe/tasks/c/vision/core/common.h" +#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h" + +#ifndef MP_EXPORT +#define MP_EXPORT __attribute__((visibility("default"))) +#endif // MP_EXPORT + +#ifdef __cplusplus +extern "C" { +#endif + +// The options for configuring a MediaPipe face landmarker task. +struct FaceLandmarkerOptions { + // Base options for configuring MediaPipe Tasks, such as specifying the model + // file with metadata, accelerator options, op resolver, etc. + struct BaseOptions base_options; + + // The running mode of the task. Default to the image mode. + // FaceLandmarker has three running modes: + // 1) The image mode for recognizing face landmarks on single image inputs. + // 2) The video mode for recognizing face landmarks on the decoded frames of a + // video. + // 3) The live stream mode for recognizing face landmarks on the live stream + // of input data, such as from camera. In this mode, the "result_callback" + // below must be specified to receive the detection results asynchronously. + RunningMode running_mode; + + // The maximum number of faces can be detected by the FaceLandmarker. + int num_faces = 1; + + // The minimum confidence score for the face detection to be considered + // successful. + float min_face_detection_confidence = 0.5; + + // The minimum confidence score of face presence score in the face landmark + // detection. + float min_face_presence_confidence = 0.5; + + // The minimum confidence score for the face tracking to be considered + // successful. + float min_tracking_confidence = 0.5; + + // Whether FaceLandmarker outputs face blendshapes classification. Face + // blendshapes are used for rendering the 3D face model. + bool output_face_blendshapes = false; + + // Whether FaceLandmarker outputs facial transformation_matrix. Facial + // transformation matrix is used to transform the face landmarks in canonical + // face to the detected face, so that users can apply face effects on the + // detected landmarks. + bool output_facial_transformation_matrixes = false; + + // The user-defined result callback for processing live stream data. + // The result callback should only be specified when the running mode is set + // to RunningMode::LIVE_STREAM. Arguments of the callback function include: + // the pointer to recognition result, the image that result was obtained + // on, the timestamp relevant to recognition results and pointer to error + // message in case of any failure. The validity of the passed arguments is + // true for the lifetime of the callback function. + // + // A caller is responsible for closing face landmarker result. + typedef void (*result_callback_fn)(FaceLandmarkerResult* result, + const MpImage& image, int64_t timestamp_ms, + char* error_msg); + result_callback_fn result_callback; +}; + +// Creates an FaceLandmarker from the provided `options`. +// Returns a pointer to the face landmarker on success. +// If an error occurs, returns `nullptr` and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT void* face_landmarker_create(struct FaceLandmarkerOptions* options, + char** error_msg); + +// Performs face landmark detection on the input `image`. Returns `0` on +// success. If an error occurs, returns an error code and sets the error +// parameter to an an error message (if `error_msg` is not `nullptr`). You must +// free the memory allocated for the error message. +MP_EXPORT int face_landmarker_detect_image(void* landmarker, + const MpImage& image, + FaceLandmarkerResult* result, + char** error_msg); + +// Performs face landmark detection on the provided video frame. +// Only use this method when the FaceLandmarker is created with the video +// running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide the video frame's timestamp (in milliseconds). The input timestamps +// must be monotonically increasing. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int face_landmarker_detect_for_video(void* landmarker, + const MpImage& image, + int64_t timestamp_ms, + FaceLandmarkerResult* result, + char** error_msg); + +// Sends live image data to face landmark detection, and the results will be +// available via the `result_callback` provided in the FaceLandmarkerOptions. +// Only use this method when the FaceLandmarker is created with the live +// stream running mode. +// The image can be of any size with format RGB or RGBA. It's required to +// provide a timestamp (in milliseconds) to indicate when the input image is +// sent to the face landmarker. The input timestamps must be monotonically +// increasing. +// The `result_callback` provides: +// - The recognition results as an FaceLandmarkerResult object. +// - The const reference to the corresponding input image that the face +// landmarker runs on. Note that the const reference to the image will no +// longer be valid when the callback returns. To access the image data +// outside of the callback, callers need to make a copy of the image. +// - The input timestamp in milliseconds. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int face_landmarker_detect_async(void* landmarker, + const MpImage& image, + int64_t timestamp_ms, + char** error_msg); + +// Frees the memory allocated inside a FaceLandmarkerResult result. +// Does not free the result pointer itself. +MP_EXPORT void face_landmarker_close_result(FaceLandmarkerResult* result); + +// Frees face landmarker. +// If an error occurs, returns an error code and sets the error parameter to an +// an error message (if `error_msg` is not `nullptr`). You must free the memory +// allocated for the error message. +MP_EXPORT int face_landmarker_close(void* landmarker, char** error_msg); + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_FACE_LANDMARKER_H_ diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h new file mode 100644 index 000000000..27d698d13 --- /dev/null +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h @@ -0,0 +1,59 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_RESULT_FACE_LANDMARKER_RESULT_H_ +#define MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_RESULT_FACE_LANDMARKER_RESULT_H_ + +#include + +#include "mediapipe/tasks/c/components/containers/category.h" +#include "mediapipe/tasks/c/components/containers/landmark.h" +#include "mediapipe/tasks/c/components/containers/matrix.h" + +#ifndef MP_EXPORT +#define MP_EXPORT __attribute__((visibility("default"))) +#endif // MP_EXPORT + +#ifdef __cplusplus +extern "C" { +#endif + +// The hand landmarker result from HandLandmarker, where each vector +// element represents a single hand detected in the image. +struct FaceLandmarkerResult { + // Optional face blendshapes results. + struct Categories* face_blendshapes; + + // The number of elements in the face_blendshapes array. + uint32_t face_blendshapes_count; + + // Detected face landmarks in normalized image coordinates. + struct NormalizedLandmarks* face_landmarks; + + // The number of elements in the face_landmarks array. + uint32_t face_landmarks_count; + + // Optional facial transformation matrix. + struct Matrix* facial_transformation_matrixes; + + // The number of elements in the facial_transformation_matrixes array. + uint32_t facial_transformation_matrixes_count; +}; + +#ifdef __cplusplus +} // extern C +#endif + +#endif // MEDIAPIPE_TASKS_C_VISION_FACE_LANDMARKER_RESULT_FACE_LANDMARKER_RESULT_H_ diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.cc b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.cc new file mode 100644 index 000000000..026719153 --- /dev/null +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.cc @@ -0,0 +1,116 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.h" + +#include + +#include "mediapipe/tasks/c/components/containers/category.h" +#include "mediapipe/tasks/c/components/containers/category_converter.h" +#include "mediapipe/tasks/c/components/containers/landmark.h" +#include "mediapipe/tasks/c/components/containers/landmark_converter.h" +#include "mediapipe/tasks/c/components/containers/matrix.h" +#include "mediapipe/tasks/c/components/containers/matrix_converter.h" +#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h" +#include "mediapipe/tasks/cc/components/containers/category.h" +#include "mediapipe/tasks/cc/components/containers/landmark.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h" + +namespace mediapipe::tasks::c::components::containers { + +using CppCategory = ::mediapipe::tasks::components::containers::Category; +using CppNormalizedLandmark = + ::mediapipe::tasks::components::containers::NormalizedLandmark; + +void CppConvertToFaceLandmarkerResult( + const ::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult& in, + FaceLandmarkerResult* out) { + out->face_landmarks_count = in.face_landmarks.size(); + out->face_landmarks = new NormalizedLandmarks[out->face_landmarks_count]; + for (uint32_t i = 0; i < out->face_landmarks_count; ++i) { + std::vector cpp_normalized_landmarks; + for (uint32_t j = 0; j < in.face_landmarks[i].landmarks.size(); ++j) { + const auto& cpp_landmark = in.face_landmarks[i].landmarks[j]; + cpp_normalized_landmarks.push_back(cpp_landmark); + } + CppConvertToNormalizedLandmarks(cpp_normalized_landmarks, + &out->face_landmarks[i]); + } + + if (in.face_blendshapes.has_value()) { + out->face_blendshapes_count = in.face_blendshapes->size(); + out->face_blendshapes = new Categories[out->face_blendshapes_count]; + + for (uint32_t i = 0; i < out->face_blendshapes_count; ++i) { + uint32_t categories_count = + in.face_blendshapes.value()[i].categories.size(); + out->face_blendshapes[i].categories_count = categories_count; + out->face_blendshapes[i].categories = new Category[categories_count]; + + for (uint32_t j = 0; j < categories_count; ++j) { + const auto& cpp_category = in.face_blendshapes.value()[i].categories[j]; + CppConvertToCategory(cpp_category, + &out->face_blendshapes[i].categories[j]); + } + } + } else { + out->face_blendshapes_count = 0; + out->face_blendshapes = nullptr; + } + + if (in.facial_transformation_matrixes.has_value()) { + out->facial_transformation_matrixes_count = + in.facial_transformation_matrixes.value().size(); + out->facial_transformation_matrixes = + new ::Matrix[out->facial_transformation_matrixes_count]; + for (uint32_t i = 0; i < out->facial_transformation_matrixes_count; ++i) { + CppConvertToMatrix(in.facial_transformation_matrixes.value()[i], + &out->facial_transformation_matrixes[i]); + } + } else { + out->facial_transformation_matrixes_count = 0; + out->facial_transformation_matrixes = nullptr; + } +} + +void CppCloseFaceLandmarkerResult(FaceLandmarkerResult* result) { + for (uint32_t i = 0; i < result->face_blendshapes_count; ++i) { + for (uint32_t j = 0; j < result->face_blendshapes[i].categories_count; + ++j) { + CppCloseCategory(&result->face_blendshapes[i].categories[j]); + } + delete[] result->face_blendshapes[i].categories; + } + delete[] result->face_blendshapes; + + for (uint32_t i = 0; i < result->face_landmarks_count; ++i) { + CppCloseNormalizedLandmarks(&result->face_landmarks[i]); + } + delete[] result->face_landmarks; + + for (uint32_t i = 0; i < result->facial_transformation_matrixes_count; ++i) { + CppCloseMatrix(&result->facial_transformation_matrixes[i]); + } + delete[] result->facial_transformation_matrixes; + + result->face_blendshapes_count = 0; + result->face_landmarks_count = 0; + result->facial_transformation_matrixes_count = 0; + result->face_blendshapes = nullptr; + result->face_landmarks = nullptr; + result->facial_transformation_matrixes = nullptr; +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.h b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.h new file mode 100644 index 000000000..dfdfa57af --- /dev/null +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.h @@ -0,0 +1,32 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_FACE_LANDMARKER_RESULT_CONVERTER_H_ +#define MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_FACE_LANDMARKER_RESULT_CONVERTER_H_ + +#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h" + +namespace mediapipe::tasks::c::components::containers { + +void CppConvertToFaceLandmarkerResult( + const mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult& in, + FaceLandmarkerResult* out); + +void CppCloseFaceLandmarkerResult(FaceLandmarkerResult* result); + +} // namespace mediapipe::tasks::c::components::containers + +#endif // MEDIAPIPE_TASKS_C_COMPONENTS_CONTAINERS_FACE_LANDMARKER_RESULT_CONVERTER_H_ diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter_test.cc b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter_test.cc new file mode 100644 index 000000000..59d3023c1 --- /dev/null +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter_test.cc @@ -0,0 +1,157 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter.h" + +#include + +#include "mediapipe/framework/formats/matrix.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/components/containers/landmark.h" +#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h" +#include "mediapipe/tasks/cc/components/containers/category.h" +#include "mediapipe/tasks/cc/components/containers/classification_result.h" +#include "mediapipe/tasks/cc/components/containers/landmark.h" +#include "mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_result.h" + +namespace mediapipe::tasks::c::components::containers { + +void InitFaceLandmarkerResult( + ::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult* + cpp_result) { + // Initialize face_landmarks + mediapipe::tasks::components::containers::NormalizedLandmark + cpp_normalized_landmark = {/* x= */ 0.1f, /* y= */ 0.2f, /* z= */ 0.3f}; + mediapipe::tasks::components::containers::NormalizedLandmarks + cpp_normalized_landmarks; + cpp_normalized_landmarks.landmarks.push_back(cpp_normalized_landmark); + cpp_result->face_landmarks.push_back(cpp_normalized_landmarks); + + // Initialize face_blendshapes + mediapipe::tasks::components::containers::Category cpp_category = { + /* index= */ 1, + /* score= */ 0.8f, + /* category_name= */ "blendshape_label_1", + /* display_name= */ "blendshape_display_name_1"}; + mediapipe::tasks::components::containers::Classifications + classifications_for_blendshapes; + classifications_for_blendshapes.categories.push_back(cpp_category); + + cpp_result->face_blendshapes = + std::vector{ + classifications_for_blendshapes}; + // cpp_result->face_blendshapes->push_back( + // classifications_for_blendshapes); + + // Initialize facial_transformation_matrixes + Eigen::MatrixXf cpp_matrix(2, 2); + cpp_matrix << 1.0f, 2.0f, 3.0f, 4.0f; + cpp_result->facial_transformation_matrixes = std::vector{cpp_matrix}; +} + +TEST(FaceLandmarkerResultConverterTest, ConvertsCustomResult) { + // Initialize a C++ FaceLandmarkerResult + ::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult cpp_result; + InitFaceLandmarkerResult(&cpp_result); + + FaceLandmarkerResult c_result; + CppConvertToFaceLandmarkerResult(cpp_result, &c_result); + + // Verify conversion of face_landmarks + EXPECT_EQ(c_result.face_landmarks_count, cpp_result.face_landmarks.size()); + for (uint32_t i = 0; i < c_result.face_landmarks_count; ++i) { + EXPECT_EQ(c_result.face_landmarks[i].landmarks_count, + cpp_result.face_landmarks[i].landmarks.size()); + for (uint32_t j = 0; j < c_result.face_landmarks[i].landmarks_count; ++j) { + const auto& cpp_landmark = cpp_result.face_landmarks[i].landmarks[j]; + EXPECT_FLOAT_EQ(c_result.face_landmarks[i].landmarks[j].x, + cpp_landmark.x); + EXPECT_FLOAT_EQ(c_result.face_landmarks[i].landmarks[j].y, + cpp_landmark.y); + EXPECT_FLOAT_EQ(c_result.face_landmarks[i].landmarks[j].z, + cpp_landmark.z); + } + } + + // Verify conversion of face_blendshapes + if (cpp_result.face_blendshapes.has_value()) { + EXPECT_EQ(c_result.face_blendshapes_count, + cpp_result.face_blendshapes.value().size()); + for (uint32_t i = 0; i < c_result.face_blendshapes_count; ++i) { + const auto& cpp_face_blendshapes = cpp_result.face_blendshapes.value(); + EXPECT_EQ(c_result.face_blendshapes[i].categories_count, + cpp_face_blendshapes[i].categories.size()); + for (uint32_t j = 0; j < c_result.face_blendshapes[i].categories_count; + ++j) { + const auto& cpp_category = cpp_face_blendshapes[i].categories[j]; + EXPECT_EQ(c_result.face_blendshapes[i].categories[j].index, + cpp_category.index); + EXPECT_FLOAT_EQ(c_result.face_blendshapes[i].categories[j].score, + cpp_category.score); + EXPECT_EQ(std::string( + c_result.face_blendshapes[i].categories[j].category_name), + cpp_category.category_name); + } + } + } + + // Verify conversion of facial_transformation_matrixes + if (cpp_result.facial_transformation_matrixes.has_value()) { + EXPECT_EQ(c_result.facial_transformation_matrixes_count, + cpp_result.facial_transformation_matrixes.value().size()); + for (uint32_t i = 0; i < c_result.facial_transformation_matrixes_count; + ++i) { + const auto& cpp_facial_transformation_matrixes = + cpp_result.facial_transformation_matrixes.value(); + // Assuming Matrix struct contains data array and dimensions + const auto& cpp_matrix = cpp_facial_transformation_matrixes[i]; + EXPECT_EQ(c_result.facial_transformation_matrixes[i].rows, + cpp_matrix.rows()); + EXPECT_EQ(c_result.facial_transformation_matrixes[i].cols, + cpp_matrix.cols()); + // Check each element of the matrix + for (long row = 0; row < cpp_matrix.rows(); ++row) { + for (long col = 0; col < cpp_matrix.cols(); ++col) { + size_t index = col * cpp_matrix.rows() + row; // Column-major index + EXPECT_FLOAT_EQ( + c_result.facial_transformation_matrixes[i].data[index], + cpp_matrix(row, col)); + } + } + } + } + + CppCloseFaceLandmarkerResult(&c_result); +} + +TEST(FaceLandmarkerResultConverterTest, FreesMemory) { + ::mediapipe::tasks::vision::face_landmarker::FaceLandmarkerResult cpp_result; + InitFaceLandmarkerResult(&cpp_result); + + FaceLandmarkerResult c_result; + CppConvertToFaceLandmarkerResult(cpp_result, &c_result); + + EXPECT_NE(c_result.face_blendshapes, nullptr); + EXPECT_NE(c_result.face_landmarks, nullptr); + EXPECT_NE(c_result.facial_transformation_matrixes, nullptr); + + CppCloseFaceLandmarkerResult(&c_result); + + EXPECT_EQ(c_result.face_blendshapes, nullptr); + EXPECT_EQ(c_result.face_landmarks, nullptr); + EXPECT_EQ(c_result.facial_transformation_matrixes, nullptr); +} + +} // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc new file mode 100644 index 000000000..d96d7f32a --- /dev/null +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc @@ -0,0 +1,291 @@ +/* Copyright 2023 The MediaPipe Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker.h" + +#include +#include +#include + +#include "absl/flags/flag.h" +#include "absl/strings/string_view.h" +#include "mediapipe/framework/deps/file_path.h" +#include "mediapipe/framework/formats/image.h" +#include "mediapipe/framework/port/gmock.h" +#include "mediapipe/framework/port/gtest.h" +#include "mediapipe/tasks/c/components/containers/landmark.h" +#include "mediapipe/tasks/c/vision/core/common.h" +#include "mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h" +#include "mediapipe/tasks/cc/vision/utils/image_utils.h" + +namespace { + +using ::mediapipe::file::JoinPath; +using ::mediapipe::tasks::vision::DecodeImageFromFile; +using testing::HasSubstr; + +constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; +constexpr char kModelName[] = "face_landmarker_v2_with_blendshapes.task"; +constexpr char kImageFile[] = "portrait.jpg"; +constexpr float kLandmarksPrecision = 0.03; +constexpr float kBlendshapesPrecision = 0.12; +constexpr float kFacialTransformationMatrixPrecision = 0.05; +constexpr int kIterations = 100; + +std::string GetFullPath(absl::string_view file_name) { + return JoinPath("./", kTestDataDirectory, file_name); +} + +void MatchesFaceLandmarkerResult(FaceLandmarkerResult* result, + const float blendshapes_precision, + const float landmark_precision, + const float matrix_precison) { + // Expects to have the same number of faces detected. + EXPECT_EQ(result->face_blendshapes_count, 1); + + // Actual blendshapes matches expected blendshapes. + EXPECT_EQ( + std::string{result->face_blendshapes[0].categories[0].category_name}, + "_neutral"); + EXPECT_NEAR(result->face_blendshapes[0].categories[0].score, 0.0f, + blendshapes_precision); + + // Actual landmarks match expected landmarks. + EXPECT_NEAR(result->face_landmarks[0].landmarks[0].x, 0.4977f, + landmark_precision); + EXPECT_NEAR(result->face_landmarks[0].landmarks[0].y, 0.2485f, + landmark_precision); + EXPECT_NEAR(result->face_landmarks[0].landmarks[0].z, -0.0305f, + landmark_precision); + + // Expects to have at least one facial transformation matrix. + EXPECT_GE(result->facial_transformation_matrixes_count, 1); + + // Actual matrix matches expected matrix. + // Assuming the expected matrix is 2x2 for demonstration. + const float expected_matrix[4] = {0.9991f, 0.0166f, -0.0374f, 0.0f}; + for (int i = 0; i < 4; ++i) { + printf(">> %f <<", result->facial_transformation_matrixes[0].data[i]); + EXPECT_NEAR(result->facial_transformation_matrixes[0].data[i], + expected_matrix[i], matrix_precison); + } +} + +TEST(FaceLandmarkerTest, ImageModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + FaceLandmarkerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* num_faces= */ 1, + /* min_face_detection_confidence= */ 0.5, + /* min_face_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + /* output_face_blendshapes = */ true, + /* output_facial_transformation_matrixes = */ true, + }; + + void* landmarker = face_landmarker_create(&options, /* error_msg */ nullptr); + EXPECT_NE(landmarker, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + FaceLandmarkerResult result; + face_landmarker_detect_image(landmarker, mp_image, &result, + /* error_msg */ nullptr); + MatchesFaceLandmarkerResult(&result, kBlendshapesPrecision, + kLandmarksPrecision, + kFacialTransformationMatrixPrecision); + face_landmarker_close_result(&result); + face_landmarker_close(landmarker, /* error_msg */ nullptr); +} + +TEST(FaceLandmarkerTest, VideoModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + FaceLandmarkerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::VIDEO, + /* num_faces= */ 1, + /* min_face_detection_confidence= */ 0.5, + /* min_face_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + /* output_face_blendshapes = */ true, + /* output_facial_transformation_matrixes = */ true, + }; + + void* landmarker = face_landmarker_create(&options, + /* error_msg */ nullptr); + EXPECT_NE(landmarker, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + FaceLandmarkerResult result; + face_landmarker_detect_for_video(landmarker, mp_image, i, &result, + /* error_msg */ nullptr); + + MatchesFaceLandmarkerResult(&result, kBlendshapesPrecision, + kLandmarksPrecision, + kFacialTransformationMatrixPrecision); + face_landmarker_close_result(&result); + } + face_landmarker_close(landmarker, /* error_msg */ nullptr); +} + +// A structure to support LiveStreamModeTest below. This structure holds a +// static method `Fn` for a callback function of C API. A `static` qualifier +// allows to take an address of the method to follow API style. Another static +// struct member is `last_timestamp` that is used to verify that current +// timestamp is greater than the previous one. +struct LiveStreamModeCallback { + static int64_t last_timestamp; + static void Fn(FaceLandmarkerResult* landmarker_result, const MpImage& image, + int64_t timestamp, char* error_msg) { + ASSERT_NE(landmarker_result, nullptr); + ASSERT_EQ(error_msg, nullptr); + MatchesFaceLandmarkerResult(landmarker_result, kBlendshapesPrecision, + kLandmarksPrecision, + kFacialTransformationMatrixPrecision); + EXPECT_GT(image.image_frame.width, 0); + EXPECT_GT(image.image_frame.height, 0); + EXPECT_GT(timestamp, last_timestamp); + ++last_timestamp; + } +}; +int64_t LiveStreamModeCallback::last_timestamp = -1; + +TEST(FaceLandmarkerTest, LiveStreamModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + + FaceLandmarkerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::LIVE_STREAM, + /* num_faces= */ 1, + /* min_face_detection_confidence= */ 0.5, + /* min_face_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + /* output_face_blendshapes = */ true, + /* output_facial_transformation_matrixes = */ true, + /* result_callback= */ LiveStreamModeCallback::Fn, + }; + + void* landmarker = face_landmarker_create(&options, /* error_msg */ + nullptr); + EXPECT_NE(landmarker, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + EXPECT_GE(face_landmarker_detect_async(landmarker, mp_image, i, + /* error_msg */ nullptr), + 0); + } + face_landmarker_close(landmarker, /* error_msg */ nullptr); + + // Due to the flow limiter, the total of outputs might be smaller than the + // number of iterations. + EXPECT_LE(LiveStreamModeCallback::last_timestamp, kIterations); + EXPECT_GT(LiveStreamModeCallback::last_timestamp, 0); +} + +TEST(FaceLandmarkerTest, InvalidArgumentHandling) { + // It is an error to set neither the asset buffer nor the path. + FaceLandmarkerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ nullptr}, + /* running_mode= */ RunningMode::IMAGE, + /* num_faces= */ 1, + /* min_face_detection_confidence= */ 0.5, + /* min_face_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + /* output_face_blendshapes = */ true, + /* output_facial_transformation_matrixes = */ true, + }; + + char* error_msg; + void* landmarker = face_landmarker_create(&options, &error_msg); + EXPECT_EQ(landmarker, nullptr); + + EXPECT_THAT( + error_msg, + HasSubstr("INVALID_ARGUMENT: BLENDSHAPES Tag and blendshapes model must " + "be both set. Get BLENDSHAPES is set: true, blendshapes model " + "is set: false [MediaPipeTasksStatus='601']")); + + free(error_msg); +} + +TEST(FaceLandmarkerTest, FailedRecognitionHandling) { + const std::string model_path = GetFullPath(kModelName); + FaceLandmarkerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* num_faces= */ 1, + /* min_face_detection_confidence= */ 0.5, + /* min_face_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + /* output_face_blendshapes = */ true, + /* output_facial_transformation_matrixes = */ true, + }; + + void* landmarker = face_landmarker_create(&options, /* error_msg */ + nullptr); + EXPECT_NE(landmarker, nullptr); + + const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}}; + FaceLandmarkerResult result; + char* error_msg; + face_landmarker_detect_image(landmarker, mp_image, &result, &error_msg); + EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet")); + free(error_msg); + face_landmarker_close(landmarker, /* error_msg */ nullptr); +} + +} // namespace diff --git a/mediapipe/tasks/testdata/vision/BUILD b/mediapipe/tasks/testdata/vision/BUILD index 2f5157309..616183c9b 100644 --- a/mediapipe/tasks/testdata/vision/BUILD +++ b/mediapipe/tasks/testdata/vision/BUILD @@ -48,6 +48,7 @@ mediapipe_files(srcs = [ "face_landmark.tflite", "face_landmarker.task", "face_landmarker_v2.task", + "face_landmarker_v2_with_blendshapes.task", "face_stylizer_color_ink.task", "fist.jpg", "fist.png", @@ -185,6 +186,7 @@ filegroup( "face_detection_short_range.tflite", "face_landmarker.task", "face_landmarker_v2.task", + "face_landmarker_v2_with_blendshapes.task", "face_stylizer_color_ink.task", "gesture_recognizer.task", "hair_segmentation.tflite", From 0200d32285731889f7a0f92fe83859a00f8d56ab Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 11 Dec 2023 05:20:48 -0800 Subject: [PATCH 2/6] Changed Recognition to Detection --- .../c/vision/face_landmarker/face_landmarker.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker.cc b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker.cc index 47f26a120..033642c7f 100644 --- a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker.cc +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker.cc @@ -98,7 +98,7 @@ FaceLandmarker* CppFaceLandmarkerCreate(const FaceLandmarkerOptions& options, char* error_msg = nullptr; if (!cpp_result.ok()) { - ABSL_LOG(ERROR) << "Recognition failed: " << cpp_result.status(); + ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status(); CppProcessError(cpp_result.status(), &error_msg); result_callback(nullptr, MpImage(), timestamp, error_msg); free(error_msg); @@ -141,7 +141,7 @@ int CppFaceLandmarkerDetect(void* landmarker, const MpImage& image, const absl::Status status = absl::InvalidArgumentError("GPU Buffer not supported yet."); - ABSL_LOG(ERROR) << "Recognition failed: " << status.message(); + ABSL_LOG(ERROR) << "Detection failed: " << status.message(); return CppProcessError(status, error_msg); } @@ -158,7 +158,7 @@ int CppFaceLandmarkerDetect(void* landmarker, const MpImage& image, auto cpp_landmarker = static_cast(landmarker); auto cpp_result = cpp_landmarker->Detect(*img); if (!cpp_result.ok()) { - ABSL_LOG(ERROR) << "Recognition failed: " << cpp_result.status(); + ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status(); return CppProcessError(cpp_result.status(), error_msg); } CppConvertToFaceLandmarkerResult(*cpp_result, result); @@ -173,7 +173,7 @@ int CppFaceLandmarkerDetectForVideo(void* landmarker, const MpImage& image, absl::Status status = absl::InvalidArgumentError("GPU Buffer not supported yet"); - ABSL_LOG(ERROR) << "Recognition failed: " << status.message(); + ABSL_LOG(ERROR) << "Detection failed: " << status.message(); return CppProcessError(status, error_msg); } @@ -190,7 +190,7 @@ int CppFaceLandmarkerDetectForVideo(void* landmarker, const MpImage& image, auto cpp_landmarker = static_cast(landmarker); auto cpp_result = cpp_landmarker->DetectForVideo(*img, timestamp_ms); if (!cpp_result.ok()) { - ABSL_LOG(ERROR) << "Recognition failed: " << cpp_result.status(); + ABSL_LOG(ERROR) << "Detection failed: " << cpp_result.status(); return CppProcessError(cpp_result.status(), error_msg); } CppConvertToFaceLandmarkerResult(*cpp_result, result); @@ -203,7 +203,7 @@ int CppFaceLandmarkerDetectAsync(void* landmarker, const MpImage& image, absl::Status status = absl::InvalidArgumentError("GPU Buffer not supported yet"); - ABSL_LOG(ERROR) << "Recognition failed: " << status.message(); + ABSL_LOG(ERROR) << "Detection failed: " << status.message(); return CppProcessError(status, error_msg); } From 42b251cb8d4808746465f9f3e3305a833eb890db Mon Sep 17 00:00:00 2001 From: Kinar Date: Mon, 11 Dec 2023 16:11:39 -0800 Subject: [PATCH 3/6] Updated tests in face_landmarker_result_converter_test --- .../face_landmarker_result_converter_test.cc | 75 +++++++++---------- 1 file changed, 34 insertions(+), 41 deletions(-) diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter_test.cc b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter_test.cc index 59d3023c1..69f8b6f47 100644 --- a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter_test.cc +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result_converter_test.cc @@ -52,8 +52,7 @@ void InitFaceLandmarkerResult( cpp_result->face_blendshapes = std::vector{ classifications_for_blendshapes}; - // cpp_result->face_blendshapes->push_back( - // classifications_for_blendshapes); + cpp_result->face_blendshapes->push_back(classifications_for_blendshapes); // Initialize facial_transformation_matrixes Eigen::MatrixXf cpp_matrix(2, 2); @@ -86,49 +85,43 @@ TEST(FaceLandmarkerResultConverterTest, ConvertsCustomResult) { } // Verify conversion of face_blendshapes - if (cpp_result.face_blendshapes.has_value()) { - EXPECT_EQ(c_result.face_blendshapes_count, - cpp_result.face_blendshapes.value().size()); - for (uint32_t i = 0; i < c_result.face_blendshapes_count; ++i) { - const auto& cpp_face_blendshapes = cpp_result.face_blendshapes.value(); - EXPECT_EQ(c_result.face_blendshapes[i].categories_count, - cpp_face_blendshapes[i].categories.size()); - for (uint32_t j = 0; j < c_result.face_blendshapes[i].categories_count; - ++j) { - const auto& cpp_category = cpp_face_blendshapes[i].categories[j]; - EXPECT_EQ(c_result.face_blendshapes[i].categories[j].index, - cpp_category.index); - EXPECT_FLOAT_EQ(c_result.face_blendshapes[i].categories[j].score, - cpp_category.score); - EXPECT_EQ(std::string( - c_result.face_blendshapes[i].categories[j].category_name), - cpp_category.category_name); - } + EXPECT_EQ(c_result.face_blendshapes_count, + cpp_result.face_blendshapes.value().size()); + for (uint32_t i = 0; i < c_result.face_blendshapes_count; ++i) { + const auto& cpp_face_blendshapes = cpp_result.face_blendshapes.value(); + EXPECT_EQ(c_result.face_blendshapes[i].categories_count, + cpp_face_blendshapes[i].categories.size()); + for (uint32_t j = 0; j < c_result.face_blendshapes[i].categories_count; + ++j) { + const auto& cpp_category = cpp_face_blendshapes[i].categories[j]; + EXPECT_EQ(c_result.face_blendshapes[i].categories[j].index, + cpp_category.index); + EXPECT_FLOAT_EQ(c_result.face_blendshapes[i].categories[j].score, + cpp_category.score); + EXPECT_EQ( + std::string(c_result.face_blendshapes[i].categories[j].category_name), + cpp_category.category_name); } } // Verify conversion of facial_transformation_matrixes - if (cpp_result.facial_transformation_matrixes.has_value()) { - EXPECT_EQ(c_result.facial_transformation_matrixes_count, - cpp_result.facial_transformation_matrixes.value().size()); - for (uint32_t i = 0; i < c_result.facial_transformation_matrixes_count; - ++i) { - const auto& cpp_facial_transformation_matrixes = - cpp_result.facial_transformation_matrixes.value(); - // Assuming Matrix struct contains data array and dimensions - const auto& cpp_matrix = cpp_facial_transformation_matrixes[i]; - EXPECT_EQ(c_result.facial_transformation_matrixes[i].rows, - cpp_matrix.rows()); - EXPECT_EQ(c_result.facial_transformation_matrixes[i].cols, - cpp_matrix.cols()); - // Check each element of the matrix - for (long row = 0; row < cpp_matrix.rows(); ++row) { - for (long col = 0; col < cpp_matrix.cols(); ++col) { - size_t index = col * cpp_matrix.rows() + row; // Column-major index - EXPECT_FLOAT_EQ( - c_result.facial_transformation_matrixes[i].data[index], - cpp_matrix(row, col)); - } + EXPECT_EQ(c_result.facial_transformation_matrixes_count, + cpp_result.facial_transformation_matrixes.value().size()); + for (uint32_t i = 0; i < c_result.facial_transformation_matrixes_count; ++i) { + const auto& cpp_facial_transformation_matrixes = + cpp_result.facial_transformation_matrixes.value(); + // Assuming Matrix struct contains data array and dimensions + const auto& cpp_matrix = cpp_facial_transformation_matrixes[i]; + EXPECT_EQ(c_result.facial_transformation_matrixes[i].rows, + cpp_matrix.rows()); + EXPECT_EQ(c_result.facial_transformation_matrixes[i].cols, + cpp_matrix.cols()); + // Check each element of the matrix + for (long row = 0; row < cpp_matrix.rows(); ++row) { + for (long col = 0; col < cpp_matrix.cols(); ++col) { + size_t index = col * cpp_matrix.rows() + row; // Column-major index + EXPECT_FLOAT_EQ(c_result.facial_transformation_matrixes[i].data[index], + cpp_matrix(row, col)); } } } From 6fab3a8b85c66dfa44723116dca27414c47d6ed3 Mon Sep 17 00:00:00 2001 From: Kinar Date: Wed, 13 Dec 2023 19:59:36 -0800 Subject: [PATCH 4/6] Simplified copying data from Eigen matrix to C-style matrix and addressed some issues --- .../components/containers/matrix_converter.cc | 12 +++++---- .../face_landmarker/face_landmarker_result.h | 14 +++++----- .../face_landmarker/face_landmarker_test.cc | 26 +++++++++---------- .../hand_landmarker/hand_landmarker_test.cc | 16 +++++++----- 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/mediapipe/tasks/c/components/containers/matrix_converter.cc b/mediapipe/tasks/c/components/containers/matrix_converter.cc index 6d823a424..9320178cb 100644 --- a/mediapipe/tasks/c/components/containers/matrix_converter.cc +++ b/mediapipe/tasks/c/components/containers/matrix_converter.cc @@ -27,11 +27,13 @@ void CppConvertToMatrix(const Eigen::MatrixXf& in, ::Matrix* out) { out->cols = in.cols(); out->data = new float[out->rows * out->cols]; - // Copy data from Eigen matrix to C matrix in column-major order - for (int col = 0; col < out->cols; ++col) { - for (int row = 0; row < out->rows; ++row) { - out->data[col * out->rows + row] = in(row, col); - } + // Copy data from Eigen matrix to C-style matrix. + // This operation copies the elements sequentially as they appear in the Eigen + // matrix's internal storage, regardless of whether it's stored in row-major + // or column-major order and ensures the integrity of data during the + // transfer. + for (int i = 0; i < out->rows * out->cols; ++i) { + out->data[i] = in.data()[i]; } } diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h index 27d698d13..0d86b7956 100644 --- a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_result.h @@ -33,19 +33,19 @@ extern "C" { // The hand landmarker result from HandLandmarker, where each vector // element represents a single hand detected in the image. struct FaceLandmarkerResult { - // Optional face blendshapes results. - struct Categories* face_blendshapes; - - // The number of elements in the face_blendshapes array. - uint32_t face_blendshapes_count; - // Detected face landmarks in normalized image coordinates. struct NormalizedLandmarks* face_landmarks; // The number of elements in the face_landmarks array. uint32_t face_landmarks_count; - // Optional facial transformation matrix. + // Optional face blendshapes results. + struct Categories* face_blendshapes; + + // The number of elements in the face_blendshapes array. + uint32_t face_blendshapes_count; + + // Optional facial transformation matrixes. struct Matrix* facial_transformation_matrixes; // The number of elements in the facial_transformation_matrixes array. diff --git a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc index d96d7f32a..fb2e1edff 100644 --- a/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc +++ b/mediapipe/tasks/c/vision/face_landmarker/face_landmarker_test.cc @@ -48,10 +48,10 @@ std::string GetFullPath(absl::string_view file_name) { return JoinPath("./", kTestDataDirectory, file_name); } -void MatchesFaceLandmarkerResult(FaceLandmarkerResult* result, - const float blendshapes_precision, - const float landmark_precision, - const float matrix_precison) { +void ExpectFaceLandmarkerResultCorrect(FaceLandmarkerResult* result, + const float blendshapes_precision, + const float landmark_precision, + const float matrix_precison) { // Expects to have the same number of faces detected. EXPECT_EQ(result->face_blendshapes_count, 1); @@ -115,9 +115,9 @@ TEST(FaceLandmarkerTest, ImageModeTest) { FaceLandmarkerResult result; face_landmarker_detect_image(landmarker, mp_image, &result, /* error_msg */ nullptr); - MatchesFaceLandmarkerResult(&result, kBlendshapesPrecision, - kLandmarksPrecision, - kFacialTransformationMatrixPrecision); + ExpectFaceLandmarkerResultCorrect(&result, kBlendshapesPrecision, + kLandmarksPrecision, + kFacialTransformationMatrixPrecision); face_landmarker_close_result(&result); face_landmarker_close(landmarker, /* error_msg */ nullptr); } @@ -157,9 +157,9 @@ TEST(FaceLandmarkerTest, VideoModeTest) { face_landmarker_detect_for_video(landmarker, mp_image, i, &result, /* error_msg */ nullptr); - MatchesFaceLandmarkerResult(&result, kBlendshapesPrecision, - kLandmarksPrecision, - kFacialTransformationMatrixPrecision); + ExpectFaceLandmarkerResultCorrect(&result, kBlendshapesPrecision, + kLandmarksPrecision, + kFacialTransformationMatrixPrecision); face_landmarker_close_result(&result); } face_landmarker_close(landmarker, /* error_msg */ nullptr); @@ -176,9 +176,9 @@ struct LiveStreamModeCallback { int64_t timestamp, char* error_msg) { ASSERT_NE(landmarker_result, nullptr); ASSERT_EQ(error_msg, nullptr); - MatchesFaceLandmarkerResult(landmarker_result, kBlendshapesPrecision, - kLandmarksPrecision, - kFacialTransformationMatrixPrecision); + ExpectFaceLandmarkerResultCorrect(landmarker_result, kBlendshapesPrecision, + kLandmarksPrecision, + kFacialTransformationMatrixPrecision); EXPECT_GT(image.image_frame.width, 0); EXPECT_GT(image.image_frame.height, 0); EXPECT_GT(timestamp, last_timestamp); diff --git a/mediapipe/tasks/c/vision/hand_landmarker/hand_landmarker_test.cc b/mediapipe/tasks/c/vision/hand_landmarker/hand_landmarker_test.cc index ed7b4646f..7cd8ec164 100644 --- a/mediapipe/tasks/c/vision/hand_landmarker/hand_landmarker_test.cc +++ b/mediapipe/tasks/c/vision/hand_landmarker/hand_landmarker_test.cc @@ -47,9 +47,9 @@ std::string GetFullPath(absl::string_view file_name) { return JoinPath("./", kTestDataDirectory, file_name); } -void MatchesHandLandmarkerResult(HandLandmarkerResult* result, - const float score_precision, - const float landmark_precision) { +void ExpectHandLandmarkerResultCorrect(HandLandmarkerResult* result, + const float score_precision, + const float landmark_precision) { // Expects to have the same number of hands detected. EXPECT_EQ(result->handedness_count, 1); @@ -104,7 +104,8 @@ TEST(HandLandmarkerTest, ImageModeTest) { HandLandmarkerResult result; hand_landmarker_detect_image(landmarker, mp_image, &result, /* error_msg */ nullptr); - MatchesHandLandmarkerResult(&result, kScorePrecision, kLandmarkPrecision); + ExpectHandLandmarkerResultCorrect(&result, kScorePrecision, + kLandmarkPrecision); hand_landmarker_close_result(&result); hand_landmarker_close(landmarker, /* error_msg */ nullptr); } @@ -141,7 +142,8 @@ TEST(HandLandmarkerTest, VideoModeTest) { hand_landmarker_detect_for_video(landmarker, mp_image, i, &result, /* error_msg */ nullptr); - MatchesHandLandmarkerResult(&result, kScorePrecision, kLandmarkPrecision); + ExpectHandLandmarkerResultCorrect(&result, kScorePrecision, + kLandmarkPrecision); hand_landmarker_close_result(&result); } hand_landmarker_close(landmarker, /* error_msg */ nullptr); @@ -158,8 +160,8 @@ struct LiveStreamModeCallback { int64_t timestamp, char* error_msg) { ASSERT_NE(landmarker_result, nullptr); ASSERT_EQ(error_msg, nullptr); - MatchesHandLandmarkerResult(landmarker_result, kScorePrecision, - kLandmarkPrecision); + ExpectHandLandmarkerResultCorrect(landmarker_result, kScorePrecision, + kLandmarkPrecision); EXPECT_GT(image.image_frame.width, 0); EXPECT_GT(image.image_frame.height, 0); EXPECT_GT(timestamp, last_timestamp); From f4bbfef67445f700399e91535435c799080f6377 Mon Sep 17 00:00:00 2001 From: Kinar Date: Thu, 14 Dec 2023 08:56:38 -0800 Subject: [PATCH 5/6] Use memcpy now for copying data and indicate how the data is stored --- .../c/components/containers/matrix_converter.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mediapipe/tasks/c/components/containers/matrix_converter.cc b/mediapipe/tasks/c/components/containers/matrix_converter.cc index 9320178cb..e3e3b7e30 100644 --- a/mediapipe/tasks/c/components/containers/matrix_converter.cc +++ b/mediapipe/tasks/c/components/containers/matrix_converter.cc @@ -27,13 +27,13 @@ void CppConvertToMatrix(const Eigen::MatrixXf& in, ::Matrix* out) { out->cols = in.cols(); out->data = new float[out->rows * out->cols]; - // Copy data from Eigen matrix to C-style matrix. - // This operation copies the elements sequentially as they appear in the Eigen - // matrix's internal storage, regardless of whether it's stored in row-major - // or column-major order and ensures the integrity of data during the - // transfer. - for (int i = 0; i < out->rows * out->cols; ++i) { - out->data[i] = in.data()[i]; + // Copies data from an Eigen matrix (default column-major) to a C-style + // matrix, preserving the sequence of elements as per the Eigen matrix's + // internal storage (column-major order by default). + if (!in.IsRowMajor) { + // Safe to use memcpy when the Eigen matrix is in its default column-major + // order. + memcpy(out->data, in.data(), sizeof(float) * out->rows * out->cols); } } From 5e75a169d3fc3d32325ab9514c8312bd92f96420 Mon Sep 17 00:00:00 2001 From: Kinar Date: Thu, 14 Dec 2023 09:02:05 -0800 Subject: [PATCH 6/6] Fix rows to columns for the field cols in struct Matrix --- mediapipe/tasks/c/components/containers/matrix.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mediapipe/tasks/c/components/containers/matrix.h b/mediapipe/tasks/c/components/containers/matrix.h index 71ec7474c..7d1de67f1 100644 --- a/mediapipe/tasks/c/components/containers/matrix.h +++ b/mediapipe/tasks/c/components/containers/matrix.h @@ -27,7 +27,7 @@ struct Matrix { // The number of rows in the matrix. long rows; - // The number of rows in the matrix. + // The number of columns in the matrix. long cols; // The matrix data.