Revised Gesture Recognizer API implementation and associated tests

This commit is contained in:
Kinar 2023-11-29 03:08:09 -08:00
parent 17c0c960be
commit 3c655e2334
7 changed files with 467 additions and 57 deletions

View File

@ -16,7 +16,6 @@ limitations under the License.
#include "mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.h"
#include <cstdint>
#include <cstdlib>
#include "mediapipe/tasks/c/components/containers/category_converter.h"
#include "mediapipe/tasks/c/components/containers/landmark_converter.h"

View File

@ -21,4 +21,102 @@ limitations under the License.
namespace mediapipe::tasks::c::components::containers {
TEST(GestureRecognizerResultConverterTest, ConvertsCustomResult) {
::mediapipe::tasks::vision::gesture_recognizer::GestureRecognizerResult
cpp_result;
// Initialize gestures
Classification classification_for_gestures;
classification_for_gestures.set_index(0);
classification_for_gestures.set_score(0.9f);
classification_for_gestures.set_label("gesture_label_1");
classification_for_gestures.set_display_name("gesture_display_name_1");
ClassificationList gestures_list;
*gestures_list.add_classification() = classification_for_gestures;
cpp_result.gestures.push_back(gestures_list);
// Initialize handedness
Classification classification_for_handedness;
classification_for_handedness.set_index(1);
classification_for_handedness.set_score(0.8f);
classification_for_handedness.set_label("handeness_label_1");
classification_for_handedness.set_display_name("handeness_display_name_1");
ClassificationList handedness_list;
*handedness_list.add_classification() = classification_for_handedness;
cpp_result.handedness.push_back(handedness_list);
// Initialize hand_landmarks
NormalizedLandmark normalized_landmark;
normalized_landmark.set_x(0.1f);
normalized_landmark.set_y(0.2f);
normalized_landmark.set_z(0.3f);
NormalizedLandmarkList normalized_landmark_list;
*normalized_landmark_list.add_landmark() = normalized_landmark;
cpp_result.hand_landmarks.push_back(normalized_landmark_list);
// Initialize hand_world_landmarks
Landmark landmark;
landmark.set_x(1.0f);
landmark.set_y(1.1f);
landmark.set_z(1.2f);
LandmarkList landmark_list;
*landmark_list.add_landmark() = landmark;
cpp_result.hand_world_landmarks.push_back(landmark_list);
GestureRecognizerResult c_result;
CppConvertToGestureRecognizerResult(cpp_result, &c_result);
// Verify conversion of gestures
EXPECT_NE(c_result.gestures, nullptr);
EXPECT_EQ(c_result.gestures_count, cpp_result.gestures.size());
for (uint32_t i = 0; i < c_result.gestures_count; ++i) {
EXPECT_EQ(c_result.gestures_categories_counts[i],
cpp_result.gestures[i].classification_size());
for (uint32_t j = 0; j < c_result.gestures_categories_counts[i]; ++j) {
auto gesture = cpp_result.gestures[i].classification(j);
EXPECT_EQ(std::string(c_result.gestures[i][j].category_name),
gesture.label());
EXPECT_FLOAT_EQ(c_result.gestures[i][j].score, gesture.score());
}
}
// Verify conversion of hand_landmarks
EXPECT_NE(c_result.hand_landmarks, nullptr);
EXPECT_EQ(c_result.hand_landmarks_count, cpp_result.hand_landmarks.size());
for (uint32_t i = 0; i < c_result.hand_landmarks_count; ++i) {
EXPECT_EQ(c_result.hand_landmarks[i].landmarks_count,
cpp_result.hand_landmarks[i].landmark_size());
for (uint32_t j = 0; j < c_result.hand_landmarks[i].landmarks_count; ++j) {
const auto& landmark = cpp_result.hand_landmarks[i].landmark(j);
EXPECT_FLOAT_EQ(c_result.hand_landmarks[i].landmarks[j].x, landmark.x());
EXPECT_FLOAT_EQ(c_result.hand_landmarks[i].landmarks[j].y, landmark.y());
EXPECT_FLOAT_EQ(c_result.hand_landmarks[i].landmarks[j].z, landmark.z());
}
}
// Verify conversion of hand_world_landmarks
EXPECT_NE(c_result.hand_world_landmarks, nullptr);
EXPECT_EQ(c_result.hand_world_landmarks_count,
cpp_result.hand_world_landmarks.size());
for (uint32_t i = 0; i < c_result.hand_world_landmarks_count; ++i) {
EXPECT_EQ(c_result.hand_world_landmarks[i].landmarks_count,
cpp_result.hand_world_landmarks[i].landmark_size());
for (uint32_t j = 0; j < c_result.hand_world_landmarks[i].landmarks_count;
++j) {
const auto& landmark = cpp_result.hand_world_landmarks[i].landmark(j);
EXPECT_FLOAT_EQ(c_result.hand_world_landmarks[i].landmarks[j].x,
landmark.x());
EXPECT_FLOAT_EQ(c_result.hand_world_landmarks[i].landmarks[j].y,
landmark.y());
EXPECT_FLOAT_EQ(c_result.hand_world_landmarks[i].landmarks[j].z,
landmark.z());
}
}
CppCloseGestureRecognizerResult(&c_result);
}
} // namespace mediapipe::tasks::c::components::containers

View File

@ -18,19 +18,13 @@ limitations under the License.
#include <cstring>
#include "mediapipe/tasks/c/components/containers/landmark.h"
typedef Landmark LandmarkC;
typedef NormalizedLandmark NormalizedLandmarkC;
typedef Landmarks LandmarksC;
typedef NormalizedLandmarks NormalizedLandmarksC;
#include "mediapipe/tasks/cc/components/containers/landmark.h"
namespace mediapipe::tasks::c::components::containers {
void CppConvertToLandmark(
const mediapipe::tasks::components::containers::Landmark& in,
LandmarkC* out) {
::Landmark* out) {
out->x = in.x;
out->y = in.y;
out->z = in.z;
@ -54,7 +48,7 @@ void CppConvertToLandmark(
void CppConvertToNormalizedLandmark(
const mediapipe::tasks::components::containers::NormalizedLandmark& in,
NormalizedLandmarkC* out) {
::NormalizedLandmark* out) {
out->x = in.x;
out->y = in.y;
out->z = in.z;
@ -78,9 +72,9 @@ void CppConvertToNormalizedLandmark(
void CppConvertToLandmarks(
const std::vector<mediapipe::tasks::components::containers::Landmark>& in,
LandmarksC* out) {
::Landmarks* out) {
out->landmarks_count = in.size();
out->landmarks = new LandmarkC[out->landmarks_count];
out->landmarks = new ::Landmark[out->landmarks_count];
for (uint32_t i = 0; i < out->landmarks_count; ++i) {
CppConvertToLandmark(in[i], &out->landmarks[i]);
}
@ -89,22 +83,22 @@ void CppConvertToLandmarks(
void CppConvertToNormalizedLandmarks(
const std::vector<
mediapipe::tasks::components::containers::NormalizedLandmark>& in,
NormalizedLandmarksC* out) {
::NormalizedLandmarks* out) {
out->landmarks_count = in.size();
out->landmarks = new NormalizedLandmarkC[out->landmarks_count];
out->landmarks = new ::NormalizedLandmark[out->landmarks_count];
for (uint32_t i = 0; i < out->landmarks_count; ++i) {
CppConvertToNormalizedLandmark(in[i], &out->landmarks[i]);
}
}
void CppCloseLandmark(LandmarkC* in) {
void CppCloseLandmark(::Landmark* in) {
if (in && in->name) {
free(in->name);
in->name = nullptr;
}
}
void CppCloseLandmarks(LandmarksC* in) {
void CppCloseLandmarks(::Landmarks* in) {
for (uint32_t i = 0; i < in->landmarks_count; ++i) {
CppCloseLandmark(&in->landmarks[i]);
}
@ -113,14 +107,14 @@ void CppCloseLandmarks(LandmarksC* in) {
in->landmarks_count = 0;
}
void CppCloseNormalizedLandmark(NormalizedLandmarkC* in) {
void CppCloseNormalizedLandmark(::NormalizedLandmark* in) {
if (in && in->name) {
free(in->name);
in->name = nullptr;
}
}
void CppCloseNormalizedLandmarks(NormalizedLandmarksC* in) {
void CppCloseNormalizedLandmarks(::NormalizedLandmarks* in) {
for (uint32_t i = 0; i < in->landmarks_count; ++i) {
CppCloseNormalizedLandmark(&in->landmarks[i]);
}

View File

@ -23,28 +23,28 @@ namespace mediapipe::tasks::c::components::containers {
void CppConvertToLandmark(
const mediapipe::tasks::components::containers::Landmark& in,
Landmark* out);
::Landmark* out);
void CppConvertToNormalizedLandmark(
const mediapipe::tasks::components::containers::NormalizedLandmark& in,
NormalizedLandmark* out);
::NormalizedLandmark* out);
void CppConvertToLandmarks(
const std::vector<mediapipe::tasks::components::containers::Landmark>& in,
Landmarks* out);
::Landmarks* out);
void CppConvertToNormalizedLandmarks(
const std::vector<
mediapipe::tasks::components::containers::NormalizedLandmark>& in,
NormalizedLandmarks* out);
::NormalizedLandmarks* out);
void CppCloseLandmark(struct Landmark* in);
void CppCloseLandmark(struct ::Landmark* in);
void CppCloseLandmarks(struct Landmarks* in);
void CppCloseLandmarks(struct ::Landmarks* in);
void CppCloseNormalizedLandmark(struct NormalizedLandmark* in);
void CppCloseNormalizedLandmark(struct ::NormalizedLandmark* in);
void CppCloseNormalizedLandmarks(struct NormalizedLandmarks* in);
void CppCloseNormalizedLandmarks(struct ::NormalizedLandmarks* in);
} // namespace mediapipe::tasks::c::components::containers

View File

@ -25,4 +25,125 @@ limitations under the License.
namespace mediapipe::tasks::c::components::containers {
TEST(LandmarkConverterTest, ConvertsCustomLandmark) {
mediapipe::tasks::components::containers::Landmark cpp_landmark = {0.1f, 0.2f,
0.3f};
::Landmark c_landmark;
CppConvertToLandmark(cpp_landmark, &c_landmark);
EXPECT_FLOAT_EQ(c_landmark.x, cpp_landmark.x);
EXPECT_FLOAT_EQ(c_landmark.y, cpp_landmark.y);
EXPECT_FLOAT_EQ(c_landmark.z, cpp_landmark.z);
CppCloseLandmark(&c_landmark);
}
TEST(LandmarksConverterTest, ConvertsCustomLandmarks) {
std::vector<mediapipe::tasks::components::containers::Landmark>
cpp_landmarks = {
{0.1f, 0.2f, 0.3f}, // First Landmark
{0.4f, 0.5f, 0.6f} // Second Landmark
};
::Landmarks c_landmarks;
CppConvertToLandmarks(cpp_landmarks, &c_landmarks);
EXPECT_EQ(c_landmarks.landmarks_count, cpp_landmarks.size());
for (size_t i = 0; i < c_landmarks.landmarks_count; ++i) {
EXPECT_FLOAT_EQ(c_landmarks.landmarks[i].x, cpp_landmarks[i].x);
EXPECT_FLOAT_EQ(c_landmarks.landmarks[i].y, cpp_landmarks[i].y);
EXPECT_FLOAT_EQ(c_landmarks.landmarks[i].z, cpp_landmarks[i].z);
}
CppCloseLandmarks(&c_landmarks);
}
TEST(NormalizedLandmarkConverterTest, ConvertsCustomNormalizedLandmark) {
mediapipe::tasks::components::containers::NormalizedLandmark
cpp_normalized_landmark = {0.7f, 0.8f, 0.9f};
::NormalizedLandmark c_normalized_landmark;
CppConvertToNormalizedLandmark(cpp_normalized_landmark,
&c_normalized_landmark);
EXPECT_FLOAT_EQ(c_normalized_landmark.x, cpp_normalized_landmark.x);
EXPECT_FLOAT_EQ(c_normalized_landmark.y, cpp_normalized_landmark.y);
EXPECT_FLOAT_EQ(c_normalized_landmark.z, cpp_normalized_landmark.z);
CppCloseNormalizedLandmark(&c_normalized_landmark);
}
TEST(NormalizedLandmarksConverterTest, ConvertsCustomNormalizedLandmarks) {
std::vector<mediapipe::tasks::components::containers::NormalizedLandmark>
cpp_normalized_landmarks = {
{0.1f, 0.2f, 0.3f}, // First NormalizedLandmark
{0.4f, 0.5f, 0.6f} // Second NormalizedLandmark
};
::NormalizedLandmarks c_normalized_landmarks;
CppConvertToNormalizedLandmarks(cpp_normalized_landmarks,
&c_normalized_landmarks);
EXPECT_EQ(c_normalized_landmarks.landmarks_count,
cpp_normalized_landmarks.size());
for (size_t i = 0; i < c_normalized_landmarks.landmarks_count; ++i) {
EXPECT_FLOAT_EQ(c_normalized_landmarks.landmarks[i].x,
cpp_normalized_landmarks[i].x);
EXPECT_FLOAT_EQ(c_normalized_landmarks.landmarks[i].y,
cpp_normalized_landmarks[i].y);
EXPECT_FLOAT_EQ(c_normalized_landmarks.landmarks[i].z,
cpp_normalized_landmarks[i].z);
}
CppCloseNormalizedLandmarks(&c_normalized_landmarks);
}
TEST(LandmarkConverterTest, FreesMemory) {
mediapipe::tasks::components::containers::Landmark cpp_landmark = {
0.1f, 0.2f, 0.3f, 0.0f, 0.0f, "foo"};
::Landmark c_landmark;
CppConvertToLandmark(cpp_landmark, &c_landmark);
EXPECT_NE(c_landmark.name, nullptr);
CppCloseLandmark(&c_landmark);
EXPECT_EQ(c_landmark.name, nullptr);
}
TEST(NormalizedLandmarkConverterTest, FreesMemory) {
mediapipe::tasks::components::containers::NormalizedLandmark cpp_landmark = {
0.1f, 0.2f, 0.3f, 0.0f, 0.0f, "foo"};
::NormalizedLandmark c_landmark;
CppConvertToNormalizedLandmark(cpp_landmark, &c_landmark);
EXPECT_NE(c_landmark.name, nullptr);
CppCloseNormalizedLandmark(&c_landmark);
EXPECT_EQ(c_landmark.name, nullptr);
}
TEST(LandmarksConverterTest, FreesMemory) {
std::vector<mediapipe::tasks::components::containers::Landmark>
cpp_landmarks = {{0.1f, 0.2f, 0.3f}, {0.4f, 0.5f, 0.6f}};
::Landmarks c_landmarks;
CppConvertToLandmarks(cpp_landmarks, &c_landmarks);
EXPECT_NE(c_landmarks.landmarks, nullptr);
CppCloseLandmarks(&c_landmarks);
EXPECT_EQ(c_landmarks.landmarks, nullptr);
}
TEST(NormalizedLandmarksConverterTest, FreesMemory) {
std::vector<mediapipe::tasks::components::containers::NormalizedLandmark>
cpp_normalized_landmarks = {{0.1f, 0.2f, 0.3f}, {0.4f, 0.5f, 0.6f}};
::NormalizedLandmarks c_normalized_landmarks;
CppConvertToNormalizedLandmarks(cpp_normalized_landmarks,
&c_normalized_landmarks);
EXPECT_NE(c_normalized_landmarks.landmarks, nullptr);
CppCloseNormalizedLandmarks(&c_normalized_landmarks);
EXPECT_EQ(c_normalized_landmarks.landmarks, nullptr);
}
} // namespace mediapipe::tasks::c::components::containers

View File

@ -82,12 +82,12 @@ struct GestureRecognizerOptions {
//
// A caller is responsible for closing gesture recognizer result.
typedef void (*result_callback_fn)(GestureRecognizerResult* result,
const MpImage image, int64_t timestamp_ms,
const MpImage& image, int64_t timestamp_ms,
char* error_msg);
result_callback_fn result_callback;
};
// Creates an GestureRecognizer from provided `options`.
// Creates an GestureRecognizer from the provided `options`.
// Returns a pointer to the gesture recognizer on success.
// If an error occurs, returns `nullptr` and sets the error parameter to an
// an error message (if `error_msg` is not `nullptr`). You must free the memory

View File

@ -36,16 +36,46 @@ using testing::HasSubstr;
constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/";
constexpr char kModelName[] = "gesture_recognizer.task";
constexpr float kPrecision = 1e-4;
constexpr float kLandmarkPrecision = 1e-3;
constexpr char kImageFile[] = "fist.jpg";
constexpr float kScorePrecision = 1e-2;
constexpr float kLandmarkPrecision = 1e-1;
constexpr int kIterations = 100;
std::string GetFullPath(absl::string_view file_name) {
return JoinPath("./", kTestDataDirectory, file_name);
}
void MatchesGestureRecognizerResult(GestureRecognizerResult* result,
const float score_precision,
const float landmark_precision) {
// Expects to have the same number of hands detected.
EXPECT_EQ(result->gestures_count, 1);
EXPECT_EQ(result->handedness_count, 1);
// Actual gesture with top score matches expected gesture.
EXPECT_EQ(std::string{result->gestures[0][0].category_name}, "Closed_Fist");
EXPECT_NEAR(result->gestures[0][0].score, 0.91f, score_precision);
// Actual handedness matches expected handedness.
EXPECT_EQ(std::string{result->handedness[0][0].category_name}, "Right");
EXPECT_NEAR(result->handedness[0][0].score, 0.9893f, score_precision);
// Actual landmarks match expected landmarks.
EXPECT_NEAR(result->hand_landmarks[0].landmarks[0].x, 0.477f,
landmark_precision);
EXPECT_NEAR(result->hand_landmarks[0].landmarks[0].y, 0.661f,
landmark_precision);
EXPECT_NEAR(result->hand_landmarks[0].landmarks[0].z, 0.0f,
landmark_precision);
EXPECT_NEAR(result->hand_world_landmarks[0].landmarks[0].x, -0.009f,
landmark_precision);
EXPECT_NEAR(result->hand_world_landmarks[0].landmarks[0].y, 0.082f,
landmark_precision);
EXPECT_NEAR(result->hand_world_landmarks[0].landmarks[0].z, 0.006f,
landmark_precision);
}
TEST(GestureRecognizerTest, ImageModeTest) {
const auto image = DecodeImageFromFile(GetFullPath("fist.jpg"));
const auto image = DecodeImageFromFile(GetFullPath(kImageFile));
ASSERT_TRUE(image.ok());
const std::string model_path = GetFullPath(kModelName);
@ -88,36 +118,204 @@ TEST(GestureRecognizerTest, ImageModeTest) {
GestureRecognizerResult result;
gesture_recognizer_recognize_image(recognizer, &mp_image, &result,
/* error_msg */ nullptr);
// Expects to have the same number of hands detected.
EXPECT_EQ(result.gestures_count, 1);
EXPECT_EQ(result.handedness_count, 1);
// Actual gesture with top score matches expected gesture.
EXPECT_EQ(std::string{result.gestures[0][0].category_name}, "Closed_Fist");
EXPECT_NEAR(result.gestures[0][0].score, 0.9000f, kPrecision);
// Actual handedness matches expected handedness.
EXPECT_EQ(std::string{result.handedness[0][0].category_name}, "Right");
EXPECT_NEAR(result.handedness[0][0].score, 0.9893f, kPrecision);
// Actual landmarks match expected landmarks.
EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].x, 0.477f,
kLandmarkPrecision);
EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].y, 0.661f,
kLandmarkPrecision);
EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].z, 0.0f,
kLandmarkPrecision);
EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].x, -0.009f,
kLandmarkPrecision);
EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].y, 0.082f,
kLandmarkPrecision);
EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].z, 0.006f,
kLandmarkPrecision);
MatchesGestureRecognizerResult(&result, kScorePrecision, kLandmarkPrecision);
gesture_recognizer_close_result(&result);
gesture_recognizer_close(recognizer, /* error_msg */ nullptr);
}
// TODO other tests
TEST(GestureRecognizerTest, VideoModeTest) {
const auto image = DecodeImageFromFile(GetFullPath(kImageFile));
ASSERT_TRUE(image.ok());
const std::string model_path = GetFullPath(kModelName);
GestureRecognizerOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* running_mode= */ RunningMode::VIDEO,
/* num_hands= */ 1,
/* min_hand_detection_confidence= */ 0.5,
/* min_hand_presence_confidence= */ 0.5,
/* min_tracking_confidence= */ 0.5,
{/* display_names_locale= */ nullptr,
/* max_results= */ -1,
/* score_threshold= */ 0.0,
/* category_allowlist= */ nullptr,
/* category_allowlist_count= */ 0,
/* category_denylist= */ nullptr,
/* category_denylist_count= */ 0},
{/* display_names_locale= */ nullptr,
/* max_results= */ -1,
/* score_threshold= */ 0.0,
/* category_allowlist= */ nullptr,
/* category_allowlist_count= */ 0,
/* category_denylist= */ nullptr,
/* category_denylist_count= */ 0}};
void* recognizer =
gesture_recognizer_create(&options, /* error_msg */ nullptr);
EXPECT_NE(recognizer, nullptr);
const auto& image_frame = image->GetImageFrameSharedPtr();
const MpImage mp_image = {
.type = MpImage::IMAGE_FRAME,
.image_frame = {.format = static_cast<ImageFormat>(image_frame->Format()),
.image_buffer = image_frame->PixelData(),
.width = image_frame->Width(),
.height = image_frame->Height()}};
for (int i = 0; i < kIterations; ++i) {
GestureRecognizerResult result;
gesture_recognizer_recognize_for_video(recognizer, &mp_image, i, &result,
/* error_msg */ nullptr);
MatchesGestureRecognizerResult(&result, kScorePrecision,
kLandmarkPrecision);
gesture_recognizer_close_result(&result);
}
gesture_recognizer_close(recognizer, /* error_msg */ nullptr);
}
// A structure to support LiveStreamModeTest below. This structure holds a
// static method `Fn` for a callback function of C API. A `static` qualifier
// allows to take an address of the method to follow API style. Another static
// struct member is `last_timestamp` that is used to verify that current
// timestamp is greater than the previous one.
struct LiveStreamModeCallback {
static int64_t last_timestamp;
static void Fn(GestureRecognizerResult* recognizer_result,
const MpImage& image, int64_t timestamp, char* error_msg) {
ASSERT_NE(recognizer_result, nullptr);
ASSERT_EQ(error_msg, nullptr);
MatchesGestureRecognizerResult(recognizer_result, kScorePrecision,
kLandmarkPrecision);
EXPECT_GT(image.image_frame.width, 0);
EXPECT_GT(image.image_frame.height, 0);
EXPECT_GT(timestamp, last_timestamp);
last_timestamp++;
}
};
int64_t LiveStreamModeCallback::last_timestamp = -1;
TEST(GestureRecognizerTest, LiveStreamModeTest) {
const auto image = DecodeImageFromFile(GetFullPath(kImageFile));
ASSERT_TRUE(image.ok());
const std::string model_path = GetFullPath(kModelName);
GestureRecognizerOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* running_mode= */ RunningMode::LIVE_STREAM,
/* num_hands= */ 1,
/* min_hand_detection_confidence= */ 0.5,
/* min_hand_presence_confidence= */ 0.5,
/* min_tracking_confidence= */ 0.5,
{/* display_names_locale= */ nullptr,
/* max_results= */ -1,
/* score_threshold= */ 0.0,
/* category_allowlist= */ nullptr,
/* category_allowlist_count= */ 0,
/* category_denylist= */ nullptr,
/* category_denylist_count= */ 0},
{/* display_names_locale= */ nullptr,
/* max_results= */ -1,
/* score_threshold= */ 0.0,
/* category_allowlist= */ nullptr,
/* category_allowlist_count= */ 0,
/* category_denylist= */ nullptr,
/* category_denylist_count= */ 0},
/* result_callback= */ LiveStreamModeCallback::Fn,
};
void* recognizer =
gesture_recognizer_create(&options, /* error_msg */ nullptr);
EXPECT_NE(recognizer, nullptr);
const auto& image_frame = image->GetImageFrameSharedPtr();
const MpImage mp_image = {
.type = MpImage::IMAGE_FRAME,
.image_frame = {.format = static_cast<ImageFormat>(image_frame->Format()),
.image_buffer = image_frame->PixelData(),
.width = image_frame->Width(),
.height = image_frame->Height()}};
for (int i = 0; i < kIterations; ++i) {
EXPECT_GE(gesture_recognizer_recognize_async(recognizer, &mp_image, i,
/* error_msg */ nullptr),
0);
}
gesture_recognizer_close(recognizer, /* error_msg */ nullptr);
// Due to the flow limiter, the total of outputs might be smaller than the
// number of iterations.
EXPECT_LE(LiveStreamModeCallback::last_timestamp, kIterations);
EXPECT_GT(LiveStreamModeCallback::last_timestamp, 0);
}
TEST(GestureRecognizerTest, InvalidArgumentHandling) {
// It is an error to set neither the asset buffer nor the path.
GestureRecognizerOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ nullptr},
/* running_mode= */ RunningMode::IMAGE,
/* num_hands= */ 1,
/* min_hand_detection_confidence= */ 0.5,
/* min_hand_presence_confidence= */ 0.5,
/* min_tracking_confidence= */ 0.5,
{},
{}};
char* error_msg;
void* recognizer = gesture_recognizer_create(&options, &error_msg);
EXPECT_EQ(recognizer, nullptr);
EXPECT_THAT(error_msg, HasSubstr("ExternalFile must specify"));
free(error_msg);
}
TEST(GestureRecognizerTest, FailedRecognitionHandling) {
const std::string model_path = GetFullPath(kModelName);
GestureRecognizerOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* running_mode= */ RunningMode::IMAGE,
/* num_hands= */ 1,
/* min_hand_detection_confidence= */ 0.5,
/* min_hand_presence_confidence= */ 0.5,
/* min_tracking_confidence= */ 0.5,
{/* display_names_locale= */ nullptr,
/* max_results= */ -1,
/* score_threshold= */ 0.0,
/* category_allowlist= */ nullptr,
/* category_allowlist_count= */ 0,
/* category_denylist= */ nullptr,
/* category_denylist_count= */ 0},
{/* display_names_locale= */ nullptr,
/* max_results= */ -1,
/* score_threshold= */ 0.0,
/* category_allowlist= */ nullptr,
/* category_allowlist_count= */ 0,
/* category_denylist= */ nullptr,
/* category_denylist_count= */ 0},
};
void* recognizer = gesture_recognizer_create(&options, /* error_msg */
nullptr);
EXPECT_NE(recognizer, nullptr);
const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}};
GestureRecognizerResult result;
char* error_msg;
gesture_recognizer_recognize_image(recognizer, &mp_image, &result,
&error_msg);
EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet"));
free(error_msg);
gesture_recognizer_close(recognizer, /* error_msg */ nullptr);
}
} // namespace