diff --git a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc index 3b2da8eab..6ac8b1370 100644 --- a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc +++ b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.cc @@ -16,7 +16,6 @@ limitations under the License. #include "mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter.h" #include -#include #include "mediapipe/tasks/c/components/containers/category_converter.h" #include "mediapipe/tasks/c/components/containers/landmark_converter.h" diff --git a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc index 551498e12..6c1f2f798 100644 --- a/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc +++ b/mediapipe/tasks/c/components/containers/gesture_recognizer_result_converter_test.cc @@ -21,4 +21,102 @@ limitations under the License. namespace mediapipe::tasks::c::components::containers { +TEST(GestureRecognizerResultConverterTest, ConvertsCustomResult) { + ::mediapipe::tasks::vision::gesture_recognizer::GestureRecognizerResult + cpp_result; + + // Initialize gestures + Classification classification_for_gestures; + classification_for_gestures.set_index(0); + classification_for_gestures.set_score(0.9f); + classification_for_gestures.set_label("gesture_label_1"); + classification_for_gestures.set_display_name("gesture_display_name_1"); + ClassificationList gestures_list; + *gestures_list.add_classification() = classification_for_gestures; + cpp_result.gestures.push_back(gestures_list); + + // Initialize handedness + Classification classification_for_handedness; + classification_for_handedness.set_index(1); + classification_for_handedness.set_score(0.8f); + classification_for_handedness.set_label("handeness_label_1"); + classification_for_handedness.set_display_name("handeness_display_name_1"); + ClassificationList handedness_list; + *handedness_list.add_classification() = classification_for_handedness; + cpp_result.handedness.push_back(handedness_list); + + // Initialize hand_landmarks + NormalizedLandmark normalized_landmark; + normalized_landmark.set_x(0.1f); + normalized_landmark.set_y(0.2f); + normalized_landmark.set_z(0.3f); + NormalizedLandmarkList normalized_landmark_list; + *normalized_landmark_list.add_landmark() = normalized_landmark; + cpp_result.hand_landmarks.push_back(normalized_landmark_list); + + // Initialize hand_world_landmarks + Landmark landmark; + landmark.set_x(1.0f); + landmark.set_y(1.1f); + landmark.set_z(1.2f); + + LandmarkList landmark_list; + *landmark_list.add_landmark() = landmark; + cpp_result.hand_world_landmarks.push_back(landmark_list); + + GestureRecognizerResult c_result; + CppConvertToGestureRecognizerResult(cpp_result, &c_result); + + // Verify conversion of gestures + EXPECT_NE(c_result.gestures, nullptr); + EXPECT_EQ(c_result.gestures_count, cpp_result.gestures.size()); + + for (uint32_t i = 0; i < c_result.gestures_count; ++i) { + EXPECT_EQ(c_result.gestures_categories_counts[i], + cpp_result.gestures[i].classification_size()); + for (uint32_t j = 0; j < c_result.gestures_categories_counts[i]; ++j) { + auto gesture = cpp_result.gestures[i].classification(j); + EXPECT_EQ(std::string(c_result.gestures[i][j].category_name), + gesture.label()); + EXPECT_FLOAT_EQ(c_result.gestures[i][j].score, gesture.score()); + } + } + + // Verify conversion of hand_landmarks + EXPECT_NE(c_result.hand_landmarks, nullptr); + EXPECT_EQ(c_result.hand_landmarks_count, cpp_result.hand_landmarks.size()); + + for (uint32_t i = 0; i < c_result.hand_landmarks_count; ++i) { + EXPECT_EQ(c_result.hand_landmarks[i].landmarks_count, + cpp_result.hand_landmarks[i].landmark_size()); + for (uint32_t j = 0; j < c_result.hand_landmarks[i].landmarks_count; ++j) { + const auto& landmark = cpp_result.hand_landmarks[i].landmark(j); + EXPECT_FLOAT_EQ(c_result.hand_landmarks[i].landmarks[j].x, landmark.x()); + EXPECT_FLOAT_EQ(c_result.hand_landmarks[i].landmarks[j].y, landmark.y()); + EXPECT_FLOAT_EQ(c_result.hand_landmarks[i].landmarks[j].z, landmark.z()); + } + } + + // Verify conversion of hand_world_landmarks + EXPECT_NE(c_result.hand_world_landmarks, nullptr); + EXPECT_EQ(c_result.hand_world_landmarks_count, + cpp_result.hand_world_landmarks.size()); + for (uint32_t i = 0; i < c_result.hand_world_landmarks_count; ++i) { + EXPECT_EQ(c_result.hand_world_landmarks[i].landmarks_count, + cpp_result.hand_world_landmarks[i].landmark_size()); + for (uint32_t j = 0; j < c_result.hand_world_landmarks[i].landmarks_count; + ++j) { + const auto& landmark = cpp_result.hand_world_landmarks[i].landmark(j); + EXPECT_FLOAT_EQ(c_result.hand_world_landmarks[i].landmarks[j].x, + landmark.x()); + EXPECT_FLOAT_EQ(c_result.hand_world_landmarks[i].landmarks[j].y, + landmark.y()); + EXPECT_FLOAT_EQ(c_result.hand_world_landmarks[i].landmarks[j].z, + landmark.z()); + } + } + + CppCloseGestureRecognizerResult(&c_result); +} + } // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/landmark_converter.cc b/mediapipe/tasks/c/components/containers/landmark_converter.cc index 5f4ab5ef2..f5643ffaa 100644 --- a/mediapipe/tasks/c/components/containers/landmark_converter.cc +++ b/mediapipe/tasks/c/components/containers/landmark_converter.cc @@ -18,19 +18,13 @@ limitations under the License. #include #include "mediapipe/tasks/c/components/containers/landmark.h" - -typedef Landmark LandmarkC; -typedef NormalizedLandmark NormalizedLandmarkC; -typedef Landmarks LandmarksC; -typedef NormalizedLandmarks NormalizedLandmarksC; - #include "mediapipe/tasks/cc/components/containers/landmark.h" namespace mediapipe::tasks::c::components::containers { void CppConvertToLandmark( const mediapipe::tasks::components::containers::Landmark& in, - LandmarkC* out) { + ::Landmark* out) { out->x = in.x; out->y = in.y; out->z = in.z; @@ -54,7 +48,7 @@ void CppConvertToLandmark( void CppConvertToNormalizedLandmark( const mediapipe::tasks::components::containers::NormalizedLandmark& in, - NormalizedLandmarkC* out) { + ::NormalizedLandmark* out) { out->x = in.x; out->y = in.y; out->z = in.z; @@ -78,9 +72,9 @@ void CppConvertToNormalizedLandmark( void CppConvertToLandmarks( const std::vector& in, - LandmarksC* out) { + ::Landmarks* out) { out->landmarks_count = in.size(); - out->landmarks = new LandmarkC[out->landmarks_count]; + out->landmarks = new ::Landmark[out->landmarks_count]; for (uint32_t i = 0; i < out->landmarks_count; ++i) { CppConvertToLandmark(in[i], &out->landmarks[i]); } @@ -89,22 +83,22 @@ void CppConvertToLandmarks( void CppConvertToNormalizedLandmarks( const std::vector< mediapipe::tasks::components::containers::NormalizedLandmark>& in, - NormalizedLandmarksC* out) { + ::NormalizedLandmarks* out) { out->landmarks_count = in.size(); - out->landmarks = new NormalizedLandmarkC[out->landmarks_count]; + out->landmarks = new ::NormalizedLandmark[out->landmarks_count]; for (uint32_t i = 0; i < out->landmarks_count; ++i) { CppConvertToNormalizedLandmark(in[i], &out->landmarks[i]); } } -void CppCloseLandmark(LandmarkC* in) { +void CppCloseLandmark(::Landmark* in) { if (in && in->name) { free(in->name); in->name = nullptr; } } -void CppCloseLandmarks(LandmarksC* in) { +void CppCloseLandmarks(::Landmarks* in) { for (uint32_t i = 0; i < in->landmarks_count; ++i) { CppCloseLandmark(&in->landmarks[i]); } @@ -113,14 +107,14 @@ void CppCloseLandmarks(LandmarksC* in) { in->landmarks_count = 0; } -void CppCloseNormalizedLandmark(NormalizedLandmarkC* in) { +void CppCloseNormalizedLandmark(::NormalizedLandmark* in) { if (in && in->name) { free(in->name); in->name = nullptr; } } -void CppCloseNormalizedLandmarks(NormalizedLandmarksC* in) { +void CppCloseNormalizedLandmarks(::NormalizedLandmarks* in) { for (uint32_t i = 0; i < in->landmarks_count; ++i) { CppCloseNormalizedLandmark(&in->landmarks[i]); } diff --git a/mediapipe/tasks/c/components/containers/landmark_converter.h b/mediapipe/tasks/c/components/containers/landmark_converter.h index f59158112..1b3626386 100644 --- a/mediapipe/tasks/c/components/containers/landmark_converter.h +++ b/mediapipe/tasks/c/components/containers/landmark_converter.h @@ -23,28 +23,28 @@ namespace mediapipe::tasks::c::components::containers { void CppConvertToLandmark( const mediapipe::tasks::components::containers::Landmark& in, - Landmark* out); + ::Landmark* out); void CppConvertToNormalizedLandmark( const mediapipe::tasks::components::containers::NormalizedLandmark& in, - NormalizedLandmark* out); + ::NormalizedLandmark* out); void CppConvertToLandmarks( const std::vector& in, - Landmarks* out); + ::Landmarks* out); void CppConvertToNormalizedLandmarks( const std::vector< mediapipe::tasks::components::containers::NormalizedLandmark>& in, - NormalizedLandmarks* out); + ::NormalizedLandmarks* out); -void CppCloseLandmark(struct Landmark* in); +void CppCloseLandmark(struct ::Landmark* in); -void CppCloseLandmarks(struct Landmarks* in); +void CppCloseLandmarks(struct ::Landmarks* in); -void CppCloseNormalizedLandmark(struct NormalizedLandmark* in); +void CppCloseNormalizedLandmark(struct ::NormalizedLandmark* in); -void CppCloseNormalizedLandmarks(struct NormalizedLandmarks* in); +void CppCloseNormalizedLandmarks(struct ::NormalizedLandmarks* in); } // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/components/containers/landmark_converter_test.cc b/mediapipe/tasks/c/components/containers/landmark_converter_test.cc index cf2b5a0e9..d6cbe3e85 100644 --- a/mediapipe/tasks/c/components/containers/landmark_converter_test.cc +++ b/mediapipe/tasks/c/components/containers/landmark_converter_test.cc @@ -25,4 +25,125 @@ limitations under the License. namespace mediapipe::tasks::c::components::containers { +TEST(LandmarkConverterTest, ConvertsCustomLandmark) { + mediapipe::tasks::components::containers::Landmark cpp_landmark = {0.1f, 0.2f, + 0.3f}; + + ::Landmark c_landmark; + CppConvertToLandmark(cpp_landmark, &c_landmark); + EXPECT_FLOAT_EQ(c_landmark.x, cpp_landmark.x); + EXPECT_FLOAT_EQ(c_landmark.y, cpp_landmark.y); + EXPECT_FLOAT_EQ(c_landmark.z, cpp_landmark.z); + CppCloseLandmark(&c_landmark); +} + +TEST(LandmarksConverterTest, ConvertsCustomLandmarks) { + std::vector + cpp_landmarks = { + {0.1f, 0.2f, 0.3f}, // First Landmark + {0.4f, 0.5f, 0.6f} // Second Landmark + }; + + ::Landmarks c_landmarks; + CppConvertToLandmarks(cpp_landmarks, &c_landmarks); + + EXPECT_EQ(c_landmarks.landmarks_count, cpp_landmarks.size()); + for (size_t i = 0; i < c_landmarks.landmarks_count; ++i) { + EXPECT_FLOAT_EQ(c_landmarks.landmarks[i].x, cpp_landmarks[i].x); + EXPECT_FLOAT_EQ(c_landmarks.landmarks[i].y, cpp_landmarks[i].y); + EXPECT_FLOAT_EQ(c_landmarks.landmarks[i].z, cpp_landmarks[i].z); + } + + CppCloseLandmarks(&c_landmarks); +} + +TEST(NormalizedLandmarkConverterTest, ConvertsCustomNormalizedLandmark) { + mediapipe::tasks::components::containers::NormalizedLandmark + cpp_normalized_landmark = {0.7f, 0.8f, 0.9f}; + + ::NormalizedLandmark c_normalized_landmark; + CppConvertToNormalizedLandmark(cpp_normalized_landmark, + &c_normalized_landmark); + + EXPECT_FLOAT_EQ(c_normalized_landmark.x, cpp_normalized_landmark.x); + EXPECT_FLOAT_EQ(c_normalized_landmark.y, cpp_normalized_landmark.y); + EXPECT_FLOAT_EQ(c_normalized_landmark.z, cpp_normalized_landmark.z); + + CppCloseNormalizedLandmark(&c_normalized_landmark); +} + +TEST(NormalizedLandmarksConverterTest, ConvertsCustomNormalizedLandmarks) { + std::vector + cpp_normalized_landmarks = { + {0.1f, 0.2f, 0.3f}, // First NormalizedLandmark + {0.4f, 0.5f, 0.6f} // Second NormalizedLandmark + }; + + ::NormalizedLandmarks c_normalized_landmarks; + CppConvertToNormalizedLandmarks(cpp_normalized_landmarks, + &c_normalized_landmarks); + + EXPECT_EQ(c_normalized_landmarks.landmarks_count, + cpp_normalized_landmarks.size()); + for (size_t i = 0; i < c_normalized_landmarks.landmarks_count; ++i) { + EXPECT_FLOAT_EQ(c_normalized_landmarks.landmarks[i].x, + cpp_normalized_landmarks[i].x); + EXPECT_FLOAT_EQ(c_normalized_landmarks.landmarks[i].y, + cpp_normalized_landmarks[i].y); + EXPECT_FLOAT_EQ(c_normalized_landmarks.landmarks[i].z, + cpp_normalized_landmarks[i].z); + } + + CppCloseNormalizedLandmarks(&c_normalized_landmarks); +} + +TEST(LandmarkConverterTest, FreesMemory) { + mediapipe::tasks::components::containers::Landmark cpp_landmark = { + 0.1f, 0.2f, 0.3f, 0.0f, 0.0f, "foo"}; + + ::Landmark c_landmark; + CppConvertToLandmark(cpp_landmark, &c_landmark); + EXPECT_NE(c_landmark.name, nullptr); + + CppCloseLandmark(&c_landmark); + EXPECT_EQ(c_landmark.name, nullptr); +} + +TEST(NormalizedLandmarkConverterTest, FreesMemory) { + mediapipe::tasks::components::containers::NormalizedLandmark cpp_landmark = { + 0.1f, 0.2f, 0.3f, 0.0f, 0.0f, "foo"}; + + ::NormalizedLandmark c_landmark; + CppConvertToNormalizedLandmark(cpp_landmark, &c_landmark); + EXPECT_NE(c_landmark.name, nullptr); + + CppCloseNormalizedLandmark(&c_landmark); + EXPECT_EQ(c_landmark.name, nullptr); +} + +TEST(LandmarksConverterTest, FreesMemory) { + std::vector + cpp_landmarks = {{0.1f, 0.2f, 0.3f}, {0.4f, 0.5f, 0.6f}}; + + ::Landmarks c_landmarks; + CppConvertToLandmarks(cpp_landmarks, &c_landmarks); + EXPECT_NE(c_landmarks.landmarks, nullptr); + + CppCloseLandmarks(&c_landmarks); + EXPECT_EQ(c_landmarks.landmarks, nullptr); +} + +TEST(NormalizedLandmarksConverterTest, FreesMemory) { + std::vector + cpp_normalized_landmarks = {{0.1f, 0.2f, 0.3f}, {0.4f, 0.5f, 0.6f}}; + + ::NormalizedLandmarks c_normalized_landmarks; + CppConvertToNormalizedLandmarks(cpp_normalized_landmarks, + &c_normalized_landmarks); + EXPECT_NE(c_normalized_landmarks.landmarks, nullptr); + + CppCloseNormalizedLandmarks(&c_normalized_landmarks); + EXPECT_EQ(c_normalized_landmarks.landmarks, nullptr); +} + } // namespace mediapipe::tasks::c::components::containers diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h index 1c2b65112..39f4a1734 100644 --- a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h +++ b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer.h @@ -82,12 +82,12 @@ struct GestureRecognizerOptions { // // A caller is responsible for closing gesture recognizer result. typedef void (*result_callback_fn)(GestureRecognizerResult* result, - const MpImage image, int64_t timestamp_ms, + const MpImage& image, int64_t timestamp_ms, char* error_msg); result_callback_fn result_callback; }; -// Creates an GestureRecognizer from provided `options`. +// Creates an GestureRecognizer from the provided `options`. // Returns a pointer to the gesture recognizer on success. // If an error occurs, returns `nullptr` and sets the error parameter to an // an error message (if `error_msg` is not `nullptr`). You must free the memory diff --git a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc index 723ff9838..ce3f5df5a 100644 --- a/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc +++ b/mediapipe/tasks/c/vision/gesture_recognizer/gesture_recognizer_test.cc @@ -36,16 +36,46 @@ using testing::HasSubstr; constexpr char kTestDataDirectory[] = "/mediapipe/tasks/testdata/vision/"; constexpr char kModelName[] = "gesture_recognizer.task"; -constexpr float kPrecision = 1e-4; -constexpr float kLandmarkPrecision = 1e-3; +constexpr char kImageFile[] = "fist.jpg"; +constexpr float kScorePrecision = 1e-2; +constexpr float kLandmarkPrecision = 1e-1; constexpr int kIterations = 100; std::string GetFullPath(absl::string_view file_name) { return JoinPath("./", kTestDataDirectory, file_name); } +void MatchesGestureRecognizerResult(GestureRecognizerResult* result, + const float score_precision, + const float landmark_precision) { + // Expects to have the same number of hands detected. + EXPECT_EQ(result->gestures_count, 1); + EXPECT_EQ(result->handedness_count, 1); + // Actual gesture with top score matches expected gesture. + EXPECT_EQ(std::string{result->gestures[0][0].category_name}, "Closed_Fist"); + EXPECT_NEAR(result->gestures[0][0].score, 0.91f, score_precision); + + // Actual handedness matches expected handedness. + EXPECT_EQ(std::string{result->handedness[0][0].category_name}, "Right"); + EXPECT_NEAR(result->handedness[0][0].score, 0.9893f, score_precision); + + // Actual landmarks match expected landmarks. + EXPECT_NEAR(result->hand_landmarks[0].landmarks[0].x, 0.477f, + landmark_precision); + EXPECT_NEAR(result->hand_landmarks[0].landmarks[0].y, 0.661f, + landmark_precision); + EXPECT_NEAR(result->hand_landmarks[0].landmarks[0].z, 0.0f, + landmark_precision); + EXPECT_NEAR(result->hand_world_landmarks[0].landmarks[0].x, -0.009f, + landmark_precision); + EXPECT_NEAR(result->hand_world_landmarks[0].landmarks[0].y, 0.082f, + landmark_precision); + EXPECT_NEAR(result->hand_world_landmarks[0].landmarks[0].z, 0.006f, + landmark_precision); +} + TEST(GestureRecognizerTest, ImageModeTest) { - const auto image = DecodeImageFromFile(GetFullPath("fist.jpg")); + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); ASSERT_TRUE(image.ok()); const std::string model_path = GetFullPath(kModelName); @@ -88,36 +118,204 @@ TEST(GestureRecognizerTest, ImageModeTest) { GestureRecognizerResult result; gesture_recognizer_recognize_image(recognizer, &mp_image, &result, /* error_msg */ nullptr); - - // Expects to have the same number of hands detected. - EXPECT_EQ(result.gestures_count, 1); - EXPECT_EQ(result.handedness_count, 1); - // Actual gesture with top score matches expected gesture. - EXPECT_EQ(std::string{result.gestures[0][0].category_name}, "Closed_Fist"); - EXPECT_NEAR(result.gestures[0][0].score, 0.9000f, kPrecision); - - // Actual handedness matches expected handedness. - EXPECT_EQ(std::string{result.handedness[0][0].category_name}, "Right"); - EXPECT_NEAR(result.handedness[0][0].score, 0.9893f, kPrecision); - - // Actual landmarks match expected landmarks. - EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].x, 0.477f, - kLandmarkPrecision); - EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].y, 0.661f, - kLandmarkPrecision); - EXPECT_NEAR(result.hand_landmarks[0].landmarks[0].z, 0.0f, - kLandmarkPrecision); - EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].x, -0.009f, - kLandmarkPrecision); - EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].y, 0.082f, - kLandmarkPrecision); - EXPECT_NEAR(result.hand_world_landmarks[0].landmarks[0].z, 0.006f, - kLandmarkPrecision); - + MatchesGestureRecognizerResult(&result, kScorePrecision, kLandmarkPrecision); gesture_recognizer_close_result(&result); gesture_recognizer_close(recognizer, /* error_msg */ nullptr); } -// TODO other tests +TEST(GestureRecognizerTest, VideoModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + GestureRecognizerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::VIDEO, + /* num_hands= */ 1, + /* min_hand_detection_confidence= */ 0.5, + /* min_hand_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}}; + + void* recognizer = + gesture_recognizer_create(&options, /* error_msg */ nullptr); + EXPECT_NE(recognizer, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + GestureRecognizerResult result; + gesture_recognizer_recognize_for_video(recognizer, &mp_image, i, &result, + /* error_msg */ nullptr); + + MatchesGestureRecognizerResult(&result, kScorePrecision, + kLandmarkPrecision); + gesture_recognizer_close_result(&result); + } + gesture_recognizer_close(recognizer, /* error_msg */ nullptr); +} + +// A structure to support LiveStreamModeTest below. This structure holds a +// static method `Fn` for a callback function of C API. A `static` qualifier +// allows to take an address of the method to follow API style. Another static +// struct member is `last_timestamp` that is used to verify that current +// timestamp is greater than the previous one. +struct LiveStreamModeCallback { + static int64_t last_timestamp; + static void Fn(GestureRecognizerResult* recognizer_result, + const MpImage& image, int64_t timestamp, char* error_msg) { + ASSERT_NE(recognizer_result, nullptr); + ASSERT_EQ(error_msg, nullptr); + MatchesGestureRecognizerResult(recognizer_result, kScorePrecision, + kLandmarkPrecision); + EXPECT_GT(image.image_frame.width, 0); + EXPECT_GT(image.image_frame.height, 0); + EXPECT_GT(timestamp, last_timestamp); + last_timestamp++; + } +}; +int64_t LiveStreamModeCallback::last_timestamp = -1; + +TEST(GestureRecognizerTest, LiveStreamModeTest) { + const auto image = DecodeImageFromFile(GetFullPath(kImageFile)); + ASSERT_TRUE(image.ok()); + + const std::string model_path = GetFullPath(kModelName); + + GestureRecognizerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::LIVE_STREAM, + /* num_hands= */ 1, + /* min_hand_detection_confidence= */ 0.5, + /* min_hand_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + /* result_callback= */ LiveStreamModeCallback::Fn, + }; + + void* recognizer = + gesture_recognizer_create(&options, /* error_msg */ nullptr); + EXPECT_NE(recognizer, nullptr); + + const auto& image_frame = image->GetImageFrameSharedPtr(); + const MpImage mp_image = { + .type = MpImage::IMAGE_FRAME, + .image_frame = {.format = static_cast(image_frame->Format()), + .image_buffer = image_frame->PixelData(), + .width = image_frame->Width(), + .height = image_frame->Height()}}; + + for (int i = 0; i < kIterations; ++i) { + EXPECT_GE(gesture_recognizer_recognize_async(recognizer, &mp_image, i, + /* error_msg */ nullptr), + 0); + } + gesture_recognizer_close(recognizer, /* error_msg */ nullptr); + + // Due to the flow limiter, the total of outputs might be smaller than the + // number of iterations. + EXPECT_LE(LiveStreamModeCallback::last_timestamp, kIterations); + EXPECT_GT(LiveStreamModeCallback::last_timestamp, 0); +} + +TEST(GestureRecognizerTest, InvalidArgumentHandling) { + // It is an error to set neither the asset buffer nor the path. + GestureRecognizerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ nullptr}, + /* running_mode= */ RunningMode::IMAGE, + /* num_hands= */ 1, + /* min_hand_detection_confidence= */ 0.5, + /* min_hand_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + {}, + {}}; + + char* error_msg; + void* recognizer = gesture_recognizer_create(&options, &error_msg); + EXPECT_EQ(recognizer, nullptr); + + EXPECT_THAT(error_msg, HasSubstr("ExternalFile must specify")); + + free(error_msg); +} + +TEST(GestureRecognizerTest, FailedRecognitionHandling) { + const std::string model_path = GetFullPath(kModelName); + GestureRecognizerOptions options = { + /* base_options= */ {/* model_asset_buffer= */ nullptr, + /* model_asset_buffer_count= */ 0, + /* model_asset_path= */ model_path.c_str()}, + /* running_mode= */ RunningMode::IMAGE, + /* num_hands= */ 1, + /* min_hand_detection_confidence= */ 0.5, + /* min_hand_presence_confidence= */ 0.5, + /* min_tracking_confidence= */ 0.5, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + {/* display_names_locale= */ nullptr, + /* max_results= */ -1, + /* score_threshold= */ 0.0, + /* category_allowlist= */ nullptr, + /* category_allowlist_count= */ 0, + /* category_denylist= */ nullptr, + /* category_denylist_count= */ 0}, + }; + + void* recognizer = gesture_recognizer_create(&options, /* error_msg */ + nullptr); + EXPECT_NE(recognizer, nullptr); + + const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}}; + GestureRecognizerResult result; + char* error_msg; + gesture_recognizer_recognize_image(recognizer, &mp_image, &result, + &error_msg); + EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet")); + free(error_msg); + gesture_recognizer_close(recognizer, /* error_msg */ nullptr); +} } // namespace