cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Drop default arguments in Image Embedder C API

Browse Source
This commit is contained in:
Kinar 2023-11-07 07:34:08 -08:00
parent 60fcfa74cc
commit 197358dfee
2 changed files with 36 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
mediapipe/tasks/c/vision/image_embedder/image_embedder.h
View File

@ -108,28 +108,28 @@ struct ImageEmbedderOptions {
// Creates an ImageEmbedder from provided `options`. // Creates an ImageEmbedder from provided `options`.
// Returns a pointer to the image embedder on success. // Returns a pointer to the image embedder on success.
// If an error occurs, returns `nullptr` and sets the error parameter to an // If an error occurs, returns `nullptr` and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory // an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message. // allocated for the error message.
MP_EXPORT void* image_embedder_create(struct ImageEmbedderOptions* options, MP_EXPORT void* image_embedder_create(struct ImageEmbedderOptions* options,
char** error_msg = nullptr); char** error_msg);
// Performs embedding extraction on the input `image`. Returns `0` on success. // Performs embedding extraction on the input `image`. Returns `0` on success.
// If an error occurs, returns an error code and sets the error parameter to an // If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory // an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message. // allocated for the error message.
MP_EXPORT int image_embedder_embed_image(void* embedder, const MpImage* image, MP_EXPORT int image_embedder_embed_image(void* embedder, const MpImage* image,
ImageEmbedderResult* result, ImageEmbedderResult* result,
char** error_msg = nullptr); char** error_msg);
MP_EXPORT int image_embedder_embed_for_video(void* embedder, MP_EXPORT int image_embedder_embed_for_video(void* embedder,
const MpImage* image, const MpImage* image,
int64_t timestamp_ms, int64_t timestamp_ms,
ImageEmbedderResult* result, ImageEmbedderResult* result,
char** error_msg = nullptr); char** error_msg);
MP_EXPORT int image_embedder_embed_async(void* embedder, const MpImage* image, MP_EXPORT int image_embedder_embed_async(void* embedder, const MpImage* image,
int64_t timestamp_ms, int64_t timestamp_ms,
char** error_msg = nullptr); char** error_msg);
// Frees the memory allocated inside a ImageEmbedderResult result. // Frees the memory allocated inside a ImageEmbedderResult result.
// Does not free the result pointer itself. // Does not free the result pointer itself.
@ -137,9 +137,9 @@ MP_EXPORT void image_embedder_close_result(ImageEmbedderResult* result);
// Frees image embedder. // Frees image embedder.
// If an error occurs, returns an error code and sets the error parameter to an // If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory // an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message. // allocated for the error message.
MP_EXPORT int image_embedder_close(void* embedder, char** error_msg = nullptr); MP_EXPORT int image_embedder_close(void* embedder, char** error_msg);
// Utility function to compute cosine similarity [1] between two embeddings. // Utility function to compute cosine similarity [1] between two embeddings.
// May return an InvalidArgumentError if e.g. the embeddings are of different // May return an InvalidArgumentError if e.g. the embeddings are of different
@ -148,7 +148,7 @@ MP_EXPORT int image_embedder_close(void* embedder, char** error_msg = nullptr);
// //
// [1]: https://en.wikipedia.org/wiki/Cosine_similarity // [1]: https://en.wikipedia.org/wiki/Cosine_similarity
MP_EXPORT int cosine_similarity(const Embedding& u, const Embedding& v, MP_EXPORT int cosine_similarity(const Embedding& u, const Embedding& v,
double* similarity, char** error_msg = nullptr); double* similarity, char** error_msg);
#ifdef __cplusplus #ifdef __cplusplus
} // extern C } // extern C

43
mediapipe/tasks/c/vision/image_embedder/image_embedder_test.cc
View File

@ -70,7 +70,8 @@ TEST(ImageEmbedderTest, ImageModeTest) {
{/* l2_normalize= */ true, {/* l2_normalize= */ true,
/* quantize= */ false}}; /* quantize= */ false}};
void* embedder = image_embedder_create(&options); void* embedder = image_embedder_create(&options,
/* error_msg */ nullptr);
EXPECT_NE(embedder, nullptr); EXPECT_NE(embedder, nullptr);
const MpImage mp_image = { const MpImage mp_image = {
@ -83,11 +84,12 @@ TEST(ImageEmbedderTest, ImageModeTest) {
.height = image->GetImageFrameSharedPtr()->Height()}}; .height = image->GetImageFrameSharedPtr()->Height()}};
ImageEmbedderResult result; ImageEmbedderResult result;
image_embedder_embed_image(embedder, &mp_image, &result); image_embedder_embed_image(embedder, &mp_image, &result,
/* error_msg */ nullptr);
CheckMobileNetV3Result(result, false); CheckMobileNetV3Result(result, false);
EXPECT_NEAR(result.embeddings[0].float_embedding[0], -0.0142344, kPrecision); EXPECT_NEAR(result.embeddings[0].float_embedding[0], -0.0142344, kPrecision);
image_embedder_close_result(&result); image_embedder_close_result(&result);
image_embedder_close(embedder); image_embedder_close(embedder, /* error_msg */ nullptr);
} }
TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) { TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) {
@ -106,7 +108,8 @@ TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) {
{/* l2_normalize= */ true, {/* l2_normalize= */ true,
/* quantize= */ false}}; /* quantize= */ false}};
void* embedder = image_embedder_create(&options); void* embedder = image_embedder_create(&options,
/* error_msg */ nullptr);
EXPECT_NE(embedder, nullptr); EXPECT_NE(embedder, nullptr);
const MpImage mp_image = { const MpImage mp_image = {
@ -129,9 +132,11 @@ TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) {
// Extract both embeddings. // Extract both embeddings.
ImageEmbedderResult image_result; ImageEmbedderResult image_result;
image_embedder_embed_image(embedder, &mp_image, &image_result); image_embedder_embed_image(embedder, &mp_image, &image_result,
/* error_msg */ nullptr);
ImageEmbedderResult crop_result; ImageEmbedderResult crop_result;
image_embedder_embed_image(embedder, &mp_crop, &crop_result); image_embedder_embed_image(embedder, &mp_crop, &crop_result,
/* error_msg */ nullptr);
// Check results. // Check results.
CheckMobileNetV3Result(image_result, false); CheckMobileNetV3Result(image_result, false);
@ -139,12 +144,12 @@ TEST(ImageEmbedderTest, SucceedsWithCosineSimilarity) {
// Check cosine similarity. // Check cosine similarity.
double similarity; double similarity;
cosine_similarity(image_result.embeddings[0], crop_result.embeddings[0], cosine_similarity(image_result.embeddings[0], crop_result.embeddings[0],
&similarity); &similarity, /* error_msg */ nullptr);
double expected_similarity = 0.925519; double expected_similarity = 0.925519;
EXPECT_LE(abs(similarity - expected_similarity), kPrecision); EXPECT_LE(abs(similarity - expected_similarity), kPrecision);
image_embedder_close_result(&image_result); image_embedder_close_result(&image_result);
image_embedder_close_result(&crop_result); image_embedder_close_result(&crop_result);
image_embedder_close(embedder); image_embedder_close(embedder, /* error_msg */ nullptr);
} }
TEST(ImageEmbedderTest, VideoModeTest) { TEST(ImageEmbedderTest, VideoModeTest) {
@ -161,7 +166,8 @@ TEST(ImageEmbedderTest, VideoModeTest) {
{/* l2_normalize= */ true, {/* l2_normalize= */ true,
/* quantize= */ false}}; /* quantize= */ false}};
void* embedder = image_embedder_create(&options); void* embedder = image_embedder_create(&options,
/* error_msg */ nullptr);
EXPECT_NE(embedder, nullptr); EXPECT_NE(embedder, nullptr);
const auto& image_frame = image->GetImageFrameSharedPtr(); const auto& image_frame = image->GetImageFrameSharedPtr();
@ -174,13 +180,14 @@ TEST(ImageEmbedderTest, VideoModeTest) {
for (int i = 0; i < kIterations; ++i) { for (int i = 0; i < kIterations; ++i) {
ImageEmbedderResult result; ImageEmbedderResult result;
image_embedder_embed_for_video(embedder, &mp_image, i, &result); image_embedder_embed_for_video(embedder, &mp_image, i, &result,
/* error_msg */ nullptr);
CheckMobileNetV3Result(result, false); CheckMobileNetV3Result(result, false);
EXPECT_NEAR(result.embeddings[0].float_embedding[0], -0.0142344, EXPECT_NEAR(result.embeddings[0].float_embedding[0], -0.0142344,
kPrecision); kPrecision);
image_embedder_close_result(&result); image_embedder_close_result(&result);
} }
image_embedder_close(embedder); image_embedder_close(embedder, /* error_msg */ nullptr);
} }
// A structure to support LiveStreamModeTest below. This structure holds a // A structure to support LiveStreamModeTest below. This structure holds a
@ -222,7 +229,8 @@ TEST(ImageEmbedderTest, LiveStreamModeTest) {
/* result_callback= */ LiveStreamModeCallback::Fn, /* result_callback= */ LiveStreamModeCallback::Fn,
}; };
void* embedder = image_embedder_create(&options); void* embedder = image_embedder_create(&options,
/* error_msg */ nullptr);
EXPECT_NE(embedder, nullptr); EXPECT_NE(embedder, nullptr);
const auto& image_frame = image->GetImageFrameSharedPtr(); const auto& image_frame = image->GetImageFrameSharedPtr();
@ -234,9 +242,11 @@ TEST(ImageEmbedderTest, LiveStreamModeTest) {
.height = image_frame->Height()}}; .height = image_frame->Height()}};
for (int i = 0; i < kIterations; ++i) { for (int i = 0; i < kIterations; ++i) {
EXPECT_GE(image_embedder_embed_async(embedder, &mp_image, i), 0); EXPECT_GE(image_embedder_embed_async(embedder, &mp_image, i,
/* error_msg */ nullptr),
0);
} }
image_embedder_close(embedder); image_embedder_close(embedder, /* error_msg */ nullptr);
// Due to the flow limiter, the total of outputs might be smaller than the // Due to the flow limiter, the total of outputs might be smaller than the
// number of iterations. // number of iterations.
@ -274,7 +284,8 @@ TEST(ImageEmbedderTest, FailedEmbeddingHandling) {
/* quantize= */ false}, /* quantize= */ false},
}; };
void* embedder = image_embedder_create(&options); void* embedder = image_embedder_create(&options,
/* error_msg */ nullptr);
EXPECT_NE(embedder, nullptr); EXPECT_NE(embedder, nullptr);
const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}}; const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}};
@ -283,7 +294,7 @@ TEST(ImageEmbedderTest, FailedEmbeddingHandling) {
image_embedder_embed_image(embedder, &mp_image, &result, &error_msg); image_embedder_embed_image(embedder, &mp_image, &result, &error_msg);
EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet.")); EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet."));
free(error_msg); free(error_msg);
image_embedder_close(embedder); image_embedder_close(embedder, /* error_msg */ nullptr);
} }
} // namespace } // namespace