cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into c-image-embedder-api

Browse Source
This commit is contained in:
Kinar R 2023-11-07 20:37:09 +05:30 committed by GitHub
commit 42a916ad4f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 74 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
mediapipe/gpu/BUILD
View File

@ -516,7 +516,6 @@ cc_library(
":gpu_buffer_storage",
":image_frame_view",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/port:ret_check",
"@com_google_absl//absl/strings:str_format",
],
)
@ -1224,13 +1223,9 @@ mediapipe_cc_test(
],
requires_full_emulation = True,
deps = [
":gl_texture_buffer",
":gl_texture_util",
":gpu_buffer_format",
":gpu_buffer_storage_ahwb",
":gpu_test_base",
"//mediapipe/framework/port:gtest_main",
"//mediapipe/framework/tool:test_util",
],
)

2
mediapipe/gpu/gpu_buffer_format.cc
View File

@ -238,7 +238,7 @@ ImageFormat::Format ImageFormatForGpuBufferFormat(GpuBufferFormat format) {
case GpuBufferFormat::kRGBAFloat128:
return ImageFormat::VEC32F4;
case GpuBufferFormat::kRGBA32:
return ImageFormat::SRGBA;
// TODO: this likely maps to ImageFormat::SRGBA
case GpuBufferFormat::kGrayHalf16:
case GpuBufferFormat::kOneComponent8Alpha:
case GpuBufferFormat::kOneComponent8Red:

5
mediapipe/tasks/c/core/base_options.h
View File

@ -22,9 +22,12 @@ extern "C" {
// Base options for MediaPipe C Tasks.
struct BaseOptions {
// The model asset file contents as a string.
// The model asset file contents as bytes.
const char* model_asset_buffer;
// The size of the model assets buffer (or `0` if not set).
const unsigned int model_asset_buffer_count;
// The path to the model asset to open and mmap in memory.
const char* model_asset_path;
};

4
mediapipe/tasks/c/core/base_options_converter.cc
View File

@ -27,7 +27,9 @@ void CppConvertToBaseOptions(const BaseOptions& in,
mediapipe::tasks::core::BaseOptions* out) {
out->model_asset_buffer =
in.model_asset_buffer
? std::make_unique<std::string>(in.model_asset_buffer)
? std::make_unique<std::string>(
in.model_asset_buffer,
in.model_asset_buffer + in.model_asset_buffer_count)
: nullptr;
out->model_asset_path =
in.model_asset_path ? std::string(in.model_asset_path) : "";

4
mediapipe/tasks/c/core/base_options_converter_test.cc
View File

@ -15,6 +15,7 @@ limitations under the License.
#include "mediapipe/tasks/c/core/base_options_converter.h"
#include <cstring>
#include <string>
#include "mediapipe/framework/port/gtest.h"
@ -28,6 +29,8 @@ constexpr char kModelAssetPath[] = "abc.tflite";
TEST(BaseOptionsConverterTest, ConvertsBaseOptionsAssetBuffer) {
BaseOptions c_base_options = {/* model_asset_buffer= */ kAssetBuffer,
/* model_asset_buffer_count= */
static_cast<unsigned int>(strlen(kAssetBuffer)),
/* model_asset_path= */ nullptr};
mediapipe::tasks::core::BaseOptions cpp_base_options = {};
@ -39,6 +42,7 @@ TEST(BaseOptionsConverterTest, ConvertsBaseOptionsAssetBuffer) {
TEST(BaseOptionsConverterTest, ConvertsBaseOptionsAssetPath) {
BaseOptions c_base_options = {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ kModelAssetPath};
mediapipe::tasks::core::BaseOptions cpp_base_options = {};

13
mediapipe/tasks/c/text/language_detector/language_detector.h
View File

@ -60,18 +60,18 @@ struct LanguageDetectorOptions {
// Creates a LanguageDetector from the provided `options`.
// Returns a pointer to the language detector on success.
// If an error occurs, returns `nullptr` and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT void* language_detector_create(
struct LanguageDetectorOptions* options, char** error_msg = nullptr);
struct LanguageDetectorOptions* options, char** error_msg);
// Performs language detection on the input `text`. Returns `0` on success.
// If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT int language_detector_detect(void* detector, const char* utf8_str,
LanguageDetectorResult* result,
char** error_msg = nullptr);
char** error_msg);
// Frees the memory allocated inside a LanguageDetectorResult result. Does not
// free the result pointer itself.
@ -79,10 +79,9 @@ MP_EXPORT void language_detector_close_result(LanguageDetectorResult* result);
// Shuts down the LanguageDetector when all the work is done. Frees all memory.
// If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT int language_detector_close(void* detector,
char** error_msg = nullptr);
MP_EXPORT int language_detector_close(void* detector, char** error_msg);
#ifdef __cplusplus
} // extern C

9
mediapipe/tasks/c/text/language_detector/language_detector_test.cc
View File

@ -44,6 +44,7 @@ TEST(LanguageDetectorTest, SmokeTest) {
std::string model_path = GetFullPath(kTestLanguageDetectorModelPath);
LanguageDetectorOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* classifier_options= */
{/* display_names_locale= */ nullptr,
@ -55,22 +56,24 @@ TEST(LanguageDetectorTest, SmokeTest) {
/* category_denylist_count= */ 0},
};
void* detector = language_detector_create(&options);
void* detector = language_detector_create(&options, /* error_msg */ nullptr);
EXPECT_NE(detector, nullptr);
LanguageDetectorResult result;
language_detector_detect(detector, kTestString, &result);
language_detector_detect(detector, kTestString, &result,
/* error_msg */ nullptr);
EXPECT_EQ(std::string(result.predictions[0].language_code), "fr");
EXPECT_NEAR(result.predictions[0].probability, 0.999781, kPrecision);
language_detector_close_result(&result);
language_detector_close(detector);
language_detector_close(detector, /* error_msg */ nullptr);
}
TEST(LanguageDetectorTest, ErrorHandling) {
// It is an error to set neither the asset buffer nor the path.
LanguageDetectorOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ nullptr},
/* classifier_options= */ {},
};

13
mediapipe/tasks/c/text/text_classifier/text_classifier.h
View File

@ -44,18 +44,18 @@ struct TextClassifierOptions {
// Creates a TextClassifier from the provided `options`.
// Returns a pointer to the text classifier on success.
// If an error occurs, returns `nullptr` and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT void* text_classifier_create(struct TextClassifierOptions* options,
char** error_msg = nullptr);
char** error_msg);
// Performs classification on the input `text`. Returns `0` on success.
// If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT int text_classifier_classify(void* classifier, const char* utf8_str,
TextClassifierResult* result,
char** error_msg = nullptr);
char** error_msg);
// Frees the memory allocated inside a TextClassifierResult result. Does not
// free the result pointer itself.
@ -63,10 +63,9 @@ MP_EXPORT void text_classifier_close_result(TextClassifierResult* result);
// Shuts down the TextClassifier when all the work is done. Frees all memory.
// If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT int text_classifier_close(void* classifier,
char** error_msg = nullptr);
MP_EXPORT int text_classifier_close(void* classifier, char** error_msg);
#ifdef __cplusplus
} // extern C

9
mediapipe/tasks/c/text/text_classifier/text_classifier_test.cc
View File

@ -43,6 +43,7 @@ TEST(TextClassifierTest, SmokeTest) {
std::string model_path = GetFullPath(kTestBertModelPath);
TextClassifierOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* classifier_options= */
{/* display_names_locale= */ nullptr,
@ -54,11 +55,12 @@ TEST(TextClassifierTest, SmokeTest) {
/* category_denylist_count= */ 0},
};
void* classifier = text_classifier_create(&options);
void* classifier = text_classifier_create(&options, /* error_msg */ nullptr);
EXPECT_NE(classifier, nullptr);
TextClassifierResult result;
text_classifier_classify(classifier, kTestString, &result);
text_classifier_classify(classifier, kTestString, &result,
/* error_msg */ nullptr);
EXPECT_EQ(result.classifications_count, 1);
EXPECT_EQ(result.classifications[0].categories_count, 2);
EXPECT_EQ(std::string{result.classifications[0].categories[0].category_name},
@ -67,13 +69,14 @@ TEST(TextClassifierTest, SmokeTest) {
kPrecision);
text_classifier_close_result(&result);
text_classifier_close(classifier);
text_classifier_close(classifier, /* error_msg */ nullptr);
}
TEST(TextClassifierTest, ErrorHandling) {
// It is an error to set neither the asset buffer nor the path.
TextClassifierOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ nullptr},
/* classifier_options= */ {},
};

7
mediapipe/tasks/c/text/text_embedder/text_embedder.h
View File

@ -47,15 +47,14 @@ struct TextEmbedderOptions {
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT void* text_embedder_create(struct TextEmbedderOptions* options,
char** error_msg = nullptr);
char** error_msg);
// Performs embedding extraction on the input `text`. Returns `0` on success.
// If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT int text_embedder_embed(void* embedder, const char* utf8_str,
TextEmbedderResult* result,
char** error_msg = nullptr);
TextEmbedderResult* result, char** error_msg);
// Frees the memory allocated inside a TextEmbedderResult result. Does not
// free the result pointer itself.
@ -65,7 +64,7 @@ MP_EXPORT void text_embedder_close_result(TextEmbedderResult* result);
// If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT int text_embedder_close(void* embedder, char** error_msg = nullptr);
MP_EXPORT int text_embedder_close(void* embedder, char** error_msg);
// Utility function to compute cosine similarity [1] between two embeddings.
// May return an InvalidArgumentError if e.g. the embeddings are of different

12
mediapipe/tasks/c/text/text_embedder/text_embedder_test.cc
View File

@ -47,21 +47,22 @@ TEST(TextEmbedderTest, SmokeTest) {
std::string model_path = GetFullPath(kTestBertModelPath);
TextEmbedderOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* embedder_options= */
{/* l2_normalize= */ false, /* quantize= */ true},
};
void* embedder = text_embedder_create(&options);
void* embedder = text_embedder_create(&options, /* error_msg */ nullptr);
EXPECT_NE(embedder, nullptr);
TextEmbedderResult result;
text_embedder_embed(embedder, kTestString0, &result);
text_embedder_embed(embedder, kTestString0, &result, /* error_msg */ nullptr);
EXPECT_EQ(result.embeddings_count, 1);
EXPECT_EQ(result.embeddings[0].values_count, 512);
text_embedder_close_result(&result);
text_embedder_close(embedder);
text_embedder_close(embedder, /* error_msg */ nullptr);
}
TEST(TextEmbedderTest, SucceedsWithCosineSimilarity) {
@ -78,9 +79,9 @@ TEST(TextEmbedderTest, SucceedsWithCosineSimilarity) {
// Extract both embeddings.
TextEmbedderResult result0;
text_embedder_embed(embedder, kTestString0, &result0);
text_embedder_embed(embedder, kTestString0, &result0, /* error_msg */ nullptr);
TextEmbedderResult result1;
text_embedder_embed(embedder, kTestString1, &result1);
text_embedder_embed(embedder, kTestString1, &result1, /* error_msg */ nullptr);
// Check cosine similarity.
double similarity;
@ -95,6 +96,7 @@ TEST(TextEmbedderTest, ErrorHandling) {
// It is an error to set neither the asset buffer nor the path.
TextEmbedderOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ nullptr},
/* embedder_options= */ {},
};

4
mediapipe/tasks/c/vision/image_classifier/BUILD
View File

@ -55,11 +55,7 @@ cc_test(
":image_classifier_lib",
"//mediapipe/framework/deps:file_path",
"//mediapipe/framework/formats:image",
"//mediapipe/framework/formats:image_frame",
"//mediapipe/framework/formats:image_frame_opencv",
"//mediapipe/framework/port:gtest",
"//mediapipe/framework/port:opencv_core",
"//mediapipe/framework/port:opencv_imgproc",
"//mediapipe/tasks/c/components/containers:category",
"//mediapipe/tasks/cc/vision/utils:image_utils",
"@com_google_absl//absl/flags:flag",

17
mediapipe/tasks/c/vision/image_classifier/image_classifier.h
View File

@ -108,30 +108,30 @@ struct ImageClassifierOptions {
// Creates an ImageClassifier from provided `options`.
// Returns a pointer to the image classifier on success.
// If an error occurs, returns `nullptr` and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT void* image_classifier_create(struct ImageClassifierOptions* options,
char** error_msg = nullptr);
char** error_msg);
// Performs image classification on the input `image`. Returns `0` on success.
// If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT int image_classifier_classify_image(void* classifier,
const MpImage* image,
ImageClassifierResult* result,
char** error_msg = nullptr);
char** error_msg);
MP_EXPORT int image_classifier_classify_for_video(void* classifier,
const MpImage* image,
int64_t timestamp_ms,
ImageClassifierResult* result,
char** error_msg = nullptr);
char** error_msg);
MP_EXPORT int image_classifier_classify_async(void* classifier,
const MpImage* image,
int64_t timestamp_ms,
char** error_msg = nullptr);
char** error_msg);
// Frees the memory allocated inside a ImageClassifierResult result.
// Does not free the result pointer itself.
@ -139,10 +139,9 @@ MP_EXPORT void image_classifier_close_result(ImageClassifierResult* result);
// Frees image classifier.
// If an error occurs, returns an error code and sets the error parameter to an
// an error message (if `error_msg` is not nullptr). You must free the memory
// an error message (if `error_msg` is not `nullptr`). You must free the memory
// allocated for the error message.
MP_EXPORT int image_classifier_close(void* classifier,
char** error_msg = nullptr);
MP_EXPORT int image_classifier_close(void* classifier, char** error_msg);
#ifdef __cplusplus
} // extern C

31
mediapipe/tasks/c/vision/image_classifier/image_classifier_test.cc
View File

@ -50,6 +50,7 @@ TEST(ImageClassifierTest, ImageModeTest) {
const std::string model_path = GetFullPath(kModelName);
ImageClassifierOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* running_mode= */ RunningMode::IMAGE,
/* classifier_options= */
@ -62,7 +63,7 @@ TEST(ImageClassifierTest, ImageModeTest) {
/* category_denylist_count= */ 0},
};
void* classifier = image_classifier_create(&options);
void* classifier = image_classifier_create(&options, /* error_msg */ nullptr);
EXPECT_NE(classifier, nullptr);
const auto& image_frame = image->GetImageFrameSharedPtr();
@ -74,7 +75,8 @@ TEST(ImageClassifierTest, ImageModeTest) {
.height = image_frame->Height()}};
ImageClassifierResult result;
image_classifier_classify_image(classifier, &mp_image, &result);
image_classifier_classify_image(classifier, &mp_image, &result,
/* error_msg */ nullptr);
EXPECT_EQ(result.classifications_count, 1);
EXPECT_EQ(result.classifications[0].categories_count, 1001);
EXPECT_EQ(std::string{result.classifications[0].categories[0].category_name},
@ -82,7 +84,7 @@ TEST(ImageClassifierTest, ImageModeTest) {
EXPECT_NEAR(result.classifications[0].categories[0].score, 0.7939f,
kPrecision);
image_classifier_close_result(&result);
image_classifier_close(classifier);
image_classifier_close(classifier, /* error_msg */ nullptr);
}
TEST(ImageClassifierTest, VideoModeTest) {
@ -92,6 +94,7 @@ TEST(ImageClassifierTest, VideoModeTest) {
const std::string model_path = GetFullPath(kModelName);
ImageClassifierOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* running_mode= */ RunningMode::VIDEO,
/* classifier_options= */
@ -105,7 +108,7 @@ TEST(ImageClassifierTest, VideoModeTest) {
/* result_callback= */ nullptr,
};
void* classifier = image_classifier_create(&options);
void* classifier = image_classifier_create(&options, /* error_msg */ nullptr);
EXPECT_NE(classifier, nullptr);
const auto& image_frame = image->GetImageFrameSharedPtr();
@ -118,7 +121,8 @@ TEST(ImageClassifierTest, VideoModeTest) {
for (int i = 0; i < kIterations; ++i) {
ImageClassifierResult result;
image_classifier_classify_for_video(classifier, &mp_image, i, &result);
image_classifier_classify_for_video(classifier, &mp_image, i, &result,
/* error_msg */ nullptr);
EXPECT_EQ(result.classifications_count, 1);
EXPECT_EQ(result.classifications[0].categories_count, 3);
EXPECT_EQ(
@ -128,7 +132,7 @@ TEST(ImageClassifierTest, VideoModeTest) {
kPrecision);
image_classifier_close_result(&result);
}
image_classifier_close(classifier);
image_classifier_close(classifier, /* error_msg */ nullptr);
}
// A structure to support LiveStreamModeTest below. This structure holds a
@ -164,6 +168,7 @@ TEST(ImageClassifierTest, LiveStreamModeTest) {
ImageClassifierOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* running_mode= */ RunningMode::LIVE_STREAM,
/* classifier_options= */
@ -177,7 +182,7 @@ TEST(ImageClassifierTest, LiveStreamModeTest) {
/* result_callback= */ LiveStreamModeCallback::Fn,
};
void* classifier = image_classifier_create(&options);
void* classifier = image_classifier_create(&options, /* error_msg */ nullptr);
EXPECT_NE(classifier, nullptr);
const auto& image_frame = image->GetImageFrameSharedPtr();
@ -189,9 +194,11 @@ TEST(ImageClassifierTest, LiveStreamModeTest) {
.height = image_frame->Height()}};
for (int i = 0; i < kIterations; ++i) {
EXPECT_GE(image_classifier_classify_async(classifier, &mp_image, i), 0);
EXPECT_GE(image_classifier_classify_async(classifier, &mp_image, i,
/* error_msg */ nullptr),
0);
}
image_classifier_close(classifier);
image_classifier_close(classifier, /* error_msg */ nullptr);
// Due to the flow limiter, the total of outputs might be smaller than the
// number of iterations.
@ -203,6 +210,7 @@ TEST(ImageClassifierTest, InvalidArgumentHandling) {
// It is an error to set neither the asset buffer nor the path.
ImageClassifierOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ nullptr},
/* classifier_options= */ {},
};
@ -220,6 +228,7 @@ TEST(ImageClassifierTest, FailedClassificationHandling) {
const std::string model_path = GetFullPath(kModelName);
ImageClassifierOptions options = {
/* base_options= */ {/* model_asset_buffer= */ nullptr,
/* model_asset_buffer_count= */ 0,
/* model_asset_path= */ model_path.c_str()},
/* running_mode= */ RunningMode::IMAGE,
/* classifier_options= */
@ -232,7 +241,7 @@ TEST(ImageClassifierTest, FailedClassificationHandling) {
/* category_denylist_count= */ 0},
};
void* classifier = image_classifier_create(&options);
void* classifier = image_classifier_create(&options, /* error_msg */ nullptr);
EXPECT_NE(classifier, nullptr);
const MpImage mp_image = {.type = MpImage::GPU_BUFFER, .gpu_buffer = {}};
@ -241,7 +250,7 @@ TEST(ImageClassifierTest, FailedClassificationHandling) {
image_classifier_classify_image(classifier, &mp_image, &result, &error_msg);
EXPECT_THAT(error_msg, HasSubstr("GPU Buffer not supported yet"));
free(error_msg);
image_classifier_close(classifier);
image_classifier_close(classifier, /* error_msg */ nullptr);
}
} // namespace