From 65e1d722eb69daf69f67325b9923ccdde7d9f152 Mon Sep 17 00:00:00 2001 From: Yuqi Li Date: Fri, 7 Oct 2022 16:44:35 -0700 Subject: [PATCH] Update external files to support file_pointer_meta. PiperOrigin-RevId: 479695361 --- .../audio_classifier/audio_classifier_test.cc | 2 +- .../tasks/cc/core/external_file_handler.cc | 22 +++++++++++++++++-- .../tasks/cc/core/proto/external_file.proto | 21 +++++++++++++++++- mediapipe/tasks/cc/metadata/utils/BUILD | 1 + .../tasks/cc/metadata/utils/zip_utils.cc | 8 +++++++ mediapipe/tasks/cc/metadata/utils/zip_utils.h | 6 +++++ .../image_classifier/image_classifier_test.cc | 2 +- .../image_embedder/image_embedder_test.cc | 2 +- .../image_segmenter/image_segmenter_test.cc | 2 +- .../object_detector/object_detector_test.cc | 2 +- .../test/vision/object_detector_test.py | 2 +- 11 files changed, 61 insertions(+), 9 deletions(-) diff --git a/mediapipe/tasks/cc/audio/audio_classifier/audio_classifier_test.cc b/mediapipe/tasks/cc/audio/audio_classifier/audio_classifier_test.cc index 4b64d2231..591d5e4f7 100644 --- a/mediapipe/tasks/cc/audio/audio_classifier/audio_classifier_test.cc +++ b/mediapipe/tasks/cc/audio/audio_classifier/audio_classifier_test.cc @@ -184,7 +184,7 @@ TEST_F(CreateFromOptionsTest, FailsWithMissingModel) { EXPECT_THAT( audio_classifier_or.status().message(), HasSubstr("ExternalFile must specify at least one of 'file_content', " - "'file_name' or 'file_descriptor_meta'.")); + "'file_name', 'file_pointer_meta' or 'file_descriptor_meta'.")); EXPECT_THAT(audio_classifier_or.status().GetPayload(kMediaPipeTasksPayload), Optional(absl::Cord(absl::StrCat( MediaPipeTasksStatus::kRunnerInitializationError)))); diff --git a/mediapipe/tasks/cc/core/external_file_handler.cc b/mediapipe/tasks/cc/core/external_file_handler.cc index 8a219bb80..33dfeca0b 100644 --- a/mediapipe/tasks/cc/core/external_file_handler.cc +++ b/mediapipe/tasks/cc/core/external_file_handler.cc @@ -92,13 +92,26 @@ absl::Status ExternalFileHandler::MapExternalFile() { #else if (!external_file_.file_content().empty()) { return absl::OkStatus(); + } else if (external_file_.has_file_pointer_meta()) { + if (external_file_.file_pointer_meta().pointer() == 0) { + return CreateStatusWithPayload( + StatusCode::kInvalidArgument, + "Need to set the file pointer in external_file.file_pointer_meta."); + } + if (external_file_.file_pointer_meta().length() <= 0) { + return CreateStatusWithPayload( + StatusCode::kInvalidArgument, + "The length of the file in external_file.file_pointer_meta should be " + "positive."); + } + return absl::OkStatus(); } if (external_file_.file_name().empty() && !external_file_.has_file_descriptor_meta()) { return CreateStatusWithPayload( StatusCode::kInvalidArgument, - "ExternalFile must specify at least one of 'file_content', 'file_name' " - "or 'file_descriptor_meta'.", + "ExternalFile must specify at least one of 'file_content', " + "'file_name', 'file_pointer_meta' or 'file_descriptor_meta'.", MediaPipeTasksStatus::kInvalidArgumentError); } // Obtain file descriptor, offset and size. @@ -196,6 +209,11 @@ absl::Status ExternalFileHandler::MapExternalFile() { absl::string_view ExternalFileHandler::GetFileContent() { if (!external_file_.file_content().empty()) { return external_file_.file_content(); + } else if (external_file_.has_file_pointer_meta()) { + void* ptr = + reinterpret_cast(external_file_.file_pointer_meta().pointer()); + return absl::string_view(static_cast(ptr), + external_file_.file_pointer_meta().length()); } else { return absl::string_view(static_cast(buffer_) + buffer_offset_ - buffer_aligned_offset_, diff --git a/mediapipe/tasks/cc/core/proto/external_file.proto b/mediapipe/tasks/cc/core/proto/external_file.proto index af4a11697..3147a2224 100644 --- a/mediapipe/tasks/cc/core/proto/external_file.proto +++ b/mediapipe/tasks/cc/core/proto/external_file.proto @@ -26,10 +26,11 @@ option java_outer_classname = "ExternalFileProto"; // (1) file contents loaded in `file_content`. // (2) file path in `file_name`. // (3) file descriptor through `file_descriptor_meta` as returned by open(2). +// (4) file pointer and length in memory through `file_pointer_meta`. // // If more than one field of these fields is provided, they are used in this // precedence order. -// Next id: 4 +// Next id: 5 message ExternalFile { // The file contents as a byte array. optional bytes file_content = 1; @@ -40,6 +41,13 @@ message ExternalFile { // The file descriptor to a file opened with open(2), with optional additional // offset and length information. optional FileDescriptorMeta file_descriptor_meta = 3; + + // The pointer points to location of a file in memory. Use the util method, + // `SetExternalFile` in [1], to configure `file_pointer_meta` from a + // `std::string_view` object. + // + // [1]: mediapipe/tasks/cc/metadata/utils/zip_utils.h + optional FilePointerMeta file_pointer_meta = 4; } // A proto defining file descriptor metadata for mapping file into memory using @@ -62,3 +70,14 @@ message FileDescriptorMeta { // offset of a given asset obtained from AssetFileDescriptor#getStartOffset(). optional int64 offset = 3; } + +// The pointer points to location of a file in memory. Make sure the file memory +// that it points locates on the same machine and it outlives this +// FilePointerMeta object. +message FilePointerMeta { + // Memory address of the file in decimal. + optional uint64 pointer = 1; + + // File length. + optional int64 length = 2; +} diff --git a/mediapipe/tasks/cc/metadata/utils/BUILD b/mediapipe/tasks/cc/metadata/utils/BUILD index bc11f9d38..881b88962 100644 --- a/mediapipe/tasks/cc/metadata/utils/BUILD +++ b/mediapipe/tasks/cc/metadata/utils/BUILD @@ -33,6 +33,7 @@ cc_library( ":zip_readonly_mem_file", "//mediapipe/framework/port:status", "//mediapipe/tasks/cc:common", + "//mediapipe/tasks/cc/core/proto:external_file_cc_proto", "@com_google_absl//absl/cleanup", "@com_google_absl//absl/container:flat_hash_map", "@com_google_absl//absl/status", diff --git a/mediapipe/tasks/cc/metadata/utils/zip_utils.cc b/mediapipe/tasks/cc/metadata/utils/zip_utils.cc index b703fccea..41d710e14 100644 --- a/mediapipe/tasks/cc/metadata/utils/zip_utils.cc +++ b/mediapipe/tasks/cc/metadata/utils/zip_utils.cc @@ -162,6 +162,14 @@ absl::Status ExtractFilesfromZipFile( return absl::OkStatus(); } +void SetExternalFile(const std::string_view& file_content, + core::proto::ExternalFile* model_file) { + auto pointer = reinterpret_cast(file_content.data()); + + model_file->mutable_file_pointer_meta()->set_pointer(pointer); + model_file->mutable_file_pointer_meta()->set_length(file_content.length()); +} + } // namespace metadata } // namespace tasks } // namespace mediapipe diff --git a/mediapipe/tasks/cc/metadata/utils/zip_utils.h b/mediapipe/tasks/cc/metadata/utils/zip_utils.h index 52ccf17dc..28708ba6a 100644 --- a/mediapipe/tasks/cc/metadata/utils/zip_utils.h +++ b/mediapipe/tasks/cc/metadata/utils/zip_utils.h @@ -20,6 +20,7 @@ limitations under the License. #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" +#include "mediapipe/tasks/cc/core/proto/external_file.pb.h" namespace mediapipe { namespace tasks { @@ -34,6 +35,11 @@ absl::Status ExtractFilesfromZipFile( const char* buffer_data, const size_t buffer_size, absl::flat_hash_map* files); +// Set file_pointer_meta in ExternalFile which is the pointer points to location +// of a file in memory by file_content. +void SetExternalFile(const std::string_view& file_content, + core::proto::ExternalFile* model_file); + } // namespace metadata } // namespace tasks } // namespace mediapipe diff --git a/mediapipe/tasks/cc/vision/image_classifier/image_classifier_test.cc b/mediapipe/tasks/cc/vision/image_classifier/image_classifier_test.cc index 070a5a034..dcb2fddfc 100644 --- a/mediapipe/tasks/cc/vision/image_classifier/image_classifier_test.cc +++ b/mediapipe/tasks/cc/vision/image_classifier/image_classifier_test.cc @@ -208,7 +208,7 @@ TEST_F(CreateTest, FailsWithMissingModel) { EXPECT_THAT( image_classifier.status().message(), HasSubstr("ExternalFile must specify at least one of 'file_content', " - "'file_name' or 'file_descriptor_meta'.")); + "'file_name', 'file_pointer_meta' or 'file_descriptor_meta'.")); EXPECT_THAT(image_classifier.status().GetPayload(kMediaPipeTasksPayload), Optional(absl::Cord(absl::StrCat( MediaPipeTasksStatus::kRunnerInitializationError)))); diff --git a/mediapipe/tasks/cc/vision/image_embedder/image_embedder_test.cc b/mediapipe/tasks/cc/vision/image_embedder/image_embedder_test.cc index 08a0d6a25..db1019b33 100644 --- a/mediapipe/tasks/cc/vision/image_embedder/image_embedder_test.cc +++ b/mediapipe/tasks/cc/vision/image_embedder/image_embedder_test.cc @@ -140,7 +140,7 @@ TEST_F(CreateTest, FailsWithMissingModel) { EXPECT_THAT( image_embedder.status().message(), HasSubstr("ExternalFile must specify at least one of 'file_content', " - "'file_name' or 'file_descriptor_meta'.")); + "'file_name', 'file_pointer_meta' or 'file_descriptor_meta'.")); EXPECT_THAT(image_embedder.status().GetPayload(kMediaPipeTasksPayload), Optional(absl::Cord(absl::StrCat( MediaPipeTasksStatus::kRunnerInitializationError)))); diff --git a/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_test.cc b/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_test.cc index 1d3f3e786..ab23a725c 100644 --- a/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_test.cc +++ b/mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_test.cc @@ -191,7 +191,7 @@ TEST_F(CreateFromOptionsTest, FailsWithMissingModel) { EXPECT_THAT( segmenter_or.status().message(), HasSubstr("ExternalFile must specify at least one of 'file_content', " - "'file_name' or 'file_descriptor_meta'.")); + "'file_name', 'file_pointer_meta' or 'file_descriptor_meta'.")); EXPECT_THAT(segmenter_or.status().GetPayload(kMediaPipeTasksPayload), Optional(absl::Cord(absl::StrCat( MediaPipeTasksStatus::kRunnerInitializationError)))); diff --git a/mediapipe/tasks/cc/vision/object_detector/object_detector_test.cc b/mediapipe/tasks/cc/vision/object_detector/object_detector_test.cc index 463c92566..bcc4c95ee 100644 --- a/mediapipe/tasks/cc/vision/object_detector/object_detector_test.cc +++ b/mediapipe/tasks/cc/vision/object_detector/object_detector_test.cc @@ -208,7 +208,7 @@ TEST_F(CreateFromOptionsTest, FailsWithMissingModel) { EXPECT_THAT( object_detector.status().message(), HasSubstr("ExternalFile must specify at least one of 'file_content', " - "'file_name' or 'file_descriptor_meta'.")); + "'file_name', 'file_pointer_meta' or 'file_descriptor_meta'.")); EXPECT_THAT(object_detector.status().GetPayload(kMediaPipeTasksPayload), Optional(absl::Cord(absl::StrCat( MediaPipeTasksStatus::kRunnerInitializationError)))); diff --git a/mediapipe/tasks/python/test/vision/object_detector_test.py b/mediapipe/tasks/python/test/vision/object_detector_test.py index 95b6bf867..d5cebd94b 100644 --- a/mediapipe/tasks/python/test/vision/object_detector_test.py +++ b/mediapipe/tasks/python/test/vision/object_detector_test.py @@ -119,7 +119,7 @@ class ObjectDetectorTest(parameterized.TestCase): with self.assertRaisesRegex( ValueError, r"ExternalFile must specify at least one of 'file_content', " - r"'file_name' or 'file_descriptor_meta'."): + r"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'."): base_options = _BaseOptions(model_asset_path='') options = _ObjectDetectorOptions(base_options=base_options) _ObjectDetector.create_from_options(options)