Update external files to support file_pointer_meta.
PiperOrigin-RevId: 479695361
This commit is contained in:
parent
b616bc4427
commit
65e1d722eb
|
@ -184,7 +184,7 @@ TEST_F(CreateFromOptionsTest, FailsWithMissingModel) {
|
|||
EXPECT_THAT(
|
||||
audio_classifier_or.status().message(),
|
||||
HasSubstr("ExternalFile must specify at least one of 'file_content', "
|
||||
"'file_name' or 'file_descriptor_meta'."));
|
||||
"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'."));
|
||||
EXPECT_THAT(audio_classifier_or.status().GetPayload(kMediaPipeTasksPayload),
|
||||
Optional(absl::Cord(absl::StrCat(
|
||||
MediaPipeTasksStatus::kRunnerInitializationError))));
|
||||
|
|
|
@ -92,13 +92,26 @@ absl::Status ExternalFileHandler::MapExternalFile() {
|
|||
#else
|
||||
if (!external_file_.file_content().empty()) {
|
||||
return absl::OkStatus();
|
||||
} else if (external_file_.has_file_pointer_meta()) {
|
||||
if (external_file_.file_pointer_meta().pointer() == 0) {
|
||||
return CreateStatusWithPayload(
|
||||
StatusCode::kInvalidArgument,
|
||||
"Need to set the file pointer in external_file.file_pointer_meta.");
|
||||
}
|
||||
if (external_file_.file_pointer_meta().length() <= 0) {
|
||||
return CreateStatusWithPayload(
|
||||
StatusCode::kInvalidArgument,
|
||||
"The length of the file in external_file.file_pointer_meta should be "
|
||||
"positive.");
|
||||
}
|
||||
return absl::OkStatus();
|
||||
}
|
||||
if (external_file_.file_name().empty() &&
|
||||
!external_file_.has_file_descriptor_meta()) {
|
||||
return CreateStatusWithPayload(
|
||||
StatusCode::kInvalidArgument,
|
||||
"ExternalFile must specify at least one of 'file_content', 'file_name' "
|
||||
"or 'file_descriptor_meta'.",
|
||||
"ExternalFile must specify at least one of 'file_content', "
|
||||
"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'.",
|
||||
MediaPipeTasksStatus::kInvalidArgumentError);
|
||||
}
|
||||
// Obtain file descriptor, offset and size.
|
||||
|
@ -196,6 +209,11 @@ absl::Status ExternalFileHandler::MapExternalFile() {
|
|||
absl::string_view ExternalFileHandler::GetFileContent() {
|
||||
if (!external_file_.file_content().empty()) {
|
||||
return external_file_.file_content();
|
||||
} else if (external_file_.has_file_pointer_meta()) {
|
||||
void* ptr =
|
||||
reinterpret_cast<void*>(external_file_.file_pointer_meta().pointer());
|
||||
return absl::string_view(static_cast<const char*>(ptr),
|
||||
external_file_.file_pointer_meta().length());
|
||||
} else {
|
||||
return absl::string_view(static_cast<const char*>(buffer_) +
|
||||
buffer_offset_ - buffer_aligned_offset_,
|
||||
|
|
|
@ -26,10 +26,11 @@ option java_outer_classname = "ExternalFileProto";
|
|||
// (1) file contents loaded in `file_content`.
|
||||
// (2) file path in `file_name`.
|
||||
// (3) file descriptor through `file_descriptor_meta` as returned by open(2).
|
||||
// (4) file pointer and length in memory through `file_pointer_meta`.
|
||||
//
|
||||
// If more than one field of these fields is provided, they are used in this
|
||||
// precedence order.
|
||||
// Next id: 4
|
||||
// Next id: 5
|
||||
message ExternalFile {
|
||||
// The file contents as a byte array.
|
||||
optional bytes file_content = 1;
|
||||
|
@ -40,6 +41,13 @@ message ExternalFile {
|
|||
// The file descriptor to a file opened with open(2), with optional additional
|
||||
// offset and length information.
|
||||
optional FileDescriptorMeta file_descriptor_meta = 3;
|
||||
|
||||
// The pointer points to location of a file in memory. Use the util method,
|
||||
// `SetExternalFile` in [1], to configure `file_pointer_meta` from a
|
||||
// `std::string_view` object.
|
||||
//
|
||||
// [1]: mediapipe/tasks/cc/metadata/utils/zip_utils.h
|
||||
optional FilePointerMeta file_pointer_meta = 4;
|
||||
}
|
||||
|
||||
// A proto defining file descriptor metadata for mapping file into memory using
|
||||
|
@ -62,3 +70,14 @@ message FileDescriptorMeta {
|
|||
// offset of a given asset obtained from AssetFileDescriptor#getStartOffset().
|
||||
optional int64 offset = 3;
|
||||
}
|
||||
|
||||
// The pointer points to location of a file in memory. Make sure the file memory
|
||||
// that it points locates on the same machine and it outlives this
|
||||
// FilePointerMeta object.
|
||||
message FilePointerMeta {
|
||||
// Memory address of the file in decimal.
|
||||
optional uint64 pointer = 1;
|
||||
|
||||
// File length.
|
||||
optional int64 length = 2;
|
||||
}
|
||||
|
|
|
@ -33,6 +33,7 @@ cc_library(
|
|||
":zip_readonly_mem_file",
|
||||
"//mediapipe/framework/port:status",
|
||||
"//mediapipe/tasks/cc:common",
|
||||
"//mediapipe/tasks/cc/core/proto:external_file_cc_proto",
|
||||
"@com_google_absl//absl/cleanup",
|
||||
"@com_google_absl//absl/container:flat_hash_map",
|
||||
"@com_google_absl//absl/status",
|
||||
|
|
|
@ -162,6 +162,14 @@ absl::Status ExtractFilesfromZipFile(
|
|||
return absl::OkStatus();
|
||||
}
|
||||
|
||||
void SetExternalFile(const std::string_view& file_content,
|
||||
core::proto::ExternalFile* model_file) {
|
||||
auto pointer = reinterpret_cast<uint64_t>(file_content.data());
|
||||
|
||||
model_file->mutable_file_pointer_meta()->set_pointer(pointer);
|
||||
model_file->mutable_file_pointer_meta()->set_length(file_content.length());
|
||||
}
|
||||
|
||||
} // namespace metadata
|
||||
} // namespace tasks
|
||||
} // namespace mediapipe
|
||||
|
|
|
@ -20,6 +20,7 @@ limitations under the License.
|
|||
|
||||
#include "absl/container/flat_hash_map.h"
|
||||
#include "absl/status/status.h"
|
||||
#include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
|
||||
|
||||
namespace mediapipe {
|
||||
namespace tasks {
|
||||
|
@ -34,6 +35,11 @@ absl::Status ExtractFilesfromZipFile(
|
|||
const char* buffer_data, const size_t buffer_size,
|
||||
absl::flat_hash_map<std::string, absl::string_view>* files);
|
||||
|
||||
// Set file_pointer_meta in ExternalFile which is the pointer points to location
|
||||
// of a file in memory by file_content.
|
||||
void SetExternalFile(const std::string_view& file_content,
|
||||
core::proto::ExternalFile* model_file);
|
||||
|
||||
} // namespace metadata
|
||||
} // namespace tasks
|
||||
} // namespace mediapipe
|
||||
|
|
|
@ -208,7 +208,7 @@ TEST_F(CreateTest, FailsWithMissingModel) {
|
|||
EXPECT_THAT(
|
||||
image_classifier.status().message(),
|
||||
HasSubstr("ExternalFile must specify at least one of 'file_content', "
|
||||
"'file_name' or 'file_descriptor_meta'."));
|
||||
"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'."));
|
||||
EXPECT_THAT(image_classifier.status().GetPayload(kMediaPipeTasksPayload),
|
||||
Optional(absl::Cord(absl::StrCat(
|
||||
MediaPipeTasksStatus::kRunnerInitializationError))));
|
||||
|
|
|
@ -140,7 +140,7 @@ TEST_F(CreateTest, FailsWithMissingModel) {
|
|||
EXPECT_THAT(
|
||||
image_embedder.status().message(),
|
||||
HasSubstr("ExternalFile must specify at least one of 'file_content', "
|
||||
"'file_name' or 'file_descriptor_meta'."));
|
||||
"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'."));
|
||||
EXPECT_THAT(image_embedder.status().GetPayload(kMediaPipeTasksPayload),
|
||||
Optional(absl::Cord(absl::StrCat(
|
||||
MediaPipeTasksStatus::kRunnerInitializationError))));
|
||||
|
|
|
@ -191,7 +191,7 @@ TEST_F(CreateFromOptionsTest, FailsWithMissingModel) {
|
|||
EXPECT_THAT(
|
||||
segmenter_or.status().message(),
|
||||
HasSubstr("ExternalFile must specify at least one of 'file_content', "
|
||||
"'file_name' or 'file_descriptor_meta'."));
|
||||
"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'."));
|
||||
EXPECT_THAT(segmenter_or.status().GetPayload(kMediaPipeTasksPayload),
|
||||
Optional(absl::Cord(absl::StrCat(
|
||||
MediaPipeTasksStatus::kRunnerInitializationError))));
|
||||
|
|
|
@ -208,7 +208,7 @@ TEST_F(CreateFromOptionsTest, FailsWithMissingModel) {
|
|||
EXPECT_THAT(
|
||||
object_detector.status().message(),
|
||||
HasSubstr("ExternalFile must specify at least one of 'file_content', "
|
||||
"'file_name' or 'file_descriptor_meta'."));
|
||||
"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'."));
|
||||
EXPECT_THAT(object_detector.status().GetPayload(kMediaPipeTasksPayload),
|
||||
Optional(absl::Cord(absl::StrCat(
|
||||
MediaPipeTasksStatus::kRunnerInitializationError))));
|
||||
|
|
|
@ -119,7 +119,7 @@ class ObjectDetectorTest(parameterized.TestCase):
|
|||
with self.assertRaisesRegex(
|
||||
ValueError,
|
||||
r"ExternalFile must specify at least one of 'file_content', "
|
||||
r"'file_name' or 'file_descriptor_meta'."):
|
||||
r"'file_name', 'file_pointer_meta' or 'file_descriptor_meta'."):
|
||||
base_options = _BaseOptions(model_asset_path='')
|
||||
options = _ObjectDetectorOptions(base_options=base_options)
|
||||
_ObjectDetector.create_from_options(options)
|
||||
|
|
Loading…
Reference in New Issue
Block a user