Internal change

PiperOrigin-RevId: 526658482
This commit is contained in:
Sebastian Schmidt 2023-04-24 08:59:16 -07:00 committed by Copybara-Service
parent abded49e5b
commit 35cf8c35f2
7 changed files with 113 additions and 38 deletions

View File

@ -61,15 +61,13 @@ cc_library(
"//mediapipe/framework/port:status", "//mediapipe/framework/port:status",
"//mediapipe/tasks/cc:common", "//mediapipe/tasks/cc:common",
"//mediapipe/tasks/cc/core/proto:external_file_cc_proto", "//mediapipe/tasks/cc/core/proto:external_file_cc_proto",
"//mediapipe/util:resource_util",
"@com_google_absl//absl/memory", "@com_google_absl//absl/memory",
"@com_google_absl//absl/status", "@com_google_absl//absl/status",
"@com_google_absl//absl/status:statusor", "@com_google_absl//absl/status:statusor",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
"@com_google_absl//absl/strings:str_format", "@com_google_absl//absl/strings:str_format",
] + select({ ],
"//mediapipe:windows": ["@bazel_tools//tools/cpp/runfiles"],
"//conditions:default": [],
}),
) )
cc_library( cc_library(

View File

@ -43,10 +43,7 @@ limitations under the License.
#include "mediapipe/framework/port/status_macros.h" #include "mediapipe/framework/port/status_macros.h"
#include "mediapipe/tasks/cc/common.h" #include "mediapipe/tasks/cc/common.h"
#include "mediapipe/tasks/cc/core/proto/external_file.pb.h" #include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
#include "mediapipe/util/resource_util.h"
#ifdef _WIN32
#include "tools/cpp/runfiles/runfiles.h"
#endif // _WIN32
namespace mediapipe { namespace mediapipe {
namespace tasks { namespace tasks {
@ -96,30 +93,6 @@ ExternalFileHandler::CreateFromExternalFile(
return handler; return handler;
} }
absl::StatusOr<std::string> PathToResourceAsFile(std::string path) {
#ifndef _WIN32
return path;
#else
std::string qualified_path = path;
if (absl::StartsWith(qualified_path, "./")) {
qualified_path = "mediapipe" + qualified_path.substr(1);
} else if (path[0] != '/') {
qualified_path = "mediapipe/" + qualified_path;
}
std::string error;
// TODO: We should ideally use `CreateForTests` when this is
// accessed from unit tests.
std::unique_ptr<::bazel::tools::cpp::runfiles::Runfiles> runfiles(
::bazel::tools::cpp::runfiles::Runfiles::Create("", &error));
if (!runfiles) {
// Return the original path when Runfiles is not available (e.g. for Python)
return path;
}
return runfiles->Rlocation(qualified_path);
#endif // _WIN32
}
absl::Status ExternalFileHandler::MapExternalFile() { absl::Status ExternalFileHandler::MapExternalFile() {
if (!external_file_.file_content().empty()) { if (!external_file_.file_content().empty()) {
return absl::OkStatus(); return absl::OkStatus();

View File

@ -25,6 +25,7 @@ cc_library(
"vocab_utils.h", "vocab_utils.h",
], ],
deps = [ deps = [
"//mediapipe/util:resource_util",
"@com_google_absl//absl/container:node_hash_map", "@com_google_absl//absl/container:node_hash_map",
"@com_google_absl//absl/strings", "@com_google_absl//absl/strings",
], ],

View File

@ -18,6 +18,7 @@ limitations under the License.
#include <fstream> #include <fstream>
#include "absl/strings/str_split.h" #include "absl/strings/str_split.h"
#include "mediapipe/util/resource_util.h"
namespace mediapipe { namespace mediapipe {
namespace tasks { namespace tasks {
@ -34,10 +35,14 @@ void ReadIStreamLineByLine(
std::string str; std::string str;
while (std::getline(*istream, str)) { while (std::getline(*istream, str)) {
if (!str.empty()) { if (!str.empty()) {
if (str.back() == '\r') { // Remove \r on Windows
line_processor(str.substr(0, str.length() - 1));
} else {
line_processor(str); line_processor(str);
} }
} }
} }
}
absl::node_hash_map<std::string, int> ReadIStreamLineSplits( absl::node_hash_map<std::string, int> ReadIStreamLineSplits(
std::istream* istream) { std::istream* istream) {
@ -64,7 +69,8 @@ std::vector<std::string> ReadIStreamByLine(std::istream* istream) {
std::vector<std::string> LoadVocabFromFile(const std::string& path_to_vocab) { std::vector<std::string> LoadVocabFromFile(const std::string& path_to_vocab) {
std::vector<std::string> vocab_from_file; std::vector<std::string> vocab_from_file;
std::ifstream in(path_to_vocab.c_str()); std::string file_name = *PathToResourceAsFile(path_to_vocab);
std::ifstream in(file_name.c_str());
return ReadIStreamByLine(&in); return ReadIStreamByLine(&in);
} }
@ -79,7 +85,8 @@ std::vector<std::string> LoadVocabFromBuffer(const char* vocab_buffer_data,
absl::node_hash_map<std::string, int> LoadVocabAndIndexFromFile( absl::node_hash_map<std::string, int> LoadVocabAndIndexFromFile(
const std::string& path_to_vocab) { const std::string& path_to_vocab) {
absl::node_hash_map<std::string, int> vocab_index_map; absl::node_hash_map<std::string, int> vocab_index_map;
std::ifstream in(path_to_vocab.c_str()); std::string file_name = *PathToResourceAsFile(path_to_vocab);
std::ifstream in(file_name.c_str());
return ReadIStreamLineSplits(&in); return ReadIStreamLineSplits(&in);
} }

View File

@ -194,6 +194,7 @@ cc_library(
"//mediapipe/framework:android_no_jni": ["resource_util_loonix.cc"], "//mediapipe/framework:android_no_jni": ["resource_util_loonix.cc"],
"//mediapipe:ios": ["resource_util_apple.cc"], "//mediapipe:ios": ["resource_util_apple.cc"],
"//mediapipe:macos": ["resource_util_default.cc"], "//mediapipe:macos": ["resource_util_default.cc"],
"//mediapipe:windows": ["resource_util_windows.cc"],
}), }),
hdrs = [ hdrs = [
"resource_util.h", "resource_util.h",
@ -232,6 +233,10 @@ cc_library(
"//mediapipe:macos": [ "//mediapipe:macos": [
"@com_google_absl//absl/flags:flag", "@com_google_absl//absl/flags:flag",
], ],
"//mediapipe:windows": [
"@bazel_tools//tools/cpp/runfiles",
"@com_google_absl//absl/flags:flag",
],
}), }),
) )

View File

@ -0,0 +1,91 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <fstream>
#include "absl/flags/flag.h"
#include "mediapipe/framework/deps/file_path.h"
#include "mediapipe/framework/port/file_helpers.h"
#include "mediapipe/framework/port/singleton.h"
#include "mediapipe/framework/port/statusor.h"
#include "tools/cpp/runfiles/runfiles.h"
ABSL_FLAG(
std::string, resource_root_dir, "",
"The absolute path to the resource directory."
"If specified, resource_root_dir will be prepended to the original path.");
namespace mediapipe {
using mediapipe::file::GetContents;
using mediapipe::file::JoinPath;
namespace {
class RunfilesHolder {
public:
// TODO: We should ideally use `CreateForTests` when this is
// accessed from unit tests.
RunfilesHolder()
: runfiles_(
::bazel::tools::cpp::runfiles::Runfiles::Create("", nullptr)) {}
std::string Rlocation(const std::string& path) {
if (!runfiles_) {
// Return the original path when Runfiles is not available (e.g. for
// Python)
return JoinPath(absl::GetFlag(FLAGS_resource_root_dir), path);
}
return runfiles_->Rlocation(path);
}
private:
std::unique_ptr<::bazel::tools::cpp::runfiles::Runfiles> runfiles_;
};
} // namespace
namespace internal {
std::string PathToResourceAsFileInternal(const std::string& path) {
return Singleton<RunfilesHolder>::get()->Rlocation(path);
}
absl::Status DefaultGetResourceContents(const std::string& path,
std::string* output,
bool read_as_binary) {
std::string resource_path = PathToResourceAsFileInternal(path);
return GetContents(path, output, read_as_binary);
}
} // namespace internal
absl::StatusOr<std::string> PathToResourceAsFile(const std::string& path) {
std::string qualified_path = path;
if (absl::StartsWith(qualified_path, "./")) {
qualified_path = "mediapipe" + qualified_path.substr(1);
} else if (path[0] != '/') {
qualified_path = "mediapipe/" + qualified_path;
}
// Try to load the file from bazel-bin. If it does not exist, fall back to the
// resource folder.
auto bazel_path = internal::PathToResourceAsFileInternal(qualified_path);
if (file::Exists(bazel_path).ok()) {
return bazel_path;
}
return JoinPath(absl::GetFlag(FLAGS_resource_root_dir), path);
}
} // namespace mediapipe