#include "mediapipe/gpu/gpu_buffer.h" #include #include #include "absl/functional/bind_front.h" #include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "mediapipe/framework/port/logging.h" #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER #include "mediapipe/objc/util.h" #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER namespace mediapipe { namespace { struct StorageTypeFormatter { void operator()(std::string* out, const std::shared_ptr& s) const { absl::StrAppend(out, s->storage_type().name()); } }; } // namespace std::string GpuBuffer::DebugString() const { return holder_ ? absl::StrCat("GpuBuffer[", width(), "x", height(), " ", format(), " as ", holder_->DebugString(), "]") : "GpuBuffer[invalid]"; } std::string GpuBuffer::StorageHolder::DebugString() const { absl::MutexLock lock(&mutex_); return absl::StrJoin(storages_, ", ", StorageTypeFormatter()); } internal::GpuBufferStorage* GpuBuffer::StorageHolder::GetStorageForView( TypeId view_provider_type, bool for_writing) const { std::shared_ptr chosen_storage; std::function()> conversion; { absl::MutexLock lock(&mutex_); // First see if any current storage supports the view. for (const auto& s : storages_) { if (s->can_down_cast_to(view_provider_type)) { chosen_storage = s; break; } } // Then try to convert existing storages to one that does. // TODO: choose best conversion. if (!chosen_storage) { for (const auto& s : storages_) { if (auto converter = internal::GpuBufferStorageRegistry::Get() .StorageConverterForViewProvider( view_provider_type, s->storage_type())) { conversion = absl::bind_front(converter, s); break; } } } } // Avoid invoking a converter or factory while holding the mutex. // Two reasons: // 1. Readers that don't need a conversion will not be blocked. // 2. We use mutexes to make sure GL contexts are not used simultaneously on // different threads, and we also rely on Mutex's deadlock detection // heuristic, which enforces a consistent mutex acquisition order. // This function is likely to be called within a GL context, and the // conversion function may in turn use a GL context, and this may cause a // false positive in the deadlock detector. // TODO: we could use Mutex::ForgetDeadlockInfo instead. if (conversion) { auto new_storage = conversion(); absl::MutexLock lock(&mutex_); // Another reader might have already completed and inserted the same // conversion. TODO: prevent this? for (const auto& s : storages_) { if (s->can_down_cast_to(view_provider_type)) { chosen_storage = s; break; } } if (!chosen_storage) { storages_.push_back(std::move(new_storage)); chosen_storage = storages_.back(); } } if (for_writing) { // This will temporarily hold storages to be released, and do so while the // lock is not held (see above). decltype(storages_) old_storages; using std::swap; if (chosen_storage) { // Discard all other storages. absl::MutexLock lock(&mutex_); swap(old_storages, storages_); storages_ = {chosen_storage}; } else { // Allocate a new storage supporting the requested view. if (auto factory = internal::GpuBufferStorageRegistry::Get() .StorageFactoryForViewProvider(view_provider_type)) { if (auto new_storage = factory(width_, height_, format_)) { absl::MutexLock lock(&mutex_); swap(old_storages, storages_); storages_ = {std::move(new_storage)}; chosen_storage = storages_.back(); } } } } // It is ok to return a non-owning storage pointer here because this object // ensures the storage's lifetime. Overwriting a GpuBuffer while readers are // active would violate this, but it's not allowed in MediaPipe. return chosen_storage ? chosen_storage.get() : nullptr; } internal::GpuBufferStorage& GpuBuffer::GetStorageForViewOrDie( TypeId view_provider_type, bool for_writing) const { auto* chosen_storage = GpuBuffer::GetStorageForView(view_provider_type, for_writing); ABSL_CHECK(chosen_storage) << "no view provider found for requested view " << view_provider_type.name() << "; storages available: " << (holder_ ? holder_->DebugString() : "invalid"); ABSL_DCHECK(chosen_storage->can_down_cast_to(view_provider_type)); return *chosen_storage; } #if !MEDIAPIPE_DISABLE_GPU #if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER CVPixelBufferRef GetCVPixelBufferRef(const GpuBuffer& buffer) { if (buffer.GetStorageForView( kTypeId>, /*for_writing=*/false) != nullptr) { return *buffer.GetReadView(); } return nullptr; } #endif // MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER #endif // !MEDIAPIPE_DISABLE_GPU } // namespace mediapipe