2019-06-17 01:03:25 +02:00
|
|
|
// Copyright 2019 The MediaPipe Authors.
|
|
|
|
//
|
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
2021-06-24 23:10:25 +02:00
|
|
|
#include <memory>
|
|
|
|
|
2022-03-21 20:07:37 +01:00
|
|
|
#include "absl/memory/memory.h"
|
2021-06-24 23:10:25 +02:00
|
|
|
#include "mediapipe/framework/formats/image_frame.h"
|
2019-06-17 01:03:25 +02:00
|
|
|
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
|
2019-09-30 19:18:09 +02:00
|
|
|
#include "mediapipe/gpu/gpu_buffer_format.h"
|
2019-06-17 01:03:25 +02:00
|
|
|
#include "mediapipe/gpu/gpu_shared_data_internal.h"
|
2022-03-21 20:07:37 +01:00
|
|
|
#include "mediapipe/gpu/image_frame_view.h"
|
2019-06-17 01:03:25 +02:00
|
|
|
|
|
|
|
namespace mediapipe {
|
|
|
|
|
|
|
|
GlCalculatorHelperImpl::GlCalculatorHelperImpl(CalculatorContext* cc,
|
|
|
|
GpuResources* gpu_resources)
|
|
|
|
: gpu_resources_(*gpu_resources) {
|
|
|
|
gl_context_ = gpu_resources_.gl_context(cc);
|
|
|
|
}
|
|
|
|
|
|
|
|
GlCalculatorHelperImpl::~GlCalculatorHelperImpl() {
|
|
|
|
RunInGlContext(
|
|
|
|
[this] {
|
|
|
|
if (framebuffer_) {
|
|
|
|
glDeleteFramebuffers(1, &framebuffer_);
|
|
|
|
framebuffer_ = 0;
|
|
|
|
}
|
2021-02-27 09:21:16 +01:00
|
|
|
return absl::OkStatus();
|
2019-06-17 01:03:25 +02:00
|
|
|
},
|
|
|
|
/*calculator_context=*/nullptr)
|
|
|
|
.IgnoreError();
|
|
|
|
}
|
|
|
|
|
|
|
|
GlContext& GlCalculatorHelperImpl::GetGlContext() const { return *gl_context_; }
|
|
|
|
|
2021-02-27 09:21:16 +01:00
|
|
|
absl::Status GlCalculatorHelperImpl::RunInGlContext(
|
|
|
|
std::function<absl::Status(void)> gl_func,
|
2019-06-17 01:03:25 +02:00
|
|
|
CalculatorContext* calculator_context) {
|
|
|
|
if (calculator_context) {
|
|
|
|
return gl_context_->Run(std::move(gl_func), calculator_context->NodeId(),
|
|
|
|
calculator_context->InputTimestamp());
|
|
|
|
} else {
|
|
|
|
return gl_context_->Run(std::move(gl_func));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void GlCalculatorHelperImpl::CreateFramebuffer() {
|
|
|
|
// Our framebuffer will have a color attachment but no depth attachment,
|
|
|
|
// so it's important that the depth test be off. It is disabled by default,
|
|
|
|
// but we wanted to be explicit.
|
|
|
|
// TODO: move this to glBindFramebuffer?
|
|
|
|
glDisable(GL_DEPTH_TEST);
|
|
|
|
glGenFramebuffers(1, &framebuffer_);
|
|
|
|
}
|
|
|
|
|
|
|
|
void GlCalculatorHelperImpl::BindFramebuffer(const GlTexture& dst) {
|
|
|
|
#ifdef __ANDROID__
|
|
|
|
// On (some?) Android devices, attaching a new texture to the frame buffer
|
|
|
|
// does not seem to detach the old one. As a result, using that texture
|
|
|
|
// for texturing can produce incorrect output. See b/32091368 for details.
|
|
|
|
// To fix this, we have to call either glBindFramebuffer with a FBO id of 0
|
|
|
|
// or glFramebufferTexture2D with a texture ID of 0.
|
|
|
|
glBindFramebuffer(GL_FRAMEBUFFER, 0);
|
|
|
|
#endif
|
|
|
|
if (!framebuffer_) {
|
|
|
|
CreateFramebuffer();
|
|
|
|
}
|
|
|
|
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
|
|
|
|
glViewport(0, 0, dst.width(), dst.height());
|
|
|
|
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, dst.target(),
|
|
|
|
dst.name(), 0);
|
|
|
|
|
|
|
|
#ifndef NDEBUG
|
|
|
|
GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
|
|
|
|
if (status != GL_FRAMEBUFFER_COMPLETE) {
|
|
|
|
VLOG(2) << "incomplete framebuffer: " << status;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2021-10-06 22:44:33 +02:00
|
|
|
GlTexture GlCalculatorHelperImpl::MapGpuBuffer(const GpuBuffer& gpu_buffer,
|
2021-10-18 21:39:29 +02:00
|
|
|
GlTextureView view) {
|
2021-10-06 22:44:33 +02:00
|
|
|
if (gpu_buffer.format() != GpuBufferFormat::kUnknown) {
|
|
|
|
// TODO: do the params need to be reset here??
|
|
|
|
glBindTexture(view.target(), view.name());
|
|
|
|
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
|
|
|
|
gpu_buffer.format(), view.plane(), GetGlVersion());
|
|
|
|
gl_context_->SetStandardTextureParams(view.target(),
|
|
|
|
info.gl_internal_format);
|
|
|
|
glBindTexture(view.target(), 0);
|
2019-09-30 19:18:09 +02:00
|
|
|
}
|
2019-06-17 01:03:25 +02:00
|
|
|
|
2021-10-06 22:44:33 +02:00
|
|
|
return GlTexture(std::move(view));
|
2019-06-17 01:03:25 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
|
|
|
|
const GpuBuffer& gpu_buffer) {
|
2021-10-18 21:39:29 +02:00
|
|
|
return CreateSourceTexture(gpu_buffer, 0);
|
2019-06-17 01:03:25 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
|
|
|
|
const GpuBuffer& gpu_buffer, int plane) {
|
2021-11-03 22:21:54 +01:00
|
|
|
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(plane));
|
2019-06-17 01:03:25 +02:00
|
|
|
}
|
|
|
|
|
2021-10-06 22:44:33 +02:00
|
|
|
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
|
|
|
|
const ImageFrame& image_frame) {
|
2022-03-21 20:07:37 +01:00
|
|
|
auto gpu_buffer = GpuBufferCopyingImageFrame(image_frame);
|
2021-11-03 22:21:54 +01:00
|
|
|
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(0));
|
2019-06-17 01:03:25 +02:00
|
|
|
}
|
|
|
|
|
2022-03-21 20:07:37 +01:00
|
|
|
GpuBuffer GlCalculatorHelperImpl::GpuBufferWithImageFrame(
|
|
|
|
std::shared_ptr<ImageFrame> image_frame) {
|
|
|
|
return GpuBuffer(
|
|
|
|
std::make_shared<GpuBufferStorageImageFrame>(std::move(image_frame)));
|
|
|
|
}
|
|
|
|
|
|
|
|
GpuBuffer GlCalculatorHelperImpl::GpuBufferCopyingImageFrame(
|
|
|
|
const ImageFrame& image_frame) {
|
|
|
|
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
|
|
|
auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame);
|
|
|
|
// Converts absl::StatusOr to absl::Status since CHECK_OK() currently only
|
|
|
|
// deals with absl::Status in MediaPipe OSS.
|
|
|
|
CHECK_OK(maybe_buffer.status());
|
|
|
|
return GpuBuffer(std::move(maybe_buffer).value());
|
|
|
|
#else
|
|
|
|
return GpuBuffer(GlTextureBuffer::Create(image_frame));
|
|
|
|
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
|
|
|
|
}
|
|
|
|
|
2021-10-06 22:44:33 +02:00
|
|
|
template <>
|
|
|
|
std::unique_ptr<ImageFrame> GlTexture::GetFrame<ImageFrame>() const {
|
2022-03-21 20:07:37 +01:00
|
|
|
view_->DoneWriting();
|
|
|
|
std::shared_ptr<const ImageFrame> view =
|
|
|
|
view_->gpu_buffer().GetReadView<ImageFrame>();
|
|
|
|
auto copy = absl::make_unique<ImageFrame>();
|
|
|
|
copy->CopyFrom(*view, ImageFrame::kDefaultAlignmentBoundary);
|
|
|
|
return copy;
|
2021-10-06 22:44:33 +02:00
|
|
|
}
|
2021-02-27 09:21:16 +01:00
|
|
|
|
2021-10-06 22:44:33 +02:00
|
|
|
template <>
|
|
|
|
std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
|
2022-03-21 20:07:37 +01:00
|
|
|
auto gpu_buffer = view_->gpu_buffer();
|
2021-10-06 22:44:33 +02:00
|
|
|
#ifdef __EMSCRIPTEN__
|
|
|
|
// When WebGL is used, the GL context may be spontaneously lost which can
|
|
|
|
// cause GpuBuffer allocations to fail. In that case, return a dummy buffer
|
|
|
|
// to allow processing of the current frame complete.
|
|
|
|
if (!gpu_buffer) {
|
|
|
|
return std::make_unique<GpuBuffer>();
|
2021-02-27 09:21:16 +01:00
|
|
|
}
|
2021-10-06 22:44:33 +02:00
|
|
|
#endif // __EMSCRIPTEN__
|
2022-03-21 20:07:37 +01:00
|
|
|
view_->DoneWriting();
|
2021-10-06 22:44:33 +02:00
|
|
|
return absl::make_unique<GpuBuffer>(gpu_buffer);
|
2019-06-17 01:03:25 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
GlTexture GlCalculatorHelperImpl::CreateDestinationTexture(
|
|
|
|
int width, int height, GpuBufferFormat format) {
|
|
|
|
if (!framebuffer_) {
|
|
|
|
CreateFramebuffer();
|
|
|
|
}
|
|
|
|
|
2021-10-18 21:39:29 +02:00
|
|
|
GpuBuffer gpu_buffer =
|
2019-06-17 01:03:25 +02:00
|
|
|
gpu_resources_.gpu_buffer_pool().GetBuffer(width, height, format);
|
2021-11-03 22:21:54 +01:00
|
|
|
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView<GlTextureView>(0));
|
2019-06-17 01:03:25 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
} // namespace mediapipe
|