mediapipe/mediapipe/gpu/gl_calculator_helper_impl_common.cc

// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <memory>

#include "absl/memory/memory.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
#include "mediapipe/gpu/image_frame_view.h"

namespace mediapipe {

GlCalculatorHelperImpl::GlCalculatorHelperImpl(CalculatorContext* cc,
                                               GpuResources* gpu_resources)
    : gpu_resources_(*gpu_resources) {
  gl_context_ = gpu_resources_.gl_context(cc);
}

GlCalculatorHelperImpl::~GlCalculatorHelperImpl() {
  RunInGlContext(
      [this] {
        if (framebuffer_) {
          glDeleteFramebuffers(1, &framebuffer_);
          framebuffer_ = 0;
        }
        return absl::OkStatus();
      },
      /*calculator_context=*/nullptr)
      .IgnoreError();
}

GlContext& GlCalculatorHelperImpl::GetGlContext() const { return *gl_context_; }

absl::Status GlCalculatorHelperImpl::RunInGlContext(
    std::function<absl::Status(void)> gl_func,
    CalculatorContext* calculator_context) {
  if (calculator_context) {
    return gl_context_->Run(std::move(gl_func), calculator_context->NodeId(),
                            calculator_context->InputTimestamp());
  } else {
    return gl_context_->Run(std::move(gl_func));
  }
}

void GlCalculatorHelperImpl::CreateFramebuffer() {
  // Our framebuffer will have a color attachment but no depth attachment,
  // so it's important that the depth test be off. It is disabled by default,
  // but we wanted to be explicit.
  // TODO: move this to glBindFramebuffer?
  glDisable(GL_DEPTH_TEST);
  glGenFramebuffers(1, &framebuffer_);
}

void GlCalculatorHelperImpl::BindFramebuffer(const GlTexture& dst) {
#ifdef __ANDROID__
  // On (some?) Android devices, attaching a new texture to the frame buffer
  // does not seem to detach the old one. As a result, using that texture
  // for texturing can produce incorrect output. See b/32091368 for details.
  // To fix this, we have to call either glBindFramebuffer with a FBO id of 0
  // or glFramebufferTexture2D with a texture ID of 0.
  glBindFramebuffer(GL_FRAMEBUFFER, 0);
#endif
  if (!framebuffer_) {
    CreateFramebuffer();
  }
  glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
  glViewport(0, 0, dst.width(), dst.height());
  glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, dst.target(),
                         dst.name(), 0);

#ifndef NDEBUG
  GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
  if (status != GL_FRAMEBUFFER_COMPLETE) {
    VLOG(2) << "incomplete framebuffer: " << status;
  }
#endif
}

GlTexture GlCalculatorHelperImpl::MapGpuBuffer(const GpuBuffer& gpu_buffer,
                                               GlTextureView view) {
  if (gpu_buffer.format() != GpuBufferFormat::kUnknown) {
    // TODO: do the params need to be reset here??
    glBindTexture(view.target(), view.name());
    GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
        gpu_buffer.format(), view.plane(), GetGlVersion());
    gl_context_->SetStandardTextureParams(view.target(),
                                          info.gl_internal_format);
    glBindTexture(view.target(), 0);
  }

  return GlTexture(std::move(view));
}

GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
    const GpuBuffer& gpu_buffer) {
  return CreateSourceTexture(gpu_buffer, 0);
}

GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
    const GpuBuffer& gpu_buffer, int plane) {
  return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(plane));
}

GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
    const ImageFrame& image_frame) {
  auto gpu_buffer = GpuBufferCopyingImageFrame(image_frame);
  return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(0));
}

GpuBuffer GlCalculatorHelperImpl::GpuBufferWithImageFrame(
    std::shared_ptr<ImageFrame> image_frame) {
  return GpuBuffer(
      std::make_shared<GpuBufferStorageImageFrame>(std::move(image_frame)));
}

GpuBuffer GlCalculatorHelperImpl::GpuBufferCopyingImageFrame(
    const ImageFrame& image_frame) {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
  auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame);
  // Converts absl::StatusOr to absl::Status since CHECK_OK() currently only
  // deals with absl::Status in MediaPipe OSS.
  CHECK_OK(maybe_buffer.status());
  return GpuBuffer(std::move(maybe_buffer).value());
#else
  return GpuBuffer(GlTextureBuffer::Create(image_frame));
#endif  // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
}

template <>
std::unique_ptr<ImageFrame> GlTexture::GetFrame<ImageFrame>() const {
  view_->DoneWriting();
  std::shared_ptr<const ImageFrame> view =
      view_->gpu_buffer().GetReadView<ImageFrame>();
  auto copy = absl::make_unique<ImageFrame>();
  copy->CopyFrom(*view, ImageFrame::kDefaultAlignmentBoundary);
  return copy;
}

template <>
std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
  auto gpu_buffer = view_->gpu_buffer();
#ifdef __EMSCRIPTEN__
  // When WebGL is used, the GL context may be spontaneously lost which can
  // cause GpuBuffer allocations to fail. In that case, return a dummy buffer
  // to allow processing of the current frame complete.
  if (!gpu_buffer) {
    return std::make_unique<GpuBuffer>();
  }
#endif  // __EMSCRIPTEN__
  view_->DoneWriting();
  return absl::make_unique<GpuBuffer>(gpu_buffer);
}

GlTexture GlCalculatorHelperImpl::CreateDestinationTexture(
    int width, int height, GpuBufferFormat format) {
  if (!framebuffer_) {
    CreateFramebuffer();
  }

  GpuBuffer gpu_buffer =
      gpu_resources_.gpu_buffer_pool().GetBuffer(width, height, format);
  return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView<GlTextureView>(0));
}

}  // namespace mediapipe