cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove GlCalculatorHelperImpl; merge with GlCalculatorHelper

Browse Source 
Originally, there were multiple implementations of GlCalculatorHelperImpl, depending on the platform and underlying GL APIs. These have all been refactored into other components, and the remaining code in this class is unified and much reduced in size. We can get rid of this implementation detail now.

PiperOrigin-RevId: 488813220
This commit is contained in:
Camillo Lugaresi 2022-11-15 18:34:35 -08:00 committed by Copybara-Service
parent 8b319e963a
commit 1979801a92
5 changed files with 146 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mediapipe/gpu/BUILD
View File

@ -749,11 +749,9 @@ cc_library(
name = "gl_calculator_helper",
srcs = [
"gl_calculator_helper.cc",
"gl_calculator_helper_impl_common.cc",
],
hdrs = [
"gl_calculator_helper.h",
"gl_calculator_helper_impl.h",
],
linkopts = select({
"//conditions:default": [],

161
mediapipe/gpu/gl_calculator_helper.cc
View File

@ -20,18 +20,32 @@
#include "mediapipe/framework/port/canonical_errors.h"
#include "mediapipe/framework/port/ret_check.h"
#include "mediapipe/framework/port/status.h"
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
#include "mediapipe/gpu/gpu_buffer.h"
#include "mediapipe/gpu/gpu_service.h"
namespace mediapipe {
// The constructor and destructor need to be defined here so that
// std::unique_ptr can see the full definition of GlCalculatorHelperImpl.
// In the header, it is an incomplete type.
GlCalculatorHelper::GlCalculatorHelper() {}
GlCalculatorHelper::~GlCalculatorHelper() {}
GlCalculatorHelper::~GlCalculatorHelper() {
if (!Initialized()) return;
RunInGlContext(
[this] {
if (framebuffer_) {
glDeleteFramebuffers(1, &framebuffer_);
framebuffer_ = 0;
}
return absl::OkStatus();
},
/*calculator_context=*/nullptr)
.IgnoreError();
}
void GlCalculatorHelper::InitializeInternal(CalculatorContext* cc,
GpuResources* gpu_resources) {
gpu_resources_ = gpu_resources;
gl_context_ = gpu_resources_->gl_context(cc);
}
absl::Status GlCalculatorHelper::Open(CalculatorContext* cc) {
CHECK(cc);
@ -39,19 +53,16 @@ absl::Status GlCalculatorHelper::Open(CalculatorContext* cc) {
RET_CHECK(gpu_service.IsAvailable())
<< "GPU service not available. Did you forget to call "
"GlCalculatorHelper::UpdateContract?";
// TODO return error from impl_ (needs two-stage init)
impl_ =
absl::make_unique<GlCalculatorHelperImpl>(cc, &gpu_service.GetObject());
InitializeInternal(cc, &gpu_service.GetObject());
return absl::OkStatus();
}
void GlCalculatorHelper::InitializeForTest(GpuSharedData* gpu_shared) {
impl_ = absl::make_unique<GlCalculatorHelperImpl>(
nullptr, gpu_shared->gpu_resources.get());
InitializeInternal(nullptr, gpu_shared->gpu_resources.get());
}
void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) {
impl_ = absl::make_unique<GlCalculatorHelperImpl>(nullptr, gpu_resources);
InitializeInternal(nullptr, gpu_resources);
}
// static
@ -88,44 +99,109 @@ absl::Status GlCalculatorHelper::SetupInputSidePackets(
return absl::OkStatus();
}
absl::Status GlCalculatorHelper::RunInGlContext(
std::function<absl::Status(void)> gl_func,
CalculatorContext* calculator_context) {
if (calculator_context) {
return gl_context_->Run(std::move(gl_func), calculator_context->NodeId(),
calculator_context->InputTimestamp());
} else {
return gl_context_->Run(std::move(gl_func));
}
}
absl::Status GlCalculatorHelper::RunInGlContext(
std::function<absl::Status(void)> gl_func) {
if (!impl_) return absl::InternalError("helper not initialized");
if (!Initialized()) return absl::InternalError("helper not initialized");
// TODO: Remove LegacyCalculatorSupport from MediaPipe OSS.
auto calculator_context =
LegacyCalculatorSupport::Scoped<CalculatorContext>::current();
return impl_->RunInGlContext(gl_func, calculator_context);
return RunInGlContext(gl_func, calculator_context);
}
GLuint GlCalculatorHelper::framebuffer() const { return impl_->framebuffer(); }
GLuint GlCalculatorHelper::framebuffer() const { return framebuffer_; }
void GlCalculatorHelper::CreateFramebuffer() {
// Our framebuffer will have a color attachment but no depth attachment,
// so it's important that the depth test be off. It is disabled by default,
// but we wanted to be explicit.
// TODO: move this to glBindFramebuffer?
glDisable(GL_DEPTH_TEST);
glGenFramebuffers(1, &framebuffer_);
}
void GlCalculatorHelper::BindFramebuffer(const GlTexture& dst) {
return impl_->BindFramebuffer(dst);
#ifdef __ANDROID__
// On (some?) Android devices, attaching a new texture to the frame buffer
// does not seem to detach the old one. As a result, using that texture
// for texturing can produce incorrect output. See b/32091368 for details.
// To fix this, we have to call either glBindFramebuffer with a FBO id of 0
// or glFramebufferTexture2D with a texture ID of 0.
glBindFramebuffer(GL_FRAMEBUFFER, 0);
#endif
if (!framebuffer_) {
CreateFramebuffer();
}
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
glViewport(0, 0, dst.width(), dst.height());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, dst.target(),
dst.name(), 0);
#ifndef NDEBUG
GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (status != GL_FRAMEBUFFER_COMPLETE) {
VLOG(2) << "incomplete framebuffer: " << status;
}
#endif
}
GlTexture GlCalculatorHelper::CreateSourceTexture(
const GpuBuffer& pixel_buffer) {
return impl_->CreateSourceTexture(pixel_buffer);
GlTexture GlCalculatorHelper::MapGpuBuffer(const GpuBuffer& gpu_buffer,
GlTextureView view) {
if (gpu_buffer.format() != GpuBufferFormat::kUnknown) {
// TODO: do the params need to be reset here??
glBindTexture(view.target(), view.name());
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
gpu_buffer.format(), view.plane(), GetGlVersion());
gl_context_->SetStandardTextureParams(view.target(),
info.gl_internal_format);
glBindTexture(view.target(), 0);
}
return GlTexture(std::move(view), gpu_buffer);
}
GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& gpu_buffer) {
return CreateSourceTexture(gpu_buffer, 0);
}
GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& gpu_buffer,
int plane) {
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(plane));
}
GlTexture GlCalculatorHelper::CreateSourceTexture(
const ImageFrame& image_frame) {
return impl_->CreateSourceTexture(image_frame);
}
GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& pixel_buffer,
int plane) {
return impl_->CreateSourceTexture(pixel_buffer, plane);
auto gpu_buffer = GpuBufferCopyingImageFrame(image_frame);
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(0));
}
GpuBuffer GlCalculatorHelper::GpuBufferWithImageFrame(
std::shared_ptr<ImageFrame> image_frame) {
return impl_->GpuBufferWithImageFrame(std::move(image_frame));
return GpuBuffer(
std::make_shared<GpuBufferStorageImageFrame>(std::move(image_frame)));
}
GpuBuffer GlCalculatorHelper::GpuBufferCopyingImageFrame(
const ImageFrame& image_frame) {
return impl_->GpuBufferCopyingImageFrame(image_frame);
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame);
// Converts absl::StatusOr to absl::Status since CHECK_OK() currently only
// deals with absl::Status in MediaPipe OSS.
CHECK_OK(maybe_buffer.status());
return GpuBuffer(std::move(maybe_buffer).value());
#else
return GpuBuffer(GlTextureBuffer::Create(image_frame));
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
}
void GlCalculatorHelper::GetGpuBufferDimensions(const GpuBuffer& pixel_buffer,
@ -136,23 +212,36 @@ void GlCalculatorHelper::GetGpuBufferDimensions(const GpuBuffer& pixel_buffer,
*height = pixel_buffer.height();
}
GlTexture GlCalculatorHelper::CreateDestinationTexture(int output_width,
int output_height,
GlTexture GlCalculatorHelper::CreateDestinationTexture(int width, int height,
GpuBufferFormat format) {
return impl_->CreateDestinationTexture(output_width, output_height, format);
if (!framebuffer_) {
CreateFramebuffer();
}
GlContext& GlCalculatorHelper::GetGlContext() const {
return impl_->GetGlContext();
}
GlVersion GlCalculatorHelper::GetGlVersion() const {
return impl_->GetGlVersion();
GpuBuffer gpu_buffer =
gpu_resources_->gpu_buffer_pool().GetBuffer(width, height, format);
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView<GlTextureView>(0));
}
GlTexture GlCalculatorHelper::CreateSourceTexture(
const mediapipe::Image& image) {
return impl_->CreateSourceTexture(image.GetGpuBuffer());
return CreateSourceTexture(image.GetGpuBuffer());
}
template <>
std::unique_ptr<ImageFrame> GlTexture::GetFrame<ImageFrame>() const {
view_->DoneWriting();
std::shared_ptr<const ImageFrame> view =
gpu_buffer_.GetReadView<ImageFrame>();
auto copy = absl::make_unique<ImageFrame>();
copy->CopyFrom(*view, ImageFrame::kDefaultAlignmentBoundary);
return copy;
}
template <>
std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
view_->DoneWriting();
return absl::make_unique<GpuBuffer>(gpu_buffer_);
}
template <>

26
mediapipe/gpu/gl_calculator_helper.h
View File

@ -33,7 +33,6 @@
namespace mediapipe {
class GlCalculatorHelperImpl;
class GlTexture;
class GpuResources;
struct GpuSharedData;
@ -161,15 +160,30 @@ class GlCalculatorHelper {
// TODO: do we need an unbind method too?
void BindFramebuffer(const GlTexture& dst);
GlContext& GetGlContext() const;
GlContext& GetGlContext() const { return *gl_context_; }
GlVersion GetGlVersion() const;
GlVersion GetGlVersion() const { return gl_context_->GetGlVersion(); }
// Check if the calculator helper has been previously initialized.
bool Initialized() { return impl_ != nullptr; }
bool Initialized() { return gpu_resources_ != nullptr; }
private:
std::unique_ptr<GlCalculatorHelperImpl> impl_;
void InitializeInternal(CalculatorContext* cc, GpuResources* gpu_resources);
absl::Status RunInGlContext(std::function<absl::Status(void)> gl_func,
CalculatorContext* calculator_context);
// Makes a GpuBuffer accessible as a texture in the GL context.
GlTexture MapGpuBuffer(const GpuBuffer& gpu_buffer, GlTextureView view);
// Create the framebuffer for rendering.
void CreateFramebuffer();
std::shared_ptr<GlContext> gl_context_;
GLuint framebuffer_ = 0;
GpuResources* gpu_resources_ = nullptr;
};
// Represents an OpenGL texture, and is a 'view' into the memory pool.
@ -204,7 +218,7 @@ class GlTexture {
explicit GlTexture(GlTextureView view, GpuBuffer gpu_buffer)
: gpu_buffer_(std::move(gpu_buffer)),
view_(std::make_shared<GlTextureView>(std::move(view))) {}
friend class GlCalculatorHelperImpl;
friend class GlCalculatorHelper;
// We store the GpuBuffer to support GetFrame, and to ensure that the storage
// outlives the view.
GpuBuffer gpu_buffer_;

82
mediapipe/gpu/gl_calculator_helper_impl.h
View File

@ -1,82 +0,0 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_
#define MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_
#include "mediapipe/gpu/gl_calculator_helper.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
#ifdef __OBJC__
#import <AVFoundation/AVFoundation.h>
#import <Foundation/Foundation.h>
#endif // __OBJC__
#ifdef __ANDROID__
#include "mediapipe/gpu/gl_texture_buffer_pool.h"
#endif
namespace mediapipe {
// This class implements the GlCalculatorHelper for iOS and Android.
// See GlCalculatorHelper for details on these methods.
class GlCalculatorHelperImpl {
public:
explicit GlCalculatorHelperImpl(CalculatorContext* cc,
GpuResources* gpu_resources);
~GlCalculatorHelperImpl();
absl::Status RunInGlContext(std::function<absl::Status(void)> gl_func,
CalculatorContext* calculator_context);
GlTexture CreateSourceTexture(const ImageFrame& image_frame);
GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer);
// Note: multi-plane support is currently only available on iOS.
GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer, int plane);
// Creates a framebuffer and returns the texture that it is bound to.
GlTexture CreateDestinationTexture(int output_width, int output_height,
GpuBufferFormat format);
GpuBuffer GpuBufferWithImageFrame(std::shared_ptr<ImageFrame> image_frame);
GpuBuffer GpuBufferCopyingImageFrame(const ImageFrame& image_frame);
GLuint framebuffer() const { return framebuffer_; }
void BindFramebuffer(const GlTexture& dst);
GlVersion GetGlVersion() const { return gl_context_->GetGlVersion(); }
GlContext& GetGlContext() const;
// For internal use.
static void ReadTexture(const GlTextureView& view, void* output, size_t size);
private:
// Makes a GpuBuffer accessible as a texture in the GL context.
GlTexture MapGpuBuffer(const GpuBuffer& gpu_buffer, GlTextureView view);
// Create the framebuffer for rendering.
void CreateFramebuffer();
std::shared_ptr<GlContext> gl_context_;
GLuint framebuffer_ = 0;
GpuResources& gpu_resources_;
};
} // namespace mediapipe
#endif // MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_

169
mediapipe/gpu/gl_calculator_helper_impl_common.cc
View File

@ -1,169 +0,0 @@
// Copyright 2019 The MediaPipe Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include "absl/memory/memory.h"
#include "mediapipe/framework/formats/image_frame.h"
#include "mediapipe/gpu/gl_calculator_helper_impl.h"
#include "mediapipe/gpu/gpu_buffer_format.h"
#include "mediapipe/gpu/gpu_shared_data_internal.h"
#include "mediapipe/gpu/image_frame_view.h"
namespace mediapipe {
GlCalculatorHelperImpl::GlCalculatorHelperImpl(CalculatorContext* cc,
GpuResources* gpu_resources)
: gpu_resources_(*gpu_resources) {
gl_context_ = gpu_resources_.gl_context(cc);
}
GlCalculatorHelperImpl::~GlCalculatorHelperImpl() {
RunInGlContext(
[this] {
if (framebuffer_) {
glDeleteFramebuffers(1, &framebuffer_);
framebuffer_ = 0;
}
return absl::OkStatus();
},
/*calculator_context=*/nullptr)
.IgnoreError();
}
GlContext& GlCalculatorHelperImpl::GetGlContext() const { return *gl_context_; }
absl::Status GlCalculatorHelperImpl::RunInGlContext(
std::function<absl::Status(void)> gl_func,
CalculatorContext* calculator_context) {
if (calculator_context) {
return gl_context_->Run(std::move(gl_func), calculator_context->NodeId(),
calculator_context->InputTimestamp());
} else {
return gl_context_->Run(std::move(gl_func));
}
}
void GlCalculatorHelperImpl::CreateFramebuffer() {
// Our framebuffer will have a color attachment but no depth attachment,
// so it's important that the depth test be off. It is disabled by default,
// but we wanted to be explicit.
// TODO: move this to glBindFramebuffer?
glDisable(GL_DEPTH_TEST);
glGenFramebuffers(1, &framebuffer_);
}
void GlCalculatorHelperImpl::BindFramebuffer(const GlTexture& dst) {
#ifdef __ANDROID__
// On (some?) Android devices, attaching a new texture to the frame buffer
// does not seem to detach the old one. As a result, using that texture
// for texturing can produce incorrect output. See b/32091368 for details.
// To fix this, we have to call either glBindFramebuffer with a FBO id of 0
// or glFramebufferTexture2D with a texture ID of 0.
glBindFramebuffer(GL_FRAMEBUFFER, 0);
#endif
if (!framebuffer_) {
CreateFramebuffer();
}
glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
glViewport(0, 0, dst.width(), dst.height());
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, dst.target(),
dst.name(), 0);
#ifndef NDEBUG
GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
if (status != GL_FRAMEBUFFER_COMPLETE) {
VLOG(2) << "incomplete framebuffer: " << status;
}
#endif
}
GlTexture GlCalculatorHelperImpl::MapGpuBuffer(const GpuBuffer& gpu_buffer,
GlTextureView view) {
if (gpu_buffer.format() != GpuBufferFormat::kUnknown) {
// TODO: do the params need to be reset here??
glBindTexture(view.target(), view.name());
GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
gpu_buffer.format(), view.plane(), GetGlVersion());
gl_context_->SetStandardTextureParams(view.target(),
info.gl_internal_format);
glBindTexture(view.target(), 0);
}
return GlTexture(std::move(view), gpu_buffer);
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const GpuBuffer& gpu_buffer) {
return CreateSourceTexture(gpu_buffer, 0);
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const GpuBuffer& gpu_buffer, int plane) {
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(plane));
}
GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
const ImageFrame& image_frame) {
auto gpu_buffer = GpuBufferCopyingImageFrame(image_frame);
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(0));
}
GpuBuffer GlCalculatorHelperImpl::GpuBufferWithImageFrame(
std::shared_ptr<ImageFrame> image_frame) {
return GpuBuffer(
std::make_shared<GpuBufferStorageImageFrame>(std::move(image_frame)));
}
GpuBuffer GlCalculatorHelperImpl::GpuBufferCopyingImageFrame(
const ImageFrame& image_frame) {
#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame);
// Converts absl::StatusOr to absl::Status since CHECK_OK() currently only
// deals with absl::Status in MediaPipe OSS.
CHECK_OK(maybe_buffer.status());
return GpuBuffer(std::move(maybe_buffer).value());
#else
return GpuBuffer(GlTextureBuffer::Create(image_frame));
#endif // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
}
template <>
std::unique_ptr<ImageFrame> GlTexture::GetFrame<ImageFrame>() const {
view_->DoneWriting();
std::shared_ptr<const ImageFrame> view =
gpu_buffer_.GetReadView<ImageFrame>();
auto copy = absl::make_unique<ImageFrame>();
copy->CopyFrom(*view, ImageFrame::kDefaultAlignmentBoundary);
return copy;
}
template <>
std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
view_->DoneWriting();
return absl::make_unique<GpuBuffer>(gpu_buffer_);
}
GlTexture GlCalculatorHelperImpl::CreateDestinationTexture(
int width, int height, GpuBufferFormat format) {
if (!framebuffer_) {
CreateFramebuffer();
}
GpuBuffer gpu_buffer =
gpu_resources_.gpu_buffer_pool().GetBuffer(width, height, format);
return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView<GlTextureView>(0));
}
} // namespace mediapipe