Remove GlCalculatorHelperImpl; merge with GlCalculatorHelper

Originally, there were multiple implementations of GlCalculatorHelperImpl, depending on the platform and underlying GL APIs. These have all been refactored into other components, and the remaining code in this class is unified and much reduced in size. We can get rid of this implementation detail now.

PiperOrigin-RevId: 488813220

mediapipe/gpu/BUILD

@@ -749,11 +749,9 @@ cc_library(
     name = "gl_calculator_helper",
     srcs = [
         "gl_calculator_helper.cc",
-        "gl_calculator_helper_impl_common.cc",
     ],
     hdrs = [
         "gl_calculator_helper.h",
-        "gl_calculator_helper_impl.h",
     ],
     linkopts = select({
         "//conditions:default": [],

mediapipe/gpu/gl_calculator_helper.cc

@@ -20,18 +20,32 @@
 #include "mediapipe/framework/port/canonical_errors.h"
 #include "mediapipe/framework/port/ret_check.h"
 #include "mediapipe/framework/port/status.h"
-#include "mediapipe/gpu/gl_calculator_helper_impl.h"
 #include "mediapipe/gpu/gpu_buffer.h"
 #include "mediapipe/gpu/gpu_service.h"
 
 namespace mediapipe {
 
-// The constructor and destructor need to be defined here so that
-// std::unique_ptr can see the full definition of GlCalculatorHelperImpl.
-// In the header, it is an incomplete type.
 GlCalculatorHelper::GlCalculatorHelper() {}
 
-GlCalculatorHelper::~GlCalculatorHelper() {}
+GlCalculatorHelper::~GlCalculatorHelper() {
+  if (!Initialized()) return;
+  RunInGlContext(
+      [this] {
+        if (framebuffer_) {
+          glDeleteFramebuffers(1, &framebuffer_);
+          framebuffer_ = 0;
+        }
+        return absl::OkStatus();
+      },
+      /*calculator_context=*/nullptr)
+      .IgnoreError();
+}
+
+void GlCalculatorHelper::InitializeInternal(CalculatorContext* cc,
+                                            GpuResources* gpu_resources) {
+  gpu_resources_ = gpu_resources;
+  gl_context_ = gpu_resources_->gl_context(cc);
+}
 
 absl::Status GlCalculatorHelper::Open(CalculatorContext* cc) {
   CHECK(cc);
@@ -39,19 +53,16 @@ absl::Status GlCalculatorHelper::Open(CalculatorContext* cc) {
   RET_CHECK(gpu_service.IsAvailable())
       << "GPU service not available. Did you forget to call "
          "GlCalculatorHelper::UpdateContract?";
-  // TODO return error from impl_ (needs two-stage init)
+  InitializeInternal(cc, &gpu_service.GetObject());
-  impl_ =
-      absl::make_unique<GlCalculatorHelperImpl>(cc, &gpu_service.GetObject());
   return absl::OkStatus();
 }
 
 void GlCalculatorHelper::InitializeForTest(GpuSharedData* gpu_shared) {
-  impl_ = absl::make_unique<GlCalculatorHelperImpl>(
+  InitializeInternal(nullptr, gpu_shared->gpu_resources.get());
-      nullptr, gpu_shared->gpu_resources.get());
 }
 
 void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) {
-  impl_ = absl::make_unique<GlCalculatorHelperImpl>(nullptr, gpu_resources);
+  InitializeInternal(nullptr, gpu_resources);
 }
 
 // static
@@ -88,44 +99,109 @@ absl::Status GlCalculatorHelper::SetupInputSidePackets(
   return absl::OkStatus();
 }
 
+absl::Status GlCalculatorHelper::RunInGlContext(
+    std::function<absl::Status(void)> gl_func,
+    CalculatorContext* calculator_context) {
+  if (calculator_context) {
+    return gl_context_->Run(std::move(gl_func), calculator_context->NodeId(),
+                            calculator_context->InputTimestamp());
+  } else {
+    return gl_context_->Run(std::move(gl_func));
+  }
+}
+
 absl::Status GlCalculatorHelper::RunInGlContext(
     std::function<absl::Status(void)> gl_func) {
-  if (!impl_) return absl::InternalError("helper not initialized");
+  if (!Initialized()) return absl::InternalError("helper not initialized");
   // TODO: Remove LegacyCalculatorSupport from MediaPipe OSS.
   auto calculator_context =
       LegacyCalculatorSupport::Scoped<CalculatorContext>::current();
-  return impl_->RunInGlContext(gl_func, calculator_context);
+  return RunInGlContext(gl_func, calculator_context);
 }
 
-GLuint GlCalculatorHelper::framebuffer() const { return impl_->framebuffer(); }
+GLuint GlCalculatorHelper::framebuffer() const { return framebuffer_; }
+
+void GlCalculatorHelper::CreateFramebuffer() {
+  // Our framebuffer will have a color attachment but no depth attachment,
+  // so it's important that the depth test be off. It is disabled by default,
+  // but we wanted to be explicit.
+  // TODO: move this to glBindFramebuffer?
+  glDisable(GL_DEPTH_TEST);
+  glGenFramebuffers(1, &framebuffer_);
+}
 
 void GlCalculatorHelper::BindFramebuffer(const GlTexture& dst) {
-  return impl_->BindFramebuffer(dst);
+#ifdef __ANDROID__
+  // On (some?) Android devices, attaching a new texture to the frame buffer
+  // does not seem to detach the old one. As a result, using that texture
+  // for texturing can produce incorrect output. See b/32091368 for details.
+  // To fix this, we have to call either glBindFramebuffer with a FBO id of 0
+  // or glFramebufferTexture2D with a texture ID of 0.
+  glBindFramebuffer(GL_FRAMEBUFFER, 0);
+#endif
+  if (!framebuffer_) {
+    CreateFramebuffer();
+  }
+  glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
+  glViewport(0, 0, dst.width(), dst.height());
+  glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, dst.target(),
+                         dst.name(), 0);
+
+#ifndef NDEBUG
+  GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
+  if (status != GL_FRAMEBUFFER_COMPLETE) {
+    VLOG(2) << "incomplete framebuffer: " << status;
+  }
+#endif
 }
 
-GlTexture GlCalculatorHelper::CreateSourceTexture(
+GlTexture GlCalculatorHelper::MapGpuBuffer(const GpuBuffer& gpu_buffer,
-    const GpuBuffer& pixel_buffer) {
+                                           GlTextureView view) {
-  return impl_->CreateSourceTexture(pixel_buffer);
+  if (gpu_buffer.format() != GpuBufferFormat::kUnknown) {
+    // TODO: do the params need to be reset here??
+    glBindTexture(view.target(), view.name());
+    GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
+        gpu_buffer.format(), view.plane(), GetGlVersion());
+    gl_context_->SetStandardTextureParams(view.target(),
+                                          info.gl_internal_format);
+    glBindTexture(view.target(), 0);
+  }
+
+  return GlTexture(std::move(view), gpu_buffer);
+}
+
+GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& gpu_buffer) {
+  return CreateSourceTexture(gpu_buffer, 0);
+}
+
+GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& gpu_buffer,
+                                                  int plane) {
+  return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(plane));
 }
 
 GlTexture GlCalculatorHelper::CreateSourceTexture(
     const ImageFrame& image_frame) {
-  return impl_->CreateSourceTexture(image_frame);
+  auto gpu_buffer = GpuBufferCopyingImageFrame(image_frame);
-}
+  return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(0));
-
-GlTexture GlCalculatorHelper::CreateSourceTexture(const GpuBuffer& pixel_buffer,
-                                                  int plane) {
-  return impl_->CreateSourceTexture(pixel_buffer, plane);
 }
 
 GpuBuffer GlCalculatorHelper::GpuBufferWithImageFrame(
     std::shared_ptr<ImageFrame> image_frame) {
-  return impl_->GpuBufferWithImageFrame(std::move(image_frame));
+  return GpuBuffer(
+      std::make_shared<GpuBufferStorageImageFrame>(std::move(image_frame)));
 }
 
 GpuBuffer GlCalculatorHelper::GpuBufferCopyingImageFrame(
     const ImageFrame& image_frame) {
-  return impl_->GpuBufferCopyingImageFrame(image_frame);
+#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
+  auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame);
+  // Converts absl::StatusOr to absl::Status since CHECK_OK() currently only
+  // deals with absl::Status in MediaPipe OSS.
+  CHECK_OK(maybe_buffer.status());
+  return GpuBuffer(std::move(maybe_buffer).value());
+#else
+  return GpuBuffer(GlTextureBuffer::Create(image_frame));
+#endif  // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
 }
 
 void GlCalculatorHelper::GetGpuBufferDimensions(const GpuBuffer& pixel_buffer,
@@ -136,23 +212,36 @@ void GlCalculatorHelper::GetGpuBufferDimensions(const GpuBuffer& pixel_buffer,
   *height = pixel_buffer.height();
 }
 
-GlTexture GlCalculatorHelper::CreateDestinationTexture(int output_width,
+GlTexture GlCalculatorHelper::CreateDestinationTexture(int width, int height,
-                                                       int output_height,
                                                        GpuBufferFormat format) {
-  return impl_->CreateDestinationTexture(output_width, output_height, format);
+  if (!framebuffer_) {
-}
+    CreateFramebuffer();
+  }
 
-GlContext& GlCalculatorHelper::GetGlContext() const {
+  GpuBuffer gpu_buffer =
-  return impl_->GetGlContext();
+      gpu_resources_->gpu_buffer_pool().GetBuffer(width, height, format);
-}
+  return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView<GlTextureView>(0));
-
-GlVersion GlCalculatorHelper::GetGlVersion() const {
-  return impl_->GetGlVersion();
 }
 
 GlTexture GlCalculatorHelper::CreateSourceTexture(
     const mediapipe::Image& image) {
-  return impl_->CreateSourceTexture(image.GetGpuBuffer());
+  return CreateSourceTexture(image.GetGpuBuffer());
+}
+
+template <>
+std::unique_ptr<ImageFrame> GlTexture::GetFrame<ImageFrame>() const {
+  view_->DoneWriting();
+  std::shared_ptr<const ImageFrame> view =
+      gpu_buffer_.GetReadView<ImageFrame>();
+  auto copy = absl::make_unique<ImageFrame>();
+  copy->CopyFrom(*view, ImageFrame::kDefaultAlignmentBoundary);
+  return copy;
+}
+
+template <>
+std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
+  view_->DoneWriting();
+  return absl::make_unique<GpuBuffer>(gpu_buffer_);
 }
 
 template <>

mediapipe/gpu/gl_calculator_helper.h

@@ -33,7 +33,6 @@
 
 namespace mediapipe {
 
-class GlCalculatorHelperImpl;
 class GlTexture;
 class GpuResources;
 struct GpuSharedData;
@@ -161,15 +160,30 @@ class GlCalculatorHelper {
   // TODO: do we need an unbind method too?
   void BindFramebuffer(const GlTexture& dst);
 
-  GlContext& GetGlContext() const;
+  GlContext& GetGlContext() const { return *gl_context_; }
 
-  GlVersion GetGlVersion() const;
+  GlVersion GetGlVersion() const { return gl_context_->GetGlVersion(); }
 
   // Check if the calculator helper has been previously initialized.
-  bool Initialized() { return impl_ != nullptr; }
+  bool Initialized() { return gpu_resources_ != nullptr; }
 
  private:
-  std::unique_ptr<GlCalculatorHelperImpl> impl_;
+  void InitializeInternal(CalculatorContext* cc, GpuResources* gpu_resources);
+
+  absl::Status RunInGlContext(std::function<absl::Status(void)> gl_func,
+                              CalculatorContext* calculator_context);
+
+  // Makes a GpuBuffer accessible as a texture in the GL context.
+  GlTexture MapGpuBuffer(const GpuBuffer& gpu_buffer, GlTextureView view);
+
+  // Create the framebuffer for rendering.
+  void CreateFramebuffer();
+
+  std::shared_ptr<GlContext> gl_context_;
+
+  GLuint framebuffer_ = 0;
+
+  GpuResources* gpu_resources_ = nullptr;
 };
 
 // Represents an OpenGL texture, and is a 'view' into the memory pool.
@@ -204,7 +218,7 @@ class GlTexture {
   explicit GlTexture(GlTextureView view, GpuBuffer gpu_buffer)
       : gpu_buffer_(std::move(gpu_buffer)),
         view_(std::make_shared<GlTextureView>(std::move(view))) {}
-  friend class GlCalculatorHelperImpl;
+  friend class GlCalculatorHelper;
   // We store the GpuBuffer to support GetFrame, and to ensure that the storage
   // outlives the view.
   GpuBuffer gpu_buffer_;

mediapipe/gpu/gl_calculator_helper_impl.h

@@ -1,82 +0,0 @@
-// Copyright 2019 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_
-#define MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_
-
-#include "mediapipe/gpu/gl_calculator_helper.h"
-#include "mediapipe/gpu/gpu_shared_data_internal.h"
-
-#ifdef __OBJC__
-#import <AVFoundation/AVFoundation.h>
-#import <Foundation/Foundation.h>
-#endif  // __OBJC__
-
-#ifdef __ANDROID__
-#include "mediapipe/gpu/gl_texture_buffer_pool.h"
-#endif
-
-namespace mediapipe {
-
-// This class implements the GlCalculatorHelper for iOS and Android.
-// See GlCalculatorHelper for details on these methods.
-class GlCalculatorHelperImpl {
- public:
-  explicit GlCalculatorHelperImpl(CalculatorContext* cc,
-                                  GpuResources* gpu_resources);
-  ~GlCalculatorHelperImpl();
-
-  absl::Status RunInGlContext(std::function<absl::Status(void)> gl_func,
-                              CalculatorContext* calculator_context);
-
-  GlTexture CreateSourceTexture(const ImageFrame& image_frame);
-  GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer);
-
-  // Note: multi-plane support is currently only available on iOS.
-  GlTexture CreateSourceTexture(const GpuBuffer& gpu_buffer, int plane);
-
-  // Creates a framebuffer and returns the texture that it is bound to.
-  GlTexture CreateDestinationTexture(int output_width, int output_height,
-                                     GpuBufferFormat format);
-
-  GpuBuffer GpuBufferWithImageFrame(std::shared_ptr<ImageFrame> image_frame);
-  GpuBuffer GpuBufferCopyingImageFrame(const ImageFrame& image_frame);
-
-  GLuint framebuffer() const { return framebuffer_; }
-  void BindFramebuffer(const GlTexture& dst);
-
-  GlVersion GetGlVersion() const { return gl_context_->GetGlVersion(); }
-
-  GlContext& GetGlContext() const;
-
-  // For internal use.
-  static void ReadTexture(const GlTextureView& view, void* output, size_t size);
-
- private:
-  // Makes a GpuBuffer accessible as a texture in the GL context.
-  GlTexture MapGpuBuffer(const GpuBuffer& gpu_buffer, GlTextureView view);
-
-  // Create the framebuffer for rendering.
-  void CreateFramebuffer();
-
-  std::shared_ptr<GlContext> gl_context_;
-
-  GLuint framebuffer_ = 0;
-
-  GpuResources& gpu_resources_;
-};
-
-}  // namespace mediapipe
-
-#endif  // MEDIAPIPE_GPU_GL_CALCULATOR_HELPER_IMPL_H_

mediapipe/gpu/gl_calculator_helper_impl_common.cc

@@ -1,169 +0,0 @@
-// Copyright 2019 The MediaPipe Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//      http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include <memory>
-
-#include "absl/memory/memory.h"
-#include "mediapipe/framework/formats/image_frame.h"
-#include "mediapipe/gpu/gl_calculator_helper_impl.h"
-#include "mediapipe/gpu/gpu_buffer_format.h"
-#include "mediapipe/gpu/gpu_shared_data_internal.h"
-#include "mediapipe/gpu/image_frame_view.h"
-
-namespace mediapipe {
-
-GlCalculatorHelperImpl::GlCalculatorHelperImpl(CalculatorContext* cc,
-                                               GpuResources* gpu_resources)
-    : gpu_resources_(*gpu_resources) {
-  gl_context_ = gpu_resources_.gl_context(cc);
-}
-
-GlCalculatorHelperImpl::~GlCalculatorHelperImpl() {
-  RunInGlContext(
-      [this] {
-        if (framebuffer_) {
-          glDeleteFramebuffers(1, &framebuffer_);
-          framebuffer_ = 0;
-        }
-        return absl::OkStatus();
-      },
-      /*calculator_context=*/nullptr)
-      .IgnoreError();
-}
-
-GlContext& GlCalculatorHelperImpl::GetGlContext() const { return *gl_context_; }
-
-absl::Status GlCalculatorHelperImpl::RunInGlContext(
-    std::function<absl::Status(void)> gl_func,
-    CalculatorContext* calculator_context) {
-  if (calculator_context) {
-    return gl_context_->Run(std::move(gl_func), calculator_context->NodeId(),
-                            calculator_context->InputTimestamp());
-  } else {
-    return gl_context_->Run(std::move(gl_func));
-  }
-}
-
-void GlCalculatorHelperImpl::CreateFramebuffer() {
-  // Our framebuffer will have a color attachment but no depth attachment,
-  // so it's important that the depth test be off. It is disabled by default,
-  // but we wanted to be explicit.
-  // TODO: move this to glBindFramebuffer?
-  glDisable(GL_DEPTH_TEST);
-  glGenFramebuffers(1, &framebuffer_);
-}
-
-void GlCalculatorHelperImpl::BindFramebuffer(const GlTexture& dst) {
-#ifdef __ANDROID__
-  // On (some?) Android devices, attaching a new texture to the frame buffer
-  // does not seem to detach the old one. As a result, using that texture
-  // for texturing can produce incorrect output. See b/32091368 for details.
-  // To fix this, we have to call either glBindFramebuffer with a FBO id of 0
-  // or glFramebufferTexture2D with a texture ID of 0.
-  glBindFramebuffer(GL_FRAMEBUFFER, 0);
-#endif
-  if (!framebuffer_) {
-    CreateFramebuffer();
-  }
-  glBindFramebuffer(GL_FRAMEBUFFER, framebuffer_);
-  glViewport(0, 0, dst.width(), dst.height());
-  glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, dst.target(),
-                         dst.name(), 0);
-
-#ifndef NDEBUG
-  GLenum status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
-  if (status != GL_FRAMEBUFFER_COMPLETE) {
-    VLOG(2) << "incomplete framebuffer: " << status;
-  }
-#endif
-}
-
-GlTexture GlCalculatorHelperImpl::MapGpuBuffer(const GpuBuffer& gpu_buffer,
-                                               GlTextureView view) {
-  if (gpu_buffer.format() != GpuBufferFormat::kUnknown) {
-    // TODO: do the params need to be reset here??
-    glBindTexture(view.target(), view.name());
-    GlTextureInfo info = GlTextureInfoForGpuBufferFormat(
-        gpu_buffer.format(), view.plane(), GetGlVersion());
-    gl_context_->SetStandardTextureParams(view.target(),
-                                          info.gl_internal_format);
-    glBindTexture(view.target(), 0);
-  }
-
-  return GlTexture(std::move(view), gpu_buffer);
-}
-
-GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
-    const GpuBuffer& gpu_buffer) {
-  return CreateSourceTexture(gpu_buffer, 0);
-}
-
-GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
-    const GpuBuffer& gpu_buffer, int plane) {
-  return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(plane));
-}
-
-GlTexture GlCalculatorHelperImpl::CreateSourceTexture(
-    const ImageFrame& image_frame) {
-  auto gpu_buffer = GpuBufferCopyingImageFrame(image_frame);
-  return MapGpuBuffer(gpu_buffer, gpu_buffer.GetReadView<GlTextureView>(0));
-}
-
-GpuBuffer GlCalculatorHelperImpl::GpuBufferWithImageFrame(
-    std::shared_ptr<ImageFrame> image_frame) {
-  return GpuBuffer(
-      std::make_shared<GpuBufferStorageImageFrame>(std::move(image_frame)));
-}
-
-GpuBuffer GlCalculatorHelperImpl::GpuBufferCopyingImageFrame(
-    const ImageFrame& image_frame) {
-#if MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
-  auto maybe_buffer = CreateCVPixelBufferCopyingImageFrame(image_frame);
-  // Converts absl::StatusOr to absl::Status since CHECK_OK() currently only
-  // deals with absl::Status in MediaPipe OSS.
-  CHECK_OK(maybe_buffer.status());
-  return GpuBuffer(std::move(maybe_buffer).value());
-#else
-  return GpuBuffer(GlTextureBuffer::Create(image_frame));
-#endif  // !MEDIAPIPE_GPU_BUFFER_USE_CV_PIXEL_BUFFER
-}
-
-template <>
-std::unique_ptr<ImageFrame> GlTexture::GetFrame<ImageFrame>() const {
-  view_->DoneWriting();
-  std::shared_ptr<const ImageFrame> view =
-      gpu_buffer_.GetReadView<ImageFrame>();
-  auto copy = absl::make_unique<ImageFrame>();
-  copy->CopyFrom(*view, ImageFrame::kDefaultAlignmentBoundary);
-  return copy;
-}
-
-template <>
-std::unique_ptr<GpuBuffer> GlTexture::GetFrame<GpuBuffer>() const {
-  view_->DoneWriting();
-  return absl::make_unique<GpuBuffer>(gpu_buffer_);
-}
-
-GlTexture GlCalculatorHelperImpl::CreateDestinationTexture(
-    int width, int height, GpuBufferFormat format) {
-  if (!framebuffer_) {
-    CreateFramebuffer();
-  }
-
-  GpuBuffer gpu_buffer =
-      gpu_resources_.gpu_buffer_pool().GetBuffer(width, height, format);
-  return MapGpuBuffer(gpu_buffer, gpu_buffer.GetWriteView<GlTextureView>(0));
-}
-
-}  // namespace mediapipe