From 0d5f35d3511bb24a4b1710cb4dd0139ae33d4c47 Mon Sep 17 00:00:00 2001 From: Sebastian Schmidt Date: Tue, 10 Oct 2023 10:20:37 -0700 Subject: [PATCH] No public description PiperOrigin-RevId: 572292160 --- mediapipe/calculators/image/BUILD | 1 - .../image/image_clone_calculator.cc | 11 +++---- .../segmentation_smoothing_calculator.cc | 20 ++++++------ .../image/warp_affine_calculator.cc | 24 +++----------- .../tensors_to_detections_calculator.cc | 22 ++++++------- .../tensors_to_segmentation_calculator.cc | 31 +++++++------------ mediapipe/gpu/gl_calculator_helper.cc | 9 ++---- mediapipe/gpu/gl_calculator_helper.h | 3 +- mediapipe/python/pybind/calculator_graph.cc | 5 --- 9 files changed, 43 insertions(+), 83 deletions(-) diff --git a/mediapipe/calculators/image/BUILD b/mediapipe/calculators/image/BUILD index e32e7165d..259012d64 100644 --- a/mediapipe/calculators/image/BUILD +++ b/mediapipe/calculators/image/BUILD @@ -786,7 +786,6 @@ cc_library( ":affine_transformation_runner_gl", "//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gpu_buffer", - "//mediapipe/gpu:gpu_service", ], }) + select({ "//mediapipe/framework/port:disable_opencv": [], diff --git a/mediapipe/calculators/image/image_clone_calculator.cc b/mediapipe/calculators/image/image_clone_calculator.cc index 563b4a4ad..6660d55b4 100644 --- a/mediapipe/calculators/image/image_clone_calculator.cc +++ b/mediapipe/calculators/image/image_clone_calculator.cc @@ -64,8 +64,7 @@ class ImageCloneCalculator : public Node { "GPU processing is disabled in build flags"); } #else - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( - cc, /*requesst_gpu_as_optional=*/true)); + MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); #endif // MEDIAPIPE_DISABLE_GPU return absl::OkStatus(); } @@ -73,6 +72,9 @@ class ImageCloneCalculator : public Node { absl::Status Open(CalculatorContext* cc) override { const auto& options = cc->Options(); output_on_gpu_ = options.output_on_gpu(); +#if !MEDIAPIPE_DISABLE_GPU + MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); +#endif // !MEDIAPIPE_DISABLE_GPU return absl::OkStatus(); } @@ -102,10 +104,6 @@ class ImageCloneCalculator : public Node { if (output_on_gpu_ && !input_on_gpu) { #if !MEDIAPIPE_DISABLE_GPU - if (!gpu_initialized_) { - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); - gpu_initialized_ = true; - } gpu_helper_.RunInGlContext([&output]() { output->ConvertToGpu(); }); #endif // !MEDIAPIPE_DISABLE_GPU } else if (!output_on_gpu_ && input_on_gpu) { @@ -120,7 +118,6 @@ class ImageCloneCalculator : public Node { bool output_on_gpu_; #if !MEDIAPIPE_DISABLE_GPU mediapipe::GlCalculatorHelper gpu_helper_; - bool gpu_initialized_ = false; #endif // !MEDIAPIPE_DISABLE_GPU }; MEDIAPIPE_REGISTER_NODE(ImageCloneCalculator); diff --git a/mediapipe/calculators/image/segmentation_smoothing_calculator.cc b/mediapipe/calculators/image/segmentation_smoothing_calculator.cc index d238975c6..1194412a6 100644 --- a/mediapipe/calculators/image/segmentation_smoothing_calculator.cc +++ b/mediapipe/calculators/image/segmentation_smoothing_calculator.cc @@ -117,8 +117,7 @@ absl::Status SegmentationSmoothingCalculator::GetContract( cc->Outputs().Tag(kOutputMaskTag).Set(); #if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( - cc, /*requesst_gpu_as_optional=*/true)); + MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); #endif // !MEDIAPIPE_DISABLE_GPU return absl::OkStatus(); @@ -131,6 +130,10 @@ absl::Status SegmentationSmoothingCalculator::Open(CalculatorContext* cc) { cc->Options(); combine_with_previous_ratio_ = options.combine_with_previous_ratio(); +#if !MEDIAPIPE_DISABLE_GPU + MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); +#endif // !MEDIAPIPE_DISABLE_GPU + return absl::OkStatus(); } @@ -151,9 +154,6 @@ absl::Status SegmentationSmoothingCalculator::Process(CalculatorContext* cc) { if (use_gpu) { #if !MEDIAPIPE_DISABLE_GPU - if (!gpu_initialized_) { - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); - } MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { if (!gpu_initialized_) { MP_RETURN_IF_ERROR(GlSetup(cc)); @@ -178,12 +178,10 @@ absl::Status SegmentationSmoothingCalculator::Process(CalculatorContext* cc) { absl::Status SegmentationSmoothingCalculator::Close(CalculatorContext* cc) { #if !MEDIAPIPE_DISABLE_GPU - if (gpu_initialized_) { - gpu_helper_.RunInGlContext([this] { - if (program_) glDeleteProgram(program_); - program_ = 0; - }); - } + gpu_helper_.RunInGlContext([this] { + if (program_) glDeleteProgram(program_); + program_ = 0; + }); #endif // !MEDIAPIPE_DISABLE_GPU return absl::OkStatus(); diff --git a/mediapipe/calculators/image/warp_affine_calculator.cc b/mediapipe/calculators/image/warp_affine_calculator.cc index c02eb9f14..1d19c661d 100644 --- a/mediapipe/calculators/image/warp_affine_calculator.cc +++ b/mediapipe/calculators/image/warp_affine_calculator.cc @@ -36,7 +36,6 @@ #if !MEDIAPIPE_DISABLE_GPU #include "mediapipe/gpu/gl_calculator_helper.h" #include "mediapipe/gpu/gpu_buffer.h" -#include "mediapipe/gpu/gpu_service.h" #endif // !MEDIAPIPE_DISABLE_GPU namespace mediapipe { @@ -107,7 +106,6 @@ class WarpAffineRunnerHolder { cc->Options().interpolation()); return gl_helper_->Open(cc); } - absl::StatusOr GetRunner() { if (!runner_) { MP_ASSIGN_OR_RETURN( @@ -144,10 +142,7 @@ class WarpAffineRunnerHolder { MP_RETURN_IF_ERROR(cpu_holder_.Open(cc)); #endif // !MEDIAPIPE_DISABLE_OPENCV #if !MEDIAPIPE_DISABLE_GPU - if (cc->Service(kGpuService).IsAvailable()) { - MP_RETURN_IF_ERROR(gpu_holder_.Open(cc)); - gpu_holder_initialized_ = true; - } + MP_RETURN_IF_ERROR(gpu_holder_.Open(cc)); #endif // !MEDIAPIPE_DISABLE_GPU return absl::OkStatus(); } @@ -156,9 +151,6 @@ class WarpAffineRunnerHolder { const AffineTransformation::Size& size, AffineTransformation::BorderMode border_mode) override { if (input.UsesGpu()) { - if (!gpu_holder_initialized_) { - return absl::UnavailableError("GPU support is not available"); - } #if !MEDIAPIPE_DISABLE_GPU MP_ASSIGN_OR_RETURN(auto* runner, gpu_holder_.GetRunner()); MP_ASSIGN_OR_RETURN( @@ -191,7 +183,6 @@ class WarpAffineRunnerHolder { #endif // !MEDIAPIPE_DISABLE_OPENCV #if !MEDIAPIPE_DISABLE_GPU WarpAffineRunnerHolder gpu_holder_; - bool gpu_holder_initialized_ = false; #endif // !MEDIAPIPE_DISABLE_GPU }; @@ -205,24 +196,20 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl { static absl::Status UpdateContract(CalculatorContract* cc) { if constexpr (std::is_same_v || std::is_same_v) { - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( - cc, /*requesst_gpu_as_optional=*/true)); + MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); } return absl::OkStatus(); } #endif // !MEDIAPIPE_DISABLE_GPU + + absl::Status Open(CalculatorContext* cc) override { return holder_.Open(cc); } + absl::Status Process(CalculatorContext* cc) override { if (InterfaceT::kInImage(cc).IsEmpty() || InterfaceT::kMatrix(cc).IsEmpty() || InterfaceT::kOutputSize(cc).IsEmpty()) { return absl::OkStatus(); } - - if (!holder_initialized_) { - MP_RETURN_IF_ERROR(holder_.Open(cc)); - holder_initialized_ = true; - } - const std::array& transform = *InterfaceT::kMatrix(cc); auto [out_width, out_height] = *InterfaceT::kOutputSize(cc); AffineTransformation::Size output_size; @@ -243,7 +230,6 @@ class WarpAffineCalculatorImpl : public mediapipe::api2::NodeImpl { private: WarpAffineRunnerHolder holder_; - bool holder_initialized_ = false; }; } // namespace diff --git a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc index 8e649c0a1..6d42226b9 100644 --- a/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_detections_calculator.cc @@ -266,8 +266,7 @@ absl::Status TensorsToDetectionsCalculator::UpdateContract( CalculatorContract* cc) { if (CanUseGpu()) { #ifndef MEDIAPIPE_DISABLE_GL_COMPUTE - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( - cc, /*requesst_gpu_as_optional=*/true)); + MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); #elif MEDIAPIPE_METAL_ENABLED MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); #endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) @@ -281,6 +280,7 @@ absl::Status TensorsToDetectionsCalculator::Open(CalculatorContext* cc) { if (CanUseGpu()) { #ifndef MEDIAPIPE_DISABLE_GL_COMPUTE + MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); #elif MEDIAPIPE_METAL_ENABLED gpu_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; RET_CHECK(gpu_helper_); @@ -676,15 +676,13 @@ absl::Status TensorsToDetectionsCalculator::ProcessGPU( absl::Status TensorsToDetectionsCalculator::Close(CalculatorContext* cc) { #ifndef MEDIAPIPE_DISABLE_GL_COMPUTE - if (gpu_inited_) { - gpu_helper_.RunInGlContext([this] { - decoded_boxes_buffer_ = nullptr; - scored_boxes_buffer_ = nullptr; - raw_anchors_buffer_ = nullptr; - glDeleteProgram(decode_program_); - glDeleteProgram(score_program_); - }); - } + gpu_helper_.RunInGlContext([this] { + decoded_boxes_buffer_ = nullptr; + scored_boxes_buffer_ = nullptr; + raw_anchors_buffer_ = nullptr; + glDeleteProgram(decode_program_); + glDeleteProgram(score_program_); + }); #elif MEDIAPIPE_METAL_ENABLED decoded_boxes_buffer_ = nullptr; scored_boxes_buffer_ = nullptr; @@ -944,7 +942,6 @@ absl::Status TensorsToDetectionsCalculator::GpuInit(CalculatorContext* cc) { break; } #ifndef MEDIAPIPE_DISABLE_GL_COMPUTE - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, output_format_flag]() -> absl::Status { // A shader to decode detection boxes. @@ -1423,6 +1420,7 @@ kernel void scoreKernel( num_classes_, max_wg_size)); } } + #endif // !defined(MEDIAPIPE_DISABLE_GL_COMPUTE) return absl::OkStatus(); diff --git a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc index 6456126ae..6b77acee4 100644 --- a/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc +++ b/mediapipe/calculators/tensor/tensors_to_segmentation_calculator.cc @@ -173,7 +173,6 @@ class TensorsToSegmentationCalculator : public CalculatorBase { #if !MEDIAPIPE_DISABLE_GPU mediapipe::GlCalculatorHelper gpu_helper_; GLuint upsample_program_; - bool gpu_initialized_ = false; #if MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 int cached_width_ = 0; int cached_height_ = 0; @@ -207,8 +206,7 @@ absl::Status TensorsToSegmentationCalculator::GetContract( if (CanUseGpu()) { #if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract( - cc, /*requesst_gpu_as_optional=*/true)); + MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); #if MEDIAPIPE_METAL_ENABLED MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); #endif // MEDIAPIPE_METAL_ENABLED @@ -220,9 +218,12 @@ absl::Status TensorsToSegmentationCalculator::GetContract( absl::Status TensorsToSegmentationCalculator::Open(CalculatorContext* cc) { cc->SetOffset(TimestampDiff(0)); + bool use_gpu = false; if (CanUseGpu()) { #if !MEDIAPIPE_DISABLE_GPU + use_gpu = true; + MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); #if MEDIAPIPE_METAL_ENABLED metal_helper_ = [[MPPMetalHelper alloc] initWithCalculatorContext:cc]; RET_CHECK(metal_helper_); @@ -232,6 +233,14 @@ absl::Status TensorsToSegmentationCalculator::Open(CalculatorContext* cc) { MP_RETURN_IF_ERROR(LoadOptions(cc)); + if (use_gpu) { +#if !MEDIAPIPE_DISABLE_GPU + MP_RETURN_IF_ERROR(InitGpu(cc)); +#else + RET_CHECK_FAIL() << "GPU processing disabled."; +#endif // !MEDIAPIPE_DISABLE_GPU + } + return absl::OkStatus(); } @@ -276,15 +285,6 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { } if (use_gpu) { -#if !MEDIAPIPE_DISABLE_GPU - if (!gpu_initialized_) { - MP_RETURN_IF_ERROR(InitGpu(cc)); - gpu_initialized_ = true; - } -#else - RET_CHECK_FAIL() << "GPU processing disabled."; -#endif // !MEDIAPIPE_DISABLE_GPU - #if !MEDIAPIPE_DISABLE_GPU MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this, cc]() -> absl::Status { MP_RETURN_IF_ERROR(ProcessGpu(cc)); @@ -306,10 +306,6 @@ absl::Status TensorsToSegmentationCalculator::Process(CalculatorContext* cc) { absl::Status TensorsToSegmentationCalculator::Close(CalculatorContext* cc) { #if !MEDIAPIPE_DISABLE_GPU - if (!gpu_initialized_) { - return absl::OkStatus(); - } - gpu_helper_.RunInGlContext([this] { if (upsample_program_) glDeleteProgram(upsample_program_); upsample_program_ = 0; @@ -639,7 +635,6 @@ absl::Status TensorsToSegmentationCalculator::LoadOptions( absl::Status TensorsToSegmentationCalculator::InitGpu(CalculatorContext* cc) { #if !MEDIAPIPE_DISABLE_GPU - MP_RETURN_IF_ERROR(gpu_helper_.Open(cc)); MP_RETURN_IF_ERROR(gpu_helper_.RunInGlContext([this]() -> absl::Status { // A shader to process a segmentation tensor into an output mask. // Currently uses 4 channels for output, and sets R+A channels as mask value. @@ -904,8 +899,6 @@ void main() { return absl::OkStatus(); })); - - gpu_initialized_ = true; #endif // !MEDIAPIPE_DISABLE_GPU return absl::OkStatus(); diff --git a/mediapipe/gpu/gl_calculator_helper.cc b/mediapipe/gpu/gl_calculator_helper.cc index 1b113f8ac..763ac387a 100644 --- a/mediapipe/gpu/gl_calculator_helper.cc +++ b/mediapipe/gpu/gl_calculator_helper.cc @@ -56,13 +56,8 @@ void GlCalculatorHelper::InitializeForTest(GpuResources* gpu_resources) { } // static -absl::Status GlCalculatorHelper::UpdateContract(CalculatorContract* cc, - bool requesst_gpu_as_optional) { - if (requesst_gpu_as_optional) { - cc->UseService(kGpuService).Optional(); - } else { - cc->UseService(kGpuService); - } +absl::Status GlCalculatorHelper::UpdateContract(CalculatorContract* cc) { + cc->UseService(kGpuService); // Allow the legacy side packet to be provided, too, for backwards // compatibility with existing graphs. It will just be ignored. auto& input_side_packets = cc->InputSidePackets(); diff --git a/mediapipe/gpu/gl_calculator_helper.h b/mediapipe/gpu/gl_calculator_helper.h index f5d98ebfe..b6430860f 100644 --- a/mediapipe/gpu/gl_calculator_helper.h +++ b/mediapipe/gpu/gl_calculator_helper.h @@ -67,8 +67,7 @@ class GlCalculatorHelper { // This method can be called from GetContract to set up the needed GPU // resources. - static absl::Status UpdateContract(CalculatorContract* cc, - bool requesst_gpu_as_optional = false); + static absl::Status UpdateContract(CalculatorContract* cc); // This method can be called from FillExpectations to set the correct types // for the shared GL input side packet(s). diff --git a/mediapipe/python/pybind/calculator_graph.cc b/mediapipe/python/pybind/calculator_graph.cc index dfff78635..b017ca38b 100644 --- a/mediapipe/python/pybind/calculator_graph.cc +++ b/mediapipe/python/pybind/calculator_graph.cc @@ -112,11 +112,6 @@ void CalculatorGraphSubmodule(pybind11::module* module) { "graph with a ValidatedGraphConfig object."); } auto calculator_graph = absl::make_unique(); - // Disable default service initialization. This allows us to use - // the CPU versions of calculators that only optionally request - // kGpuService. - RaisePyErrorIfNotOk( - calculator_graph->DisallowServiceDefaultInitialization()); RaisePyErrorIfNotOk(calculator_graph->Initialize(graph_config_proto)); return calculator_graph.release(); }),