From 89857f33a2b645338188b73129ea99d591258b3c Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 13 Mar 2023 16:41:03 -0700 Subject: [PATCH] Make ImageToTensorCalculator use kGpuService optionally PiperOrigin-RevId: 516358053 --- mediapipe/calculators/tensor/BUILD | 16 ++-- .../tensor/image_to_tensor_calculator.cc | 4 +- .../tensor/image_to_tensor_calculator_test.cc | 78 +++++++++++++++++++ 3 files changed, 92 insertions(+), 6 deletions(-) diff --git a/mediapipe/calculators/tensor/BUILD b/mediapipe/calculators/tensor/BUILD index 7a29c3af8..a76b75494 100644 --- a/mediapipe/calculators/tensor/BUILD +++ b/mediapipe/calculators/tensor/BUILD @@ -997,17 +997,20 @@ cc_library( ":image_to_tensor_converter_gl_buffer", "//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gpu_buffer", + "//mediapipe/gpu:gpu_service", ], "//mediapipe:apple": [ ":image_to_tensor_converter_metal", "//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:MPPMetalHelper", "//mediapipe/gpu:gpu_buffer", + "//mediapipe/gpu:gpu_service", ], "//conditions:default": [ ":image_to_tensor_converter_gl_buffer", "//mediapipe/gpu:gl_calculator_helper", "//mediapipe/gpu:gpu_buffer", + "//mediapipe/gpu:gpu_service", ], }), ) @@ -1045,6 +1048,10 @@ cc_test( ":image_to_tensor_calculator", ":image_to_tensor_converter", ":image_to_tensor_utils", + "@com_google_absl//absl/flags:flag", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:str_format", "//mediapipe/framework:calculator_framework", "//mediapipe/framework:calculator_runner", "//mediapipe/framework/deps:file_path", @@ -1061,11 +1068,10 @@ cc_test( "//mediapipe/framework/port:opencv_imgproc", "//mediapipe/framework/port:parse_text_proto", "//mediapipe/util:image_test_utils", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:str_format", - ], + ] + select({ + "//mediapipe:apple": [], + "//conditions:default": ["//mediapipe/gpu:gl_context"], + }), ) cc_library( diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc index 499b497b0..d15d35086 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_calculator.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_calculator.cc @@ -45,9 +45,11 @@ #elif MEDIAPIPE_OPENGL_ES_VERSION >= MEDIAPIPE_OPENGL_ES_31 #include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_buffer.h" #include "mediapipe/gpu/gl_calculator_helper.h" +#include "mediapipe/gpu/gpu_service.h" #else #include "mediapipe/calculators/tensor/image_to_tensor_converter_gl_texture.h" #include "mediapipe/gpu/gl_calculator_helper.h" +#include "mediapipe/gpu/gpu_service.h" #endif // MEDIAPIPE_METAL_ENABLED #endif // !MEDIAPIPE_DISABLE_GPU @@ -147,7 +149,7 @@ class ImageToTensorCalculator : public Node { #if MEDIAPIPE_METAL_ENABLED MP_RETURN_IF_ERROR([MPPMetalHelper updateContract:cc]); #else - MP_RETURN_IF_ERROR(mediapipe::GlCalculatorHelper::UpdateContract(cc)); + cc->UseService(kGpuService).Optional(); #endif // MEDIAPIPE_METAL_ENABLED #endif // MEDIAPIPE_DISABLE_GPU diff --git a/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc b/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc index ed7d93886..3795b1fa0 100644 --- a/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc +++ b/mediapipe/calculators/tensor/image_to_tensor_calculator_test.cc @@ -41,6 +41,10 @@ #include "mediapipe/framework/port/status_matchers.h" #include "mediapipe/util/image_test_utils.h" +#if !MEDIAPIPE_DISABLE_GPU && !MEDIAPIPE_METAL_ENABLED +#include "mediapipe/gpu/gl_context.h" +#endif // !MEDIAPIPE_DISABLE_GPU && !MEDIAPIPE_METAL_ENABLED + namespace mediapipe { namespace { @@ -507,5 +511,79 @@ TEST(ImageToTensorCalculatorTest, NoOpExceptRangeAndUseInputImageDims) { /*tensor_width=*/std::nullopt, /*tensor_height=*/std::nullopt, /*keep_aspect=*/false, BorderMode::kZero, roi); } + +TEST(ImageToTensorCalculatorTest, CanBeUsedWithoutGpuServiceSet) { + auto graph_config = + mediapipe::ParseTextProtoOrDie(R"pb( + input_stream: "input_image" + node { + calculator: "ImageToTensorCalculator" + input_stream: "IMAGE:input_image" + output_stream: "TENSORS:tensor" + options { + [mediapipe.ImageToTensorCalculatorOptions.ext] { + output_tensor_float_range { min: 0.0f max: 1.0f } + } + } + } + )pb"); + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config)); + MP_ASSERT_OK(graph.DisallowServiceDefaultInitialization()); + MP_ASSERT_OK(graph.StartRun({})); + auto image_frame = + std::make_shared(ImageFormat::SRGBA, 128, 256, 4); + Image image = Image(std::move(image_frame)); + Packet packet = MakePacket(std::move(image)); + MP_ASSERT_OK( + graph.AddPacketToInputStream("input_image", packet.At(Timestamp(1)))); + MP_ASSERT_OK(graph.WaitUntilIdle()); + MP_ASSERT_OK(graph.CloseAllPacketSources()); + MP_ASSERT_OK(graph.WaitUntilDone()); +} + +#if !MEDIAPIPE_DISABLE_GPU && !MEDIAPIPE_METAL_ENABLED + +TEST(ImageToTensorCalculatorTest, + FailsGracefullyWhenGpuServiceNeededButNotAvailable) { + auto graph_config = + mediapipe::ParseTextProtoOrDie(R"pb( + input_stream: "input_image" + node { + calculator: "ImageToTensorCalculator" + input_stream: "IMAGE:input_image" + output_stream: "TENSORS:tensor" + options { + [mediapipe.ImageToTensorCalculatorOptions.ext] { + output_tensor_float_range { min: 0.0f max: 1.0f } + } + } + } + )pb"); + CalculatorGraph graph; + MP_ASSERT_OK(graph.Initialize(graph_config)); + MP_ASSERT_OK(graph.DisallowServiceDefaultInitialization()); + MP_ASSERT_OK(graph.StartRun({})); + + MP_ASSERT_OK_AND_ASSIGN(auto context, + GlContext::Create(nullptr, /*create_thread=*/true)); + Packet packet; + context->Run([&packet]() { + auto image_frame = + std::make_shared(ImageFormat::SRGBA, 128, 256, 4); + Image image = Image(std::move(image_frame)); + // Ensure image is available on GPU to force ImageToTensorCalculator to + // run on GPU. + ASSERT_TRUE(image.ConvertToGpu()); + packet = MakePacket(std::move(image)); + }); + MP_ASSERT_OK( + graph.AddPacketToInputStream("input_image", packet.At(Timestamp(1)))); + EXPECT_THAT(graph.WaitUntilIdle(), + StatusIs(absl::StatusCode::kInternal, + HasSubstr("GPU service not available"))); +} +#endif // !MEDIAPIPE_DISABLE_GPU && !MEDIAPIPE_METAL_ENABLED + } // namespace } // namespace mediapipe