GPU_ORIGIN configurable through base options proto.

PiperOrigin-RevId: 573251085

mediapipe/tasks/cc/components/processors/BUILD

@@ -122,6 +122,7 @@ cc_library(
         "//mediapipe/tasks/cc/components/processors/proto:image_preprocessing_graph_options_cc_proto",
         "//mediapipe/tasks/cc/core:model_resources",
         "//mediapipe/tasks/cc/core/proto:acceleration_cc_proto",
+        "//mediapipe/tasks/cc/core/proto:base_options_cc_proto",
         "//mediapipe/tasks/cc/vision/utils:image_tensor_specs",
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:statusor",

mediapipe/tasks/cc/components/processors/image_preprocessing_graph.cc

@@ -36,6 +36,7 @@ limitations under the License.
 #include "mediapipe/tasks/cc/components/processors/proto/image_preprocessing_graph_options.pb.h"
 #include "mediapipe/tasks/cc/core/model_resources.h"
 #include "mediapipe/tasks/cc/core/proto/acceleration.pb.h"
+#include "mediapipe/tasks/cc/core/proto/base_options.pb.h"
 #include "mediapipe/tasks/cc/vision/utils/image_tensor_specs.h"
 #include "tensorflow/lite/schema/schema_generated.h"
 
@@ -73,7 +74,7 @@ struct ImagePreprocessingOutputStreams {
 
 // Fills in the ImageToTensorCalculatorOptions based on the ImageTensorSpecs.
 absl::Status ConfigureImageToTensorCalculator(
-    const ImageTensorSpecs& image_tensor_specs,
+    const ImageTensorSpecs& image_tensor_specs, GpuOrigin::Mode gpu_origin,
     mediapipe::ImageToTensorCalculatorOptions* options) {
   options->set_output_tensor_width(image_tensor_specs.image_width);
   options->set_output_tensor_height(image_tensor_specs.image_height);
@@ -109,7 +110,7 @@ absl::Status ConfigureImageToTensorCalculator(
   }
   // TODO: need to support different GPU origin on different
   // platforms or applications.
-  options->set_gpu_origin(mediapipe::GpuOrigin::TOP_LEFT);
+  options->set_gpu_origin(gpu_origin);
   return absl::OkStatus();
 }
 
@@ -125,10 +126,19 @@ bool DetermineImagePreprocessingGpuBackend(
 absl::Status ConfigureImagePreprocessingGraph(
     const ModelResources& model_resources, bool use_gpu,
     proto::ImagePreprocessingGraphOptions* options) {
+  return ConfigureImagePreprocessingGraph(model_resources, use_gpu,
+                                          GpuOrigin::TOP_LEFT, options);
+}
+
+absl::Status ConfigureImagePreprocessingGraph(
+    const ModelResources& model_resources, bool use_gpu,
+    GpuOrigin::Mode gpu_origin,
+    proto::ImagePreprocessingGraphOptions* options) {
   MP_ASSIGN_OR_RETURN(auto image_tensor_specs,
                       vision::BuildInputImageTensorSpecs(model_resources));
   MP_RETURN_IF_ERROR(ConfigureImageToTensorCalculator(
-      image_tensor_specs, options->mutable_image_to_tensor_options()));
+      image_tensor_specs, gpu_origin,
+      options->mutable_image_to_tensor_options()));
   // The GPU backend isn't able to process int data. If the input tensor is
   // quantized, forces the image preprocessing graph to use CPU backend.
   if (use_gpu && image_tensor_specs.tensor_type != tflite::TensorType_UINT8) {

mediapipe/tasks/cc/components/processors/image_preprocessing_graph.h

@@ -17,6 +17,7 @@ limitations under the License.
 #define MEDIAPIPE_TASKS_CC_COMPONENTS_PROCESSORS_IMAGE_PREPROCESSING_GRAPH_H_
 
 #include "absl/status/status.h"
+#include "mediapipe/gpu/gpu_origin.pb.h"
 #include "mediapipe/tasks/cc/components/processors/proto/image_preprocessing_graph_options.pb.h"
 #include "mediapipe/tasks/cc/core/model_resources.h"
 #include "mediapipe/tasks/cc/core/proto/acceleration.pb.h"
@@ -62,6 +63,12 @@ namespace processors {
 //   IMAGE - Image @Optional
 //     The image that has the pixel data stored on the target storage (CPU vs
 //     GPU).
+absl::Status ConfigureImagePreprocessingGraph(
+    const core::ModelResources& model_resources, bool use_gpu,
+    ::mediapipe::GpuOrigin::Mode gpu_origin,
+    proto::ImagePreprocessingGraphOptions* options);
+
+// A convenient function of the above. gpu_origin is set to TOP_LEFT by default.
 absl::Status ConfigureImagePreprocessingGraph(
     const core::ModelResources& model_resources, bool use_gpu,
     proto::ImagePreprocessingGraphOptions* options);

mediapipe/tasks/cc/components/processors/image_preprocessing_graph_test.cc

@@ -31,6 +31,7 @@ limitations under the License.
 #include "mediapipe/framework/port/gmock.h"
 #include "mediapipe/framework/port/gtest.h"
 #include "mediapipe/framework/port/status_matchers.h"
+#include "mediapipe/gpu/gpu_origin.pb.h"
 #include "mediapipe/tasks/cc/components/processors/proto/image_preprocessing_graph_options.pb.h"
 #include "mediapipe/tasks/cc/core/model_resources.h"
 #include "mediapipe/tasks/cc/core/proto/acceleration.pb.h"
@@ -230,6 +231,25 @@ TEST_F(ConfigureTest, SucceedsWithFloatModelGpuBackend) {
                                 backend: GPU_BACKEND)pb"));
 }
 
+TEST_F(ConfigureTest, SucceedsGpuOriginConventional) {
+  MP_ASSERT_OK_AND_ASSIGN(
+      auto model_resources,
+      CreateModelResourcesForModel(kMobileNetFloatWithMetadata));
+
+  proto::ImagePreprocessingGraphOptions options;
+  MP_EXPECT_OK(ConfigureImagePreprocessingGraph(
+      *model_resources, true, mediapipe::GpuOrigin::CONVENTIONAL, &options));
+
+  EXPECT_THAT(options, EqualsProto(
+                           R"pb(image_to_tensor_options {
+                                  output_tensor_width: 224
+                                  output_tensor_height: 224
+                                  output_tensor_float_range { min: -1 max: 1 }
+                                  gpu_origin: CONVENTIONAL
+                                }
+                                backend: GPU_BACKEND)pb"));
+}
+
 TEST_F(ConfigureTest, FailsWithFloatModelWithoutMetadata) {
   MP_ASSERT_OK_AND_ASSIGN(
       auto model_resources,

mediapipe/tasks/cc/core/proto/BUILD

@@ -46,6 +46,7 @@ mediapipe_proto_library(
     deps = [
         ":acceleration_proto",
         ":external_file_proto",
+        "//mediapipe/gpu:gpu_origin_proto",
     ],
 )
 

mediapipe/tasks/cc/core/proto/base_options.proto

@@ -17,6 +17,7 @@ syntax = "proto2";
 
 package mediapipe.tasks.core.proto;
 
+import "mediapipe/gpu/gpu_origin.proto";
 import "mediapipe/tasks/cc/core/proto/acceleration.proto";
 import "mediapipe/tasks/cc/core/proto/external_file.proto";
 
@@ -24,7 +25,7 @@ option java_package = "com.google.mediapipe.tasks.core.proto";
 option java_outer_classname = "BaseOptionsProto";
 
 // Base options for mediapipe tasks.
-// Next Id: 4
+// Next Id: 5
 message BaseOptions {
   // The external model asset, as a single standalone TFLite file. It could be
   // packed with TFLite Model Metadata[1] and associated files if exist. Fail to
@@ -40,4 +41,7 @@ message BaseOptions {
 
   // Acceleration setting to use available delegate on the device.
   optional Acceleration acceleration = 3;
+
+  // Gpu origin for calculators with gpu supported.
+  optional mediapipe.GpuOrigin.Mode gpu_origin = 4 [default = TOP_LEFT];
 }

mediapipe/tasks/cc/vision/face_detector/face_detector_graph.cc

@@ -225,7 +225,7 @@ class FaceDetectorGraph : public core::ModelTaskGraph {
         components::processors::DetermineImagePreprocessingGpuBackend(
             subgraph_options.base_options().acceleration());
     MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph(
-        model_resources, use_gpu,
+        model_resources, use_gpu, subgraph_options.base_options().gpu_origin(),
         &preprocessing.GetOptions<
             components::processors::proto::ImagePreprocessingGraphOptions>()));
     auto& image_to_tensor_options =

mediapipe/tasks/cc/vision/face_landmarker/face_landmarker_graph.cc

@@ -134,6 +134,9 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
       ->CopyFrom(options->base_options().acceleration());
   face_detector_graph_options->mutable_base_options()->set_use_stream_mode(
       options->base_options().use_stream_mode());
+  face_detector_graph_options->mutable_base_options()->set_gpu_origin(
+      options->base_options().gpu_origin());
+
   auto* face_landmarks_detector_graph_options =
       options->mutable_face_landmarks_detector_graph_options();
   if (!face_landmarks_detector_graph_options->base_options()
@@ -151,6 +154,8 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
       ->CopyFrom(options->base_options().acceleration());
   face_landmarks_detector_graph_options->mutable_base_options()
       ->set_use_stream_mode(options->base_options().use_stream_mode());
+  face_landmarks_detector_graph_options->mutable_base_options()->set_gpu_origin(
+      options->base_options().gpu_origin());
 
   absl::StatusOr<absl::string_view> face_blendshape_model =
       resources.GetFile(kFaceBlendshapeTFLiteName);

mediapipe/tasks/cc/vision/face_landmarker/face_landmarks_detector_graph.cc

@@ -266,7 +266,7 @@ class SingleFaceLandmarksDetectorGraph : public core::ModelTaskGraph {
         components::processors::DetermineImagePreprocessingGpuBackend(
             subgraph_options.base_options().acceleration());
     MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph(
-        model_resources, use_gpu,
+        model_resources, use_gpu, subgraph_options.base_options().gpu_origin(),
         &preprocessing.GetOptions<tasks::components::processors::proto::
                                       ImagePreprocessingGraphOptions>()));
     image_in >> preprocessing.In(kImageTag);

mediapipe/tasks/cc/vision/face_stylizer/face_stylizer_graph.cc

@@ -395,7 +395,7 @@ class FaceStylizerGraph : public core::ModelTaskGraph {
         components::processors::DetermineImagePreprocessingGpuBackend(
             task_options.base_options().acceleration());
     MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph(
-        *model_resources, use_gpu,
+        *model_resources, use_gpu, task_options.base_options().gpu_origin(),
         &preprocessing.GetOptions<tasks::components::processors::proto::
                                       ImagePreprocessingGraphOptions>()));
     auto& image_to_tensor_options =

mediapipe/tasks/cc/vision/gesture_recognizer/gesture_recognizer_graph.cc

@@ -131,6 +131,11 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
   }
   hand_gesture_recognizer_graph_options->mutable_base_options()
       ->set_use_stream_mode(options->base_options().use_stream_mode());
+
+  hand_landmarker_graph_options->mutable_base_options()->set_gpu_origin(
+      options->base_options().gpu_origin());
+  hand_gesture_recognizer_graph_options->mutable_base_options()->set_gpu_origin(
+      options->base_options().gpu_origin());
   return absl::OkStatus();
 }
 

mediapipe/tasks/cc/vision/hand_detector/hand_detector_graph.cc

@@ -242,7 +242,7 @@ class HandDetectorGraph : public core::ModelTaskGraph {
         components::processors::DetermineImagePreprocessingGpuBackend(
             subgraph_options.base_options().acceleration());
     MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph(
-        model_resources, use_gpu,
+        model_resources, use_gpu, subgraph_options.base_options().gpu_origin(),
         &preprocessing.GetOptions<
             components::processors::proto::ImagePreprocessingGraphOptions>()));
     image_in >> preprocessing.In("IMAGE");

mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarker_graph.cc

@@ -125,6 +125,11 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
       ->CopyFrom(options->base_options().acceleration());
   hand_landmarks_detector_graph_options->mutable_base_options()
       ->set_use_stream_mode(options->base_options().use_stream_mode());
+
+  hand_detector_graph_options->mutable_base_options()->set_gpu_origin(
+      options->base_options().gpu_origin());
+  hand_landmarks_detector_graph_options->mutable_base_options()->set_gpu_origin(
+      options->base_options().gpu_origin());
   return absl::OkStatus();
 }
 }  // namespace

mediapipe/tasks/cc/vision/hand_landmarker/hand_landmarks_detector_graph.cc

@@ -256,7 +256,7 @@ class SingleHandLandmarksDetectorGraph : public core::ModelTaskGraph {
         components::processors::DetermineImagePreprocessingGpuBackend(
             subgraph_options.base_options().acceleration());
     MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph(
-        model_resources, use_gpu,
+        model_resources, use_gpu, subgraph_options.base_options().gpu_origin(),
         &preprocessing.GetOptions<tasks::components::processors::proto::
                                       ImagePreprocessingGraphOptions>()));
     image_in >> preprocessing.In("IMAGE");

mediapipe/tasks/cc/vision/image_classifier/image_classifier_graph.cc

@@ -142,7 +142,7 @@ class ImageClassifierGraph : public core::ModelTaskGraph {
         components::processors::DetermineImagePreprocessingGpuBackend(
             task_options.base_options().acceleration());
     MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph(
-        model_resources, use_gpu,
+        model_resources, use_gpu, task_options.base_options().gpu_origin(),
         &preprocessing.GetOptions<tasks::components::processors::proto::
                                       ImagePreprocessingGraphOptions>()));
     image_in >> preprocessing.In(kImageTag);

mediapipe/tasks/cc/vision/image_embedder/image_embedder_graph.cc

@@ -137,7 +137,7 @@ class ImageEmbedderGraph : public core::ModelTaskGraph {
         components::processors::DetermineImagePreprocessingGpuBackend(
             task_options.base_options().acceleration());
     MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph(
-        model_resources, use_gpu,
+        model_resources, use_gpu, task_options.base_options().gpu_origin(),
         &preprocessing.GetOptions<tasks::components::processors::proto::
                                       ImagePreprocessingGraphOptions>()));
     image_in >> preprocessing.In(kImageTag);

mediapipe/tasks/cc/vision/image_segmenter/image_segmenter_graph.cc

@@ -255,8 +255,8 @@ void ConfigureTensorConverterCalculator(
 // the tflite model.
 absl::StatusOr<ImageAndTensorsOnDevice> ConvertImageToTensors(
     Source<Image> image_in, Source<NormalizedRect> norm_rect_in, bool use_gpu,
-    bool is_hair_segmentation, const core::ModelResources& model_resources,
+    const core::proto::BaseOptions& base_options, bool is_hair_segmentation,
-    Graph& graph) {
+    const core::ModelResources& model_resources, Graph& graph) {
   MP_ASSIGN_OR_RETURN(const tflite::Tensor* tflite_input_tensor,
                       GetInputTensor(model_resources));
   if (tflite_input_tensor->shape()->size() != 4) {
@@ -279,7 +279,7 @@ absl::StatusOr<ImageAndTensorsOnDevice> ConvertImageToTensors(
     auto& preprocessing = graph.AddNode(
         "mediapipe.tasks.components.processors.ImagePreprocessingGraph");
     MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph(
-        model_resources, use_gpu,
+        model_resources, use_gpu, base_options.gpu_origin(),
         &preprocessing.GetOptions<tasks::components::processors::proto::
                                       ImagePreprocessingGraphOptions>()));
     image_in >> preprocessing.In(kImageTag);
@@ -518,7 +518,8 @@ class ImageSegmenterGraph : public core::ModelTaskGraph {
     MP_ASSIGN_OR_RETURN(
         auto image_and_tensors,
         ConvertImageToTensors(image_in, norm_rect_in, use_gpu,
-                              is_hair_segmentation, model_resources, graph));
+                              task_options.base_options(), is_hair_segmentation,
+                              model_resources, graph));
     // Adds inference subgraph and connects its input stream to the output
     // tensors produced by the ImageToTensorCalculator.
     auto& inference = AddInference(

mediapipe/tasks/cc/vision/object_detector/object_detector_graph.cc

@@ -192,7 +192,7 @@ class ObjectDetectorGraph : public core::ModelTaskGraph {
         components::processors::DetermineImagePreprocessingGpuBackend(
             task_options.base_options().acceleration());
     MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph(
-        model_resources, use_gpu,
+        model_resources, use_gpu, task_options.base_options().gpu_origin(),
         &preprocessing.GetOptions<tasks::components::processors::proto::
                                       ImagePreprocessingGraphOptions>()));
     image_in >> preprocessing.In(kImageTag);

mediapipe/tasks/cc/vision/pose_detector/pose_detector_graph.cc

@@ -240,7 +240,7 @@ class PoseDetectorGraph : public core::ModelTaskGraph {
         components::processors::DetermineImagePreprocessingGpuBackend(
             subgraph_options.base_options().acceleration());
     MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph(
-        model_resources, use_gpu,
+        model_resources, use_gpu, subgraph_options.base_options().gpu_origin(),
         &preprocessing.GetOptions<
             components::processors::proto::ImagePreprocessingGraphOptions>()));
     auto& image_to_tensor_options =

mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarker_graph.cc

@@ -134,6 +134,11 @@ absl::Status SetSubTaskBaseOptions(const ModelAssetBundleResources& resources,
   pose_landmarks_detector_graph_options->mutable_base_options()
       ->set_use_stream_mode(options->base_options().use_stream_mode());
 
+  pose_detector_graph_options->mutable_base_options()->set_gpu_origin(
+      options->base_options().gpu_origin());
+  pose_landmarks_detector_graph_options->mutable_base_options()->set_gpu_origin(
+      options->base_options().gpu_origin());
+
   return absl::OkStatus();
 }
 

mediapipe/tasks/cc/vision/pose_landmarker/pose_landmarks_detector_graph.cc

@@ -347,7 +347,7 @@ class SinglePoseLandmarksDetectorGraph : public core::ModelTaskGraph {
         components::processors::DetermineImagePreprocessingGpuBackend(
             subgraph_options.base_options().acceleration());
     MP_RETURN_IF_ERROR(components::processors::ConfigureImagePreprocessingGraph(
-        model_resources, use_gpu,
+        model_resources, use_gpu, subgraph_options.base_options().gpu_origin(),
         &preprocessing.GetOptions<tasks::components::processors::proto::
                                       ImagePreprocessingGraphOptions>()));
     image_in >> preprocessing.In(kImageTag);

mediapipe/tasks/web/core/BUILD

@@ -66,6 +66,7 @@ mediapipe_ts_library(
         ":task_runner",
         ":task_runner_test_utils",
         "//mediapipe/calculators/tensor:inference_calculator_jspb_proto",
+        "//mediapipe/gpu:gpu_origin_jspb_proto",
         "//mediapipe/tasks/cc/core/proto:base_options_jspb_proto",
         "//mediapipe/web/graph_runner:graph_runner_ts",
     ],

mediapipe/tasks/web/core/task_runner_test.ts

@@ -17,6 +17,7 @@ import 'jasmine';
 
 // Placeholder for internal dependency on encodeByteArray
 import {InferenceCalculatorOptions} from '../../../calculators/tensor/inference_calculator_pb';
+import {GpuOrigin as GpuOriginProto} from '../../../gpu/gpu_origin_pb';
 import {BaseOptions as BaseOptionsProto} from '../../../tasks/cc/core/proto/base_options_pb';
 import {TaskRunner} from '../../../tasks/web/core/task_runner';
 import {createSpyWasmModule, SpyWasmModule} from '../../../tasks/web/core/task_runner_test_utils';
@@ -111,6 +112,7 @@ describe('TaskRunner', () => {
       tflite: {},
       nnapi: undefined,
     },
+    gpuOrigin: GpuOriginProto.Mode.TOP_LEFT,
   };
   const mockBytesResultWithGpuDelegate = {
     ...mockBytesResult,
@@ -146,6 +148,7 @@ describe('TaskRunner', () => {
       tflite: {},
       nnapi: undefined,
     },
+    gpuOrigin: GpuOriginProto.Mode.TOP_LEFT,
   };
 
   let fetchSpy: jasmine.Spy;