Merge pull request #3975 from priankakariatyml:ios-base-options-helpers

PiperOrigin-RevId: 501073613

mediapipe/tasks/ios/core/BUILD

@@ -89,7 +89,9 @@ objc_library(
     ],
     deps = [
         "//mediapipe/framework:calculator_cc_proto",
+        "//mediapipe/tasks/cc/core:mediapipe_builtin_op_resolver",
         "//mediapipe/tasks/cc/core:task_runner",
         "//mediapipe/tasks/ios/common/utils:MPPCommonUtils",
+        "//third_party/apple_frameworks:Foundation",
     ],
 )

mediapipe/tasks/ios/core/sources/MPPTaskRunner.h

@@ -20,23 +20,65 @@
 NS_ASSUME_NONNULL_BEGIN
 
 /**
- * This class is used to create and call appropriate methods on the C++ Task Runner.
+ * This class is used to create and call appropriate methods on the C++ Task Runner to initialize,
+ * execute and terminate any MediaPipe task.
+ *
+ * An instance of the newly created C++ task runner will be stored until this class is destroyed.
+ * When methods are called for processing (performing inference), closing etc., on this class,
+ * internally the appropriate methods will be called on the C++ task runner instance to execute the
+ * appropriate actions. For each type of task, a subclass of this class must be defined to add any
+ * additional functionality. For eg:, vision tasks must create an `MPPVisionTaskRunner` and provide
+ * additional functionality. An instance of `MPPVisionTaskRunner` can in turn be used by the each
+ * vision task for creation and execution of the task. Please see the documentation for the C++ Task
+ * Runner for more details on how the taks runner operates.
  */
 @interface MPPTaskRunner : NSObject
 
 /**
- * Initializes a new `MPPTaskRunner` with the mediapipe task graph config proto.
+ * Initializes a new `MPPTaskRunner` with the MediaPipe calculator configuration proto and an
+ * optional C++ packets callback.
+ *
+ * You can pass `nullptr` for `packetsCallback` in case the mode of operation requested by the user
+ * is synchronous.
+ *
+ * If the task is operating in asynchronous mode, any iOS MediaPipe task that uses the
+ * `MPPTaskRunner` must define a C++ callback function to obtain the results of inference
+ * asynchronously and deliver the results to the user. To accomplish this, the callback function
+ * should in turn invoke the block provided by the user in the task options supplied to create the
+ * task. Please see the documentation of the C++ Task Runner for more information on the synchronous
+ * and asynchronous modes of operation.
  *
  * @param graphConfig A mediapipe task graph config proto.
+ * @param packetsCallback An optional C++ callback function that takes a list of output packets as
+ * the input argument. If provided, the callback must in turn call the block provided by the user in
+ * the appropriate task options.
  *
- * @return An instance of `MPPTaskRunner` initialized to the given graph config proto.
+ * @return An instance of `MPPTaskRunner` initialized to the given graph config proto and optional
+ * packetsCallback.
  */
 - (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig
+                              packetsCallback:
+                                  (mediapipe::tasks::core::PacketsCallback)packetsCallback
                                         error:(NSError **)error NS_DESIGNATED_INITIALIZER;
 
+/**
+ * A synchronous method for processing batch data or offline streaming data. This method is designed
+ * for processing either batch data such as unrelated images and texts or offline streaming data
+ * such as the decoded frames from a video file or audio file. The call blocks the current
+ * thread until a failure status or a successful result is returned. If the input packets have no
+ * timestamp, an internal timestamp will be assigend per invocation. Otherwise, when the timestamp
+ * is set in the input packets, the caller must ensure that the input packet timestamps are greater
+ * than the timestamps of the previous invocation. This method is thread-unsafe and it is the
+ * caller's responsibility to synchronize access to this method across multiple threads and to
+ * ensure that the input packet timestamps are in order.
+ */
 - (absl::StatusOr<mediapipe::tasks::core::PacketMap>)process:
     (const mediapipe::tasks::core::PacketMap &)packetMap;
 
+/**
+ * Shuts down the C++ task runner. After the runner is closed, any calls that send input data to the
+ * runner are illegal and will receive errors.
+ */
 - (absl::Status)close;
 
 - (instancetype)init NS_UNAVAILABLE;

mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm

@@ -15,9 +15,13 @@
 #import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h"
 #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h"
 
+#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h"
+
 namespace {
 using ::mediapipe::CalculatorGraphConfig;
+using ::mediapipe::tasks::core::MediaPipeBuiltinOpResolver;
 using ::mediapipe::tasks::core::PacketMap;
+using ::mediapipe::tasks::core::PacketsCallback;
 using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner;
 }  // namespace
 
@@ -30,15 +34,17 @@ using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner;
 @implementation MPPTaskRunner
 
 - (instancetype)initWithCalculatorGraphConfig:(CalculatorGraphConfig)graphConfig
+                              packetsCallback:(PacketsCallback)packetsCallback
                                         error:(NSError **)error {
   self = [super init];
   if (self) {
-    auto taskRunnerResult = TaskRunnerCpp::Create(std::move(graphConfig));
+    auto taskRunnerResult = TaskRunnerCpp::Create(std::move(graphConfig),
+                                                  absl::make_unique<MediaPipeBuiltinOpResolver>(),
+                                                  std::move(packetsCallback));
 
     if (![MPPCommonUtils checkCppError:taskRunnerResult.status() toError:error]) {
       return nil;
     }
-
     _cppTaskRunner = std::move(taskRunnerResult.value());
   }
   return self;

mediapipe/tasks/ios/core/utils/BUILD

@@ -0,0 +1,29 @@
+# Copyright 2023 The MediaPipe Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+objc_library(
+    name = "MPPBaseOptionsHelpers",
+    srcs = ["sources/MPPBaseOptions+Helpers.mm"],
+    hdrs = ["sources/MPPBaseOptions+Helpers.h"],
+    deps = [
+        "//mediapipe/tasks/cc/core/proto:acceleration_cc_proto",
+        "//mediapipe/tasks/cc/core/proto:base_options_cc_proto",
+        "//mediapipe/tasks/cc/core/proto:external_file_cc_proto",
+        "//mediapipe/tasks/ios/core:MPPBaseOptions",
+    ],
+)

mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h

@@ -0,0 +1,26 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/tasks/cc/core/proto/base_options.pb.h"
+#import "mediapipe/tasks/ios/core/sources/MPPBaseOptions.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface MPPBaseOptions (Helpers)
+
+- (void)copyToProto:(mediapipe::tasks::core::proto::BaseOptions *)baseOptionsProto;
+
+@end
+
+NS_ASSUME_NONNULL_END

mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.mm

@@ -0,0 +1,47 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "mediapipe/tasks/cc/core/proto/acceleration.pb.h"
+#include "mediapipe/tasks/cc/core/proto/external_file.pb.h"
+#import "mediapipe/tasks/ios/core/utils/sources/MPPBaseOptions+Helpers.h"
+
+namespace {
+using BaseOptionsProto = ::mediapipe::tasks::core::proto::BaseOptions;
+}
+
+@implementation MPPBaseOptions (Helpers)
+
+- (void)copyToProto:(BaseOptionsProto *)baseOptionsProto {
+  baseOptionsProto->Clear();
+
+  if (self.modelAssetPath) {
+    baseOptionsProto->mutable_model_asset()->set_file_name(self.modelAssetPath.UTF8String);
+  }
+
+  switch (self.delegate) {
+    case MPPDelegateCPU: {
+      baseOptionsProto->mutable_acceleration()->mutable_tflite();
+      break;
+    }
+    case MPPDelegateGPU: {
+      // TODO: Provide an implementation for GPU Delegate.
+      [NSException raise:@"Invalid value for delegate" format:@"GPU Delegate is not implemented."];
+      break;
+    }
+    default:
+      break;
+  }
+}
+
+@end

mediapipe/tasks/ios/text/core/BUILD

@@ -0,0 +1,31 @@
+# Copyright 2023 The MediaPipe Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package(default_visibility = ["//mediapipe/tasks:internal"])
+
+licenses(["notice"])
+
+objc_library(
+    name = "MPPTextTaskRunner",
+    srcs = ["sources/MPPTextTaskRunner.mm"],
+    hdrs = ["sources/MPPTextTaskRunner.h"],
+    copts = [
+        "-ObjC++",
+        "-std=c++17",
+    ],
+    deps = [
+        "//mediapipe/tasks/ios/core:MPPTaskRunner",
+        "//third_party/apple_frameworks:Foundation",
+    ],
+)

mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h

@@ -0,0 +1,43 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import <Foundation/Foundation.h>
+#import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h"
+
+NS_ASSUME_NONNULL_BEGIN
+
+/**
+ * This class is used to create and call appropriate methods on the C++ Task Runner to initialize,
+ * execute and terminate any MediaPipe text task.
+ */
+@interface MPPTextTaskRunner : MPPTaskRunner
+
+/**
+ * Initializes a new `MPPTextTaskRunner` with the MediaPipe calculator config proto.
+ *
+ * @param graphConfig A MediaPipe calculator config proto.
+ *
+ * @return An instance of `MPPTextTaskRunner` initialized to the given MediaPipe calculator config
+ * proto.
+ */
+- (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig
+                                        error:(NSError **)error;
+
+- (instancetype)init NS_UNAVAILABLE;
+
++ (instancetype)new NS_UNAVAILABLE;
+
+@end
+
+NS_ASSUME_NONNULL_END

mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.mm

@@ -0,0 +1,38 @@
+// Copyright 2023 The MediaPipe Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#import "mediapipe/tasks/ios/text/core/sources/MPPTextTaskRunner.h"
+
+namespace {
+using ::mediapipe::CalculatorGraphConfig;
+}  // namespace
+
+@implementation MPPTextTaskRunner
+
+- (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig
+                              packetsCallback:
+                                  (mediapipe::tasks::core::PacketsCallback)packetsCallback
+                                        error:(NSError **)error {
+  self = [super initWithCalculatorGraphConfig:graphConfig
+                              packetsCallback:packetsCallback
+                                        error:error];
+  return self;
+}
+
+- (instancetype)initWithCalculatorGraphConfig:(CalculatorGraphConfig)graphConfig
+                                        error:(NSError **)error {
+  return [self initWithCalculatorGraphConfig:graphConfig packetsCallback:nullptr error:error];
+}
+
+@end