From 8ba81de20be221edc5aec535350648fb8e6ac4d1 Mon Sep 17 00:00:00 2001 From: Prianka Liz Kariat Date: Thu, 5 Jan 2023 18:31:56 +0530 Subject: [PATCH] Added packets callback provision and built in op resolver in MPPTaskRunner --- mediapipe/tasks/ios/core/BUILD | 2 + .../tasks/ios/core/sources/MPPTaskRunner.h | 46 +++++++++++++++++-- .../tasks/ios/core/sources/MPPTaskRunner.mm | 10 +++- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/mediapipe/tasks/ios/core/BUILD b/mediapipe/tasks/ios/core/BUILD index 434d20085..e4797c34d 100644 --- a/mediapipe/tasks/ios/core/BUILD +++ b/mediapipe/tasks/ios/core/BUILD @@ -90,6 +90,8 @@ objc_library( deps = [ "//mediapipe/framework:calculator_cc_proto", "//mediapipe/tasks/cc/core:task_runner", + "//mediapipe/tasks/cc/core:mediapipe_builtin_op_resolver", "//mediapipe/tasks/ios/common/utils:MPPCommonUtils", + "@org_tensorflow//tensorflow/lite/core/api:op_resolver", ], ) diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h index 2b9f2ecdb..97255234f 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.h @@ -20,23 +20,63 @@ NS_ASSUME_NONNULL_BEGIN /** - * This class is used to create and call appropriate methods on the C++ Task Runner. + * This class is used to create and call appropriate methods on the C++ Task Runner to initialize, + * execute and terminate any Mediapipe task. + * + * An instance of the newly created C++ task runner will + * be stored until this class is destroyed. When methods are called for processing (performing + * inference), closing etc., on this class, internally the appropriate methods will be called on the + * C++ task runner instance to execute the appropriate actions. For each type of task, a subclass of + * this class must be defined to add any additional functionality. For eg:, vision tasks must create + * an `MPPVisionTaskRunner` and provide additional functionality. An instance of + * `MPPVisionTaskRunner` can in turn be used by the each vision task for creation and execution of + * the task. Please see the documentation for the C++ Task Runner for more details on how the taks + * runner operates. */ @interface MPPTaskRunner : NSObject /** - * Initializes a new `MPPTaskRunner` with the mediapipe task graph config proto. + * Initializes a new `MPPTaskRunner` with the mediapipe task graph config proto and an optional C++ + * packets callback. + * + * You can pass `nullptr` for `packetsCallback` in case the mode of operation + * requested by the user is synchronous. + * + * If the task is operating in asynchronous mode, any iOS Mediapipe task that uses the `MPPTaskRunner` + * must define a C++ callback function to obtain the results of inference asynchronously and deliver + * the results to the user. To accomplish this, callback function will in turn invoke the block + * provided by the user in the task options supplied to create the task. + * Please see the documentation of the C++ Task Runner for more information on the synchronous and + * asynchronous modes of operation. * * @param graphConfig A mediapipe task graph config proto. * - * @return An instance of `MPPTaskRunner` initialized to the given graph config proto. + * @param packetsCallback An optional C++ callback function that takes a list of output packets as + * the input argument. If provided, the callback must in turn call the block provided by the user in + * the appropriate task options. + * + * @return An instance of `MPPTaskRunner` initialized to the given graph config proto and optional + * packetsCallback. */ - (instancetype)initWithCalculatorGraphConfig:(mediapipe::CalculatorGraphConfig)graphConfig + packetsCallback: + (mediapipe::tasks::core::PacketsCallback)packetsCallback error:(NSError **)error NS_DESIGNATED_INITIALIZER; +/** A synchronous method for processing batch data or offline streaming data. This method is +designed for processing either batch data such as unrelated images and texts or offline streaming +data such as the decoded frames from a video file and an audio file. The call blocks the current +thread until a failure status or a successful result is returned. If the input packets have no +timestamp, an internal timestamp will be assigend per invocation. Otherwise, when the timestamp is +set in the input packets, the caller must ensure that the input packet timestamps are greater than +the timestamps of the previous invocation. This method is thread-unsafe and it is the caller's +responsibility to synchronize access to this method across multiple threads and to ensure that the +input packet timestamps are in order.*/ - (absl::StatusOr)process: (const mediapipe::tasks::core::PacketMap &)packetMap; +/** Shuts down the C++ task runner. After the runner is closed, any calls that send input data to + * the runner are illegal and will receive errors. */ - (absl::Status)close; - (instancetype)init NS_UNAVAILABLE; diff --git a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm index c5c307fd5..fd3f780fa 100644 --- a/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm +++ b/mediapipe/tasks/ios/core/sources/MPPTaskRunner.mm @@ -13,11 +13,15 @@ // limitations under the License. #import "mediapipe/tasks/ios/core/sources/MPPTaskRunner.h" +#include "mediapipe/tasks/cc/core/mediapipe_builtin_op_resolver.h" #import "mediapipe/tasks/ios/common/utils/sources/MPPCommonUtils.h" +#include "tensorflow/lite/core/api/op_resolver.h" namespace { using ::mediapipe::CalculatorGraphConfig; +using ::mediapipe::tasks::core::MediaPipeBuiltinOpResolver; using ::mediapipe::tasks::core::PacketMap; +using ::mediapipe::tasks::core::PacketsCallback; using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; } // namespace @@ -30,15 +34,17 @@ using TaskRunnerCpp = ::mediapipe::tasks::core::TaskRunner; @implementation MPPTaskRunner - (instancetype)initWithCalculatorGraphConfig:(CalculatorGraphConfig)graphConfig + packetsCallback:(PacketsCallback)packetsCallback error:(NSError **)error { self = [super init]; if (self) { - auto taskRunnerResult = TaskRunnerCpp::Create(std::move(graphConfig)); + auto taskRunnerResult = TaskRunnerCpp::Create(std::move(graphConfig), + absl::make_unique(), + std::move(packetsCallback)); if (![MPPCommonUtils checkCppError:taskRunnerResult.status() toError:error]) { return nil; } - _cppTaskRunner = std::move(taskRunnerResult.value()); } return self;