From 02bd0d95e7ff4074104a6a8383ed575a3155feb0 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Mon, 11 Sep 2023 17:42:31 -0700 Subject: [PATCH] Splitting GraphRunner into public API declared interfaces and private TS impls PiperOrigin-RevId: 564551973 --- mediapipe/web/graph_runner/BUILD | 52 +- mediapipe/web/graph_runner/graph_runner.ts | 673 +++--------------- .../web/graph_runner/graph_runner_api.d.ts | 583 +++++++++++++++ .../graph_runner_factory_api.d.ts | 79 ++ .../web/graph_runner/listener_types.d.ts | 23 + mediapipe/web/graph_runner/wasm_module.d.ts | 111 +++ 6 files changed, 927 insertions(+), 594 deletions(-) create mode 100644 mediapipe/web/graph_runner/graph_runner_api.d.ts create mode 100644 mediapipe/web/graph_runner/graph_runner_factory_api.d.ts create mode 100644 mediapipe/web/graph_runner/listener_types.d.ts create mode 100644 mediapipe/web/graph_runner/wasm_module.d.ts diff --git a/mediapipe/web/graph_runner/BUILD b/mediapipe/web/graph_runner/BUILD index 6fc6379f5..8f30297bb 100644 --- a/mediapipe/web/graph_runner/BUILD +++ b/mediapipe/web/graph_runner/BUILD @@ -1,19 +1,67 @@ # The TypeScript graph runner used by all MediaPipe Web tasks. -load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library") +load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_declaration", "mediapipe_ts_library") load("@npm//@bazel/jasmine:index.bzl", "jasmine_node_test") package(default_visibility = [ "//mediapipe/tasks:internal", ]) +# Public types for GraphRunner listeners +mediapipe_ts_declaration( + name = "listener_types", + srcs = [ + "listener_types.d.ts", + ], + visibility = ["//visibility:private"], +) + +# Public API for GraphRunner +mediapipe_ts_declaration( + name = "graph_runner_api", + srcs = [ + "graph_runner_api.d.ts", + ], + deps = [ + ":listener_types", + ":wasm_module_private", + ], +) + +# Public API for GraphRunner creation factory functions +mediapipe_ts_library( + name = "graph_runner_factory_api", + srcs = [ + "graph_runner_factory_api.d.ts", + ], + deps = [ + ":graph_runner_api", + ":wasm_module_private", + ], +) + +# Private declarations for the Wasm module +mediapipe_ts_declaration( + name = "wasm_module_private", + srcs = ["wasm_module.d.ts"], + visibility = ["//visibility:private"], + deps = [":listener_types"], +) + +# Internal GraphRunner implementation mediapipe_ts_library( name = "graph_runner_ts", srcs = [ ":graph_runner.ts", ], allow_unoptimized_namespaces = True, - deps = [":platform_utils"], + deps = [ + ":graph_runner_api", + ":graph_runner_factory_api", + ":listener_types", + ":platform_utils", + ":wasm_module_private", + ], ) mediapipe_ts_library( diff --git a/mediapipe/web/graph_runner/graph_runner.ts b/mediapipe/web/graph_runner/graph_runner.ts index 5f222603f..3ef4d4aa5 100644 --- a/mediapipe/web/graph_runner/graph_runner.ts +++ b/mediapipe/web/graph_runner/graph_runner.ts @@ -3,40 +3,27 @@ import {isWebKit} from '../../web/graph_runner/platform_utils'; // Placeholder for internal dependency on trusted resource url -// This file can serve as a common interface for most simple TypeScript -// libraries-- additionally, it can hook automatically into wasm_mediapipe_demo -// to autogenerate simple TS APIs from demos for instantaneous 1P integrations. +import {GraphRunnerApi, ImageSource} from './graph_runner_api'; +import {CreateGraphRunnerApi, CreateMediaPipeLibApi, FileLocator, WasmMediaPipeConstructor} from './graph_runner_factory_api'; +import {EmptyPacketListener, ErrorListener, SimpleListener, VectorListener} from './listener_types'; +import {WasmModule} from './wasm_module'; -/** - * Simple interface for allowing users to set the directory where internal - * wasm-loading and asset-loading code looks (e.g. for .wasm and .data file - * locations). - */ -export declare interface FileLocator { - locateFile: (filename: string) => string; - mainScriptUrlOrBlob?: string; -} +export {ReturnType} from './graph_runner_factory_api'; +// This file contains the internal implementations behind the public APIs +// declared in "graph_runner_api.d.ts" and "graph_runner_factory_api.d.ts". -/** - * A listener that receives the contents of a non-empty MediaPipe packet and - * its timestamp. - */ -export type SimpleListener = (data: T, timestamp: number) => void; - -/** - * A listener that receives the current MediaPipe packet timestamp. This is - * invoked even for empty packet. - */ -export type EmptyPacketListener = (timestamp: number) => void; - -/** - * A listener that receives a single element of vector-returning output packet. - * Receives one element at a time (in order). Once all elements are processed, - * the listener is invoked with `data` set to `unknown` and `done` set to true. - * Intended for internal usage. - */ -export type VectorListener = (data: T, done: boolean, timestamp: number) => - void; +// First we re-export all of our imported public types/defines, so that users +// can import everything they need from here directly. +export { + EmptyPacketListener, + ErrorListener, + FileLocator, + ImageSource, + SimpleListener, + VectorListener, + WasmMediaPipeConstructor, + WasmModule, +}; /** * A listener that receives the CalculatorGraphConfig in binary encoding. @@ -49,114 +36,6 @@ export type CalculatorGraphConfigListener = (graphConfig: Uint8Array) => void; */ export const CALCULATOR_GRAPH_CONFIG_LISTENER_NAME = '__graph_config__'; -/** - * Declarations for Emscripten's WebAssembly Module behavior, so TS compiler - * doesn't break our JS/C++ bridge. - */ -export declare interface WasmModule { - canvas: HTMLCanvasElement|OffscreenCanvas|null; - HEAPU8: Uint8Array; - HEAPU32: Uint32Array; - HEAPF32: Float32Array; - HEAPF64: Float64Array; - FS_createDataFile: - (parent: string, name: string, data: Uint8Array, canRead: boolean, - canWrite: boolean, canOwn: boolean) => void; - FS_createPath: - (parent: string, name: string, canRead: boolean, - canWrite: boolean) => void; - FS_unlink(path: string): void; - gpuOriginForWebTexturesIsBottomLeft?: boolean; - - errorListener?: ErrorListener; - _bindTextureToCanvas: () => boolean; - _changeBinaryGraph: (size: number, dataPtr: number) => void; - _changeTextGraph: (size: number, dataPtr: number) => void; - _closeGraph: () => void; - _free: (ptr: number) => void; - _malloc: (size: number) => number; - _processFrame: (width: number, height: number, timestamp: number) => void; - _setAutoRenderToScreen: (enabled: boolean) => void; - _waitUntilIdle: () => void; - - // Exposed so that clients of this lib can access this field - dataFileDownloads?: {[url: string]: {loaded: number, total: number}}; - - // Wasm Module multistream entrypoints. Require - // gl_graph_runner_internal_multi_input as a build dependency. - stringToNewUTF8: (data: string) => number; - _bindTextureToStream: (streamNamePtr: number) => void; - _addBoundTextureToStream: - (streamNamePtr: number, width: number, height: number, - timestamp: number) => void; - _addBoolToInputStream: - (data: boolean, streamNamePtr: number, timestamp: number) => void; - _addDoubleToInputStream: - (data: number, streamNamePtr: number, timestamp: number) => void; - _addFloatToInputStream: - (data: number, streamNamePtr: number, timestamp: number) => void; - _addIntToInputStream: - (data: number, streamNamePtr: number, timestamp: number) => void; - _addStringToInputStream: - (dataPtr: number, streamNamePtr: number, timestamp: number) => void; - _addFlatHashMapToInputStream: - (keysPtr: number, valuesPtr: number, count: number, streamNamePtr: number, - timestamp: number) => void; - _addProtoToInputStream: - (dataPtr: number, dataSize: number, protoNamePtr: number, - streamNamePtr: number, timestamp: number) => void; - _addEmptyPacketToInputStream: - (streamNamePtr: number, timestamp: number) => void; - - // Input side packets - _addBoolToInputSidePacket: (data: boolean, streamNamePtr: number) => void; - _addDoubleToInputSidePacket: (data: number, streamNamePtr: number) => void; - _addFloatToInputSidePacket: (data: number, streamNamePtr: number) => void; - _addIntToInputSidePacket: (data: number, streamNamePtr: number) => void; - _addStringToInputSidePacket: (dataPtr: number, streamNamePtr: number) => void; - _addProtoToInputSidePacket: - (dataPtr: number, dataSize: number, protoNamePtr: number, - streamNamePtr: number) => void; - - // Map of output streams to packet listeners. Also built as part of - // gl_graph_runner_internal_multi_input. - simpleListeners?: - Record|VectorListener>; - // Map of output streams to empty packet listeners. - emptyPacketListeners?: Record; - _attachBoolListener: (streamNamePtr: number) => void; - _attachBoolVectorListener: (streamNamePtr: number) => void; - _attachDoubleListener: (streamNamePtr: number) => void; - _attachDoubleVectorListener: (streamNamePtr: number) => void; - _attachFloatListener: (streamNamePtr: number) => void; - _attachFloatVectorListener: (streamNamePtr: number) => void; - _attachIntListener: (streamNamePtr: number) => void; - _attachIntVectorListener: (streamNamePtr: number) => void; - _attachStringListener: (streamNamePtr: number) => void; - _attachStringVectorListener: (streamNamePtr: number) => void; - _attachProtoListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void; - _attachProtoVectorListener: - (streamNamePtr: number, makeDeepCopy?: boolean) => void; - - // Require dependency ":gl_graph_runner_audio_out" - _attachAudioListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void; - - // Require dependency ":gl_graph_runner_audio" - _addAudioToInputStream: (dataPtr: number, numChannels: number, - numSamples: number, streamNamePtr: number, timestamp: number) => void; - _configureAudio: (channels: number, samples: number, sampleRate: number, - streamNamePtr: number, headerNamePtr: number) => void; - - // Get the graph configuration and invoke the listener configured under - // streamNamePtr - _getGraphConfig: (streamNamePtr: number, makeDeepCopy?: boolean) => void; - - // TODO: Refactor to just use a few numbers (perhaps refactor away - // from gl_graph_runner_internal.cc entirely to use something a little more - // streamlined; new version is _processFrame above). - _processGl: (frameDataPtr: number) => number; -} - // Global declarations, for tapping into Window for Wasm blob running declare global { interface Window { @@ -173,32 +52,13 @@ declare global { */ declare function importScripts(...urls: Array): void; -/** - * Valid types of image sources which we can run our GraphRunner over. - */ -export type ImageSource = - HTMLCanvasElement|HTMLVideoElement|HTMLImageElement|ImageData|ImageBitmap; - - -/** A listener that will be invoked with an absl::StatusCode and message. */ -export type ErrorListener = (code: number, message: string) => void; - -/** - * Internal type of constructors used for initializing GraphRunner and - * subclasses. - */ -export type WasmMediaPipeConstructor = - (new ( - module: WasmModule, canvas?: HTMLCanvasElement|OffscreenCanvas|null) => - LibType); - /** * Simple class to run an arbitrary image-in/image-out MediaPipe graph (i.e. * as created by wasm_mediapipe_demo BUILD macro), and either render results * into canvas, or else return the output WebGLTexture. Takes a WebAssembly * Module. */ -export class GraphRunner { +export class GraphRunner implements GraphRunnerApi { // TODO: These should be protected/private, but are left exposed for // now so that we can use proper TS mixins with this class as a base. This // should be somewhat fixed when we create our .d.ts files. @@ -241,11 +101,7 @@ export class GraphRunner { } } - /** - * Convenience helper to load a MediaPipe graph from a file and pass it to - * setGraph. - * @param graphFile The url of the MediaPipe graph file to load. - */ + /** {@override GraphRunnerApi} */ async initializeGraph(graphFile: string): Promise { // Fetch and set graph const response = await fetch(graphFile); @@ -255,26 +111,12 @@ export class GraphRunner { this.setGraph(new Uint8Array(graphData), isBinary); } - /** - * Convenience helper for calling setGraph with a string representing a text - * proto config. - * @param graphConfig The text proto graph config, expected to be a string in - * default JavaScript UTF-16 format. - */ + /** {@override GraphRunnerApi} */ setGraphFromString(graphConfig: string): void { this.setGraph((new TextEncoder()).encode(graphConfig), false); } - /** - * Takes the raw data from a MediaPipe graph, and passes it to C++ to be run - * over the video stream. Will replace the previously running MediaPipe graph, - * if there is one. - * @param graphData The raw MediaPipe graph data, either in binary - * protobuffer format (.binarypb), or else in raw text format (.pbtxt or - * .textproto). - * @param isBinary This should be set to true if the graph is in - * binary format, and false if it is in human-readable text format. - */ + /** {@override GraphRunnerApi} */ setGraph(graphData: Uint8Array, isBinary: boolean): void { const size = graphData.length; const dataPtr = this.wasmModule._malloc(size); @@ -287,24 +129,7 @@ export class GraphRunner { this.wasmModule._free(dataPtr); } - /** - * Configures the current graph to handle audio processing in a certain way - * for all its audio input streams. Additionally can configure audio headers - * (both input side packets as well as input stream headers), but these - * configurations only take effect if called before the graph is set/started. - * @param numChannels The number of channels of audio input. Only 1 - * is supported for now. - * @param numSamples The number of samples that are taken in each - * audio capture. - * @param sampleRate The rate, in Hz, of the sampling. - * @param streamName The optional name of the input stream to additionally - * configure with audio information. This configuration only occurs before - * the graph is set/started. If unset, a default stream name will be used. - * @param headerName The optional name of the header input side packet to - * additionally configure with audio information. This configuration only - * occurs before the graph is set/started. If unset, a default header name - * will be used. - */ + /** {@override GraphRunnerApi} */ configureAudio(numChannels: number, numSamples: number, sampleRate: number, streamName?: string, headerName?: string) { if (!this.wasmModule._configureAudio) { @@ -322,55 +147,17 @@ export class GraphRunner { }); } - /** - * Allows disabling automatic canvas resizing, in case clients want to control - * control this. - * @param resize True will re-enable automatic canvas resizing, while false - * will disable the feature. - */ + /** {@override GraphRunnerApi} */ setAutoResizeCanvas(resize: boolean): void { this.autoResizeCanvas = resize; } - /** - * Allows disabling the automatic render-to-screen code, in case clients don't - * need/want this. In particular, this removes the requirement for pipelines - * to have access to GPU resources, as well as the requirement for graphs to - * have "input_frames_gpu" and "output_frames_gpu" streams defined, so pure - * CPU pipelines and non-video pipelines can be created. - * NOTE: This only affects future graph initializations (via setGraph or - * initializeGraph), and does NOT affect the currently running graph, so - * calls to this should be made *before* setGraph/initializeGraph for the - * graph file being targeted. - * @param enabled True will re-enable automatic render-to-screen code and - * cause GPU resources to once again be requested, while false will - * disable the feature. - */ + /** {@override GraphRunnerApi} */ setAutoRenderToScreen(enabled: boolean): void { this.wasmModule._setAutoRenderToScreen(enabled); } - /** - * Overrides the vertical orientation for input GpuBuffers and the automatic - * render-to-screen code. The default for our OpenGL code on other platforms - * (Android, Linux) is to use a bottom-left origin. But the default for WebGL - * is to use a top-left origin. We use WebGL default normally, and many - * calculators and graphs have platform-specific code to handle the resulting - * orientation flip. However, in order to be able to use a single graph on all - * platforms without alterations, it may be useful to send images into a web - * graph using the OpenGL orientation. Users can call this function with - * `bottomLeftIsOrigin = true` in order to enforce an orientation for all - * GpuBuffer inputs which is consistent with OpenGL on other platforms. - * This call will also vertically flip the automatic render-to-screen code as - * well, so that webcam input (for example) will render properly when passed - * through the graph still. - * NOTE: This will immediately affect GpuBuffer inputs, but must be called - * *before* graph start in order to affect the automatic render-to-screen - * code! - * @param bottomLeftIsOrigin True will flip our input GpuBuffers and auto - * render-to-screen to match the classic OpenGL orientation, while false will - * disable this feature to match the default WebGL orientation. - */ + /** {@override GraphRunnerApi} */ setGpuBufferVerticalFlip(bottomLeftIsOrigin: boolean): void { this.wasmModule.gpuOriginForWebTexturesIsBottomLeft = bottomLeftIsOrigin; } @@ -436,7 +223,8 @@ export class GraphRunner { * Takes the raw data from a JS image source, and sends it to C++ to be * processed, waiting synchronously for the response. Note that we will resize * our GL canvas to fit the input, so input size should only change - * infrequently. + * infrequently. NOTE: This call has been deprecated in favor of + * `addGpuBufferToStream`. * @param imageSource An image source to process. * @param timestamp The timestamp of the current frame, in ms. * @return texture? The WebGL texture reference, if one was produced. @@ -554,26 +342,12 @@ export class GraphRunner { }; } - /** - * Attaches a listener that will be invoked when the MediaPipe framework - * returns an error. - */ + /** {@override GraphRunnerApi} */ attachErrorListener(callbackFcn: (code: number, message: string) => void) { this.wasmModule.errorListener = callbackFcn; } - /** - * Attaches a listener that will be invoked when the MediaPipe framework - * receives an empty packet on the provided output stream. This can be used - * to receive the latest output timestamp. - * - * Empty packet listeners are only active if there is a corresponding packet - * listener. - * - * @param outputStreamName The name of the graph output stream to receive - * empty packets from. - * @param callbackFcn The callback to receive the timestamp. - */ + /** {@override GraphRunnerApi} */ attachEmptyPacketListener( outputStreamName: string, callbackFcn: EmptyPacketListener) { this.wasmModule.emptyPacketListeners = @@ -581,15 +355,7 @@ export class GraphRunner { this.wasmModule.emptyPacketListeners[outputStreamName] = callbackFcn; } - /** - * Takes the raw data from a JS audio capture array, and sends it to C++ to be - * processed. - * @param audioData An array of raw audio capture data, like - * from a call to getChannelData on an AudioBuffer. - * @param streamName The name of the MediaPipe graph stream to add the audio - * data to. - * @param timestamp The timestamp of the current frame, in ms. - */ + /** {@override GraphRunnerApi} */ addAudioToStream( audioData: Float32Array, streamName: string, timestamp: number) { // numChannels and numSamples being 0 will cause defaults to be used, @@ -597,22 +363,7 @@ export class GraphRunner { this.addAudioToStreamWithShape(audioData, 0, 0, streamName, timestamp); } - /** - * Takes the raw data from a JS audio capture array, and sends it to C++ to be - * processed, shaping the audioData array into an audio matrix according to - * the numChannels and numSamples parameters. - * @param audioData An array of raw audio capture data, like - * from a call to getChannelData on an AudioBuffer. - * @param numChannels The number of audio channels this data represents. If 0 - * is passed, then the value will be taken from the last call to - * configureAudio. - * @param numSamples The number of audio samples captured in this data packet. - * If 0 is passed, then the value will be taken from the last call to - * configureAudio. - * @param streamName The name of the MediaPipe graph stream to add the audio - * data to. - * @param timestamp The timestamp of the current frame, in ms. - */ + /** {@override GraphRunnerApi} */ addAudioToStreamWithShape( audioData: Float32Array, numChannels: number, numSamples: number, streamName: string, timestamp: number) { @@ -633,19 +384,7 @@ export class GraphRunner { }); } - /** - * Takes the relevant information from the HTML video or image element, and - * passes it into the WebGL-based graph for processing on the given stream at - * the given timestamp. Can be used for additional auxiliary GpuBuffer input - * streams. Processing will not occur until a blocking call (like - * processVideoGl or finishProcessing) is made. For use with - * 'gl_graph_runner_internal_multi_input'. - * @param imageSource Reference to the video frame we wish to add into our - * graph. - * @param streamName The name of the MediaPipe graph stream to add the frame - * to. - * @param timestamp The timestamp of the input frame, in ms. - */ + /** {@override GraphRunnerApi} */ addGpuBufferToStream( imageSource: ImageSource, streamName: string, timestamp: number): void { this.wrapStringPtr(streamName, (streamNamePtr: number) => { @@ -656,36 +395,21 @@ export class GraphRunner { }); } - /** - * Sends a boolean packet into the specified stream at the given timestamp. - * @param data The boolean data to send. - * @param streamName The name of the graph input stream to send data into. - * @param timestamp The timestamp of the input data, in ms. - */ + /** {@override GraphRunnerApi} */ addBoolToStream(data: boolean, streamName: string, timestamp: number): void { this.wrapStringPtr(streamName, (streamNamePtr: number) => { this.wasmModule._addBoolToInputStream(data, streamNamePtr, timestamp); }); } - /** - * Sends a double packet into the specified stream at the given timestamp. - * @param data The double data to send. - * @param streamName The name of the graph input stream to send data into. - * @param timestamp The timestamp of the input data, in ms. - */ + /** {@override GraphRunnerApi} */ addDoubleToStream(data: number, streamName: string, timestamp: number): void { this.wrapStringPtr(streamName, (streamNamePtr: number) => { this.wasmModule._addDoubleToInputStream(data, streamNamePtr, timestamp); }); } - /** - * Sends a float packet into the specified stream at the given timestamp. - * @param data The float data to send. - * @param streamName The name of the graph input stream to send data into. - * @param timestamp The timestamp of the input data, in ms. - */ + /** {@override GraphRunnerApi} */ addFloatToStream(data: number, streamName: string, timestamp: number): void { this.wrapStringPtr(streamName, (streamNamePtr: number) => { // NOTE: _addFloatToStream and _addIntToStream are reserved for JS @@ -695,24 +419,14 @@ export class GraphRunner { }); } - /** - * Sends an integer packet into the specified stream at the given timestamp. - * @param data The integer data to send. - * @param streamName The name of the graph input stream to send data into. - * @param timestamp The timestamp of the input data, in ms. - */ + /** {@override GraphRunnerApi} */ addIntToStream(data: number, streamName: string, timestamp: number): void { this.wrapStringPtr(streamName, (streamNamePtr: number) => { this.wasmModule._addIntToInputStream(data, streamNamePtr, timestamp); }); } - /** - * Sends a string packet into the specified stream at the given timestamp. - * @param data The string data to send. - * @param streamName The name of the graph input stream to send data into. - * @param timestamp The timestamp of the input data, in ms. - */ + /** {@override GraphRunnerApi} */ addStringToStream(data: string, streamName: string, timestamp: number): void { this.wrapStringPtr(streamName, (streamNamePtr: number) => { this.wrapStringPtr(data, (dataPtr: number) => { @@ -722,14 +436,7 @@ export class GraphRunner { }); } - /** - * Sends a Record packet into the specified stream at the - * given timestamp. - * @param data The records to send (will become a - * std::flat_hash_map, streamName: string, timestamp: number): void { @@ -744,16 +451,7 @@ export class GraphRunner { }); } - /** - * Sends a serialized protobuffer packet into the specified stream at the - * given timestamp, to be parsed into the specified protobuffer type. - * @param data The binary (serialized) raw protobuffer data. - * @param protoType The C++ namespaced type this protobuffer data corresponds - * to (e.g. "foo.Bar"). It will be converted to this type when output as a - * packet into the graph. - * @param streamName The name of the graph input stream to send data into. - * @param timestamp The timestamp of the input data, in ms. - */ + /** {@override GraphRunnerApi} */ addProtoToStream( data: Uint8Array, protoType: string, streamName: string, timestamp: number): void { @@ -770,74 +468,42 @@ export class GraphRunner { }); } - /** - * Sends an empty packet into the specified stream at the given timestamp, - * effectively advancing that input stream's timestamp bounds without - * sending additional data packets. - * @param streamName The name of the graph input stream to send the empty - * packet into. - * @param timestamp The timestamp of the empty packet, in ms. - */ + /** {@override GraphRunnerApi} */ addEmptyPacketToStream(streamName: string, timestamp: number): void { this.wrapStringPtr(streamName, (streamNamePtr: number) => { this.wasmModule._addEmptyPacketToInputStream(streamNamePtr, timestamp); }); } - /** - * Attaches a boolean packet to the specified input_side_packet. - * @param data The boolean data to send. - * @param sidePacketName The name of the graph input side packet to send data - * into. - */ + /** {@override GraphRunnerApi} */ addBoolToInputSidePacket(data: boolean, sidePacketName: string): void { this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { this.wasmModule._addBoolToInputSidePacket(data, sidePacketNamePtr); }); } - /** - * Attaches a double packet to the specified input_side_packet. - * @param data The double data to send. - * @param sidePacketName The name of the graph input side packet to send data - * into. - */ + /** {@override GraphRunnerApi} */ addDoubleToInputSidePacket(data: number, sidePacketName: string): void { this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { this.wasmModule._addDoubleToInputSidePacket(data, sidePacketNamePtr); }); } - /** - * Attaches a float packet to the specified input_side_packet. - * @param data The float data to send. - * @param sidePacketName The name of the graph input side packet to send data - * into. - */ + /** {@override GraphRunnerApi} */ addFloatToInputSidePacket(data: number, sidePacketName: string): void { this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { this.wasmModule._addFloatToInputSidePacket(data, sidePacketNamePtr); }); } - /** - * Attaches a integer packet to the specified input_side_packet. - * @param data The integer data to send. - * @param sidePacketName The name of the graph input side packet to send data - * into. - */ + /** {@override GraphRunnerApi} */ addIntToInputSidePacket(data: number, sidePacketName: string): void { this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { this.wasmModule._addIntToInputSidePacket(data, sidePacketNamePtr); }); } - /** - * Attaches a string packet to the specified input_side_packet. - * @param data The string data to send. - * @param sidePacketName The name of the graph input side packet to send data - * into. - */ + /** {@override GraphRunnerApi} */ addStringToInputSidePacket(data: string, sidePacketName: string): void { this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { this.wrapStringPtr(data, (dataPtr: number) => { @@ -846,14 +512,7 @@ export class GraphRunner { }); } - /** - * Attaches a serialized proto packet to the specified input_side_packet. - * @param data The binary (serialized) raw protobuffer data. - * @param protoType The C++ namespaced type this protobuffer data corresponds - * to. It will be converted to this type for use in the graph. - * @param sidePacketName The name of the graph input side packet to send data - * into. - */ + /** {@override GraphRunnerApi} */ addProtoToInputSidePacket( data: Uint8Array, protoType: string, sidePacketName: string): void { this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { @@ -869,15 +528,7 @@ export class GraphRunner { }); } - /** - * Attaches a boolean packet listener to the specified output_stream. - * @param outputStreamName The name of the graph output stream to grab boolean - * data from. - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that the data is only guaranteed to exist for the - * duration of the callback, and the callback will be called inline, so it - * should not perform overly complicated (or any async) behavior. - */ + /** {@override GraphRunnerApi} */ attachBoolListener( outputStreamName: string, callbackFcn: SimpleListener): void { // Set up our TS listener to receive any packets for this stream. @@ -889,15 +540,7 @@ export class GraphRunner { }); } - /** - * Attaches a bool[] packet listener to the specified output_stream. - * @param outputStreamName The name of the graph output stream to grab - * std::vector data from. - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that the data is only guaranteed to exist for the - * duration of the callback, and the callback will be called inline, so it - * should not perform overly complicated (or any async) behavior. - */ + /** {@override GraphRunnerApi} */ attachBoolVectorListener( outputStreamName: string, callbackFcn: SimpleListener): void { // Set up our TS listener to receive any packets for this stream. @@ -909,15 +552,7 @@ export class GraphRunner { }); } - /** - * Attaches an int packet listener to the specified output_stream. - * @param outputStreamName The name of the graph output stream to grab int - * data from. - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that the data is only guaranteed to exist for the - * duration of the callback, and the callback will be called inline, so it - * should not perform overly complicated (or any async) behavior. - */ + /** {@override GraphRunnerApi} */ attachIntListener( outputStreamName: string, callbackFcn: SimpleListener): void { // Set up our TS listener to receive any packets for this stream. @@ -929,15 +564,7 @@ export class GraphRunner { }); } - /** - * Attaches an int[] packet listener to the specified output_stream. - * @param outputStreamName The name of the graph output stream to grab - * std::vector data from. - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that the data is only guaranteed to exist for the - * duration of the callback, and the callback will be called inline, so it - * should not perform overly complicated (or any async) behavior. - */ + /** {@override GraphRunnerApi} */ attachIntVectorListener( outputStreamName: string, callbackFcn: SimpleListener): void { // Set up our TS listener to receive any packets for this stream. @@ -949,15 +576,7 @@ export class GraphRunner { }); } - /** - * Attaches a double packet listener to the specified output_stream. - * @param outputStreamName The name of the graph output stream to grab double - * data from. - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that the data is only guaranteed to exist for the - * duration of the callback, and the callback will be called inline, so it - * should not perform overly complicated (or any async) behavior. - */ + /** {@override GraphRunnerApi} */ attachDoubleListener( outputStreamName: string, callbackFcn: SimpleListener): void { // Set up our TS listener to receive any packets for this stream. @@ -969,15 +588,7 @@ export class GraphRunner { }); } - /** - * Attaches a double[] packet listener to the specified output_stream. - * @param outputStreamName The name of the graph output stream to grab - * std::vector data from. - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that the data is only guaranteed to exist for the - * duration of the callback, and the callback will be called inline, so it - * should not perform overly complicated (or any async) behavior. - */ + /** {@override GraphRunnerApi} */ attachDoubleVectorListener( outputStreamName: string, callbackFcn: SimpleListener): void { // Set up our TS listener to receive any packets for this stream. @@ -989,15 +600,7 @@ export class GraphRunner { }); } - /** - * Attaches a float packet listener to the specified output_stream. - * @param outputStreamName The name of the graph output stream to grab float - * data from. - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that the data is only guaranteed to exist for the - * duration of the callback, and the callback will be called inline, so it - * should not perform overly complicated (or any async) behavior. - */ + /** {@override GraphRunnerApi} */ attachFloatListener( outputStreamName: string, callbackFcn: SimpleListener): void { // Set up our TS listener to receive any packets for this stream. @@ -1009,15 +612,7 @@ export class GraphRunner { }); } - /** - * Attaches a float[] packet listener to the specified output_stream. - * @param outputStreamName The name of the graph output stream to grab - * std::vector data from. - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that the data is only guaranteed to exist for the - * duration of the callback, and the callback will be called inline, so it - * should not perform overly complicated (or any async) behavior. - */ + /** {@override GraphRunnerApi} */ attachFloatVectorListener( outputStreamName: string, callbackFcn: SimpleListener): void { // Set up our TS listener to receive any packets for this stream. @@ -1029,15 +624,7 @@ export class GraphRunner { }); } - /** - * Attaches a string packet listener to the specified output_stream. - * @param outputStreamName The name of the graph output stream to grab string - * data from. - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that the data is only guaranteed to exist for the - * duration of the callback, and the callback will be called inline, so it - * should not perform overly complicated (or any async) behavior. - */ + /** {@override GraphRunnerApi} */ attachStringListener( outputStreamName: string, callbackFcn: SimpleListener): void { // Set up our TS listener to receive any packets for this stream. @@ -1049,15 +636,7 @@ export class GraphRunner { }); } - /** - * Attaches a string[] packet listener to the specified output_stream. - * @param outputStreamName The name of the graph output stream to grab - * std::vector data from. - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that the data is only guaranteed to exist for the - * duration of the callback, and the callback will be called inline, so it - * should not perform overly complicated (or any async) behavior. - */ + /** {@override GraphRunnerApi} */ attachStringVectorListener( outputStreamName: string, callbackFcn: SimpleListener): void { // Set up our TS listener to receive any packets for this stream. @@ -1069,25 +648,7 @@ export class GraphRunner { }); } - /** - * Attaches a serialized proto packet listener to the specified output_stream. - * @param outputStreamName The name of the graph output stream to grab binary - * serialized proto data from (in Uint8Array format). - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that by default the data is only guaranteed to - * exist for the duration of the callback, and the callback will be called - * inline, so it should not perform overly complicated (or any async) - * behavior. If the proto data needs to be able to outlive the call, you - * may set the optional makeDeepCopy parameter to true, or can manually - * deep-copy the data yourself. - * @param makeDeepCopy Optional convenience parameter which, if set to true, - * will override the default memory management behavior and make a deep - * copy of the underlying data, rather than just returning a view into the - * C++-managed memory. At the cost of a data copy, this allows the - * returned data to outlive the callback lifetime (and it will be cleaned - * up automatically by JS garbage collection whenever the user is finished - * with it). - */ + /** {@override GraphRunnerApi} */ attachProtoListener( outputStreamName: string, callbackFcn: SimpleListener, makeDeepCopy?: boolean): void { @@ -1102,26 +663,7 @@ export class GraphRunner { }); } - /** - * Attaches a listener for an array of serialized proto packets to the - * specified output_stream. - * @param outputStreamName The name of the graph output stream to grab a - * vector of binary serialized proto data from (in Uint8Array[] format). - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that by default the data is only guaranteed to - * exist for the duration of the callback, and the callback will be called - * inline, so it should not perform overly complicated (or any async) - * behavior. If the proto data needs to be able to outlive the call, you - * may set the optional makeDeepCopy parameter to true, or can manually - * deep-copy the data yourself. - * @param makeDeepCopy Optional convenience parameter which, if set to true, - * will override the default memory management behavior and make a deep - * copy of the underlying data, rather than just returning a view into the - * C++-managed memory. At the cost of a data copy, this allows the - * returned data to outlive the callback lifetime (and it will be cleaned - * up automatically by JS garbage collection whenever the user is finished - * with it). - */ + /** {@override GraphRunnerApi} */ attachProtoVectorListener( outputStreamName: string, callbackFcn: SimpleListener, makeDeepCopy?: boolean): void { @@ -1136,26 +678,7 @@ export class GraphRunner { }); } - /** - * Attaches an audio packet listener to the specified output_stream, to be - * given a Float32Array as output. - * @param outputStreamName The name of the graph output stream to grab audio - * data from. - * @param callbackFcn The function that will be called back with the data, as - * it is received. Note that the data is only guaranteed to exist for the - * duration of the callback, and the callback will be called inline, so it - * should not perform overly complicated (or any async) behavior. If the - * audio data needs to be able to outlive the call, you may set the - * optional makeDeepCopy parameter to true, or can manually deep-copy the - * data yourself. - * @param makeDeepCopy Optional convenience parameter which, if set to true, - * will override the default memory management behavior and make a deep - * copy of the underlying data, rather than just returning a view into the - * C++-managed memory. At the cost of a data copy, this allows the - * returned data to outlive the callback lifetime (and it will be cleaned - * up automatically by JS garbage collection whenever the user is finished - * with it). - */ + /** {@override GraphRunnerApi} */ attachAudioListener( outputStreamName: string, callbackFcn: SimpleListener, makeDeepCopy?: boolean): void { @@ -1181,19 +704,12 @@ export class GraphRunner { }); } - /** - * Forces all queued-up packets to be pushed through the MediaPipe graph as - * far as possible, performing all processing until no more processing can be - * done. - */ + /** {@override GraphRunnerApi} */ finishProcessing(): void { this.wasmModule._waitUntilIdle(); } - /** - * Closes the input streams and all calculators for this graph and frees up - * any C++ resources. The graph will not be usable once closed. - */ + /** {@override GraphRunnerApi} */ closeGraph(): void { this.wasmModule._closeGraph(); this.wasmModule.simpleListeners = undefined; @@ -1221,38 +737,13 @@ async function runScript(scriptUrl: string|string) { } } -/** - * Helper type macro for use with createMediaPipeLib. Allows us to easily infer - * the type of a mixin-extended GraphRunner. Example usage: - * const GraphRunnerConstructor = - * SupportImage(SupportSerialization(GraphRunner)); - * let mediaPipe: ReturnType; - * ... - * mediaPipe = await createMediaPipeLib(GraphRunnerConstructor, ...); - */ -// tslint:disable-next-line:no-any -export type ReturnType = T extends (...args: unknown[]) => infer R ? R : any; - -/** - * Global function to initialize Wasm blob and load runtime assets for a - * specialized MediaPipe library. This allows us to create a requested - * subclass inheriting from GraphRunner. - * @param constructorFcn The name of the class to instantiate via "new". - * @param wasmLoaderScript Url for the wasm-runner script; produced by the build - * process. - * @param assetLoaderScript Url for the asset-loading script; produced by the - * build process. - * @param fileLocator A function to override the file locations for assets - * loaded by the MediaPipe library. - * @return promise A promise which will resolve when initialization has - * completed successfully. - */ -export async function createMediaPipeLib( +/** {@override CreateMediaPipeLibApi} */ +export const createMediaPipeLib: CreateMediaPipeLibApi = async( constructorFcn: WasmMediaPipeConstructor, wasmLoaderScript?: string|null, assetLoaderScript?: string|null, glCanvas?: HTMLCanvasElement|OffscreenCanvas|null, - fileLocator?: FileLocator): Promise { + fileLocator?: FileLocator): Promise => { // Run wasm-loader script here if (wasmLoaderScript) { await runScript(wasmLoaderScript); @@ -1288,26 +779,24 @@ export async function createMediaPipeLib( // Don't reuse factory or module seed self.ModuleFactory = self.Module = undefined; return new constructorFcn(module, glCanvas); +}; + +// We extend the CreateGraphRunnerApi interface here for now so that by default +// callers of `createGraphRunner` will be given a `GraphRunner` rather than a +// `GraphRunnerApi`. +interface CreateGraphRunnerImplType extends CreateGraphRunnerApi { + (wasmLoaderScript?: string, + assetLoaderScript?: string, + glCanvas?: HTMLCanvasElement|OffscreenCanvas|null, + fileLocator?: FileLocator): Promise; } -/** - * Global function to initialize Wasm blob and load runtime assets for a generic - * MediaPipe library. - * @param wasmLoaderScript Url for the wasm-runner script; produced by the build - * process. - * @param assetLoaderScript Url for the asset-loading script; produced by the - * build process. - * @param fileLocator A function to override the file locations for assets - * loaded by the MediaPipe library. - * @return promise A promise which will resolve when initialization has - * completed successfully. - */ -export async function createGraphRunner( +/** {@override CreateGraphRunnerApi} */ +export const createGraphRunner: CreateGraphRunnerImplType = async( wasmLoaderScript?: string, assetLoaderScript?: string, glCanvas?: HTMLCanvasElement|OffscreenCanvas|null, - fileLocator?: FileLocator): Promise { + fileLocator?: FileLocator): Promise => { return createMediaPipeLib( - GraphRunner, wasmLoaderScript, assetLoaderScript, glCanvas, - fileLocator); -} + GraphRunner, wasmLoaderScript, assetLoaderScript, glCanvas, fileLocator); +}; diff --git a/mediapipe/web/graph_runner/graph_runner_api.d.ts b/mediapipe/web/graph_runner/graph_runner_api.d.ts new file mode 100644 index 000000000..cb5172491 --- /dev/null +++ b/mediapipe/web/graph_runner/graph_runner_api.d.ts @@ -0,0 +1,583 @@ +import {EmptyPacketListener, ErrorListener, SimpleListener, VectorListener} from './listener_types'; +import {WasmModule} from './wasm_module'; + +/** + * This file can serve as a common interface for most MediaPipe-based TypeScript + * libraries. Additionally, it can hook automatically into `wasm_mediapipe_demo` + * to allow for easy migrations from old pure JS demos. See `mediapipe_ts_demo` + * and `wasm_mediapipe_files` BUILD rules. Public API is described below in more + * detail, public API for GraphRunner factory functions is described in + * "graph_runner_factory_api.d.ts", and the same for listener callbacks + * can be found in "listenertypes.d.ts". An actual implementation is coded in + * `graph_runner.ts`, and that can be built against directly. The purpose of + * this file is primarily to enforce a common interface and provide better + * documentation on the design of the API as a whole. + */ + +// We re-export all of our imported public listener types, so that users can +// import everything they need for the public interfaces directly from here. +export { + EmptyPacketListener, + ErrorListener, + SimpleListener, + VectorListener, +}; + +/** + * Valid types of image sources which we can run our GraphRunner over. + */ +export type ImageSource = + HTMLCanvasElement|HTMLVideoElement|HTMLImageElement|ImageData|ImageBitmap; + +/** + * Simple interface for a class to run an arbitrary MediaPipe graph on web, and + * either render results into canvas, or else stream output into attached + * listeners. Takes a WebAssembly Module and an optional canvas for rendering. + * Standard implementation is `GraphRunner`. + * There are three categories of functions: + * - Those which add inputs into the graph (named `add*ToStream` or + * `add*ToInputSidePacket`). + * - Those which listen for outputs from the graph (named `attach*Listener`). + * - Those concerned with the graph running itself (runner settings, loading + * the graph, destruction, and `finishProcessing`). + * The expected ordering of one-time initialization calls should be: + * 1. Construction + * 2. Adding any input side packets and attaching any output listeners + * 3. Initializing/setting the graph + * After this, for the main loop, the user would add packets to streams for a + * given frame and then call `finishProcessing()`. + * Example usage pattern would be: + * ``` + * // We assume we have already constructed a GraphRunner called graphRunner. + * // Generally we would do this via a helpful factory method like + * // `createGraphRunner` or some variant thereof. + * const graphRunner: GraphRunner; + * + * // Initialization code: + * graphRunner.addBoolToInputSidePacket(true, 'some_input_bool_side_packet'); + * graphRunner.attachStringVectorListener('some_output_string_vec_stream', + * (data: string[], timestamp: number) => { console.log(data); }); + * await graphRunner.initializeGraph('path/to/graph_file.pbtxt'); + * + * // Main loop code (run every frame): + * const frameTimestamp = performance.now(); + * graphRunner.addStringToStream( + * 'Hello World', 'some_input_string_stream', frameTimestamp); + * graphRunner.addFloatToStream( + * 3.7, 'some_input_float_stream', frameTimestamp); + * graphRunner.finishProcessing(); + * + * // When done (no need to call if the user is just expected to close the + * // browser tab): + * graphRunner.closeGraph(); + * ``` + */ +export interface GraphRunnerApi { + /** + * Fetches a MediaPipe graph from a URL string pointing to the graph file. + * Will then set the graph using the results, replacing the previously running + * MediaPipe graph, if there is one. If the graph file's name ends with + * ".pbtxt" with ".textproto", it will assume text-formatted, and otherwise + * will assume a binary format for the proto. + * @param graphFile The url of the MediaPipe graph file to load. + */ + initializeGraph(graphFile: string): Promise; + + /** + * Convenience helper for loading a MediaPipe graph from a string representing + * a text proto config. Useful for graph files which are expected to be edited + * locally, web-side. Will replace the previously running MediaPipe graph, + * if there is one. + * @param graphConfig The text proto graph config, expected to be a string in + * default JavaScript UTF-16 format. + */ + setGraphFromString(graphConfig: string): void; + + /** + * Takes the raw data from a MediaPipe graph, and passes it to C++ to be run + * over the video stream. Will replace the previously running MediaPipe graph, + * if there is one. + * @param graphData The raw MediaPipe graph data, either in binary + * protobuffer format (.binarypb), or else in raw text format (.pbtxt or + * .textproto). + * @param isBinary This should be set to true if the graph is in + * binary format, and false if it is in human-readable text format. + */ + setGraph(graphData: Uint8Array, isBinary: boolean): void; + + /** + * Configures the current graph to handle audio processing in a certain way + * for all its audio input streams. Additionally can configure audio headers + * (both input side packets as well as input stream headers), but these + * configurations only take effect if called before the graph is set/started. + * @param numChannels The number of channels of audio input. Only 1 + * is supported for now. + * @param numSamples The number of samples that are taken in each + * audio capture. + * @param sampleRate The rate, in Hz, of the sampling. + * @param streamName The optional name of the input stream to additionally + * configure with audio information. This configuration only occurs before + * the graph is set/started. If unset, a default stream name will be used. + * @param headerName The optional name of the header input side packet to + * additionally configure with audio information. This configuration only + * occurs before the graph is set/started. If unset, a default header name + * will be used. + */ + configureAudio( + numChannels: number, numSamples: number, sampleRate: number, + streamName?: string, headerName?: string): void; + + /** + * Allows disabling automatic canvas resizing, in case clients want to control + * control this. By default, the canvas will be resized to the size of the + * last GPU image input (see `addGpuBufferToStream`). + * @param resize True will re-enable automatic canvas resizing, while false + * will disable the feature. + */ + setAutoResizeCanvas(resize: boolean): void; + + /** + * Allows disabling the automatic render-to-screen code, in case clients don't + * need/want this. In particular, this removes the requirement for pipelines + * to have access to GPU resources, as well as the requirement for graphs to + * have an "output_frames_gpu" stream defined, so pure CPU pipelines and + * non-video pipelines can be created. + * NOTE: This only affects future graph initializations (via `setGraph`` or + * `initializeGraph``), and does NOT affect the currently running graph, + * so calls to this should be made *before* `setGraph`/`initializeGraph` + * for the graph file being targeted. + * @param enabled True will re-enable automatic render-to-screen code and + * cause GPU resources to once again be requested, while false will + * disable the feature. + */ + setAutoRenderToScreen(enabled: boolean): void; + + /** + * Overrides the vertical orientation for input GpuBuffers and the automatic + * render-to-screen code. The default for our OpenGL code on other platforms + * (Android, Linux) is to use a bottom-left origin. But the default for WebGL + * is to use a top-left origin. We use WebGL default normally, and many + * calculators and graphs have platform-specific code to handle the resulting + * orientation flip. However, in order to be able to use a single graph on all + * platforms without alterations, it may be useful to send images into a web + * graph using the OpenGL orientation. Users can call this function with + * `bottomLeftIsOrigin = true` in order to enforce an orientation for all + * GpuBuffer inputs which is consistent with OpenGL on other platforms. + * This call will also vertically flip the automatic render-to-screen code as + * well, so that webcam input (for example) will render properly when passed + * through the graph still. + * NOTE: This will immediately affect GpuBuffer inputs, but must be called + * *before* graph start in order to affect the automatic render-to-screen + * code! + * @param bottomLeftIsOrigin True will flip our input GpuBuffers and auto + * render-to-screen to match the classic OpenGL orientation, while false will + * disable this feature to match the default WebGL orientation. + */ + setGpuBufferVerticalFlip(bottomLeftIsOrigin: boolean): void; + + /** + * Attaches a listener that will be invoked when the MediaPipe web framework + * returns an error. + */ + attachErrorListener(callbackFcn: ErrorListener): void; + + /** + * Attaches a listener that will be invoked when the MediaPipe framework + * receives an empty packet on the provided output stream. This can be used + * to receive the latest output timestamp. + * + * Empty packet listeners are only active if there is a corresponding packet + * listener. + * + * @param outputStreamName The name of the graph output stream to receive + * empty packets from. + * @param callbackFcn The callback to receive the timestamp. + */ + attachEmptyPacketListener( + outputStreamName: string, callbackFcn: EmptyPacketListener): void; + + /** + * Takes the raw data from a JS audio capture array, and sends it to C++ to be + * processed. + * @param audioData An array of raw audio capture data, like + * from a call to getChannelData on an AudioBuffer. + * @param streamName The name of the MediaPipe graph stream to add the audio + * data to. + * @param timestamp The timestamp of the current frame, in ms. + */ + addAudioToStream( + audioData: Float32Array, streamName: string, timestamp: number): void; + + /** + * Takes the raw data from a JS audio capture array, and sends it to C++ to be + * processed, shaping the audioData array into an audio matrix according to + * the numChannels and numSamples parameters. + * @param audioData An array of raw audio capture data, like + * from a call to getChannelData on an AudioBuffer. + * @param numChannels The number of audio channels this data represents. If 0 + * is passed, then the value will be taken from the last call to + * configureAudio. + * @param numSamples The number of audio samples captured in this data packet. + * If 0 is passed, then the value will be taken from the last call to + * configureAudio. + * @param streamName The name of the MediaPipe graph stream to add the audio + * data to. + * @param timestamp The timestamp of the current frame, in ms. + */ + addAudioToStreamWithShape( + audioData: Float32Array, numChannels: number, numSamples: number, + streamName: string, timestamp: number): void; + + /** + * Takes the relevant information from the HTML video or image element, and + * passes it into the WebGL-based graph for processing on the given stream at + * the given timestamp. Can be used for additional auxiliary GpuBuffer input + * streams. Like all `add*ToStream` calls, processing will not occur until a + * blocking call like `finishProcessing` or the deprecated `processGl` is + * made. For use with 'gl_graph_runner_internal_multi_input'. + * @param imageSource Reference to the video frame we wish to add into our + * graph. + * @param streamName The name of the MediaPipe graph stream to add the frame + * to. + * @param timestamp The timestamp of the input frame, in ms. + */ + addGpuBufferToStream( + imageSource: ImageSource, streamName: string, timestamp: number): void; + + /** + * Sends a boolean packet into the specified stream at the given timestamp. + * @param data The boolean data to send. + * @param streamName The name of the graph input stream to send data into. + * @param timestamp The timestamp of the input data, in ms. + */ + addBoolToStream(data: boolean, streamName: string, timestamp: number): void; + + /** + * Sends a double packet into the specified stream at the given timestamp. + * @param data The double data to send. + * @param streamName The name of the graph input stream to send data into. + * @param timestamp The timestamp of the input data, in ms. + */ + addDoubleToStream(data: number, streamName: string, timestamp: number): void; + + /** + * Sends a float packet into the specified stream at the given timestamp. + * @param data The float data to send. + * @param streamName The name of the graph input stream to send data into. + * @param timestamp The timestamp of the input data, in ms. + */ + addFloatToStream(data: number, streamName: string, timestamp: number): void; + + /** + * Sends an integer packet into the specified stream at the given timestamp. + * @param data The integer data to send. + * @param streamName The name of the graph input stream to send data into. + * @param timestamp The timestamp of the input data, in ms. + */ + addIntToStream(data: number, streamName: string, timestamp: number): void; + + /** + * Sends a string packet into the specified stream at the given timestamp. + * @param data The string data to send. + * @param streamName The name of the graph input stream to send data into. + * @param timestamp The timestamp of the input data, in ms. + */ + addStringToStream(data: string, streamName: string, timestamp: number): void; + + /** + * Sends a Record packet into the specified stream at the + * given timestamp. + * @param data The records to send (will become a + * std::flat_hash_map, streamName: string, + timestamp: number): void; + + /** + * Sends a serialized protobuffer packet into the specified stream at the + * given timestamp, to be parsed into the specified protobuffer type. + * @param data The binary (serialized) raw protobuffer data. + * @param protoType The C++ namespaced type this protobuffer data corresponds + * to (e.g. "foo.Bar"). It will be converted to this type when output as a + * packet into the graph. + * @param streamName The name of the graph input stream to send data into. + * @param timestamp The timestamp of the input data, in ms. + */ + addProtoToStream( + data: Uint8Array, protoType: string, streamName: string, + timestamp: number): void; + + /** + * Sends an empty packet into the specified stream at the given timestamp, + * effectively advancing that input stream's timestamp bounds without + * sending additional data packets. + * @param streamName The name of the graph input stream to send the empty + * packet into. + * @param timestamp The timestamp of the empty packet, in ms. + */ + addEmptyPacketToStream(streamName: string, timestamp: number): void; + + /** + * Attaches a boolean packet to the specified input_side_packet. + * @param data The boolean data to send. + * @param sidePacketName The name of the graph input side packet to send data + * into. + */ + addBoolToInputSidePacket(data: boolean, sidePacketName: string): void; + + /** + * Attaches a double packet to the specified input_side_packet. + * @param data The double data to send. + * @param sidePacketName The name of the graph input side packet to send data + * into. + */ + addDoubleToInputSidePacket(data: number, sidePacketName: string): void; + + /** + * Attaches a float packet to the specified input_side_packet. + * @param data The float data to send. + * @param sidePacketName The name of the graph input side packet to send data + * into. + */ + addFloatToInputSidePacket(data: number, sidePacketName: string): void; + + /** + * Attaches a integer packet to the specified input_side_packet. + * @param data The integer data to send. + * @param sidePacketName The name of the graph input side packet to send data + * into. + */ + addIntToInputSidePacket(data: number, sidePacketName: string): void; + + /** + * Attaches a string packet to the specified input_side_packet. + * @param data The string data to send. + * @param sidePacketName The name of the graph input side packet to send data + * into. + */ + addStringToInputSidePacket(data: string, sidePacketName: string): void; + + /** + * Attaches a serialized proto packet to the specified input_side_packet. + * @param data The binary (serialized) raw protobuffer data. + * @param protoType The C++ namespaced type this protobuffer data corresponds + * to. It will be converted to this type for use in the graph. + * @param sidePacketName The name of the graph input side packet to send data + * into. + */ + addProtoToInputSidePacket( + data: Uint8Array, protoType: string, sidePacketName: string): void; + + /** + * Attaches a boolean packet listener to the specified output_stream. + * @param outputStreamName The name of the graph output stream to grab boolean + * data from. + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that the data is only guaranteed to exist for the + * duration of the callback, and the callback will be called inline, so it + * should not perform overly complicated (or any async) behavior. + */ + attachBoolListener( + outputStreamName: string, callbackFcn: SimpleListener): void; + + /** + * Attaches a bool[] packet listener to the specified output_stream. + * @param outputStreamName The name of the graph output stream to grab + * std::vector data from. + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that the data is only guaranteed to exist for the + * duration of the callback, and the callback will be called inline, so it + * should not perform overly complicated (or any async) behavior. + */ + attachBoolVectorListener( + outputStreamName: string, callbackFcn: SimpleListener): void; + + /** + * Attaches an int packet listener to the specified output_stream. + * @param outputStreamName The name of the graph output stream to grab int + * data from. + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that the data is only guaranteed to exist for the + * duration of the callback, and the callback will be called inline, so it + * should not perform overly complicated (or any async) behavior. + */ + attachIntListener( + outputStreamName: string, callbackFcn: SimpleListener): void; + + /** + * Attaches an int[] packet listener to the specified output_stream. + * @param outputStreamName The name of the graph output stream to grab + * std::vector data from. + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that the data is only guaranteed to exist for the + * duration of the callback, and the callback will be called inline, so it + * should not perform overly complicated (or any async) behavior. + */ + attachIntVectorListener( + outputStreamName: string, callbackFcn: SimpleListener): void; + + /** + * Attaches a double packet listener to the specified output_stream. + * @param outputStreamName The name of the graph output stream to grab double + * data from. + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that the data is only guaranteed to exist for the + * duration of the callback, and the callback will be called inline, so it + * should not perform overly complicated (or any async) behavior. + */ + attachDoubleListener( + outputStreamName: string, callbackFcn: SimpleListener): void; + + /** + * Attaches a double[] packet listener to the specified output_stream. + * @param outputStreamName The name of the graph output stream to grab + * std::vector data from. + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that the data is only guaranteed to exist for the + * duration of the callback, and the callback will be called inline, so it + * should not perform overly complicated (or any async) behavior. + */ + attachDoubleVectorListener( + outputStreamName: string, callbackFcn: SimpleListener): void; + + /** + * Attaches a float packet listener to the specified output_stream. + * @param outputStreamName The name of the graph output stream to grab float + * data from. + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that the data is only guaranteed to exist for the + * duration of the callback, and the callback will be called inline, so it + * should not perform overly complicated (or any async) behavior. + */ + attachFloatListener( + outputStreamName: string, callbackFcn: SimpleListener): void; + + /** + * Attaches a float[] packet listener to the specified output_stream. + * @param outputStreamName The name of the graph output stream to grab + * std::vector data from. + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that the data is only guaranteed to exist for the + * duration of the callback, and the callback will be called inline, so it + * should not perform overly complicated (or any async) behavior. + */ + attachFloatVectorListener( + outputStreamName: string, callbackFcn: SimpleListener): void; + + /** + * Attaches a string packet listener to the specified output_stream. + * @param outputStreamName The name of the graph output stream to grab string + * data from. + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that the data is only guaranteed to exist for the + * duration of the callback, and the callback will be called inline, so it + * should not perform overly complicated (or any async) behavior. + */ + attachStringListener( + outputStreamName: string, callbackFcn: SimpleListener): void; + + /** + * Attaches a string[] packet listener to the specified output_stream. + * @param outputStreamName The name of the graph output stream to grab + * std::vector data from. + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that the data is only guaranteed to exist for the + * duration of the callback, and the callback will be called inline, so it + * should not perform overly complicated (or any async) behavior. + */ + attachStringVectorListener( + outputStreamName: string, callbackFcn: SimpleListener): void; + + /** + * Attaches a serialized proto packet listener to the specified output_stream. + * @param outputStreamName The name of the graph output stream to grab binary + * serialized proto data from (in Uint8Array format). + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that by default the data is only guaranteed to + * exist for the duration of the callback, and the callback will be called + * inline, so it should not perform overly complicated (or any async) + * behavior. If the proto data needs to be able to outlive the call, you + * may set the optional makeDeepCopy parameter to true, or can manually + * deep-copy the data yourself. + * @param makeDeepCopy Optional convenience parameter which, if set to true, + * will override the default memory management behavior and make a deep + * copy of the underlying data, rather than just returning a view into the + * C++-managed memory. At the cost of a data copy, this allows the + * returned data to outlive the callback lifetime (and it will be cleaned + * up automatically by JS garbage collection whenever the user is finished + * with it). + */ + attachProtoListener( + outputStreamName: string, callbackFcn: SimpleListener, + makeDeepCopy?: boolean): void; + + /** + * Attaches a listener for an array of serialized proto packets to the + * specified output_stream. + * @param outputStreamName The name of the graph output stream to grab a + * vector of binary serialized proto data from (in Uint8Array[] format). + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that by default the data is only guaranteed to + * exist for the duration of the callback, and the callback will be called + * inline, so it should not perform overly complicated (or any async) + * behavior. If the proto data needs to be able to outlive the call, you + * may set the optional makeDeepCopy parameter to true, or can manually + * deep-copy the data yourself. + * @param makeDeepCopy Optional convenience parameter which, if set to true, + * will override the default memory management behavior and make a deep + * copy of the underlying data, rather than just returning a view into the + * C++-managed memory. At the cost of a data copy, this allows the + * returned data to outlive the callback lifetime (and it will be cleaned + * up automatically by JS garbage collection whenever the user is finished + * with it). + */ + attachProtoVectorListener( + outputStreamName: string, callbackFcn: SimpleListener, + makeDeepCopy?: boolean): void; + + /** + * Attaches an audio packet listener to the specified output_stream, to be + * given a Float32Array as output. Requires wasm build dependency + * "gl_graph_runner_audio_out". + * @param outputStreamName The name of the graph output stream to grab audio + * data from. + * @param callbackFcn The function that will be called back with the data, as + * it is received. Note that the data is only guaranteed to exist for the + * duration of the callback, and the callback will be called inline, so it + * should not perform overly complicated (or any async) behavior. If the + * audio data needs to be able to outlive the call, you may set the + * optional makeDeepCopy parameter to true, or can manually deep-copy the + * data yourself. + * @param makeDeepCopy Optional convenience parameter which, if set to true, + * will override the default memory management behavior and make a deep + * copy of the underlying data, rather than just returning a view into the + * C++-managed memory. At the cost of a data copy, this allows the + * returned data to outlive the callback lifetime (and it will be cleaned + * up automatically by JS garbage collection whenever the user is finished + * with it). + */ + attachAudioListener( + outputStreamName: string, callbackFcn: SimpleListener, + makeDeepCopy?: boolean): void; + + /** + * Forces all queued-up packets to be pushed through the MediaPipe graph as + * far as possible, performing all processing until no more processing can be + * done. This is fully synchronous and by default single-threaded, so the + * calling thread will be blocked until processing completes. This must be + * called once for every frame, as `add*ToStream` calls merely queue up + * packets to the appropriate streams for processing, but processing does not + * occur until this function is called. Any listeners receiving output will + * be called back before this operation completes. + */ + finishProcessing(): void; + + /** + * Closes the input streams and all calculators for this graph and frees up + * any C++ resources. The graph will not be usable once closed! + */ + closeGraph(): void; +} diff --git a/mediapipe/web/graph_runner/graph_runner_factory_api.d.ts b/mediapipe/web/graph_runner/graph_runner_factory_api.d.ts new file mode 100644 index 000000000..67732abaf --- /dev/null +++ b/mediapipe/web/graph_runner/graph_runner_factory_api.d.ts @@ -0,0 +1,79 @@ +// Placeholder for internal dependency on trusted resource url + +import {GraphRunnerApi} from './graph_runner_api'; +import {WasmModule} from './wasm_module'; + +/** + * Simple interface for allowing users to set the directory where internal + * wasm-loading and asset-loading code looks (e.g. for .wasm and .data file + * locations). + */ +export declare interface FileLocator { + locateFile: (filename: string) => string; + mainScriptUrlOrBlob?: string; +} + +/** + * Helper type macro for use with createMediaPipeLib. Allows us to easily infer + * the type of a mixin-extended GraphRunner. Example usage: + * const GraphRunnerConstructor = + * SupportImage(SupportSerialization(GraphRunner)); + * let mediaPipe: ReturnType; + * ... + * mediaPipe = await createMediaPipeLib(GraphRunnerConstructor, ...); + */ +// tslint:disable-next-line:no-any +export type ReturnType = T extends(...args: unknown[]) => infer R ? R : any; + +/** + * Internal type of constructors used for initializing GraphRunner and + * subclasses. + */ +export type WasmMediaPipeConstructor = + (new ( + module: WasmModule, canvas?: HTMLCanvasElement|OffscreenCanvas|null) => + LibType); + +/** + * Global function interface to initialize Wasm blob and load runtime assets for + * a specialized MediaPipe library. This allows us to create a requested + * subclass inheriting from GraphRunner. Standard implementation is + * `createMediaPipeLib`. + * @param constructorFcn The name of the class to instantiate via "new". + * @param wasmLoaderScript Url for the wasm-runner script; produced by the build + * process. + * @param assetLoaderScript Url for the asset-loading script; produced by the + * build process. + * @param fileLocator A function to override the file locations for assets + * loaded by the MediaPipe library. + * @return promise A promise which will resolve when initialization has + * completed successfully. + */ +export interface CreateMediaPipeLibApi { + ( + constructorFcn: WasmMediaPipeConstructor, + wasmLoaderScript?: string|null, + assetLoaderScript?: string|null, + glCanvas?: HTMLCanvasElement|OffscreenCanvas|null, + fileLocator?: FileLocator): Promise; +} + +/** + * Global function interface to initialize Wasm blob and load runtime assets for + * a generic MediaPipe library. Standard implementation is + * `createGraphRunner`. + * @param wasmLoaderScript Url for the wasm-runner script; produced by the build + * process. + * @param assetLoaderScript Url for the asset-loading script; produced by the + * build process. + * @param fileLocator A function to override the file locations for assets + * loaded by the MediaPipe library. + * @return promise A promise which will resolve when initialization has + * completed successfully. + */ +export interface CreateGraphRunnerApi { + (wasmLoaderScript?: string, + assetLoaderScript?: string, + glCanvas?: HTMLCanvasElement|OffscreenCanvas|null, + fileLocator?: FileLocator): Promise; +} diff --git a/mediapipe/web/graph_runner/listener_types.d.ts b/mediapipe/web/graph_runner/listener_types.d.ts new file mode 100644 index 000000000..81af40382 --- /dev/null +++ b/mediapipe/web/graph_runner/listener_types.d.ts @@ -0,0 +1,23 @@ +/** + * A listener that receives the contents of a non-empty MediaPipe packet and + * its timestamp. + */ +export type SimpleListener = (data: T, timestamp: number) => void; + +/** + * A listener that receives the current MediaPipe packet timestamp. This is + * invoked even for empty packet. + */ +export type EmptyPacketListener = (timestamp: number) => void; + +/** + * A listener that receives a single element of vector-returning output packet. + * Receives one element at a time (in order). Once all elements are processed, + * the listener is invoked with `data` set to `unknown` and `done` set to true. + * Intended for internal usage. + */ +export type VectorListener = (data: T, done: boolean, timestamp: number) => + void; + +/** A listener that will be invoked with an absl::StatusCode and message. */ +export type ErrorListener = (code: number, message: string) => void; diff --git a/mediapipe/web/graph_runner/wasm_module.d.ts b/mediapipe/web/graph_runner/wasm_module.d.ts new file mode 100644 index 000000000..1dad39a53 --- /dev/null +++ b/mediapipe/web/graph_runner/wasm_module.d.ts @@ -0,0 +1,111 @@ +import {EmptyPacketListener, ErrorListener, SimpleListener, VectorListener} from './listener_types'; + +/** + * Declarations for Emscripten's WebAssembly Module behavior, so TS compiler + * doesn't break our various JS/C++ bridges. For internal usage. + */ +export declare interface WasmModule { + canvas: HTMLCanvasElement|OffscreenCanvas|null; + HEAPU8: Uint8Array; + HEAPU32: Uint32Array; + HEAPF32: Float32Array; + HEAPF64: Float64Array; + FS_createDataFile: + (parent: string, name: string, data: Uint8Array, canRead: boolean, + canWrite: boolean, canOwn: boolean) => void; + FS_createPath: + (parent: string, name: string, canRead: boolean, + canWrite: boolean) => void; + FS_unlink(path: string): void; + gpuOriginForWebTexturesIsBottomLeft?: boolean; + + errorListener?: ErrorListener; + _bindTextureToCanvas: () => boolean; + _changeBinaryGraph: (size: number, dataPtr: number) => void; + _changeTextGraph: (size: number, dataPtr: number) => void; + _closeGraph: () => void; + _free: (ptr: number) => void; + _malloc: (size: number) => number; + _processFrame: (width: number, height: number, timestamp: number) => void; + _setAutoRenderToScreen: (enabled: boolean) => void; + _waitUntilIdle: () => void; + + // Exposed so that clients of this lib can access this field + dataFileDownloads?: {[url: string]: {loaded: number, total: number}}; + + // Wasm Module multistream entrypoints. Require + // gl_graph_runner_internal_multi_input as a build dependency. + stringToNewUTF8: (data: string) => number; + _bindTextureToStream: (streamNamePtr: number) => void; + _addBoundTextureToStream: + (streamNamePtr: number, width: number, height: number, + timestamp: number) => void; + _addBoolToInputStream: + (data: boolean, streamNamePtr: number, timestamp: number) => void; + _addDoubleToInputStream: + (data: number, streamNamePtr: number, timestamp: number) => void; + _addFloatToInputStream: + (data: number, streamNamePtr: number, timestamp: number) => void; + _addIntToInputStream: + (data: number, streamNamePtr: number, timestamp: number) => void; + _addStringToInputStream: + (dataPtr: number, streamNamePtr: number, timestamp: number) => void; + _addFlatHashMapToInputStream: + (keysPtr: number, valuesPtr: number, count: number, streamNamePtr: number, + timestamp: number) => void; + _addProtoToInputStream: + (dataPtr: number, dataSize: number, protoNamePtr: number, + streamNamePtr: number, timestamp: number) => void; + _addEmptyPacketToInputStream: + (streamNamePtr: number, timestamp: number) => void; + + // Input side packets + _addBoolToInputSidePacket: (data: boolean, streamNamePtr: number) => void; + _addDoubleToInputSidePacket: (data: number, streamNamePtr: number) => void; + _addFloatToInputSidePacket: (data: number, streamNamePtr: number) => void; + _addIntToInputSidePacket: (data: number, streamNamePtr: number) => void; + _addStringToInputSidePacket: (dataPtr: number, streamNamePtr: number) => void; + _addProtoToInputSidePacket: + (dataPtr: number, dataSize: number, protoNamePtr: number, + streamNamePtr: number) => void; + + // Map of output streams to packet listeners. Also built as part of + // gl_graph_runner_internal_multi_input. + simpleListeners?: + Record|VectorListener>; + // Map of output streams to empty packet listeners. + emptyPacketListeners?: Record; + _attachBoolListener: (streamNamePtr: number) => void; + _attachBoolVectorListener: (streamNamePtr: number) => void; + _attachDoubleListener: (streamNamePtr: number) => void; + _attachDoubleVectorListener: (streamNamePtr: number) => void; + _attachFloatListener: (streamNamePtr: number) => void; + _attachFloatVectorListener: (streamNamePtr: number) => void; + _attachIntListener: (streamNamePtr: number) => void; + _attachIntVectorListener: (streamNamePtr: number) => void; + _attachStringListener: (streamNamePtr: number) => void; + _attachStringVectorListener: (streamNamePtr: number) => void; + _attachProtoListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void; + _attachProtoVectorListener: + (streamNamePtr: number, makeDeepCopy?: boolean) => void; + + // Require dependency ":gl_graph_runner_audio_out" + _attachAudioListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void; + + // Require dependency ":gl_graph_runner_audio" + _addAudioToInputStream: + (dataPtr: number, numChannels: number, numSamples: number, + streamNamePtr: number, timestamp: number) => void; + _configureAudio: + (channels: number, samples: number, sampleRate: number, + streamNamePtr: number, headerNamePtr: number) => void; + + // Get the graph configuration and invoke the listener configured under + // streamNamePtr + _getGraphConfig: (streamNamePtr: number, makeDeepCopy?: boolean) => void; + + // TODO: Refactor to just use a few numbers (perhaps refactor away + // from gl_graph_runner_internal.cc entirely to use something a little more + // streamlined; new version is _processFrame above). + _processGl: (frameDataPtr: number) => number; +}