cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Splitting GraphRunner into public API declared interfaces and private TS impls

Browse Source 
PiperOrigin-RevId: 564551973
This commit is contained in:
MediaPipe Team 2023-09-11 17:42:31 -07:00 committed by Copybara-Service
parent 0fec532ebe
commit 02bd0d95e7
6 changed files with 927 additions and 594 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

52
mediapipe/web/graph_runner/BUILD
View File

@ -1,19 +1,67 @@
# The TypeScript graph runner used by all MediaPipe Web tasks.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_declaration", "mediapipe_ts_library")
load("@npm//@bazel/jasmine:index.bzl", "jasmine_node_test")
package(default_visibility = [
"//mediapipe/tasks:internal",
])
# Public types for GraphRunner listeners
mediapipe_ts_declaration(
name = "listener_types",
srcs = [
"listener_types.d.ts",
],
visibility = ["//visibility:private"],
)
# Public API for GraphRunner
mediapipe_ts_declaration(
name = "graph_runner_api",
srcs = [
"graph_runner_api.d.ts",
],
deps = [
":listener_types",
":wasm_module_private",
],
)
# Public API for GraphRunner creation factory functions
mediapipe_ts_library(
name = "graph_runner_factory_api",
srcs = [
"graph_runner_factory_api.d.ts",
],
deps = [
":graph_runner_api",
":wasm_module_private",
],
)
# Private declarations for the Wasm module
mediapipe_ts_declaration(
name = "wasm_module_private",
srcs = ["wasm_module.d.ts"],
visibility = ["//visibility:private"],
deps = [":listener_types"],
)
# Internal GraphRunner implementation
mediapipe_ts_library(
name = "graph_runner_ts",
srcs = [
":graph_runner.ts",
],
allow_unoptimized_namespaces = True,
deps = [":platform_utils"],
deps = [
":graph_runner_api",
":graph_runner_factory_api",
":listener_types",
":platform_utils",
":wasm_module_private",
],
)
mediapipe_ts_library(

673
mediapipe/web/graph_runner/graph_runner.ts
View File

@ -3,40 +3,27 @@
import {isWebKit} from '../../web/graph_runner/platform_utils';
// Placeholder for internal dependency on trusted resource url
// This file can serve as a common interface for most simple TypeScript
// libraries-- additionally, it can hook automatically into wasm_mediapipe_demo
// to autogenerate simple TS APIs from demos for instantaneous 1P integrations.
import {GraphRunnerApi, ImageSource} from './graph_runner_api';
import {CreateGraphRunnerApi, CreateMediaPipeLibApi, FileLocator, WasmMediaPipeConstructor} from './graph_runner_factory_api';
import {EmptyPacketListener, ErrorListener, SimpleListener, VectorListener} from './listener_types';
import {WasmModule} from './wasm_module';
/**
* Simple interface for allowing users to set the directory where internal
* wasm-loading and asset-loading code looks (e.g. for .wasm and .data file
* locations).
*/
export declare interface FileLocator {
locateFile: (filename: string) => string;
mainScriptUrlOrBlob?: string;
}
export {ReturnType} from './graph_runner_factory_api';
// This file contains the internal implementations behind the public APIs
// declared in "graph_runner_api.d.ts" and "graph_runner_factory_api.d.ts".
/**
* A listener that receives the contents of a non-empty MediaPipe packet and
* its timestamp.
*/
export type SimpleListener<T> = (data: T, timestamp: number) => void;
/**
* A listener that receives the current MediaPipe packet timestamp. This is
* invoked even for empty packet.
*/
export type EmptyPacketListener = (timestamp: number) => void;
/**
* A listener that receives a single element of vector-returning output packet.
* Receives one element at a time (in order). Once all elements are processed,
* the listener is invoked with `data` set to `unknown` and `done` set to true.
* Intended for internal usage.
*/
export type VectorListener<T> = (data: T, done: boolean, timestamp: number) =>
void;
// First we re-export all of our imported public types/defines, so that users
// can import everything they need from here directly.
export {
EmptyPacketListener,
ErrorListener,
FileLocator,
ImageSource,
SimpleListener,
VectorListener,
WasmMediaPipeConstructor,
WasmModule,
};
/**
* A listener that receives the CalculatorGraphConfig in binary encoding.
@ -49,114 +36,6 @@ export type CalculatorGraphConfigListener = (graphConfig: Uint8Array) => void;
*/
export const CALCULATOR_GRAPH_CONFIG_LISTENER_NAME = '__graph_config__';
/**
* Declarations for Emscripten's WebAssembly Module behavior, so TS compiler
* doesn't break our JS/C++ bridge.
*/
export declare interface WasmModule {
canvas: HTMLCanvasElement|OffscreenCanvas|null;
HEAPU8: Uint8Array;
HEAPU32: Uint32Array;
HEAPF32: Float32Array;
HEAPF64: Float64Array;
FS_createDataFile:
(parent: string, name: string, data: Uint8Array, canRead: boolean,
canWrite: boolean, canOwn: boolean) => void;
FS_createPath:
(parent: string, name: string, canRead: boolean,
canWrite: boolean) => void;
FS_unlink(path: string): void;
gpuOriginForWebTexturesIsBottomLeft?: boolean;
errorListener?: ErrorListener;
_bindTextureToCanvas: () => boolean;
_changeBinaryGraph: (size: number, dataPtr: number) => void;
_changeTextGraph: (size: number, dataPtr: number) => void;
_closeGraph: () => void;
_free: (ptr: number) => void;
_malloc: (size: number) => number;
_processFrame: (width: number, height: number, timestamp: number) => void;
_setAutoRenderToScreen: (enabled: boolean) => void;
_waitUntilIdle: () => void;
// Exposed so that clients of this lib can access this field
dataFileDownloads?: {[url: string]: {loaded: number, total: number}};
// Wasm Module multistream entrypoints. Require
// gl_graph_runner_internal_multi_input as a build dependency.
stringToNewUTF8: (data: string) => number;
_bindTextureToStream: (streamNamePtr: number) => void;
_addBoundTextureToStream:
(streamNamePtr: number, width: number, height: number,
timestamp: number) => void;
_addBoolToInputStream:
(data: boolean, streamNamePtr: number, timestamp: number) => void;
_addDoubleToInputStream:
(data: number, streamNamePtr: number, timestamp: number) => void;
_addFloatToInputStream:
(data: number, streamNamePtr: number, timestamp: number) => void;
_addIntToInputStream:
(data: number, streamNamePtr: number, timestamp: number) => void;
_addStringToInputStream:
(dataPtr: number, streamNamePtr: number, timestamp: number) => void;
_addFlatHashMapToInputStream:
(keysPtr: number, valuesPtr: number, count: number, streamNamePtr: number,
timestamp: number) => void;
_addProtoToInputStream:
(dataPtr: number, dataSize: number, protoNamePtr: number,
streamNamePtr: number, timestamp: number) => void;
_addEmptyPacketToInputStream:
(streamNamePtr: number, timestamp: number) => void;
// Input side packets
_addBoolToInputSidePacket: (data: boolean, streamNamePtr: number) => void;
_addDoubleToInputSidePacket: (data: number, streamNamePtr: number) => void;
_addFloatToInputSidePacket: (data: number, streamNamePtr: number) => void;
_addIntToInputSidePacket: (data: number, streamNamePtr: number) => void;
_addStringToInputSidePacket: (dataPtr: number, streamNamePtr: number) => void;
_addProtoToInputSidePacket:
(dataPtr: number, dataSize: number, protoNamePtr: number,
streamNamePtr: number) => void;
// Map of output streams to packet listeners. Also built as part of
// gl_graph_runner_internal_multi_input.
simpleListeners?:
Record<string, SimpleListener<unknown>|VectorListener<unknown>>;
// Map of output streams to empty packet listeners.
emptyPacketListeners?: Record<string, EmptyPacketListener>;
_attachBoolListener: (streamNamePtr: number) => void;
_attachBoolVectorListener: (streamNamePtr: number) => void;
_attachDoubleListener: (streamNamePtr: number) => void;
_attachDoubleVectorListener: (streamNamePtr: number) => void;
_attachFloatListener: (streamNamePtr: number) => void;
_attachFloatVectorListener: (streamNamePtr: number) => void;
_attachIntListener: (streamNamePtr: number) => void;
_attachIntVectorListener: (streamNamePtr: number) => void;
_attachStringListener: (streamNamePtr: number) => void;
_attachStringVectorListener: (streamNamePtr: number) => void;
_attachProtoListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
_attachProtoVectorListener:
(streamNamePtr: number, makeDeepCopy?: boolean) => void;
// Require dependency ":gl_graph_runner_audio_out"
_attachAudioListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
// Require dependency ":gl_graph_runner_audio"
_addAudioToInputStream: (dataPtr: number, numChannels: number,
numSamples: number, streamNamePtr: number, timestamp: number) => void;
_configureAudio: (channels: number, samples: number, sampleRate: number,
streamNamePtr: number, headerNamePtr: number) => void;
// Get the graph configuration and invoke the listener configured under
// streamNamePtr
_getGraphConfig: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
// TODO: Refactor to just use a few numbers (perhaps refactor away
// from gl_graph_runner_internal.cc entirely to use something a little more
// streamlined; new version is _processFrame above).
_processGl: (frameDataPtr: number) => number;
}
// Global declarations, for tapping into Window for Wasm blob running
declare global {
interface Window {
@ -173,32 +52,13 @@ declare global {
*/
declare function importScripts(...urls: Array<string|URL>): void;
/**
* Valid types of image sources which we can run our GraphRunner over.
*/
export type ImageSource =
HTMLCanvasElement|HTMLVideoElement|HTMLImageElement|ImageData|ImageBitmap;
/** A listener that will be invoked with an absl::StatusCode and message. */
export type ErrorListener = (code: number, message: string) => void;
/**
* Internal type of constructors used for initializing GraphRunner and
* subclasses.
*/
export type WasmMediaPipeConstructor<LibType> =
(new (
module: WasmModule, canvas?: HTMLCanvasElement|OffscreenCanvas|null) =>
LibType);
/**
* Simple class to run an arbitrary image-in/image-out MediaPipe graph (i.e.
* as created by wasm_mediapipe_demo BUILD macro), and either render results
* into canvas, or else return the output WebGLTexture. Takes a WebAssembly
* Module.
*/
export class GraphRunner {
export class GraphRunner implements GraphRunnerApi {
// TODO: These should be protected/private, but are left exposed for
// now so that we can use proper TS mixins with this class as a base. This
// should be somewhat fixed when we create our .d.ts files.
@ -241,11 +101,7 @@ export class GraphRunner {
}
}
/**
* Convenience helper to load a MediaPipe graph from a file and pass it to
* setGraph.
* @param graphFile The url of the MediaPipe graph file to load.
*/
/** {@override GraphRunnerApi} */
async initializeGraph(graphFile: string): Promise<void> {
// Fetch and set graph
const response = await fetch(graphFile);
@ -255,26 +111,12 @@ export class GraphRunner {
this.setGraph(new Uint8Array(graphData), isBinary);
}
/**
* Convenience helper for calling setGraph with a string representing a text
* proto config.
* @param graphConfig The text proto graph config, expected to be a string in
* default JavaScript UTF-16 format.
*/
/** {@override GraphRunnerApi} */
setGraphFromString(graphConfig: string): void {
this.setGraph((new TextEncoder()).encode(graphConfig), false);
}
/**
* Takes the raw data from a MediaPipe graph, and passes it to C++ to be run
* over the video stream. Will replace the previously running MediaPipe graph,
* if there is one.
* @param graphData The raw MediaPipe graph data, either in binary
* protobuffer format (.binarypb), or else in raw text format (.pbtxt or
* .textproto).
* @param isBinary This should be set to true if the graph is in
* binary format, and false if it is in human-readable text format.
*/
/** {@override GraphRunnerApi} */
setGraph(graphData: Uint8Array, isBinary: boolean): void {
const size = graphData.length;
const dataPtr = this.wasmModule._malloc(size);
@ -287,24 +129,7 @@ export class GraphRunner {
this.wasmModule._free(dataPtr);
}
/**
* Configures the current graph to handle audio processing in a certain way
* for all its audio input streams. Additionally can configure audio headers
* (both input side packets as well as input stream headers), but these
* configurations only take effect if called before the graph is set/started.
* @param numChannels The number of channels of audio input. Only 1
* is supported for now.
* @param numSamples The number of samples that are taken in each
* audio capture.
* @param sampleRate The rate, in Hz, of the sampling.
* @param streamName The optional name of the input stream to additionally
* configure with audio information. This configuration only occurs before
* the graph is set/started. If unset, a default stream name will be used.
* @param headerName The optional name of the header input side packet to
* additionally configure with audio information. This configuration only
* occurs before the graph is set/started. If unset, a default header name
* will be used.
*/
/** {@override GraphRunnerApi} */
configureAudio(numChannels: number, numSamples: number, sampleRate: number,
streamName?: string, headerName?: string) {
if (!this.wasmModule._configureAudio) {
@ -322,55 +147,17 @@ export class GraphRunner {
});
}
/**
* Allows disabling automatic canvas resizing, in case clients want to control
* control this.
* @param resize True will re-enable automatic canvas resizing, while false
* will disable the feature.
*/
/** {@override GraphRunnerApi} */
setAutoResizeCanvas(resize: boolean): void {
this.autoResizeCanvas = resize;
}
/**
* Allows disabling the automatic render-to-screen code, in case clients don't
* need/want this. In particular, this removes the requirement for pipelines
* to have access to GPU resources, as well as the requirement for graphs to
* have "input_frames_gpu" and "output_frames_gpu" streams defined, so pure
* CPU pipelines and non-video pipelines can be created.
* NOTE: This only affects future graph initializations (via setGraph or
* initializeGraph), and does NOT affect the currently running graph, so
* calls to this should be made *before* setGraph/initializeGraph for the
* graph file being targeted.
* @param enabled True will re-enable automatic render-to-screen code and
* cause GPU resources to once again be requested, while false will
* disable the feature.
*/
/** {@override GraphRunnerApi} */
setAutoRenderToScreen(enabled: boolean): void {
this.wasmModule._setAutoRenderToScreen(enabled);
}
/**
* Overrides the vertical orientation for input GpuBuffers and the automatic
* render-to-screen code. The default for our OpenGL code on other platforms
* (Android, Linux) is to use a bottom-left origin. But the default for WebGL
* is to use a top-left origin. We use WebGL default normally, and many
* calculators and graphs have platform-specific code to handle the resulting
* orientation flip. However, in order to be able to use a single graph on all
* platforms without alterations, it may be useful to send images into a web
* graph using the OpenGL orientation. Users can call this function with
* `bottomLeftIsOrigin = true` in order to enforce an orientation for all
* GpuBuffer inputs which is consistent with OpenGL on other platforms.
* This call will also vertically flip the automatic render-to-screen code as
* well, so that webcam input (for example) will render properly when passed
* through the graph still.
* NOTE: This will immediately affect GpuBuffer inputs, but must be called
* *before* graph start in order to affect the automatic render-to-screen
* code!
* @param bottomLeftIsOrigin True will flip our input GpuBuffers and auto
* render-to-screen to match the classic OpenGL orientation, while false will
* disable this feature to match the default WebGL orientation.
*/
/** {@override GraphRunnerApi} */
setGpuBufferVerticalFlip(bottomLeftIsOrigin: boolean): void {
this.wasmModule.gpuOriginForWebTexturesIsBottomLeft = bottomLeftIsOrigin;
}
@ -436,7 +223,8 @@ export class GraphRunner {
* Takes the raw data from a JS image source, and sends it to C++ to be
* processed, waiting synchronously for the response. Note that we will resize
* our GL canvas to fit the input, so input size should only change
* infrequently.
* infrequently. NOTE: This call has been deprecated in favor of
* `addGpuBufferToStream`.
* @param imageSource An image source to process.
* @param timestamp The timestamp of the current frame, in ms.
* @return texture? The WebGL texture reference, if one was produced.
@ -554,26 +342,12 @@ export class GraphRunner {
};
}
/**
* Attaches a listener that will be invoked when the MediaPipe framework
* returns an error.
*/
/** {@override GraphRunnerApi} */
attachErrorListener(callbackFcn: (code: number, message: string) => void) {
this.wasmModule.errorListener = callbackFcn;
}
/**
* Attaches a listener that will be invoked when the MediaPipe framework
* receives an empty packet on the provided output stream. This can be used
* to receive the latest output timestamp.
*
* Empty packet listeners are only active if there is a corresponding packet
* listener.
*
* @param outputStreamName The name of the graph output stream to receive
* empty packets from.
* @param callbackFcn The callback to receive the timestamp.
*/
/** {@override GraphRunnerApi} */
attachEmptyPacketListener(
outputStreamName: string, callbackFcn: EmptyPacketListener) {
this.wasmModule.emptyPacketListeners =
@ -581,15 +355,7 @@ export class GraphRunner {
this.wasmModule.emptyPacketListeners[outputStreamName] = callbackFcn;
}
/**
* Takes the raw data from a JS audio capture array, and sends it to C++ to be
* processed.
* @param audioData An array of raw audio capture data, like
* from a call to getChannelData on an AudioBuffer.
* @param streamName The name of the MediaPipe graph stream to add the audio
* data to.
* @param timestamp The timestamp of the current frame, in ms.
*/
/** {@override GraphRunnerApi} */
addAudioToStream(
audioData: Float32Array, streamName: string, timestamp: number) {
// numChannels and numSamples being 0 will cause defaults to be used,
@ -597,22 +363,7 @@ export class GraphRunner {
this.addAudioToStreamWithShape(audioData, 0, 0, streamName, timestamp);
}
/**
* Takes the raw data from a JS audio capture array, and sends it to C++ to be
* processed, shaping the audioData array into an audio matrix according to
* the numChannels and numSamples parameters.
* @param audioData An array of raw audio capture data, like
* from a call to getChannelData on an AudioBuffer.
* @param numChannels The number of audio channels this data represents. If 0
* is passed, then the value will be taken from the last call to
* configureAudio.
* @param numSamples The number of audio samples captured in this data packet.
* If 0 is passed, then the value will be taken from the last call to
* configureAudio.
* @param streamName The name of the MediaPipe graph stream to add the audio
* data to.
* @param timestamp The timestamp of the current frame, in ms.
*/
/** {@override GraphRunnerApi} */
addAudioToStreamWithShape(
audioData: Float32Array, numChannels: number, numSamples: number,
streamName: string, timestamp: number) {
@ -633,19 +384,7 @@ export class GraphRunner {
});
}
/**
* Takes the relevant information from the HTML video or image element, and
* passes it into the WebGL-based graph for processing on the given stream at
* the given timestamp. Can be used for additional auxiliary GpuBuffer input
* streams. Processing will not occur until a blocking call (like
* processVideoGl or finishProcessing) is made. For use with
* 'gl_graph_runner_internal_multi_input'.
* @param imageSource Reference to the video frame we wish to add into our
* graph.
* @param streamName The name of the MediaPipe graph stream to add the frame
* to.
* @param timestamp The timestamp of the input frame, in ms.
*/
/** {@override GraphRunnerApi} */
addGpuBufferToStream(
imageSource: ImageSource, streamName: string, timestamp: number): void {
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
@ -656,36 +395,21 @@ export class GraphRunner {
});
}
/**
* Sends a boolean packet into the specified stream at the given timestamp.
* @param data The boolean data to send.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
/** {@override GraphRunnerApi} */
addBoolToStream(data: boolean, streamName: string, timestamp: number): void {
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
this.wasmModule._addBoolToInputStream(data, streamNamePtr, timestamp);
});
}
/**
* Sends a double packet into the specified stream at the given timestamp.
* @param data The double data to send.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
/** {@override GraphRunnerApi} */
addDoubleToStream(data: number, streamName: string, timestamp: number): void {
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
this.wasmModule._addDoubleToInputStream(data, streamNamePtr, timestamp);
});
}
/**
* Sends a float packet into the specified stream at the given timestamp.
* @param data The float data to send.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
/** {@override GraphRunnerApi} */
addFloatToStream(data: number, streamName: string, timestamp: number): void {
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
// NOTE: _addFloatToStream and _addIntToStream are reserved for JS
@ -695,24 +419,14 @@ export class GraphRunner {
});
}
/**
* Sends an integer packet into the specified stream at the given timestamp.
* @param data The integer data to send.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
/** {@override GraphRunnerApi} */
addIntToStream(data: number, streamName: string, timestamp: number): void {
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
this.wasmModule._addIntToInputStream(data, streamNamePtr, timestamp);
});
}
/**
* Sends a string packet into the specified stream at the given timestamp.
* @param data The string data to send.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
/** {@override GraphRunnerApi} */
addStringToStream(data: string, streamName: string, timestamp: number): void {
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
this.wrapStringPtr(data, (dataPtr: number) => {
@ -722,14 +436,7 @@ export class GraphRunner {
});
}
/**
* Sends a Record<string, string> packet into the specified stream at the
* given timestamp.
* @param data The records to send (will become a
* std::flat_hash_map<std::string, std::string).
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
/** {@override GraphRunnerApi} */
addStringRecordToStream(
data: Record<string, string>, streamName: string,
timestamp: number): void {
@ -744,16 +451,7 @@ export class GraphRunner {
});
}
/**
* Sends a serialized protobuffer packet into the specified stream at the
* given timestamp, to be parsed into the specified protobuffer type.
* @param data The binary (serialized) raw protobuffer data.
* @param protoType The C++ namespaced type this protobuffer data corresponds
* to (e.g. "foo.Bar"). It will be converted to this type when output as a
* packet into the graph.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
/** {@override GraphRunnerApi} */
addProtoToStream(
data: Uint8Array, protoType: string, streamName: string,
timestamp: number): void {
@ -770,74 +468,42 @@ export class GraphRunner {
});
}
/**
* Sends an empty packet into the specified stream at the given timestamp,
* effectively advancing that input stream's timestamp bounds without
* sending additional data packets.
* @param streamName The name of the graph input stream to send the empty
* packet into.
* @param timestamp The timestamp of the empty packet, in ms.
*/
/** {@override GraphRunnerApi} */
addEmptyPacketToStream(streamName: string, timestamp: number): void {
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
this.wasmModule._addEmptyPacketToInputStream(streamNamePtr, timestamp);
});
}
/**
* Attaches a boolean packet to the specified input_side_packet.
* @param data The boolean data to send.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
/** {@override GraphRunnerApi} */
addBoolToInputSidePacket(data: boolean, sidePacketName: string): void {
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
this.wasmModule._addBoolToInputSidePacket(data, sidePacketNamePtr);
});
}
/**
* Attaches a double packet to the specified input_side_packet.
* @param data The double data to send.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
/** {@override GraphRunnerApi} */
addDoubleToInputSidePacket(data: number, sidePacketName: string): void {
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
this.wasmModule._addDoubleToInputSidePacket(data, sidePacketNamePtr);
});
}
/**
* Attaches a float packet to the specified input_side_packet.
* @param data The float data to send.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
/** {@override GraphRunnerApi} */
addFloatToInputSidePacket(data: number, sidePacketName: string): void {
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
this.wasmModule._addFloatToInputSidePacket(data, sidePacketNamePtr);
});
}
/**
* Attaches a integer packet to the specified input_side_packet.
* @param data The integer data to send.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
/** {@override GraphRunnerApi} */
addIntToInputSidePacket(data: number, sidePacketName: string): void {
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
this.wasmModule._addIntToInputSidePacket(data, sidePacketNamePtr);
});
}
/**
* Attaches a string packet to the specified input_side_packet.
* @param data The string data to send.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
/** {@override GraphRunnerApi} */
addStringToInputSidePacket(data: string, sidePacketName: string): void {
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
this.wrapStringPtr(data, (dataPtr: number) => {
@ -846,14 +512,7 @@ export class GraphRunner {
});
}
/**
* Attaches a serialized proto packet to the specified input_side_packet.
* @param data The binary (serialized) raw protobuffer data.
* @param protoType The C++ namespaced type this protobuffer data corresponds
* to. It will be converted to this type for use in the graph.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
/** {@override GraphRunnerApi} */
addProtoToInputSidePacket(
data: Uint8Array, protoType: string, sidePacketName: string): void {
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
@ -869,15 +528,7 @@ export class GraphRunner {
});
}
/**
* Attaches a boolean packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab boolean
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
/** {@override GraphRunnerApi} */
attachBoolListener(
outputStreamName: string, callbackFcn: SimpleListener<boolean>): void {
// Set up our TS listener to receive any packets for this stream.
@ -889,15 +540,7 @@ export class GraphRunner {
});
}
/**
* Attaches a bool[] packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab
* std::vector<bool> data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
/** {@override GraphRunnerApi} */
attachBoolVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<boolean[]>): void {
// Set up our TS listener to receive any packets for this stream.
@ -909,15 +552,7 @@ export class GraphRunner {
});
}
/**
* Attaches an int packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab int
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
/** {@override GraphRunnerApi} */
attachIntListener(
outputStreamName: string, callbackFcn: SimpleListener<number>): void {
// Set up our TS listener to receive any packets for this stream.
@ -929,15 +564,7 @@ export class GraphRunner {
});
}
/**
* Attaches an int[] packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab
* std::vector<int> data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
/** {@override GraphRunnerApi} */
attachIntVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void {
// Set up our TS listener to receive any packets for this stream.
@ -949,15 +576,7 @@ export class GraphRunner {
});
}
/**
* Attaches a double packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab double
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
/** {@override GraphRunnerApi} */
attachDoubleListener(
outputStreamName: string, callbackFcn: SimpleListener<number>): void {
// Set up our TS listener to receive any packets for this stream.
@ -969,15 +588,7 @@ export class GraphRunner {
});
}
/**
* Attaches a double[] packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab
* std::vector<double> data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
/** {@override GraphRunnerApi} */
attachDoubleVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void {
// Set up our TS listener to receive any packets for this stream.
@ -989,15 +600,7 @@ export class GraphRunner {
});
}
/**
* Attaches a float packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab float
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
/** {@override GraphRunnerApi} */
attachFloatListener(
outputStreamName: string, callbackFcn: SimpleListener<number>): void {
// Set up our TS listener to receive any packets for this stream.
@ -1009,15 +612,7 @@ export class GraphRunner {
});
}
/**
* Attaches a float[] packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab
* std::vector<float> data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
/** {@override GraphRunnerApi} */
attachFloatVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void {
// Set up our TS listener to receive any packets for this stream.
@ -1029,15 +624,7 @@ export class GraphRunner {
});
}
/**
* Attaches a string packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab string
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
/** {@override GraphRunnerApi} */
attachStringListener(
outputStreamName: string, callbackFcn: SimpleListener<string>): void {
// Set up our TS listener to receive any packets for this stream.
@ -1049,15 +636,7 @@ export class GraphRunner {
});
}
/**
* Attaches a string[] packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab
* std::vector<std::string> data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
/** {@override GraphRunnerApi} */
attachStringVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<string[]>): void {
// Set up our TS listener to receive any packets for this stream.
@ -1069,25 +648,7 @@ export class GraphRunner {
});
}
/**
* Attaches a serialized proto packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab binary
* serialized proto data from (in Uint8Array format).
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that by default the data is only guaranteed to
* exist for the duration of the callback, and the callback will be called
* inline, so it should not perform overly complicated (or any async)
* behavior. If the proto data needs to be able to outlive the call, you
* may set the optional makeDeepCopy parameter to true, or can manually
* deep-copy the data yourself.
* @param makeDeepCopy Optional convenience parameter which, if set to true,
* will override the default memory management behavior and make a deep
* copy of the underlying data, rather than just returning a view into the
* C++-managed memory. At the cost of a data copy, this allows the
* returned data to outlive the callback lifetime (and it will be cleaned
* up automatically by JS garbage collection whenever the user is finished
* with it).
*/
/** {@override GraphRunnerApi} */
attachProtoListener(
outputStreamName: string, callbackFcn: SimpleListener<Uint8Array>,
makeDeepCopy?: boolean): void {
@ -1102,26 +663,7 @@ export class GraphRunner {
});
}
/**
* Attaches a listener for an array of serialized proto packets to the
* specified output_stream.
* @param outputStreamName The name of the graph output stream to grab a
* vector of binary serialized proto data from (in Uint8Array[] format).
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that by default the data is only guaranteed to
* exist for the duration of the callback, and the callback will be called
* inline, so it should not perform overly complicated (or any async)
* behavior. If the proto data needs to be able to outlive the call, you
* may set the optional makeDeepCopy parameter to true, or can manually
* deep-copy the data yourself.
* @param makeDeepCopy Optional convenience parameter which, if set to true,
* will override the default memory management behavior and make a deep
* copy of the underlying data, rather than just returning a view into the
* C++-managed memory. At the cost of a data copy, this allows the
* returned data to outlive the callback lifetime (and it will be cleaned
* up automatically by JS garbage collection whenever the user is finished
* with it).
*/
/** {@override GraphRunnerApi} */
attachProtoVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<Uint8Array[]>,
makeDeepCopy?: boolean): void {
@ -1136,26 +678,7 @@ export class GraphRunner {
});
}
/**
* Attaches an audio packet listener to the specified output_stream, to be
* given a Float32Array as output.
* @param outputStreamName The name of the graph output stream to grab audio
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior. If the
* audio data needs to be able to outlive the call, you may set the
* optional makeDeepCopy parameter to true, or can manually deep-copy the
* data yourself.
* @param makeDeepCopy Optional convenience parameter which, if set to true,
* will override the default memory management behavior and make a deep
* copy of the underlying data, rather than just returning a view into the
* C++-managed memory. At the cost of a data copy, this allows the
* returned data to outlive the callback lifetime (and it will be cleaned
* up automatically by JS garbage collection whenever the user is finished
* with it).
*/
/** {@override GraphRunnerApi} */
attachAudioListener(
outputStreamName: string, callbackFcn: SimpleListener<Float32Array>,
makeDeepCopy?: boolean): void {
@ -1181,19 +704,12 @@ export class GraphRunner {
});
}
/**
* Forces all queued-up packets to be pushed through the MediaPipe graph as
* far as possible, performing all processing until no more processing can be
* done.
*/
/** {@override GraphRunnerApi} */
finishProcessing(): void {
this.wasmModule._waitUntilIdle();
}
/**
* Closes the input streams and all calculators for this graph and frees up
* any C++ resources. The graph will not be usable once closed.
*/
/** {@override GraphRunnerApi} */
closeGraph(): void {
this.wasmModule._closeGraph();
this.wasmModule.simpleListeners = undefined;
@ -1221,38 +737,13 @@ async function runScript(scriptUrl: string|string) {
}
}
/**
* Helper type macro for use with createMediaPipeLib. Allows us to easily infer
* the type of a mixin-extended GraphRunner. Example usage:
* const GraphRunnerConstructor =
* SupportImage(SupportSerialization(GraphRunner));
* let mediaPipe: ReturnType<typeof GraphRunnerConstructor>;
* ...
* mediaPipe = await createMediaPipeLib(GraphRunnerConstructor, ...);
*/
// tslint:disable-next-line:no-any
export type ReturnType<T> = T extends (...args: unknown[]) => infer R ? R : any;
/**
* Global function to initialize Wasm blob and load runtime assets for a
* specialized MediaPipe library. This allows us to create a requested
* subclass inheriting from GraphRunner.
* @param constructorFcn The name of the class to instantiate via "new".
* @param wasmLoaderScript Url for the wasm-runner script; produced by the build
* process.
* @param assetLoaderScript Url for the asset-loading script; produced by the
* build process.
* @param fileLocator A function to override the file locations for assets
* loaded by the MediaPipe library.
* @return promise A promise which will resolve when initialization has
* completed successfully.
*/
export async function createMediaPipeLib<LibType>(
/** {@override CreateMediaPipeLibApi} */
export const createMediaPipeLib: CreateMediaPipeLibApi = async<LibType>(
constructorFcn: WasmMediaPipeConstructor<LibType>,
wasmLoaderScript?: string|null,
assetLoaderScript?: string|null,
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
fileLocator?: FileLocator): Promise<LibType> {
fileLocator?: FileLocator): Promise<LibType> => {
// Run wasm-loader script here
if (wasmLoaderScript) {
await runScript(wasmLoaderScript);
@ -1288,26 +779,24 @@ export async function createMediaPipeLib<LibType>(
// Don't reuse factory or module seed
self.ModuleFactory = self.Module = undefined;
return new constructorFcn(module, glCanvas);
};
// We extend the CreateGraphRunnerApi interface here for now so that by default
// callers of `createGraphRunner` will be given a `GraphRunner` rather than a
// `GraphRunnerApi`.
interface CreateGraphRunnerImplType extends CreateGraphRunnerApi {
(wasmLoaderScript?: string,
assetLoaderScript?: string,
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
fileLocator?: FileLocator): Promise<GraphRunner>;
}
/**
* Global function to initialize Wasm blob and load runtime assets for a generic
* MediaPipe library.
* @param wasmLoaderScript Url for the wasm-runner script; produced by the build
* process.
* @param assetLoaderScript Url for the asset-loading script; produced by the
* build process.
* @param fileLocator A function to override the file locations for assets
* loaded by the MediaPipe library.
* @return promise A promise which will resolve when initialization has
* completed successfully.
*/
export async function createGraphRunner(
/** {@override CreateGraphRunnerApi} */
export const createGraphRunner: CreateGraphRunnerImplType = async(
wasmLoaderScript?: string,
assetLoaderScript?: string,
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
fileLocator?: FileLocator): Promise<GraphRunner> {
fileLocator?: FileLocator): Promise<GraphRunner> => {
return createMediaPipeLib(
GraphRunner, wasmLoaderScript, assetLoaderScript, glCanvas,
fileLocator);
}
GraphRunner, wasmLoaderScript, assetLoaderScript, glCanvas, fileLocator);
};

583
mediapipe/web/graph_runner/graph_runner_api.d.ts vendored Normal file
View File

@ -0,0 +1,583 @@
import {EmptyPacketListener, ErrorListener, SimpleListener, VectorListener} from './listener_types';
import {WasmModule} from './wasm_module';
/**
* This file can serve as a common interface for most MediaPipe-based TypeScript
* libraries. Additionally, it can hook automatically into `wasm_mediapipe_demo`
* to allow for easy migrations from old pure JS demos. See `mediapipe_ts_demo`
* and `wasm_mediapipe_files` BUILD rules. Public API is described below in more
* detail, public API for GraphRunner factory functions is described in
* "graph_runner_factory_api.d.ts", and the same for listener callbacks
* can be found in "listenertypes.d.ts". An actual implementation is coded in
* `graph_runner.ts`, and that can be built against directly. The purpose of
* this file is primarily to enforce a common interface and provide better
* documentation on the design of the API as a whole.
*/
// We re-export all of our imported public listener types, so that users can
// import everything they need for the public interfaces directly from here.
export {
EmptyPacketListener,
ErrorListener,
SimpleListener,
VectorListener,
};
/**
* Valid types of image sources which we can run our GraphRunner over.
*/
export type ImageSource =
HTMLCanvasElement|HTMLVideoElement|HTMLImageElement|ImageData|ImageBitmap;
/**
* Simple interface for a class to run an arbitrary MediaPipe graph on web, and
* either render results into canvas, or else stream output into attached
* listeners. Takes a WebAssembly Module and an optional canvas for rendering.
* Standard implementation is `GraphRunner`.
* There are three categories of functions:
* - Those which add inputs into the graph (named `add*ToStream` or
* `add*ToInputSidePacket`).
* - Those which listen for outputs from the graph (named `attach*Listener`).
* - Those concerned with the graph running itself (runner settings, loading
* the graph, destruction, and `finishProcessing`).
* The expected ordering of one-time initialization calls should be:
* 1. Construction
* 2. Adding any input side packets and attaching any output listeners
* 3. Initializing/setting the graph
* After this, for the main loop, the user would add packets to streams for a
* given frame and then call `finishProcessing()`.
* Example usage pattern would be:
* ```
* // We assume we have already constructed a GraphRunner called graphRunner.
* // Generally we would do this via a helpful factory method like
* // `createGraphRunner` or some variant thereof.
* const graphRunner: GraphRunner;
*
* // Initialization code:
* graphRunner.addBoolToInputSidePacket(true, 'some_input_bool_side_packet');
* graphRunner.attachStringVectorListener('some_output_string_vec_stream',
* (data: string[], timestamp: number) => { console.log(data); });
* await graphRunner.initializeGraph('path/to/graph_file.pbtxt');
*
* // Main loop code (run every frame):
* const frameTimestamp = performance.now();
* graphRunner.addStringToStream(
* 'Hello World', 'some_input_string_stream', frameTimestamp);
* graphRunner.addFloatToStream(
* 3.7, 'some_input_float_stream', frameTimestamp);
* graphRunner.finishProcessing();
*
* // When done (no need to call if the user is just expected to close the
* // browser tab):
* graphRunner.closeGraph();
* ```
*/
export interface GraphRunnerApi {
/**
* Fetches a MediaPipe graph from a URL string pointing to the graph file.
* Will then set the graph using the results, replacing the previously running
* MediaPipe graph, if there is one. If the graph file's name ends with
* ".pbtxt" with ".textproto", it will assume text-formatted, and otherwise
* will assume a binary format for the proto.
* @param graphFile The url of the MediaPipe graph file to load.
*/
initializeGraph(graphFile: string): Promise<void>;
/**
* Convenience helper for loading a MediaPipe graph from a string representing
* a text proto config. Useful for graph files which are expected to be edited
* locally, web-side. Will replace the previously running MediaPipe graph,
* if there is one.
* @param graphConfig The text proto graph config, expected to be a string in
* default JavaScript UTF-16 format.
*/
setGraphFromString(graphConfig: string): void;
/**
* Takes the raw data from a MediaPipe graph, and passes it to C++ to be run
* over the video stream. Will replace the previously running MediaPipe graph,
* if there is one.
* @param graphData The raw MediaPipe graph data, either in binary
* protobuffer format (.binarypb), or else in raw text format (.pbtxt or
* .textproto).
* @param isBinary This should be set to true if the graph is in
* binary format, and false if it is in human-readable text format.
*/
setGraph(graphData: Uint8Array, isBinary: boolean): void;
/**
* Configures the current graph to handle audio processing in a certain way
* for all its audio input streams. Additionally can configure audio headers
* (both input side packets as well as input stream headers), but these
* configurations only take effect if called before the graph is set/started.
* @param numChannels The number of channels of audio input. Only 1
* is supported for now.
* @param numSamples The number of samples that are taken in each
* audio capture.
* @param sampleRate The rate, in Hz, of the sampling.
* @param streamName The optional name of the input stream to additionally
* configure with audio information. This configuration only occurs before
* the graph is set/started. If unset, a default stream name will be used.
* @param headerName The optional name of the header input side packet to
* additionally configure with audio information. This configuration only
* occurs before the graph is set/started. If unset, a default header name
* will be used.
*/
configureAudio(
numChannels: number, numSamples: number, sampleRate: number,
streamName?: string, headerName?: string): void;
/**
* Allows disabling automatic canvas resizing, in case clients want to control
* control this. By default, the canvas will be resized to the size of the
* last GPU image input (see `addGpuBufferToStream`).
* @param resize True will re-enable automatic canvas resizing, while false
* will disable the feature.
*/
setAutoResizeCanvas(resize: boolean): void;
/**
* Allows disabling the automatic render-to-screen code, in case clients don't
* need/want this. In particular, this removes the requirement for pipelines
* to have access to GPU resources, as well as the requirement for graphs to
* have an "output_frames_gpu" stream defined, so pure CPU pipelines and
* non-video pipelines can be created.
* NOTE: This only affects future graph initializations (via `setGraph`` or
* `initializeGraph``), and does NOT affect the currently running graph,
* so calls to this should be made *before* `setGraph`/`initializeGraph`
* for the graph file being targeted.
* @param enabled True will re-enable automatic render-to-screen code and
* cause GPU resources to once again be requested, while false will
* disable the feature.
*/
setAutoRenderToScreen(enabled: boolean): void;
/**
* Overrides the vertical orientation for input GpuBuffers and the automatic
* render-to-screen code. The default for our OpenGL code on other platforms
* (Android, Linux) is to use a bottom-left origin. But the default for WebGL
* is to use a top-left origin. We use WebGL default normally, and many
* calculators and graphs have platform-specific code to handle the resulting
* orientation flip. However, in order to be able to use a single graph on all
* platforms without alterations, it may be useful to send images into a web
* graph using the OpenGL orientation. Users can call this function with
* `bottomLeftIsOrigin = true` in order to enforce an orientation for all
* GpuBuffer inputs which is consistent with OpenGL on other platforms.
* This call will also vertically flip the automatic render-to-screen code as
* well, so that webcam input (for example) will render properly when passed
* through the graph still.
* NOTE: This will immediately affect GpuBuffer inputs, but must be called
* *before* graph start in order to affect the automatic render-to-screen
* code!
* @param bottomLeftIsOrigin True will flip our input GpuBuffers and auto
* render-to-screen to match the classic OpenGL orientation, while false will
* disable this feature to match the default WebGL orientation.
*/
setGpuBufferVerticalFlip(bottomLeftIsOrigin: boolean): void;
/**
* Attaches a listener that will be invoked when the MediaPipe web framework
* returns an error.
*/
attachErrorListener(callbackFcn: ErrorListener): void;
/**
* Attaches a listener that will be invoked when the MediaPipe framework
* receives an empty packet on the provided output stream. This can be used
* to receive the latest output timestamp.
*
* Empty packet listeners are only active if there is a corresponding packet
* listener.
*
* @param outputStreamName The name of the graph output stream to receive
* empty packets from.
* @param callbackFcn The callback to receive the timestamp.
*/
attachEmptyPacketListener(
outputStreamName: string, callbackFcn: EmptyPacketListener): void;
/**
* Takes the raw data from a JS audio capture array, and sends it to C++ to be
* processed.
* @param audioData An array of raw audio capture data, like
* from a call to getChannelData on an AudioBuffer.
* @param streamName The name of the MediaPipe graph stream to add the audio
* data to.
* @param timestamp The timestamp of the current frame, in ms.
*/
addAudioToStream(
audioData: Float32Array, streamName: string, timestamp: number): void;
/**
* Takes the raw data from a JS audio capture array, and sends it to C++ to be
* processed, shaping the audioData array into an audio matrix according to
* the numChannels and numSamples parameters.
* @param audioData An array of raw audio capture data, like
* from a call to getChannelData on an AudioBuffer.
* @param numChannels The number of audio channels this data represents. If 0
* is passed, then the value will be taken from the last call to
* configureAudio.
* @param numSamples The number of audio samples captured in this data packet.
* If 0 is passed, then the value will be taken from the last call to
* configureAudio.
* @param streamName The name of the MediaPipe graph stream to add the audio
* data to.
* @param timestamp The timestamp of the current frame, in ms.
*/
addAudioToStreamWithShape(
audioData: Float32Array, numChannels: number, numSamples: number,
streamName: string, timestamp: number): void;
/**
* Takes the relevant information from the HTML video or image element, and
* passes it into the WebGL-based graph for processing on the given stream at
* the given timestamp. Can be used for additional auxiliary GpuBuffer input
* streams. Like all `add*ToStream` calls, processing will not occur until a
* blocking call like `finishProcessing` or the deprecated `processGl` is
* made. For use with 'gl_graph_runner_internal_multi_input'.
* @param imageSource Reference to the video frame we wish to add into our
* graph.
* @param streamName The name of the MediaPipe graph stream to add the frame
* to.
* @param timestamp The timestamp of the input frame, in ms.
*/
addGpuBufferToStream(
imageSource: ImageSource, streamName: string, timestamp: number): void;
/**
* Sends a boolean packet into the specified stream at the given timestamp.
* @param data The boolean data to send.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
addBoolToStream(data: boolean, streamName: string, timestamp: number): void;
/**
* Sends a double packet into the specified stream at the given timestamp.
* @param data The double data to send.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
addDoubleToStream(data: number, streamName: string, timestamp: number): void;
/**
* Sends a float packet into the specified stream at the given timestamp.
* @param data The float data to send.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
addFloatToStream(data: number, streamName: string, timestamp: number): void;
/**
* Sends an integer packet into the specified stream at the given timestamp.
* @param data The integer data to send.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
addIntToStream(data: number, streamName: string, timestamp: number): void;
/**
* Sends a string packet into the specified stream at the given timestamp.
* @param data The string data to send.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
addStringToStream(data: string, streamName: string, timestamp: number): void;
/**
* Sends a Record<string, string> packet into the specified stream at the
* given timestamp.
* @param data The records to send (will become a
* std::flat_hash_map<std::string, std::string).
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
addStringRecordToStream(
data: Record<string, string>, streamName: string,
timestamp: number): void;
/**
* Sends a serialized protobuffer packet into the specified stream at the
* given timestamp, to be parsed into the specified protobuffer type.
* @param data The binary (serialized) raw protobuffer data.
* @param protoType The C++ namespaced type this protobuffer data corresponds
* to (e.g. "foo.Bar"). It will be converted to this type when output as a
* packet into the graph.
* @param streamName The name of the graph input stream to send data into.
* @param timestamp The timestamp of the input data, in ms.
*/
addProtoToStream(
data: Uint8Array, protoType: string, streamName: string,
timestamp: number): void;
/**
* Sends an empty packet into the specified stream at the given timestamp,
* effectively advancing that input stream's timestamp bounds without
* sending additional data packets.
* @param streamName The name of the graph input stream to send the empty
* packet into.
* @param timestamp The timestamp of the empty packet, in ms.
*/
addEmptyPacketToStream(streamName: string, timestamp: number): void;
/**
* Attaches a boolean packet to the specified input_side_packet.
* @param data The boolean data to send.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
addBoolToInputSidePacket(data: boolean, sidePacketName: string): void;
/**
* Attaches a double packet to the specified input_side_packet.
* @param data The double data to send.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
addDoubleToInputSidePacket(data: number, sidePacketName: string): void;
/**
* Attaches a float packet to the specified input_side_packet.
* @param data The float data to send.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
addFloatToInputSidePacket(data: number, sidePacketName: string): void;
/**
* Attaches a integer packet to the specified input_side_packet.
* @param data The integer data to send.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
addIntToInputSidePacket(data: number, sidePacketName: string): void;
/**
* Attaches a string packet to the specified input_side_packet.
* @param data The string data to send.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
addStringToInputSidePacket(data: string, sidePacketName: string): void;
/**
* Attaches a serialized proto packet to the specified input_side_packet.
* @param data The binary (serialized) raw protobuffer data.
* @param protoType The C++ namespaced type this protobuffer data corresponds
* to. It will be converted to this type for use in the graph.
* @param sidePacketName The name of the graph input side packet to send data
* into.
*/
addProtoToInputSidePacket(
data: Uint8Array, protoType: string, sidePacketName: string): void;
/**
* Attaches a boolean packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab boolean
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
attachBoolListener(
outputStreamName: string, callbackFcn: SimpleListener<boolean>): void;
/**
* Attaches a bool[] packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab
* std::vector<bool> data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
attachBoolVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<boolean[]>): void;
/**
* Attaches an int packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab int
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
attachIntListener(
outputStreamName: string, callbackFcn: SimpleListener<number>): void;
/**
* Attaches an int[] packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab
* std::vector<int> data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
attachIntVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void;
/**
* Attaches a double packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab double
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
attachDoubleListener(
outputStreamName: string, callbackFcn: SimpleListener<number>): void;
/**
* Attaches a double[] packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab
* std::vector<double> data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
attachDoubleVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void;
/**
* Attaches a float packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab float
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
attachFloatListener(
outputStreamName: string, callbackFcn: SimpleListener<number>): void;
/**
* Attaches a float[] packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab
* std::vector<float> data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
attachFloatVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void;
/**
* Attaches a string packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab string
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
attachStringListener(
outputStreamName: string, callbackFcn: SimpleListener<string>): void;
/**
* Attaches a string[] packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab
* std::vector<std::string> data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior.
*/
attachStringVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<string[]>): void;
/**
* Attaches a serialized proto packet listener to the specified output_stream.
* @param outputStreamName The name of the graph output stream to grab binary
* serialized proto data from (in Uint8Array format).
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that by default the data is only guaranteed to
* exist for the duration of the callback, and the callback will be called
* inline, so it should not perform overly complicated (or any async)
* behavior. If the proto data needs to be able to outlive the call, you
* may set the optional makeDeepCopy parameter to true, or can manually
* deep-copy the data yourself.
* @param makeDeepCopy Optional convenience parameter which, if set to true,
* will override the default memory management behavior and make a deep
* copy of the underlying data, rather than just returning a view into the
* C++-managed memory. At the cost of a data copy, this allows the
* returned data to outlive the callback lifetime (and it will be cleaned
* up automatically by JS garbage collection whenever the user is finished
* with it).
*/
attachProtoListener(
outputStreamName: string, callbackFcn: SimpleListener<Uint8Array>,
makeDeepCopy?: boolean): void;
/**
* Attaches a listener for an array of serialized proto packets to the
* specified output_stream.
* @param outputStreamName The name of the graph output stream to grab a
* vector of binary serialized proto data from (in Uint8Array[] format).
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that by default the data is only guaranteed to
* exist for the duration of the callback, and the callback will be called
* inline, so it should not perform overly complicated (or any async)
* behavior. If the proto data needs to be able to outlive the call, you
* may set the optional makeDeepCopy parameter to true, or can manually
* deep-copy the data yourself.
* @param makeDeepCopy Optional convenience parameter which, if set to true,
* will override the default memory management behavior and make a deep
* copy of the underlying data, rather than just returning a view into the
* C++-managed memory. At the cost of a data copy, this allows the
* returned data to outlive the callback lifetime (and it will be cleaned
* up automatically by JS garbage collection whenever the user is finished
* with it).
*/
attachProtoVectorListener(
outputStreamName: string, callbackFcn: SimpleListener<Uint8Array[]>,
makeDeepCopy?: boolean): void;
/**
* Attaches an audio packet listener to the specified output_stream, to be
* given a Float32Array as output. Requires wasm build dependency
* "gl_graph_runner_audio_out".
* @param outputStreamName The name of the graph output stream to grab audio
* data from.
* @param callbackFcn The function that will be called back with the data, as
* it is received. Note that the data is only guaranteed to exist for the
* duration of the callback, and the callback will be called inline, so it
* should not perform overly complicated (or any async) behavior. If the
* audio data needs to be able to outlive the call, you may set the
* optional makeDeepCopy parameter to true, or can manually deep-copy the
* data yourself.
* @param makeDeepCopy Optional convenience parameter which, if set to true,
* will override the default memory management behavior and make a deep
* copy of the underlying data, rather than just returning a view into the
* C++-managed memory. At the cost of a data copy, this allows the
* returned data to outlive the callback lifetime (and it will be cleaned
* up automatically by JS garbage collection whenever the user is finished
* with it).
*/
attachAudioListener(
outputStreamName: string, callbackFcn: SimpleListener<Float32Array>,
makeDeepCopy?: boolean): void;
/**
* Forces all queued-up packets to be pushed through the MediaPipe graph as
* far as possible, performing all processing until no more processing can be
* done. This is fully synchronous and by default single-threaded, so the
* calling thread will be blocked until processing completes. This must be
* called once for every frame, as `add*ToStream` calls merely queue up
* packets to the appropriate streams for processing, but processing does not
* occur until this function is called. Any listeners receiving output will
* be called back before this operation completes.
*/
finishProcessing(): void;
/**
* Closes the input streams and all calculators for this graph and frees up
* any C++ resources. The graph will not be usable once closed!
*/
closeGraph(): void;
}

79
mediapipe/web/graph_runner/graph_runner_factory_api.d.ts vendored Normal file
View File

@ -0,0 +1,79 @@
// Placeholder for internal dependency on trusted resource url
import {GraphRunnerApi} from './graph_runner_api';
import {WasmModule} from './wasm_module';
/**
* Simple interface for allowing users to set the directory where internal
* wasm-loading and asset-loading code looks (e.g. for .wasm and .data file
* locations).
*/
export declare interface FileLocator {
locateFile: (filename: string) => string;
mainScriptUrlOrBlob?: string;
}
/**
* Helper type macro for use with createMediaPipeLib. Allows us to easily infer
* the type of a mixin-extended GraphRunner. Example usage:
* const GraphRunnerConstructor =
* SupportImage(SupportSerialization(GraphRunner));
* let mediaPipe: ReturnType<typeof GraphRunnerConstructor>;
* ...
* mediaPipe = await createMediaPipeLib(GraphRunnerConstructor, ...);
*/
// tslint:disable-next-line:no-any
export type ReturnType<T> = T extends(...args: unknown[]) => infer R ? R : any;
/**
* Internal type of constructors used for initializing GraphRunner and
* subclasses.
*/
export type WasmMediaPipeConstructor<LibType> =
(new (
module: WasmModule, canvas?: HTMLCanvasElement|OffscreenCanvas|null) =>
LibType);
/**
* Global function interface to initialize Wasm blob and load runtime assets for
* a specialized MediaPipe library. This allows us to create a requested
* subclass inheriting from GraphRunner. Standard implementation is
* `createMediaPipeLib<LibType>`.
* @param constructorFcn The name of the class to instantiate via "new".
* @param wasmLoaderScript Url for the wasm-runner script; produced by the build
* process.
* @param assetLoaderScript Url for the asset-loading script; produced by the
* build process.
* @param fileLocator A function to override the file locations for assets
* loaded by the MediaPipe library.
* @return promise A promise which will resolve when initialization has
* completed successfully.
*/
export interface CreateMediaPipeLibApi {
<LibType>(
constructorFcn: WasmMediaPipeConstructor<LibType>,
wasmLoaderScript?: string|null,
assetLoaderScript?: string|null,
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
fileLocator?: FileLocator): Promise<LibType>;
}
/**
* Global function interface to initialize Wasm blob and load runtime assets for
* a generic MediaPipe library. Standard implementation is
* `createGraphRunner`.
* @param wasmLoaderScript Url for the wasm-runner script; produced by the build
* process.
* @param assetLoaderScript Url for the asset-loading script; produced by the
* build process.
* @param fileLocator A function to override the file locations for assets
* loaded by the MediaPipe library.
* @return promise A promise which will resolve when initialization has
* completed successfully.
*/
export interface CreateGraphRunnerApi {
(wasmLoaderScript?: string,
assetLoaderScript?: string,
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
fileLocator?: FileLocator): Promise<GraphRunnerApi>;
}

23
mediapipe/web/graph_runner/listener_types.d.ts vendored Normal file
View File

@ -0,0 +1,23 @@
/**
* A listener that receives the contents of a non-empty MediaPipe packet and
* its timestamp.
*/
export type SimpleListener<T> = (data: T, timestamp: number) => void;
/**
* A listener that receives the current MediaPipe packet timestamp. This is
* invoked even for empty packet.
*/
export type EmptyPacketListener = (timestamp: number) => void;
/**
* A listener that receives a single element of vector-returning output packet.
* Receives one element at a time (in order). Once all elements are processed,
* the listener is invoked with `data` set to `unknown` and `done` set to true.
* Intended for internal usage.
*/
export type VectorListener<T> = (data: T, done: boolean, timestamp: number) =>
void;
/** A listener that will be invoked with an absl::StatusCode and message. */
export type ErrorListener = (code: number, message: string) => void;

111
mediapipe/web/graph_runner/wasm_module.d.ts vendored Normal file
View File

@ -0,0 +1,111 @@
import {EmptyPacketListener, ErrorListener, SimpleListener, VectorListener} from './listener_types';
/**
* Declarations for Emscripten's WebAssembly Module behavior, so TS compiler
* doesn't break our various JS/C++ bridges. For internal usage.
*/
export declare interface WasmModule {
canvas: HTMLCanvasElement|OffscreenCanvas|null;
HEAPU8: Uint8Array;
HEAPU32: Uint32Array;
HEAPF32: Float32Array;
HEAPF64: Float64Array;
FS_createDataFile:
(parent: string, name: string, data: Uint8Array, canRead: boolean,
canWrite: boolean, canOwn: boolean) => void;
FS_createPath:
(parent: string, name: string, canRead: boolean,
canWrite: boolean) => void;
FS_unlink(path: string): void;
gpuOriginForWebTexturesIsBottomLeft?: boolean;
errorListener?: ErrorListener;
_bindTextureToCanvas: () => boolean;
_changeBinaryGraph: (size: number, dataPtr: number) => void;
_changeTextGraph: (size: number, dataPtr: number) => void;
_closeGraph: () => void;
_free: (ptr: number) => void;
_malloc: (size: number) => number;
_processFrame: (width: number, height: number, timestamp: number) => void;
_setAutoRenderToScreen: (enabled: boolean) => void;
_waitUntilIdle: () => void;
// Exposed so that clients of this lib can access this field
dataFileDownloads?: {[url: string]: {loaded: number, total: number}};
// Wasm Module multistream entrypoints. Require
// gl_graph_runner_internal_multi_input as a build dependency.
stringToNewUTF8: (data: string) => number;
_bindTextureToStream: (streamNamePtr: number) => void;
_addBoundTextureToStream:
(streamNamePtr: number, width: number, height: number,
timestamp: number) => void;
_addBoolToInputStream:
(data: boolean, streamNamePtr: number, timestamp: number) => void;
_addDoubleToInputStream:
(data: number, streamNamePtr: number, timestamp: number) => void;
_addFloatToInputStream:
(data: number, streamNamePtr: number, timestamp: number) => void;
_addIntToInputStream:
(data: number, streamNamePtr: number, timestamp: number) => void;
_addStringToInputStream:
(dataPtr: number, streamNamePtr: number, timestamp: number) => void;
_addFlatHashMapToInputStream:
(keysPtr: number, valuesPtr: number, count: number, streamNamePtr: number,
timestamp: number) => void;
_addProtoToInputStream:
(dataPtr: number, dataSize: number, protoNamePtr: number,
streamNamePtr: number, timestamp: number) => void;
_addEmptyPacketToInputStream:
(streamNamePtr: number, timestamp: number) => void;
// Input side packets
_addBoolToInputSidePacket: (data: boolean, streamNamePtr: number) => void;
_addDoubleToInputSidePacket: (data: number, streamNamePtr: number) => void;
_addFloatToInputSidePacket: (data: number, streamNamePtr: number) => void;
_addIntToInputSidePacket: (data: number, streamNamePtr: number) => void;
_addStringToInputSidePacket: (dataPtr: number, streamNamePtr: number) => void;
_addProtoToInputSidePacket:
(dataPtr: number, dataSize: number, protoNamePtr: number,
streamNamePtr: number) => void;
// Map of output streams to packet listeners. Also built as part of
// gl_graph_runner_internal_multi_input.
simpleListeners?:
Record<string, SimpleListener<unknown>|VectorListener<unknown>>;
// Map of output streams to empty packet listeners.
emptyPacketListeners?: Record<string, EmptyPacketListener>;
_attachBoolListener: (streamNamePtr: number) => void;
_attachBoolVectorListener: (streamNamePtr: number) => void;
_attachDoubleListener: (streamNamePtr: number) => void;
_attachDoubleVectorListener: (streamNamePtr: number) => void;
_attachFloatListener: (streamNamePtr: number) => void;
_attachFloatVectorListener: (streamNamePtr: number) => void;
_attachIntListener: (streamNamePtr: number) => void;
_attachIntVectorListener: (streamNamePtr: number) => void;
_attachStringListener: (streamNamePtr: number) => void;
_attachStringVectorListener: (streamNamePtr: number) => void;
_attachProtoListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
_attachProtoVectorListener:
(streamNamePtr: number, makeDeepCopy?: boolean) => void;
// Require dependency ":gl_graph_runner_audio_out"
_attachAudioListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
// Require dependency ":gl_graph_runner_audio"
_addAudioToInputStream:
(dataPtr: number, numChannels: number, numSamples: number,
streamNamePtr: number, timestamp: number) => void;
_configureAudio:
(channels: number, samples: number, sampleRate: number,
streamNamePtr: number, headerNamePtr: number) => void;
// Get the graph configuration and invoke the listener configured under
// streamNamePtr
_getGraphConfig: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
// TODO: Refactor to just use a few numbers (perhaps refactor away
// from gl_graph_runner_internal.cc entirely to use something a little more
// streamlined; new version is _processFrame above).
_processGl: (frameDataPtr: number) => number;
}