Splitting GraphRunner into public API declared interfaces and private TS impls
PiperOrigin-RevId: 564551973
This commit is contained in:
parent
0fec532ebe
commit
02bd0d95e7
|
@ -1,19 +1,67 @@
|
||||||
# The TypeScript graph runner used by all MediaPipe Web tasks.
|
# The TypeScript graph runner used by all MediaPipe Web tasks.
|
||||||
|
|
||||||
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
|
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_declaration", "mediapipe_ts_library")
|
||||||
load("@npm//@bazel/jasmine:index.bzl", "jasmine_node_test")
|
load("@npm//@bazel/jasmine:index.bzl", "jasmine_node_test")
|
||||||
|
|
||||||
package(default_visibility = [
|
package(default_visibility = [
|
||||||
"//mediapipe/tasks:internal",
|
"//mediapipe/tasks:internal",
|
||||||
])
|
])
|
||||||
|
|
||||||
|
# Public types for GraphRunner listeners
|
||||||
|
mediapipe_ts_declaration(
|
||||||
|
name = "listener_types",
|
||||||
|
srcs = [
|
||||||
|
"listener_types.d.ts",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:private"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Public API for GraphRunner
|
||||||
|
mediapipe_ts_declaration(
|
||||||
|
name = "graph_runner_api",
|
||||||
|
srcs = [
|
||||||
|
"graph_runner_api.d.ts",
|
||||||
|
],
|
||||||
|
deps = [
|
||||||
|
":listener_types",
|
||||||
|
":wasm_module_private",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Public API for GraphRunner creation factory functions
|
||||||
|
mediapipe_ts_library(
|
||||||
|
name = "graph_runner_factory_api",
|
||||||
|
srcs = [
|
||||||
|
"graph_runner_factory_api.d.ts",
|
||||||
|
],
|
||||||
|
deps = [
|
||||||
|
":graph_runner_api",
|
||||||
|
":wasm_module_private",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Private declarations for the Wasm module
|
||||||
|
mediapipe_ts_declaration(
|
||||||
|
name = "wasm_module_private",
|
||||||
|
srcs = ["wasm_module.d.ts"],
|
||||||
|
visibility = ["//visibility:private"],
|
||||||
|
deps = [":listener_types"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# Internal GraphRunner implementation
|
||||||
mediapipe_ts_library(
|
mediapipe_ts_library(
|
||||||
name = "graph_runner_ts",
|
name = "graph_runner_ts",
|
||||||
srcs = [
|
srcs = [
|
||||||
":graph_runner.ts",
|
":graph_runner.ts",
|
||||||
],
|
],
|
||||||
allow_unoptimized_namespaces = True,
|
allow_unoptimized_namespaces = True,
|
||||||
deps = [":platform_utils"],
|
deps = [
|
||||||
|
":graph_runner_api",
|
||||||
|
":graph_runner_factory_api",
|
||||||
|
":listener_types",
|
||||||
|
":platform_utils",
|
||||||
|
":wasm_module_private",
|
||||||
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
mediapipe_ts_library(
|
mediapipe_ts_library(
|
||||||
|
|
|
@ -3,40 +3,27 @@
|
||||||
import {isWebKit} from '../../web/graph_runner/platform_utils';
|
import {isWebKit} from '../../web/graph_runner/platform_utils';
|
||||||
// Placeholder for internal dependency on trusted resource url
|
// Placeholder for internal dependency on trusted resource url
|
||||||
|
|
||||||
// This file can serve as a common interface for most simple TypeScript
|
import {GraphRunnerApi, ImageSource} from './graph_runner_api';
|
||||||
// libraries-- additionally, it can hook automatically into wasm_mediapipe_demo
|
import {CreateGraphRunnerApi, CreateMediaPipeLibApi, FileLocator, WasmMediaPipeConstructor} from './graph_runner_factory_api';
|
||||||
// to autogenerate simple TS APIs from demos for instantaneous 1P integrations.
|
import {EmptyPacketListener, ErrorListener, SimpleListener, VectorListener} from './listener_types';
|
||||||
|
import {WasmModule} from './wasm_module';
|
||||||
|
|
||||||
/**
|
export {ReturnType} from './graph_runner_factory_api';
|
||||||
* Simple interface for allowing users to set the directory where internal
|
// This file contains the internal implementations behind the public APIs
|
||||||
* wasm-loading and asset-loading code looks (e.g. for .wasm and .data file
|
// declared in "graph_runner_api.d.ts" and "graph_runner_factory_api.d.ts".
|
||||||
* locations).
|
|
||||||
*/
|
|
||||||
export declare interface FileLocator {
|
|
||||||
locateFile: (filename: string) => string;
|
|
||||||
mainScriptUrlOrBlob?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
// First we re-export all of our imported public types/defines, so that users
|
||||||
* A listener that receives the contents of a non-empty MediaPipe packet and
|
// can import everything they need from here directly.
|
||||||
* its timestamp.
|
export {
|
||||||
*/
|
EmptyPacketListener,
|
||||||
export type SimpleListener<T> = (data: T, timestamp: number) => void;
|
ErrorListener,
|
||||||
|
FileLocator,
|
||||||
/**
|
ImageSource,
|
||||||
* A listener that receives the current MediaPipe packet timestamp. This is
|
SimpleListener,
|
||||||
* invoked even for empty packet.
|
VectorListener,
|
||||||
*/
|
WasmMediaPipeConstructor,
|
||||||
export type EmptyPacketListener = (timestamp: number) => void;
|
WasmModule,
|
||||||
|
};
|
||||||
/**
|
|
||||||
* A listener that receives a single element of vector-returning output packet.
|
|
||||||
* Receives one element at a time (in order). Once all elements are processed,
|
|
||||||
* the listener is invoked with `data` set to `unknown` and `done` set to true.
|
|
||||||
* Intended for internal usage.
|
|
||||||
*/
|
|
||||||
export type VectorListener<T> = (data: T, done: boolean, timestamp: number) =>
|
|
||||||
void;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A listener that receives the CalculatorGraphConfig in binary encoding.
|
* A listener that receives the CalculatorGraphConfig in binary encoding.
|
||||||
|
@ -49,114 +36,6 @@ export type CalculatorGraphConfigListener = (graphConfig: Uint8Array) => void;
|
||||||
*/
|
*/
|
||||||
export const CALCULATOR_GRAPH_CONFIG_LISTENER_NAME = '__graph_config__';
|
export const CALCULATOR_GRAPH_CONFIG_LISTENER_NAME = '__graph_config__';
|
||||||
|
|
||||||
/**
|
|
||||||
* Declarations for Emscripten's WebAssembly Module behavior, so TS compiler
|
|
||||||
* doesn't break our JS/C++ bridge.
|
|
||||||
*/
|
|
||||||
export declare interface WasmModule {
|
|
||||||
canvas: HTMLCanvasElement|OffscreenCanvas|null;
|
|
||||||
HEAPU8: Uint8Array;
|
|
||||||
HEAPU32: Uint32Array;
|
|
||||||
HEAPF32: Float32Array;
|
|
||||||
HEAPF64: Float64Array;
|
|
||||||
FS_createDataFile:
|
|
||||||
(parent: string, name: string, data: Uint8Array, canRead: boolean,
|
|
||||||
canWrite: boolean, canOwn: boolean) => void;
|
|
||||||
FS_createPath:
|
|
||||||
(parent: string, name: string, canRead: boolean,
|
|
||||||
canWrite: boolean) => void;
|
|
||||||
FS_unlink(path: string): void;
|
|
||||||
gpuOriginForWebTexturesIsBottomLeft?: boolean;
|
|
||||||
|
|
||||||
errorListener?: ErrorListener;
|
|
||||||
_bindTextureToCanvas: () => boolean;
|
|
||||||
_changeBinaryGraph: (size: number, dataPtr: number) => void;
|
|
||||||
_changeTextGraph: (size: number, dataPtr: number) => void;
|
|
||||||
_closeGraph: () => void;
|
|
||||||
_free: (ptr: number) => void;
|
|
||||||
_malloc: (size: number) => number;
|
|
||||||
_processFrame: (width: number, height: number, timestamp: number) => void;
|
|
||||||
_setAutoRenderToScreen: (enabled: boolean) => void;
|
|
||||||
_waitUntilIdle: () => void;
|
|
||||||
|
|
||||||
// Exposed so that clients of this lib can access this field
|
|
||||||
dataFileDownloads?: {[url: string]: {loaded: number, total: number}};
|
|
||||||
|
|
||||||
// Wasm Module multistream entrypoints. Require
|
|
||||||
// gl_graph_runner_internal_multi_input as a build dependency.
|
|
||||||
stringToNewUTF8: (data: string) => number;
|
|
||||||
_bindTextureToStream: (streamNamePtr: number) => void;
|
|
||||||
_addBoundTextureToStream:
|
|
||||||
(streamNamePtr: number, width: number, height: number,
|
|
||||||
timestamp: number) => void;
|
|
||||||
_addBoolToInputStream:
|
|
||||||
(data: boolean, streamNamePtr: number, timestamp: number) => void;
|
|
||||||
_addDoubleToInputStream:
|
|
||||||
(data: number, streamNamePtr: number, timestamp: number) => void;
|
|
||||||
_addFloatToInputStream:
|
|
||||||
(data: number, streamNamePtr: number, timestamp: number) => void;
|
|
||||||
_addIntToInputStream:
|
|
||||||
(data: number, streamNamePtr: number, timestamp: number) => void;
|
|
||||||
_addStringToInputStream:
|
|
||||||
(dataPtr: number, streamNamePtr: number, timestamp: number) => void;
|
|
||||||
_addFlatHashMapToInputStream:
|
|
||||||
(keysPtr: number, valuesPtr: number, count: number, streamNamePtr: number,
|
|
||||||
timestamp: number) => void;
|
|
||||||
_addProtoToInputStream:
|
|
||||||
(dataPtr: number, dataSize: number, protoNamePtr: number,
|
|
||||||
streamNamePtr: number, timestamp: number) => void;
|
|
||||||
_addEmptyPacketToInputStream:
|
|
||||||
(streamNamePtr: number, timestamp: number) => void;
|
|
||||||
|
|
||||||
// Input side packets
|
|
||||||
_addBoolToInputSidePacket: (data: boolean, streamNamePtr: number) => void;
|
|
||||||
_addDoubleToInputSidePacket: (data: number, streamNamePtr: number) => void;
|
|
||||||
_addFloatToInputSidePacket: (data: number, streamNamePtr: number) => void;
|
|
||||||
_addIntToInputSidePacket: (data: number, streamNamePtr: number) => void;
|
|
||||||
_addStringToInputSidePacket: (dataPtr: number, streamNamePtr: number) => void;
|
|
||||||
_addProtoToInputSidePacket:
|
|
||||||
(dataPtr: number, dataSize: number, protoNamePtr: number,
|
|
||||||
streamNamePtr: number) => void;
|
|
||||||
|
|
||||||
// Map of output streams to packet listeners. Also built as part of
|
|
||||||
// gl_graph_runner_internal_multi_input.
|
|
||||||
simpleListeners?:
|
|
||||||
Record<string, SimpleListener<unknown>|VectorListener<unknown>>;
|
|
||||||
// Map of output streams to empty packet listeners.
|
|
||||||
emptyPacketListeners?: Record<string, EmptyPacketListener>;
|
|
||||||
_attachBoolListener: (streamNamePtr: number) => void;
|
|
||||||
_attachBoolVectorListener: (streamNamePtr: number) => void;
|
|
||||||
_attachDoubleListener: (streamNamePtr: number) => void;
|
|
||||||
_attachDoubleVectorListener: (streamNamePtr: number) => void;
|
|
||||||
_attachFloatListener: (streamNamePtr: number) => void;
|
|
||||||
_attachFloatVectorListener: (streamNamePtr: number) => void;
|
|
||||||
_attachIntListener: (streamNamePtr: number) => void;
|
|
||||||
_attachIntVectorListener: (streamNamePtr: number) => void;
|
|
||||||
_attachStringListener: (streamNamePtr: number) => void;
|
|
||||||
_attachStringVectorListener: (streamNamePtr: number) => void;
|
|
||||||
_attachProtoListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
|
||||||
_attachProtoVectorListener:
|
|
||||||
(streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
|
||||||
|
|
||||||
// Require dependency ":gl_graph_runner_audio_out"
|
|
||||||
_attachAudioListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
|
||||||
|
|
||||||
// Require dependency ":gl_graph_runner_audio"
|
|
||||||
_addAudioToInputStream: (dataPtr: number, numChannels: number,
|
|
||||||
numSamples: number, streamNamePtr: number, timestamp: number) => void;
|
|
||||||
_configureAudio: (channels: number, samples: number, sampleRate: number,
|
|
||||||
streamNamePtr: number, headerNamePtr: number) => void;
|
|
||||||
|
|
||||||
// Get the graph configuration and invoke the listener configured under
|
|
||||||
// streamNamePtr
|
|
||||||
_getGraphConfig: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
|
||||||
|
|
||||||
// TODO: Refactor to just use a few numbers (perhaps refactor away
|
|
||||||
// from gl_graph_runner_internal.cc entirely to use something a little more
|
|
||||||
// streamlined; new version is _processFrame above).
|
|
||||||
_processGl: (frameDataPtr: number) => number;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Global declarations, for tapping into Window for Wasm blob running
|
// Global declarations, for tapping into Window for Wasm blob running
|
||||||
declare global {
|
declare global {
|
||||||
interface Window {
|
interface Window {
|
||||||
|
@ -173,32 +52,13 @@ declare global {
|
||||||
*/
|
*/
|
||||||
declare function importScripts(...urls: Array<string|URL>): void;
|
declare function importScripts(...urls: Array<string|URL>): void;
|
||||||
|
|
||||||
/**
|
|
||||||
* Valid types of image sources which we can run our GraphRunner over.
|
|
||||||
*/
|
|
||||||
export type ImageSource =
|
|
||||||
HTMLCanvasElement|HTMLVideoElement|HTMLImageElement|ImageData|ImageBitmap;
|
|
||||||
|
|
||||||
|
|
||||||
/** A listener that will be invoked with an absl::StatusCode and message. */
|
|
||||||
export type ErrorListener = (code: number, message: string) => void;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Internal type of constructors used for initializing GraphRunner and
|
|
||||||
* subclasses.
|
|
||||||
*/
|
|
||||||
export type WasmMediaPipeConstructor<LibType> =
|
|
||||||
(new (
|
|
||||||
module: WasmModule, canvas?: HTMLCanvasElement|OffscreenCanvas|null) =>
|
|
||||||
LibType);
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Simple class to run an arbitrary image-in/image-out MediaPipe graph (i.e.
|
* Simple class to run an arbitrary image-in/image-out MediaPipe graph (i.e.
|
||||||
* as created by wasm_mediapipe_demo BUILD macro), and either render results
|
* as created by wasm_mediapipe_demo BUILD macro), and either render results
|
||||||
* into canvas, or else return the output WebGLTexture. Takes a WebAssembly
|
* into canvas, or else return the output WebGLTexture. Takes a WebAssembly
|
||||||
* Module.
|
* Module.
|
||||||
*/
|
*/
|
||||||
export class GraphRunner {
|
export class GraphRunner implements GraphRunnerApi {
|
||||||
// TODO: These should be protected/private, but are left exposed for
|
// TODO: These should be protected/private, but are left exposed for
|
||||||
// now so that we can use proper TS mixins with this class as a base. This
|
// now so that we can use proper TS mixins with this class as a base. This
|
||||||
// should be somewhat fixed when we create our .d.ts files.
|
// should be somewhat fixed when we create our .d.ts files.
|
||||||
|
@ -241,11 +101,7 @@ export class GraphRunner {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Convenience helper to load a MediaPipe graph from a file and pass it to
|
|
||||||
* setGraph.
|
|
||||||
* @param graphFile The url of the MediaPipe graph file to load.
|
|
||||||
*/
|
|
||||||
async initializeGraph(graphFile: string): Promise<void> {
|
async initializeGraph(graphFile: string): Promise<void> {
|
||||||
// Fetch and set graph
|
// Fetch and set graph
|
||||||
const response = await fetch(graphFile);
|
const response = await fetch(graphFile);
|
||||||
|
@ -255,26 +111,12 @@ export class GraphRunner {
|
||||||
this.setGraph(new Uint8Array(graphData), isBinary);
|
this.setGraph(new Uint8Array(graphData), isBinary);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Convenience helper for calling setGraph with a string representing a text
|
|
||||||
* proto config.
|
|
||||||
* @param graphConfig The text proto graph config, expected to be a string in
|
|
||||||
* default JavaScript UTF-16 format.
|
|
||||||
*/
|
|
||||||
setGraphFromString(graphConfig: string): void {
|
setGraphFromString(graphConfig: string): void {
|
||||||
this.setGraph((new TextEncoder()).encode(graphConfig), false);
|
this.setGraph((new TextEncoder()).encode(graphConfig), false);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Takes the raw data from a MediaPipe graph, and passes it to C++ to be run
|
|
||||||
* over the video stream. Will replace the previously running MediaPipe graph,
|
|
||||||
* if there is one.
|
|
||||||
* @param graphData The raw MediaPipe graph data, either in binary
|
|
||||||
* protobuffer format (.binarypb), or else in raw text format (.pbtxt or
|
|
||||||
* .textproto).
|
|
||||||
* @param isBinary This should be set to true if the graph is in
|
|
||||||
* binary format, and false if it is in human-readable text format.
|
|
||||||
*/
|
|
||||||
setGraph(graphData: Uint8Array, isBinary: boolean): void {
|
setGraph(graphData: Uint8Array, isBinary: boolean): void {
|
||||||
const size = graphData.length;
|
const size = graphData.length;
|
||||||
const dataPtr = this.wasmModule._malloc(size);
|
const dataPtr = this.wasmModule._malloc(size);
|
||||||
|
@ -287,24 +129,7 @@ export class GraphRunner {
|
||||||
this.wasmModule._free(dataPtr);
|
this.wasmModule._free(dataPtr);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Configures the current graph to handle audio processing in a certain way
|
|
||||||
* for all its audio input streams. Additionally can configure audio headers
|
|
||||||
* (both input side packets as well as input stream headers), but these
|
|
||||||
* configurations only take effect if called before the graph is set/started.
|
|
||||||
* @param numChannels The number of channels of audio input. Only 1
|
|
||||||
* is supported for now.
|
|
||||||
* @param numSamples The number of samples that are taken in each
|
|
||||||
* audio capture.
|
|
||||||
* @param sampleRate The rate, in Hz, of the sampling.
|
|
||||||
* @param streamName The optional name of the input stream to additionally
|
|
||||||
* configure with audio information. This configuration only occurs before
|
|
||||||
* the graph is set/started. If unset, a default stream name will be used.
|
|
||||||
* @param headerName The optional name of the header input side packet to
|
|
||||||
* additionally configure with audio information. This configuration only
|
|
||||||
* occurs before the graph is set/started. If unset, a default header name
|
|
||||||
* will be used.
|
|
||||||
*/
|
|
||||||
configureAudio(numChannels: number, numSamples: number, sampleRate: number,
|
configureAudio(numChannels: number, numSamples: number, sampleRate: number,
|
||||||
streamName?: string, headerName?: string) {
|
streamName?: string, headerName?: string) {
|
||||||
if (!this.wasmModule._configureAudio) {
|
if (!this.wasmModule._configureAudio) {
|
||||||
|
@ -322,55 +147,17 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Allows disabling automatic canvas resizing, in case clients want to control
|
|
||||||
* control this.
|
|
||||||
* @param resize True will re-enable automatic canvas resizing, while false
|
|
||||||
* will disable the feature.
|
|
||||||
*/
|
|
||||||
setAutoResizeCanvas(resize: boolean): void {
|
setAutoResizeCanvas(resize: boolean): void {
|
||||||
this.autoResizeCanvas = resize;
|
this.autoResizeCanvas = resize;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Allows disabling the automatic render-to-screen code, in case clients don't
|
|
||||||
* need/want this. In particular, this removes the requirement for pipelines
|
|
||||||
* to have access to GPU resources, as well as the requirement for graphs to
|
|
||||||
* have "input_frames_gpu" and "output_frames_gpu" streams defined, so pure
|
|
||||||
* CPU pipelines and non-video pipelines can be created.
|
|
||||||
* NOTE: This only affects future graph initializations (via setGraph or
|
|
||||||
* initializeGraph), and does NOT affect the currently running graph, so
|
|
||||||
* calls to this should be made *before* setGraph/initializeGraph for the
|
|
||||||
* graph file being targeted.
|
|
||||||
* @param enabled True will re-enable automatic render-to-screen code and
|
|
||||||
* cause GPU resources to once again be requested, while false will
|
|
||||||
* disable the feature.
|
|
||||||
*/
|
|
||||||
setAutoRenderToScreen(enabled: boolean): void {
|
setAutoRenderToScreen(enabled: boolean): void {
|
||||||
this.wasmModule._setAutoRenderToScreen(enabled);
|
this.wasmModule._setAutoRenderToScreen(enabled);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Overrides the vertical orientation for input GpuBuffers and the automatic
|
|
||||||
* render-to-screen code. The default for our OpenGL code on other platforms
|
|
||||||
* (Android, Linux) is to use a bottom-left origin. But the default for WebGL
|
|
||||||
* is to use a top-left origin. We use WebGL default normally, and many
|
|
||||||
* calculators and graphs have platform-specific code to handle the resulting
|
|
||||||
* orientation flip. However, in order to be able to use a single graph on all
|
|
||||||
* platforms without alterations, it may be useful to send images into a web
|
|
||||||
* graph using the OpenGL orientation. Users can call this function with
|
|
||||||
* `bottomLeftIsOrigin = true` in order to enforce an orientation for all
|
|
||||||
* GpuBuffer inputs which is consistent with OpenGL on other platforms.
|
|
||||||
* This call will also vertically flip the automatic render-to-screen code as
|
|
||||||
* well, so that webcam input (for example) will render properly when passed
|
|
||||||
* through the graph still.
|
|
||||||
* NOTE: This will immediately affect GpuBuffer inputs, but must be called
|
|
||||||
* *before* graph start in order to affect the automatic render-to-screen
|
|
||||||
* code!
|
|
||||||
* @param bottomLeftIsOrigin True will flip our input GpuBuffers and auto
|
|
||||||
* render-to-screen to match the classic OpenGL orientation, while false will
|
|
||||||
* disable this feature to match the default WebGL orientation.
|
|
||||||
*/
|
|
||||||
setGpuBufferVerticalFlip(bottomLeftIsOrigin: boolean): void {
|
setGpuBufferVerticalFlip(bottomLeftIsOrigin: boolean): void {
|
||||||
this.wasmModule.gpuOriginForWebTexturesIsBottomLeft = bottomLeftIsOrigin;
|
this.wasmModule.gpuOriginForWebTexturesIsBottomLeft = bottomLeftIsOrigin;
|
||||||
}
|
}
|
||||||
|
@ -436,7 +223,8 @@ export class GraphRunner {
|
||||||
* Takes the raw data from a JS image source, and sends it to C++ to be
|
* Takes the raw data from a JS image source, and sends it to C++ to be
|
||||||
* processed, waiting synchronously for the response. Note that we will resize
|
* processed, waiting synchronously for the response. Note that we will resize
|
||||||
* our GL canvas to fit the input, so input size should only change
|
* our GL canvas to fit the input, so input size should only change
|
||||||
* infrequently.
|
* infrequently. NOTE: This call has been deprecated in favor of
|
||||||
|
* `addGpuBufferToStream`.
|
||||||
* @param imageSource An image source to process.
|
* @param imageSource An image source to process.
|
||||||
* @param timestamp The timestamp of the current frame, in ms.
|
* @param timestamp The timestamp of the current frame, in ms.
|
||||||
* @return texture? The WebGL texture reference, if one was produced.
|
* @return texture? The WebGL texture reference, if one was produced.
|
||||||
|
@ -554,26 +342,12 @@ export class GraphRunner {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a listener that will be invoked when the MediaPipe framework
|
|
||||||
* returns an error.
|
|
||||||
*/
|
|
||||||
attachErrorListener(callbackFcn: (code: number, message: string) => void) {
|
attachErrorListener(callbackFcn: (code: number, message: string) => void) {
|
||||||
this.wasmModule.errorListener = callbackFcn;
|
this.wasmModule.errorListener = callbackFcn;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a listener that will be invoked when the MediaPipe framework
|
|
||||||
* receives an empty packet on the provided output stream. This can be used
|
|
||||||
* to receive the latest output timestamp.
|
|
||||||
*
|
|
||||||
* Empty packet listeners are only active if there is a corresponding packet
|
|
||||||
* listener.
|
|
||||||
*
|
|
||||||
* @param outputStreamName The name of the graph output stream to receive
|
|
||||||
* empty packets from.
|
|
||||||
* @param callbackFcn The callback to receive the timestamp.
|
|
||||||
*/
|
|
||||||
attachEmptyPacketListener(
|
attachEmptyPacketListener(
|
||||||
outputStreamName: string, callbackFcn: EmptyPacketListener) {
|
outputStreamName: string, callbackFcn: EmptyPacketListener) {
|
||||||
this.wasmModule.emptyPacketListeners =
|
this.wasmModule.emptyPacketListeners =
|
||||||
|
@ -581,15 +355,7 @@ export class GraphRunner {
|
||||||
this.wasmModule.emptyPacketListeners[outputStreamName] = callbackFcn;
|
this.wasmModule.emptyPacketListeners[outputStreamName] = callbackFcn;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Takes the raw data from a JS audio capture array, and sends it to C++ to be
|
|
||||||
* processed.
|
|
||||||
* @param audioData An array of raw audio capture data, like
|
|
||||||
* from a call to getChannelData on an AudioBuffer.
|
|
||||||
* @param streamName The name of the MediaPipe graph stream to add the audio
|
|
||||||
* data to.
|
|
||||||
* @param timestamp The timestamp of the current frame, in ms.
|
|
||||||
*/
|
|
||||||
addAudioToStream(
|
addAudioToStream(
|
||||||
audioData: Float32Array, streamName: string, timestamp: number) {
|
audioData: Float32Array, streamName: string, timestamp: number) {
|
||||||
// numChannels and numSamples being 0 will cause defaults to be used,
|
// numChannels and numSamples being 0 will cause defaults to be used,
|
||||||
|
@ -597,22 +363,7 @@ export class GraphRunner {
|
||||||
this.addAudioToStreamWithShape(audioData, 0, 0, streamName, timestamp);
|
this.addAudioToStreamWithShape(audioData, 0, 0, streamName, timestamp);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Takes the raw data from a JS audio capture array, and sends it to C++ to be
|
|
||||||
* processed, shaping the audioData array into an audio matrix according to
|
|
||||||
* the numChannels and numSamples parameters.
|
|
||||||
* @param audioData An array of raw audio capture data, like
|
|
||||||
* from a call to getChannelData on an AudioBuffer.
|
|
||||||
* @param numChannels The number of audio channels this data represents. If 0
|
|
||||||
* is passed, then the value will be taken from the last call to
|
|
||||||
* configureAudio.
|
|
||||||
* @param numSamples The number of audio samples captured in this data packet.
|
|
||||||
* If 0 is passed, then the value will be taken from the last call to
|
|
||||||
* configureAudio.
|
|
||||||
* @param streamName The name of the MediaPipe graph stream to add the audio
|
|
||||||
* data to.
|
|
||||||
* @param timestamp The timestamp of the current frame, in ms.
|
|
||||||
*/
|
|
||||||
addAudioToStreamWithShape(
|
addAudioToStreamWithShape(
|
||||||
audioData: Float32Array, numChannels: number, numSamples: number,
|
audioData: Float32Array, numChannels: number, numSamples: number,
|
||||||
streamName: string, timestamp: number) {
|
streamName: string, timestamp: number) {
|
||||||
|
@ -633,19 +384,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Takes the relevant information from the HTML video or image element, and
|
|
||||||
* passes it into the WebGL-based graph for processing on the given stream at
|
|
||||||
* the given timestamp. Can be used for additional auxiliary GpuBuffer input
|
|
||||||
* streams. Processing will not occur until a blocking call (like
|
|
||||||
* processVideoGl or finishProcessing) is made. For use with
|
|
||||||
* 'gl_graph_runner_internal_multi_input'.
|
|
||||||
* @param imageSource Reference to the video frame we wish to add into our
|
|
||||||
* graph.
|
|
||||||
* @param streamName The name of the MediaPipe graph stream to add the frame
|
|
||||||
* to.
|
|
||||||
* @param timestamp The timestamp of the input frame, in ms.
|
|
||||||
*/
|
|
||||||
addGpuBufferToStream(
|
addGpuBufferToStream(
|
||||||
imageSource: ImageSource, streamName: string, timestamp: number): void {
|
imageSource: ImageSource, streamName: string, timestamp: number): void {
|
||||||
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
||||||
|
@ -656,36 +395,21 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Sends a boolean packet into the specified stream at the given timestamp.
|
|
||||||
* @param data The boolean data to send.
|
|
||||||
* @param streamName The name of the graph input stream to send data into.
|
|
||||||
* @param timestamp The timestamp of the input data, in ms.
|
|
||||||
*/
|
|
||||||
addBoolToStream(data: boolean, streamName: string, timestamp: number): void {
|
addBoolToStream(data: boolean, streamName: string, timestamp: number): void {
|
||||||
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
||||||
this.wasmModule._addBoolToInputStream(data, streamNamePtr, timestamp);
|
this.wasmModule._addBoolToInputStream(data, streamNamePtr, timestamp);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Sends a double packet into the specified stream at the given timestamp.
|
|
||||||
* @param data The double data to send.
|
|
||||||
* @param streamName The name of the graph input stream to send data into.
|
|
||||||
* @param timestamp The timestamp of the input data, in ms.
|
|
||||||
*/
|
|
||||||
addDoubleToStream(data: number, streamName: string, timestamp: number): void {
|
addDoubleToStream(data: number, streamName: string, timestamp: number): void {
|
||||||
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
||||||
this.wasmModule._addDoubleToInputStream(data, streamNamePtr, timestamp);
|
this.wasmModule._addDoubleToInputStream(data, streamNamePtr, timestamp);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Sends a float packet into the specified stream at the given timestamp.
|
|
||||||
* @param data The float data to send.
|
|
||||||
* @param streamName The name of the graph input stream to send data into.
|
|
||||||
* @param timestamp The timestamp of the input data, in ms.
|
|
||||||
*/
|
|
||||||
addFloatToStream(data: number, streamName: string, timestamp: number): void {
|
addFloatToStream(data: number, streamName: string, timestamp: number): void {
|
||||||
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
||||||
// NOTE: _addFloatToStream and _addIntToStream are reserved for JS
|
// NOTE: _addFloatToStream and _addIntToStream are reserved for JS
|
||||||
|
@ -695,24 +419,14 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Sends an integer packet into the specified stream at the given timestamp.
|
|
||||||
* @param data The integer data to send.
|
|
||||||
* @param streamName The name of the graph input stream to send data into.
|
|
||||||
* @param timestamp The timestamp of the input data, in ms.
|
|
||||||
*/
|
|
||||||
addIntToStream(data: number, streamName: string, timestamp: number): void {
|
addIntToStream(data: number, streamName: string, timestamp: number): void {
|
||||||
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
||||||
this.wasmModule._addIntToInputStream(data, streamNamePtr, timestamp);
|
this.wasmModule._addIntToInputStream(data, streamNamePtr, timestamp);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Sends a string packet into the specified stream at the given timestamp.
|
|
||||||
* @param data The string data to send.
|
|
||||||
* @param streamName The name of the graph input stream to send data into.
|
|
||||||
* @param timestamp The timestamp of the input data, in ms.
|
|
||||||
*/
|
|
||||||
addStringToStream(data: string, streamName: string, timestamp: number): void {
|
addStringToStream(data: string, streamName: string, timestamp: number): void {
|
||||||
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
||||||
this.wrapStringPtr(data, (dataPtr: number) => {
|
this.wrapStringPtr(data, (dataPtr: number) => {
|
||||||
|
@ -722,14 +436,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Sends a Record<string, string> packet into the specified stream at the
|
|
||||||
* given timestamp.
|
|
||||||
* @param data The records to send (will become a
|
|
||||||
* std::flat_hash_map<std::string, std::string).
|
|
||||||
* @param streamName The name of the graph input stream to send data into.
|
|
||||||
* @param timestamp The timestamp of the input data, in ms.
|
|
||||||
*/
|
|
||||||
addStringRecordToStream(
|
addStringRecordToStream(
|
||||||
data: Record<string, string>, streamName: string,
|
data: Record<string, string>, streamName: string,
|
||||||
timestamp: number): void {
|
timestamp: number): void {
|
||||||
|
@ -744,16 +451,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Sends a serialized protobuffer packet into the specified stream at the
|
|
||||||
* given timestamp, to be parsed into the specified protobuffer type.
|
|
||||||
* @param data The binary (serialized) raw protobuffer data.
|
|
||||||
* @param protoType The C++ namespaced type this protobuffer data corresponds
|
|
||||||
* to (e.g. "foo.Bar"). It will be converted to this type when output as a
|
|
||||||
* packet into the graph.
|
|
||||||
* @param streamName The name of the graph input stream to send data into.
|
|
||||||
* @param timestamp The timestamp of the input data, in ms.
|
|
||||||
*/
|
|
||||||
addProtoToStream(
|
addProtoToStream(
|
||||||
data: Uint8Array, protoType: string, streamName: string,
|
data: Uint8Array, protoType: string, streamName: string,
|
||||||
timestamp: number): void {
|
timestamp: number): void {
|
||||||
|
@ -770,74 +468,42 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Sends an empty packet into the specified stream at the given timestamp,
|
|
||||||
* effectively advancing that input stream's timestamp bounds without
|
|
||||||
* sending additional data packets.
|
|
||||||
* @param streamName The name of the graph input stream to send the empty
|
|
||||||
* packet into.
|
|
||||||
* @param timestamp The timestamp of the empty packet, in ms.
|
|
||||||
*/
|
|
||||||
addEmptyPacketToStream(streamName: string, timestamp: number): void {
|
addEmptyPacketToStream(streamName: string, timestamp: number): void {
|
||||||
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
|
||||||
this.wasmModule._addEmptyPacketToInputStream(streamNamePtr, timestamp);
|
this.wasmModule._addEmptyPacketToInputStream(streamNamePtr, timestamp);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a boolean packet to the specified input_side_packet.
|
|
||||||
* @param data The boolean data to send.
|
|
||||||
* @param sidePacketName The name of the graph input side packet to send data
|
|
||||||
* into.
|
|
||||||
*/
|
|
||||||
addBoolToInputSidePacket(data: boolean, sidePacketName: string): void {
|
addBoolToInputSidePacket(data: boolean, sidePacketName: string): void {
|
||||||
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
||||||
this.wasmModule._addBoolToInputSidePacket(data, sidePacketNamePtr);
|
this.wasmModule._addBoolToInputSidePacket(data, sidePacketNamePtr);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a double packet to the specified input_side_packet.
|
|
||||||
* @param data The double data to send.
|
|
||||||
* @param sidePacketName The name of the graph input side packet to send data
|
|
||||||
* into.
|
|
||||||
*/
|
|
||||||
addDoubleToInputSidePacket(data: number, sidePacketName: string): void {
|
addDoubleToInputSidePacket(data: number, sidePacketName: string): void {
|
||||||
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
||||||
this.wasmModule._addDoubleToInputSidePacket(data, sidePacketNamePtr);
|
this.wasmModule._addDoubleToInputSidePacket(data, sidePacketNamePtr);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a float packet to the specified input_side_packet.
|
|
||||||
* @param data The float data to send.
|
|
||||||
* @param sidePacketName The name of the graph input side packet to send data
|
|
||||||
* into.
|
|
||||||
*/
|
|
||||||
addFloatToInputSidePacket(data: number, sidePacketName: string): void {
|
addFloatToInputSidePacket(data: number, sidePacketName: string): void {
|
||||||
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
||||||
this.wasmModule._addFloatToInputSidePacket(data, sidePacketNamePtr);
|
this.wasmModule._addFloatToInputSidePacket(data, sidePacketNamePtr);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a integer packet to the specified input_side_packet.
|
|
||||||
* @param data The integer data to send.
|
|
||||||
* @param sidePacketName The name of the graph input side packet to send data
|
|
||||||
* into.
|
|
||||||
*/
|
|
||||||
addIntToInputSidePacket(data: number, sidePacketName: string): void {
|
addIntToInputSidePacket(data: number, sidePacketName: string): void {
|
||||||
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
||||||
this.wasmModule._addIntToInputSidePacket(data, sidePacketNamePtr);
|
this.wasmModule._addIntToInputSidePacket(data, sidePacketNamePtr);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a string packet to the specified input_side_packet.
|
|
||||||
* @param data The string data to send.
|
|
||||||
* @param sidePacketName The name of the graph input side packet to send data
|
|
||||||
* into.
|
|
||||||
*/
|
|
||||||
addStringToInputSidePacket(data: string, sidePacketName: string): void {
|
addStringToInputSidePacket(data: string, sidePacketName: string): void {
|
||||||
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
||||||
this.wrapStringPtr(data, (dataPtr: number) => {
|
this.wrapStringPtr(data, (dataPtr: number) => {
|
||||||
|
@ -846,14 +512,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a serialized proto packet to the specified input_side_packet.
|
|
||||||
* @param data The binary (serialized) raw protobuffer data.
|
|
||||||
* @param protoType The C++ namespaced type this protobuffer data corresponds
|
|
||||||
* to. It will be converted to this type for use in the graph.
|
|
||||||
* @param sidePacketName The name of the graph input side packet to send data
|
|
||||||
* into.
|
|
||||||
*/
|
|
||||||
addProtoToInputSidePacket(
|
addProtoToInputSidePacket(
|
||||||
data: Uint8Array, protoType: string, sidePacketName: string): void {
|
data: Uint8Array, protoType: string, sidePacketName: string): void {
|
||||||
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => {
|
||||||
|
@ -869,15 +528,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a boolean packet listener to the specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab boolean
|
|
||||||
* data from.
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that the data is only guaranteed to exist for the
|
|
||||||
* duration of the callback, and the callback will be called inline, so it
|
|
||||||
* should not perform overly complicated (or any async) behavior.
|
|
||||||
*/
|
|
||||||
attachBoolListener(
|
attachBoolListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<boolean>): void {
|
outputStreamName: string, callbackFcn: SimpleListener<boolean>): void {
|
||||||
// Set up our TS listener to receive any packets for this stream.
|
// Set up our TS listener to receive any packets for this stream.
|
||||||
|
@ -889,15 +540,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a bool[] packet listener to the specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab
|
|
||||||
* std::vector<bool> data from.
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that the data is only guaranteed to exist for the
|
|
||||||
* duration of the callback, and the callback will be called inline, so it
|
|
||||||
* should not perform overly complicated (or any async) behavior.
|
|
||||||
*/
|
|
||||||
attachBoolVectorListener(
|
attachBoolVectorListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<boolean[]>): void {
|
outputStreamName: string, callbackFcn: SimpleListener<boolean[]>): void {
|
||||||
// Set up our TS listener to receive any packets for this stream.
|
// Set up our TS listener to receive any packets for this stream.
|
||||||
|
@ -909,15 +552,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches an int packet listener to the specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab int
|
|
||||||
* data from.
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that the data is only guaranteed to exist for the
|
|
||||||
* duration of the callback, and the callback will be called inline, so it
|
|
||||||
* should not perform overly complicated (or any async) behavior.
|
|
||||||
*/
|
|
||||||
attachIntListener(
|
attachIntListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<number>): void {
|
outputStreamName: string, callbackFcn: SimpleListener<number>): void {
|
||||||
// Set up our TS listener to receive any packets for this stream.
|
// Set up our TS listener to receive any packets for this stream.
|
||||||
|
@ -929,15 +564,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches an int[] packet listener to the specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab
|
|
||||||
* std::vector<int> data from.
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that the data is only guaranteed to exist for the
|
|
||||||
* duration of the callback, and the callback will be called inline, so it
|
|
||||||
* should not perform overly complicated (or any async) behavior.
|
|
||||||
*/
|
|
||||||
attachIntVectorListener(
|
attachIntVectorListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void {
|
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void {
|
||||||
// Set up our TS listener to receive any packets for this stream.
|
// Set up our TS listener to receive any packets for this stream.
|
||||||
|
@ -949,15 +576,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a double packet listener to the specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab double
|
|
||||||
* data from.
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that the data is only guaranteed to exist for the
|
|
||||||
* duration of the callback, and the callback will be called inline, so it
|
|
||||||
* should not perform overly complicated (or any async) behavior.
|
|
||||||
*/
|
|
||||||
attachDoubleListener(
|
attachDoubleListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<number>): void {
|
outputStreamName: string, callbackFcn: SimpleListener<number>): void {
|
||||||
// Set up our TS listener to receive any packets for this stream.
|
// Set up our TS listener to receive any packets for this stream.
|
||||||
|
@ -969,15 +588,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a double[] packet listener to the specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab
|
|
||||||
* std::vector<double> data from.
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that the data is only guaranteed to exist for the
|
|
||||||
* duration of the callback, and the callback will be called inline, so it
|
|
||||||
* should not perform overly complicated (or any async) behavior.
|
|
||||||
*/
|
|
||||||
attachDoubleVectorListener(
|
attachDoubleVectorListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void {
|
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void {
|
||||||
// Set up our TS listener to receive any packets for this stream.
|
// Set up our TS listener to receive any packets for this stream.
|
||||||
|
@ -989,15 +600,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a float packet listener to the specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab float
|
|
||||||
* data from.
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that the data is only guaranteed to exist for the
|
|
||||||
* duration of the callback, and the callback will be called inline, so it
|
|
||||||
* should not perform overly complicated (or any async) behavior.
|
|
||||||
*/
|
|
||||||
attachFloatListener(
|
attachFloatListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<number>): void {
|
outputStreamName: string, callbackFcn: SimpleListener<number>): void {
|
||||||
// Set up our TS listener to receive any packets for this stream.
|
// Set up our TS listener to receive any packets for this stream.
|
||||||
|
@ -1009,15 +612,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a float[] packet listener to the specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab
|
|
||||||
* std::vector<float> data from.
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that the data is only guaranteed to exist for the
|
|
||||||
* duration of the callback, and the callback will be called inline, so it
|
|
||||||
* should not perform overly complicated (or any async) behavior.
|
|
||||||
*/
|
|
||||||
attachFloatVectorListener(
|
attachFloatVectorListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void {
|
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void {
|
||||||
// Set up our TS listener to receive any packets for this stream.
|
// Set up our TS listener to receive any packets for this stream.
|
||||||
|
@ -1029,15 +624,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a string packet listener to the specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab string
|
|
||||||
* data from.
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that the data is only guaranteed to exist for the
|
|
||||||
* duration of the callback, and the callback will be called inline, so it
|
|
||||||
* should not perform overly complicated (or any async) behavior.
|
|
||||||
*/
|
|
||||||
attachStringListener(
|
attachStringListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<string>): void {
|
outputStreamName: string, callbackFcn: SimpleListener<string>): void {
|
||||||
// Set up our TS listener to receive any packets for this stream.
|
// Set up our TS listener to receive any packets for this stream.
|
||||||
|
@ -1049,15 +636,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a string[] packet listener to the specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab
|
|
||||||
* std::vector<std::string> data from.
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that the data is only guaranteed to exist for the
|
|
||||||
* duration of the callback, and the callback will be called inline, so it
|
|
||||||
* should not perform overly complicated (or any async) behavior.
|
|
||||||
*/
|
|
||||||
attachStringVectorListener(
|
attachStringVectorListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<string[]>): void {
|
outputStreamName: string, callbackFcn: SimpleListener<string[]>): void {
|
||||||
// Set up our TS listener to receive any packets for this stream.
|
// Set up our TS listener to receive any packets for this stream.
|
||||||
|
@ -1069,25 +648,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a serialized proto packet listener to the specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab binary
|
|
||||||
* serialized proto data from (in Uint8Array format).
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that by default the data is only guaranteed to
|
|
||||||
* exist for the duration of the callback, and the callback will be called
|
|
||||||
* inline, so it should not perform overly complicated (or any async)
|
|
||||||
* behavior. If the proto data needs to be able to outlive the call, you
|
|
||||||
* may set the optional makeDeepCopy parameter to true, or can manually
|
|
||||||
* deep-copy the data yourself.
|
|
||||||
* @param makeDeepCopy Optional convenience parameter which, if set to true,
|
|
||||||
* will override the default memory management behavior and make a deep
|
|
||||||
* copy of the underlying data, rather than just returning a view into the
|
|
||||||
* C++-managed memory. At the cost of a data copy, this allows the
|
|
||||||
* returned data to outlive the callback lifetime (and it will be cleaned
|
|
||||||
* up automatically by JS garbage collection whenever the user is finished
|
|
||||||
* with it).
|
|
||||||
*/
|
|
||||||
attachProtoListener(
|
attachProtoListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<Uint8Array>,
|
outputStreamName: string, callbackFcn: SimpleListener<Uint8Array>,
|
||||||
makeDeepCopy?: boolean): void {
|
makeDeepCopy?: boolean): void {
|
||||||
|
@ -1102,26 +663,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches a listener for an array of serialized proto packets to the
|
|
||||||
* specified output_stream.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab a
|
|
||||||
* vector of binary serialized proto data from (in Uint8Array[] format).
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that by default the data is only guaranteed to
|
|
||||||
* exist for the duration of the callback, and the callback will be called
|
|
||||||
* inline, so it should not perform overly complicated (or any async)
|
|
||||||
* behavior. If the proto data needs to be able to outlive the call, you
|
|
||||||
* may set the optional makeDeepCopy parameter to true, or can manually
|
|
||||||
* deep-copy the data yourself.
|
|
||||||
* @param makeDeepCopy Optional convenience parameter which, if set to true,
|
|
||||||
* will override the default memory management behavior and make a deep
|
|
||||||
* copy of the underlying data, rather than just returning a view into the
|
|
||||||
* C++-managed memory. At the cost of a data copy, this allows the
|
|
||||||
* returned data to outlive the callback lifetime (and it will be cleaned
|
|
||||||
* up automatically by JS garbage collection whenever the user is finished
|
|
||||||
* with it).
|
|
||||||
*/
|
|
||||||
attachProtoVectorListener(
|
attachProtoVectorListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<Uint8Array[]>,
|
outputStreamName: string, callbackFcn: SimpleListener<Uint8Array[]>,
|
||||||
makeDeepCopy?: boolean): void {
|
makeDeepCopy?: boolean): void {
|
||||||
|
@ -1136,26 +678,7 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Attaches an audio packet listener to the specified output_stream, to be
|
|
||||||
* given a Float32Array as output.
|
|
||||||
* @param outputStreamName The name of the graph output stream to grab audio
|
|
||||||
* data from.
|
|
||||||
* @param callbackFcn The function that will be called back with the data, as
|
|
||||||
* it is received. Note that the data is only guaranteed to exist for the
|
|
||||||
* duration of the callback, and the callback will be called inline, so it
|
|
||||||
* should not perform overly complicated (or any async) behavior. If the
|
|
||||||
* audio data needs to be able to outlive the call, you may set the
|
|
||||||
* optional makeDeepCopy parameter to true, or can manually deep-copy the
|
|
||||||
* data yourself.
|
|
||||||
* @param makeDeepCopy Optional convenience parameter which, if set to true,
|
|
||||||
* will override the default memory management behavior and make a deep
|
|
||||||
* copy of the underlying data, rather than just returning a view into the
|
|
||||||
* C++-managed memory. At the cost of a data copy, this allows the
|
|
||||||
* returned data to outlive the callback lifetime (and it will be cleaned
|
|
||||||
* up automatically by JS garbage collection whenever the user is finished
|
|
||||||
* with it).
|
|
||||||
*/
|
|
||||||
attachAudioListener(
|
attachAudioListener(
|
||||||
outputStreamName: string, callbackFcn: SimpleListener<Float32Array>,
|
outputStreamName: string, callbackFcn: SimpleListener<Float32Array>,
|
||||||
makeDeepCopy?: boolean): void {
|
makeDeepCopy?: boolean): void {
|
||||||
|
@ -1181,19 +704,12 @@ export class GraphRunner {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Forces all queued-up packets to be pushed through the MediaPipe graph as
|
|
||||||
* far as possible, performing all processing until no more processing can be
|
|
||||||
* done.
|
|
||||||
*/
|
|
||||||
finishProcessing(): void {
|
finishProcessing(): void {
|
||||||
this.wasmModule._waitUntilIdle();
|
this.wasmModule._waitUntilIdle();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override GraphRunnerApi} */
|
||||||
* Closes the input streams and all calculators for this graph and frees up
|
|
||||||
* any C++ resources. The graph will not be usable once closed.
|
|
||||||
*/
|
|
||||||
closeGraph(): void {
|
closeGraph(): void {
|
||||||
this.wasmModule._closeGraph();
|
this.wasmModule._closeGraph();
|
||||||
this.wasmModule.simpleListeners = undefined;
|
this.wasmModule.simpleListeners = undefined;
|
||||||
|
@ -1221,38 +737,13 @@ async function runScript(scriptUrl: string|string) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override CreateMediaPipeLibApi} */
|
||||||
* Helper type macro for use with createMediaPipeLib. Allows us to easily infer
|
export const createMediaPipeLib: CreateMediaPipeLibApi = async<LibType>(
|
||||||
* the type of a mixin-extended GraphRunner. Example usage:
|
|
||||||
* const GraphRunnerConstructor =
|
|
||||||
* SupportImage(SupportSerialization(GraphRunner));
|
|
||||||
* let mediaPipe: ReturnType<typeof GraphRunnerConstructor>;
|
|
||||||
* ...
|
|
||||||
* mediaPipe = await createMediaPipeLib(GraphRunnerConstructor, ...);
|
|
||||||
*/
|
|
||||||
// tslint:disable-next-line:no-any
|
|
||||||
export type ReturnType<T> = T extends (...args: unknown[]) => infer R ? R : any;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Global function to initialize Wasm blob and load runtime assets for a
|
|
||||||
* specialized MediaPipe library. This allows us to create a requested
|
|
||||||
* subclass inheriting from GraphRunner.
|
|
||||||
* @param constructorFcn The name of the class to instantiate via "new".
|
|
||||||
* @param wasmLoaderScript Url for the wasm-runner script; produced by the build
|
|
||||||
* process.
|
|
||||||
* @param assetLoaderScript Url for the asset-loading script; produced by the
|
|
||||||
* build process.
|
|
||||||
* @param fileLocator A function to override the file locations for assets
|
|
||||||
* loaded by the MediaPipe library.
|
|
||||||
* @return promise A promise which will resolve when initialization has
|
|
||||||
* completed successfully.
|
|
||||||
*/
|
|
||||||
export async function createMediaPipeLib<LibType>(
|
|
||||||
constructorFcn: WasmMediaPipeConstructor<LibType>,
|
constructorFcn: WasmMediaPipeConstructor<LibType>,
|
||||||
wasmLoaderScript?: string|null,
|
wasmLoaderScript?: string|null,
|
||||||
assetLoaderScript?: string|null,
|
assetLoaderScript?: string|null,
|
||||||
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
|
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
|
||||||
fileLocator?: FileLocator): Promise<LibType> {
|
fileLocator?: FileLocator): Promise<LibType> => {
|
||||||
// Run wasm-loader script here
|
// Run wasm-loader script here
|
||||||
if (wasmLoaderScript) {
|
if (wasmLoaderScript) {
|
||||||
await runScript(wasmLoaderScript);
|
await runScript(wasmLoaderScript);
|
||||||
|
@ -1288,26 +779,24 @@ export async function createMediaPipeLib<LibType>(
|
||||||
// Don't reuse factory or module seed
|
// Don't reuse factory or module seed
|
||||||
self.ModuleFactory = self.Module = undefined;
|
self.ModuleFactory = self.Module = undefined;
|
||||||
return new constructorFcn(module, glCanvas);
|
return new constructorFcn(module, glCanvas);
|
||||||
|
};
|
||||||
|
|
||||||
|
// We extend the CreateGraphRunnerApi interface here for now so that by default
|
||||||
|
// callers of `createGraphRunner` will be given a `GraphRunner` rather than a
|
||||||
|
// `GraphRunnerApi`.
|
||||||
|
interface CreateGraphRunnerImplType extends CreateGraphRunnerApi {
|
||||||
|
(wasmLoaderScript?: string,
|
||||||
|
assetLoaderScript?: string,
|
||||||
|
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
|
||||||
|
fileLocator?: FileLocator): Promise<GraphRunner>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/** {@override CreateGraphRunnerApi} */
|
||||||
* Global function to initialize Wasm blob and load runtime assets for a generic
|
export const createGraphRunner: CreateGraphRunnerImplType = async(
|
||||||
* MediaPipe library.
|
|
||||||
* @param wasmLoaderScript Url for the wasm-runner script; produced by the build
|
|
||||||
* process.
|
|
||||||
* @param assetLoaderScript Url for the asset-loading script; produced by the
|
|
||||||
* build process.
|
|
||||||
* @param fileLocator A function to override the file locations for assets
|
|
||||||
* loaded by the MediaPipe library.
|
|
||||||
* @return promise A promise which will resolve when initialization has
|
|
||||||
* completed successfully.
|
|
||||||
*/
|
|
||||||
export async function createGraphRunner(
|
|
||||||
wasmLoaderScript?: string,
|
wasmLoaderScript?: string,
|
||||||
assetLoaderScript?: string,
|
assetLoaderScript?: string,
|
||||||
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
|
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
|
||||||
fileLocator?: FileLocator): Promise<GraphRunner> {
|
fileLocator?: FileLocator): Promise<GraphRunner> => {
|
||||||
return createMediaPipeLib(
|
return createMediaPipeLib(
|
||||||
GraphRunner, wasmLoaderScript, assetLoaderScript, glCanvas,
|
GraphRunner, wasmLoaderScript, assetLoaderScript, glCanvas, fileLocator);
|
||||||
fileLocator);
|
};
|
||||||
}
|
|
||||||
|
|
583
mediapipe/web/graph_runner/graph_runner_api.d.ts
vendored
Normal file
583
mediapipe/web/graph_runner/graph_runner_api.d.ts
vendored
Normal file
|
@ -0,0 +1,583 @@
|
||||||
|
import {EmptyPacketListener, ErrorListener, SimpleListener, VectorListener} from './listener_types';
|
||||||
|
import {WasmModule} from './wasm_module';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This file can serve as a common interface for most MediaPipe-based TypeScript
|
||||||
|
* libraries. Additionally, it can hook automatically into `wasm_mediapipe_demo`
|
||||||
|
* to allow for easy migrations from old pure JS demos. See `mediapipe_ts_demo`
|
||||||
|
* and `wasm_mediapipe_files` BUILD rules. Public API is described below in more
|
||||||
|
* detail, public API for GraphRunner factory functions is described in
|
||||||
|
* "graph_runner_factory_api.d.ts", and the same for listener callbacks
|
||||||
|
* can be found in "listenertypes.d.ts". An actual implementation is coded in
|
||||||
|
* `graph_runner.ts`, and that can be built against directly. The purpose of
|
||||||
|
* this file is primarily to enforce a common interface and provide better
|
||||||
|
* documentation on the design of the API as a whole.
|
||||||
|
*/
|
||||||
|
|
||||||
|
// We re-export all of our imported public listener types, so that users can
|
||||||
|
// import everything they need for the public interfaces directly from here.
|
||||||
|
export {
|
||||||
|
EmptyPacketListener,
|
||||||
|
ErrorListener,
|
||||||
|
SimpleListener,
|
||||||
|
VectorListener,
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Valid types of image sources which we can run our GraphRunner over.
|
||||||
|
*/
|
||||||
|
export type ImageSource =
|
||||||
|
HTMLCanvasElement|HTMLVideoElement|HTMLImageElement|ImageData|ImageBitmap;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Simple interface for a class to run an arbitrary MediaPipe graph on web, and
|
||||||
|
* either render results into canvas, or else stream output into attached
|
||||||
|
* listeners. Takes a WebAssembly Module and an optional canvas for rendering.
|
||||||
|
* Standard implementation is `GraphRunner`.
|
||||||
|
* There are three categories of functions:
|
||||||
|
* - Those which add inputs into the graph (named `add*ToStream` or
|
||||||
|
* `add*ToInputSidePacket`).
|
||||||
|
* - Those which listen for outputs from the graph (named `attach*Listener`).
|
||||||
|
* - Those concerned with the graph running itself (runner settings, loading
|
||||||
|
* the graph, destruction, and `finishProcessing`).
|
||||||
|
* The expected ordering of one-time initialization calls should be:
|
||||||
|
* 1. Construction
|
||||||
|
* 2. Adding any input side packets and attaching any output listeners
|
||||||
|
* 3. Initializing/setting the graph
|
||||||
|
* After this, for the main loop, the user would add packets to streams for a
|
||||||
|
* given frame and then call `finishProcessing()`.
|
||||||
|
* Example usage pattern would be:
|
||||||
|
* ```
|
||||||
|
* // We assume we have already constructed a GraphRunner called graphRunner.
|
||||||
|
* // Generally we would do this via a helpful factory method like
|
||||||
|
* // `createGraphRunner` or some variant thereof.
|
||||||
|
* const graphRunner: GraphRunner;
|
||||||
|
*
|
||||||
|
* // Initialization code:
|
||||||
|
* graphRunner.addBoolToInputSidePacket(true, 'some_input_bool_side_packet');
|
||||||
|
* graphRunner.attachStringVectorListener('some_output_string_vec_stream',
|
||||||
|
* (data: string[], timestamp: number) => { console.log(data); });
|
||||||
|
* await graphRunner.initializeGraph('path/to/graph_file.pbtxt');
|
||||||
|
*
|
||||||
|
* // Main loop code (run every frame):
|
||||||
|
* const frameTimestamp = performance.now();
|
||||||
|
* graphRunner.addStringToStream(
|
||||||
|
* 'Hello World', 'some_input_string_stream', frameTimestamp);
|
||||||
|
* graphRunner.addFloatToStream(
|
||||||
|
* 3.7, 'some_input_float_stream', frameTimestamp);
|
||||||
|
* graphRunner.finishProcessing();
|
||||||
|
*
|
||||||
|
* // When done (no need to call if the user is just expected to close the
|
||||||
|
* // browser tab):
|
||||||
|
* graphRunner.closeGraph();
|
||||||
|
* ```
|
||||||
|
*/
|
||||||
|
export interface GraphRunnerApi {
|
||||||
|
/**
|
||||||
|
* Fetches a MediaPipe graph from a URL string pointing to the graph file.
|
||||||
|
* Will then set the graph using the results, replacing the previously running
|
||||||
|
* MediaPipe graph, if there is one. If the graph file's name ends with
|
||||||
|
* ".pbtxt" with ".textproto", it will assume text-formatted, and otherwise
|
||||||
|
* will assume a binary format for the proto.
|
||||||
|
* @param graphFile The url of the MediaPipe graph file to load.
|
||||||
|
*/
|
||||||
|
initializeGraph(graphFile: string): Promise<void>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convenience helper for loading a MediaPipe graph from a string representing
|
||||||
|
* a text proto config. Useful for graph files which are expected to be edited
|
||||||
|
* locally, web-side. Will replace the previously running MediaPipe graph,
|
||||||
|
* if there is one.
|
||||||
|
* @param graphConfig The text proto graph config, expected to be a string in
|
||||||
|
* default JavaScript UTF-16 format.
|
||||||
|
*/
|
||||||
|
setGraphFromString(graphConfig: string): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Takes the raw data from a MediaPipe graph, and passes it to C++ to be run
|
||||||
|
* over the video stream. Will replace the previously running MediaPipe graph,
|
||||||
|
* if there is one.
|
||||||
|
* @param graphData The raw MediaPipe graph data, either in binary
|
||||||
|
* protobuffer format (.binarypb), or else in raw text format (.pbtxt or
|
||||||
|
* .textproto).
|
||||||
|
* @param isBinary This should be set to true if the graph is in
|
||||||
|
* binary format, and false if it is in human-readable text format.
|
||||||
|
*/
|
||||||
|
setGraph(graphData: Uint8Array, isBinary: boolean): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Configures the current graph to handle audio processing in a certain way
|
||||||
|
* for all its audio input streams. Additionally can configure audio headers
|
||||||
|
* (both input side packets as well as input stream headers), but these
|
||||||
|
* configurations only take effect if called before the graph is set/started.
|
||||||
|
* @param numChannels The number of channels of audio input. Only 1
|
||||||
|
* is supported for now.
|
||||||
|
* @param numSamples The number of samples that are taken in each
|
||||||
|
* audio capture.
|
||||||
|
* @param sampleRate The rate, in Hz, of the sampling.
|
||||||
|
* @param streamName The optional name of the input stream to additionally
|
||||||
|
* configure with audio information. This configuration only occurs before
|
||||||
|
* the graph is set/started. If unset, a default stream name will be used.
|
||||||
|
* @param headerName The optional name of the header input side packet to
|
||||||
|
* additionally configure with audio information. This configuration only
|
||||||
|
* occurs before the graph is set/started. If unset, a default header name
|
||||||
|
* will be used.
|
||||||
|
*/
|
||||||
|
configureAudio(
|
||||||
|
numChannels: number, numSamples: number, sampleRate: number,
|
||||||
|
streamName?: string, headerName?: string): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Allows disabling automatic canvas resizing, in case clients want to control
|
||||||
|
* control this. By default, the canvas will be resized to the size of the
|
||||||
|
* last GPU image input (see `addGpuBufferToStream`).
|
||||||
|
* @param resize True will re-enable automatic canvas resizing, while false
|
||||||
|
* will disable the feature.
|
||||||
|
*/
|
||||||
|
setAutoResizeCanvas(resize: boolean): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Allows disabling the automatic render-to-screen code, in case clients don't
|
||||||
|
* need/want this. In particular, this removes the requirement for pipelines
|
||||||
|
* to have access to GPU resources, as well as the requirement for graphs to
|
||||||
|
* have an "output_frames_gpu" stream defined, so pure CPU pipelines and
|
||||||
|
* non-video pipelines can be created.
|
||||||
|
* NOTE: This only affects future graph initializations (via `setGraph`` or
|
||||||
|
* `initializeGraph``), and does NOT affect the currently running graph,
|
||||||
|
* so calls to this should be made *before* `setGraph`/`initializeGraph`
|
||||||
|
* for the graph file being targeted.
|
||||||
|
* @param enabled True will re-enable automatic render-to-screen code and
|
||||||
|
* cause GPU resources to once again be requested, while false will
|
||||||
|
* disable the feature.
|
||||||
|
*/
|
||||||
|
setAutoRenderToScreen(enabled: boolean): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Overrides the vertical orientation for input GpuBuffers and the automatic
|
||||||
|
* render-to-screen code. The default for our OpenGL code on other platforms
|
||||||
|
* (Android, Linux) is to use a bottom-left origin. But the default for WebGL
|
||||||
|
* is to use a top-left origin. We use WebGL default normally, and many
|
||||||
|
* calculators and graphs have platform-specific code to handle the resulting
|
||||||
|
* orientation flip. However, in order to be able to use a single graph on all
|
||||||
|
* platforms without alterations, it may be useful to send images into a web
|
||||||
|
* graph using the OpenGL orientation. Users can call this function with
|
||||||
|
* `bottomLeftIsOrigin = true` in order to enforce an orientation for all
|
||||||
|
* GpuBuffer inputs which is consistent with OpenGL on other platforms.
|
||||||
|
* This call will also vertically flip the automatic render-to-screen code as
|
||||||
|
* well, so that webcam input (for example) will render properly when passed
|
||||||
|
* through the graph still.
|
||||||
|
* NOTE: This will immediately affect GpuBuffer inputs, but must be called
|
||||||
|
* *before* graph start in order to affect the automatic render-to-screen
|
||||||
|
* code!
|
||||||
|
* @param bottomLeftIsOrigin True will flip our input GpuBuffers and auto
|
||||||
|
* render-to-screen to match the classic OpenGL orientation, while false will
|
||||||
|
* disable this feature to match the default WebGL orientation.
|
||||||
|
*/
|
||||||
|
setGpuBufferVerticalFlip(bottomLeftIsOrigin: boolean): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a listener that will be invoked when the MediaPipe web framework
|
||||||
|
* returns an error.
|
||||||
|
*/
|
||||||
|
attachErrorListener(callbackFcn: ErrorListener): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a listener that will be invoked when the MediaPipe framework
|
||||||
|
* receives an empty packet on the provided output stream. This can be used
|
||||||
|
* to receive the latest output timestamp.
|
||||||
|
*
|
||||||
|
* Empty packet listeners are only active if there is a corresponding packet
|
||||||
|
* listener.
|
||||||
|
*
|
||||||
|
* @param outputStreamName The name of the graph output stream to receive
|
||||||
|
* empty packets from.
|
||||||
|
* @param callbackFcn The callback to receive the timestamp.
|
||||||
|
*/
|
||||||
|
attachEmptyPacketListener(
|
||||||
|
outputStreamName: string, callbackFcn: EmptyPacketListener): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Takes the raw data from a JS audio capture array, and sends it to C++ to be
|
||||||
|
* processed.
|
||||||
|
* @param audioData An array of raw audio capture data, like
|
||||||
|
* from a call to getChannelData on an AudioBuffer.
|
||||||
|
* @param streamName The name of the MediaPipe graph stream to add the audio
|
||||||
|
* data to.
|
||||||
|
* @param timestamp The timestamp of the current frame, in ms.
|
||||||
|
*/
|
||||||
|
addAudioToStream(
|
||||||
|
audioData: Float32Array, streamName: string, timestamp: number): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Takes the raw data from a JS audio capture array, and sends it to C++ to be
|
||||||
|
* processed, shaping the audioData array into an audio matrix according to
|
||||||
|
* the numChannels and numSamples parameters.
|
||||||
|
* @param audioData An array of raw audio capture data, like
|
||||||
|
* from a call to getChannelData on an AudioBuffer.
|
||||||
|
* @param numChannels The number of audio channels this data represents. If 0
|
||||||
|
* is passed, then the value will be taken from the last call to
|
||||||
|
* configureAudio.
|
||||||
|
* @param numSamples The number of audio samples captured in this data packet.
|
||||||
|
* If 0 is passed, then the value will be taken from the last call to
|
||||||
|
* configureAudio.
|
||||||
|
* @param streamName The name of the MediaPipe graph stream to add the audio
|
||||||
|
* data to.
|
||||||
|
* @param timestamp The timestamp of the current frame, in ms.
|
||||||
|
*/
|
||||||
|
addAudioToStreamWithShape(
|
||||||
|
audioData: Float32Array, numChannels: number, numSamples: number,
|
||||||
|
streamName: string, timestamp: number): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Takes the relevant information from the HTML video or image element, and
|
||||||
|
* passes it into the WebGL-based graph for processing on the given stream at
|
||||||
|
* the given timestamp. Can be used for additional auxiliary GpuBuffer input
|
||||||
|
* streams. Like all `add*ToStream` calls, processing will not occur until a
|
||||||
|
* blocking call like `finishProcessing` or the deprecated `processGl` is
|
||||||
|
* made. For use with 'gl_graph_runner_internal_multi_input'.
|
||||||
|
* @param imageSource Reference to the video frame we wish to add into our
|
||||||
|
* graph.
|
||||||
|
* @param streamName The name of the MediaPipe graph stream to add the frame
|
||||||
|
* to.
|
||||||
|
* @param timestamp The timestamp of the input frame, in ms.
|
||||||
|
*/
|
||||||
|
addGpuBufferToStream(
|
||||||
|
imageSource: ImageSource, streamName: string, timestamp: number): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sends a boolean packet into the specified stream at the given timestamp.
|
||||||
|
* @param data The boolean data to send.
|
||||||
|
* @param streamName The name of the graph input stream to send data into.
|
||||||
|
* @param timestamp The timestamp of the input data, in ms.
|
||||||
|
*/
|
||||||
|
addBoolToStream(data: boolean, streamName: string, timestamp: number): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sends a double packet into the specified stream at the given timestamp.
|
||||||
|
* @param data The double data to send.
|
||||||
|
* @param streamName The name of the graph input stream to send data into.
|
||||||
|
* @param timestamp The timestamp of the input data, in ms.
|
||||||
|
*/
|
||||||
|
addDoubleToStream(data: number, streamName: string, timestamp: number): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sends a float packet into the specified stream at the given timestamp.
|
||||||
|
* @param data The float data to send.
|
||||||
|
* @param streamName The name of the graph input stream to send data into.
|
||||||
|
* @param timestamp The timestamp of the input data, in ms.
|
||||||
|
*/
|
||||||
|
addFloatToStream(data: number, streamName: string, timestamp: number): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sends an integer packet into the specified stream at the given timestamp.
|
||||||
|
* @param data The integer data to send.
|
||||||
|
* @param streamName The name of the graph input stream to send data into.
|
||||||
|
* @param timestamp The timestamp of the input data, in ms.
|
||||||
|
*/
|
||||||
|
addIntToStream(data: number, streamName: string, timestamp: number): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sends a string packet into the specified stream at the given timestamp.
|
||||||
|
* @param data The string data to send.
|
||||||
|
* @param streamName The name of the graph input stream to send data into.
|
||||||
|
* @param timestamp The timestamp of the input data, in ms.
|
||||||
|
*/
|
||||||
|
addStringToStream(data: string, streamName: string, timestamp: number): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sends a Record<string, string> packet into the specified stream at the
|
||||||
|
* given timestamp.
|
||||||
|
* @param data The records to send (will become a
|
||||||
|
* std::flat_hash_map<std::string, std::string).
|
||||||
|
* @param streamName The name of the graph input stream to send data into.
|
||||||
|
* @param timestamp The timestamp of the input data, in ms.
|
||||||
|
*/
|
||||||
|
addStringRecordToStream(
|
||||||
|
data: Record<string, string>, streamName: string,
|
||||||
|
timestamp: number): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sends a serialized protobuffer packet into the specified stream at the
|
||||||
|
* given timestamp, to be parsed into the specified protobuffer type.
|
||||||
|
* @param data The binary (serialized) raw protobuffer data.
|
||||||
|
* @param protoType The C++ namespaced type this protobuffer data corresponds
|
||||||
|
* to (e.g. "foo.Bar"). It will be converted to this type when output as a
|
||||||
|
* packet into the graph.
|
||||||
|
* @param streamName The name of the graph input stream to send data into.
|
||||||
|
* @param timestamp The timestamp of the input data, in ms.
|
||||||
|
*/
|
||||||
|
addProtoToStream(
|
||||||
|
data: Uint8Array, protoType: string, streamName: string,
|
||||||
|
timestamp: number): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sends an empty packet into the specified stream at the given timestamp,
|
||||||
|
* effectively advancing that input stream's timestamp bounds without
|
||||||
|
* sending additional data packets.
|
||||||
|
* @param streamName The name of the graph input stream to send the empty
|
||||||
|
* packet into.
|
||||||
|
* @param timestamp The timestamp of the empty packet, in ms.
|
||||||
|
*/
|
||||||
|
addEmptyPacketToStream(streamName: string, timestamp: number): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a boolean packet to the specified input_side_packet.
|
||||||
|
* @param data The boolean data to send.
|
||||||
|
* @param sidePacketName The name of the graph input side packet to send data
|
||||||
|
* into.
|
||||||
|
*/
|
||||||
|
addBoolToInputSidePacket(data: boolean, sidePacketName: string): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a double packet to the specified input_side_packet.
|
||||||
|
* @param data The double data to send.
|
||||||
|
* @param sidePacketName The name of the graph input side packet to send data
|
||||||
|
* into.
|
||||||
|
*/
|
||||||
|
addDoubleToInputSidePacket(data: number, sidePacketName: string): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a float packet to the specified input_side_packet.
|
||||||
|
* @param data The float data to send.
|
||||||
|
* @param sidePacketName The name of the graph input side packet to send data
|
||||||
|
* into.
|
||||||
|
*/
|
||||||
|
addFloatToInputSidePacket(data: number, sidePacketName: string): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a integer packet to the specified input_side_packet.
|
||||||
|
* @param data The integer data to send.
|
||||||
|
* @param sidePacketName The name of the graph input side packet to send data
|
||||||
|
* into.
|
||||||
|
*/
|
||||||
|
addIntToInputSidePacket(data: number, sidePacketName: string): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a string packet to the specified input_side_packet.
|
||||||
|
* @param data The string data to send.
|
||||||
|
* @param sidePacketName The name of the graph input side packet to send data
|
||||||
|
* into.
|
||||||
|
*/
|
||||||
|
addStringToInputSidePacket(data: string, sidePacketName: string): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a serialized proto packet to the specified input_side_packet.
|
||||||
|
* @param data The binary (serialized) raw protobuffer data.
|
||||||
|
* @param protoType The C++ namespaced type this protobuffer data corresponds
|
||||||
|
* to. It will be converted to this type for use in the graph.
|
||||||
|
* @param sidePacketName The name of the graph input side packet to send data
|
||||||
|
* into.
|
||||||
|
*/
|
||||||
|
addProtoToInputSidePacket(
|
||||||
|
data: Uint8Array, protoType: string, sidePacketName: string): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a boolean packet listener to the specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab boolean
|
||||||
|
* data from.
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that the data is only guaranteed to exist for the
|
||||||
|
* duration of the callback, and the callback will be called inline, so it
|
||||||
|
* should not perform overly complicated (or any async) behavior.
|
||||||
|
*/
|
||||||
|
attachBoolListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<boolean>): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a bool[] packet listener to the specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab
|
||||||
|
* std::vector<bool> data from.
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that the data is only guaranteed to exist for the
|
||||||
|
* duration of the callback, and the callback will be called inline, so it
|
||||||
|
* should not perform overly complicated (or any async) behavior.
|
||||||
|
*/
|
||||||
|
attachBoolVectorListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<boolean[]>): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches an int packet listener to the specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab int
|
||||||
|
* data from.
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that the data is only guaranteed to exist for the
|
||||||
|
* duration of the callback, and the callback will be called inline, so it
|
||||||
|
* should not perform overly complicated (or any async) behavior.
|
||||||
|
*/
|
||||||
|
attachIntListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<number>): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches an int[] packet listener to the specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab
|
||||||
|
* std::vector<int> data from.
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that the data is only guaranteed to exist for the
|
||||||
|
* duration of the callback, and the callback will be called inline, so it
|
||||||
|
* should not perform overly complicated (or any async) behavior.
|
||||||
|
*/
|
||||||
|
attachIntVectorListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a double packet listener to the specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab double
|
||||||
|
* data from.
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that the data is only guaranteed to exist for the
|
||||||
|
* duration of the callback, and the callback will be called inline, so it
|
||||||
|
* should not perform overly complicated (or any async) behavior.
|
||||||
|
*/
|
||||||
|
attachDoubleListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<number>): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a double[] packet listener to the specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab
|
||||||
|
* std::vector<double> data from.
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that the data is only guaranteed to exist for the
|
||||||
|
* duration of the callback, and the callback will be called inline, so it
|
||||||
|
* should not perform overly complicated (or any async) behavior.
|
||||||
|
*/
|
||||||
|
attachDoubleVectorListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a float packet listener to the specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab float
|
||||||
|
* data from.
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that the data is only guaranteed to exist for the
|
||||||
|
* duration of the callback, and the callback will be called inline, so it
|
||||||
|
* should not perform overly complicated (or any async) behavior.
|
||||||
|
*/
|
||||||
|
attachFloatListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<number>): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a float[] packet listener to the specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab
|
||||||
|
* std::vector<float> data from.
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that the data is only guaranteed to exist for the
|
||||||
|
* duration of the callback, and the callback will be called inline, so it
|
||||||
|
* should not perform overly complicated (or any async) behavior.
|
||||||
|
*/
|
||||||
|
attachFloatVectorListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<number[]>): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a string packet listener to the specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab string
|
||||||
|
* data from.
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that the data is only guaranteed to exist for the
|
||||||
|
* duration of the callback, and the callback will be called inline, so it
|
||||||
|
* should not perform overly complicated (or any async) behavior.
|
||||||
|
*/
|
||||||
|
attachStringListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<string>): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a string[] packet listener to the specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab
|
||||||
|
* std::vector<std::string> data from.
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that the data is only guaranteed to exist for the
|
||||||
|
* duration of the callback, and the callback will be called inline, so it
|
||||||
|
* should not perform overly complicated (or any async) behavior.
|
||||||
|
*/
|
||||||
|
attachStringVectorListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<string[]>): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a serialized proto packet listener to the specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab binary
|
||||||
|
* serialized proto data from (in Uint8Array format).
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that by default the data is only guaranteed to
|
||||||
|
* exist for the duration of the callback, and the callback will be called
|
||||||
|
* inline, so it should not perform overly complicated (or any async)
|
||||||
|
* behavior. If the proto data needs to be able to outlive the call, you
|
||||||
|
* may set the optional makeDeepCopy parameter to true, or can manually
|
||||||
|
* deep-copy the data yourself.
|
||||||
|
* @param makeDeepCopy Optional convenience parameter which, if set to true,
|
||||||
|
* will override the default memory management behavior and make a deep
|
||||||
|
* copy of the underlying data, rather than just returning a view into the
|
||||||
|
* C++-managed memory. At the cost of a data copy, this allows the
|
||||||
|
* returned data to outlive the callback lifetime (and it will be cleaned
|
||||||
|
* up automatically by JS garbage collection whenever the user is finished
|
||||||
|
* with it).
|
||||||
|
*/
|
||||||
|
attachProtoListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<Uint8Array>,
|
||||||
|
makeDeepCopy?: boolean): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches a listener for an array of serialized proto packets to the
|
||||||
|
* specified output_stream.
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab a
|
||||||
|
* vector of binary serialized proto data from (in Uint8Array[] format).
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that by default the data is only guaranteed to
|
||||||
|
* exist for the duration of the callback, and the callback will be called
|
||||||
|
* inline, so it should not perform overly complicated (or any async)
|
||||||
|
* behavior. If the proto data needs to be able to outlive the call, you
|
||||||
|
* may set the optional makeDeepCopy parameter to true, or can manually
|
||||||
|
* deep-copy the data yourself.
|
||||||
|
* @param makeDeepCopy Optional convenience parameter which, if set to true,
|
||||||
|
* will override the default memory management behavior and make a deep
|
||||||
|
* copy of the underlying data, rather than just returning a view into the
|
||||||
|
* C++-managed memory. At the cost of a data copy, this allows the
|
||||||
|
* returned data to outlive the callback lifetime (and it will be cleaned
|
||||||
|
* up automatically by JS garbage collection whenever the user is finished
|
||||||
|
* with it).
|
||||||
|
*/
|
||||||
|
attachProtoVectorListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<Uint8Array[]>,
|
||||||
|
makeDeepCopy?: boolean): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Attaches an audio packet listener to the specified output_stream, to be
|
||||||
|
* given a Float32Array as output. Requires wasm build dependency
|
||||||
|
* "gl_graph_runner_audio_out".
|
||||||
|
* @param outputStreamName The name of the graph output stream to grab audio
|
||||||
|
* data from.
|
||||||
|
* @param callbackFcn The function that will be called back with the data, as
|
||||||
|
* it is received. Note that the data is only guaranteed to exist for the
|
||||||
|
* duration of the callback, and the callback will be called inline, so it
|
||||||
|
* should not perform overly complicated (or any async) behavior. If the
|
||||||
|
* audio data needs to be able to outlive the call, you may set the
|
||||||
|
* optional makeDeepCopy parameter to true, or can manually deep-copy the
|
||||||
|
* data yourself.
|
||||||
|
* @param makeDeepCopy Optional convenience parameter which, if set to true,
|
||||||
|
* will override the default memory management behavior and make a deep
|
||||||
|
* copy of the underlying data, rather than just returning a view into the
|
||||||
|
* C++-managed memory. At the cost of a data copy, this allows the
|
||||||
|
* returned data to outlive the callback lifetime (and it will be cleaned
|
||||||
|
* up automatically by JS garbage collection whenever the user is finished
|
||||||
|
* with it).
|
||||||
|
*/
|
||||||
|
attachAudioListener(
|
||||||
|
outputStreamName: string, callbackFcn: SimpleListener<Float32Array>,
|
||||||
|
makeDeepCopy?: boolean): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Forces all queued-up packets to be pushed through the MediaPipe graph as
|
||||||
|
* far as possible, performing all processing until no more processing can be
|
||||||
|
* done. This is fully synchronous and by default single-threaded, so the
|
||||||
|
* calling thread will be blocked until processing completes. This must be
|
||||||
|
* called once for every frame, as `add*ToStream` calls merely queue up
|
||||||
|
* packets to the appropriate streams for processing, but processing does not
|
||||||
|
* occur until this function is called. Any listeners receiving output will
|
||||||
|
* be called back before this operation completes.
|
||||||
|
*/
|
||||||
|
finishProcessing(): void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Closes the input streams and all calculators for this graph and frees up
|
||||||
|
* any C++ resources. The graph will not be usable once closed!
|
||||||
|
*/
|
||||||
|
closeGraph(): void;
|
||||||
|
}
|
79
mediapipe/web/graph_runner/graph_runner_factory_api.d.ts
vendored
Normal file
79
mediapipe/web/graph_runner/graph_runner_factory_api.d.ts
vendored
Normal file
|
@ -0,0 +1,79 @@
|
||||||
|
// Placeholder for internal dependency on trusted resource url
|
||||||
|
|
||||||
|
import {GraphRunnerApi} from './graph_runner_api';
|
||||||
|
import {WasmModule} from './wasm_module';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Simple interface for allowing users to set the directory where internal
|
||||||
|
* wasm-loading and asset-loading code looks (e.g. for .wasm and .data file
|
||||||
|
* locations).
|
||||||
|
*/
|
||||||
|
export declare interface FileLocator {
|
||||||
|
locateFile: (filename: string) => string;
|
||||||
|
mainScriptUrlOrBlob?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper type macro for use with createMediaPipeLib. Allows us to easily infer
|
||||||
|
* the type of a mixin-extended GraphRunner. Example usage:
|
||||||
|
* const GraphRunnerConstructor =
|
||||||
|
* SupportImage(SupportSerialization(GraphRunner));
|
||||||
|
* let mediaPipe: ReturnType<typeof GraphRunnerConstructor>;
|
||||||
|
* ...
|
||||||
|
* mediaPipe = await createMediaPipeLib(GraphRunnerConstructor, ...);
|
||||||
|
*/
|
||||||
|
// tslint:disable-next-line:no-any
|
||||||
|
export type ReturnType<T> = T extends(...args: unknown[]) => infer R ? R : any;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Internal type of constructors used for initializing GraphRunner and
|
||||||
|
* subclasses.
|
||||||
|
*/
|
||||||
|
export type WasmMediaPipeConstructor<LibType> =
|
||||||
|
(new (
|
||||||
|
module: WasmModule, canvas?: HTMLCanvasElement|OffscreenCanvas|null) =>
|
||||||
|
LibType);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Global function interface to initialize Wasm blob and load runtime assets for
|
||||||
|
* a specialized MediaPipe library. This allows us to create a requested
|
||||||
|
* subclass inheriting from GraphRunner. Standard implementation is
|
||||||
|
* `createMediaPipeLib<LibType>`.
|
||||||
|
* @param constructorFcn The name of the class to instantiate via "new".
|
||||||
|
* @param wasmLoaderScript Url for the wasm-runner script; produced by the build
|
||||||
|
* process.
|
||||||
|
* @param assetLoaderScript Url for the asset-loading script; produced by the
|
||||||
|
* build process.
|
||||||
|
* @param fileLocator A function to override the file locations for assets
|
||||||
|
* loaded by the MediaPipe library.
|
||||||
|
* @return promise A promise which will resolve when initialization has
|
||||||
|
* completed successfully.
|
||||||
|
*/
|
||||||
|
export interface CreateMediaPipeLibApi {
|
||||||
|
<LibType>(
|
||||||
|
constructorFcn: WasmMediaPipeConstructor<LibType>,
|
||||||
|
wasmLoaderScript?: string|null,
|
||||||
|
assetLoaderScript?: string|null,
|
||||||
|
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
|
||||||
|
fileLocator?: FileLocator): Promise<LibType>;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Global function interface to initialize Wasm blob and load runtime assets for
|
||||||
|
* a generic MediaPipe library. Standard implementation is
|
||||||
|
* `createGraphRunner`.
|
||||||
|
* @param wasmLoaderScript Url for the wasm-runner script; produced by the build
|
||||||
|
* process.
|
||||||
|
* @param assetLoaderScript Url for the asset-loading script; produced by the
|
||||||
|
* build process.
|
||||||
|
* @param fileLocator A function to override the file locations for assets
|
||||||
|
* loaded by the MediaPipe library.
|
||||||
|
* @return promise A promise which will resolve when initialization has
|
||||||
|
* completed successfully.
|
||||||
|
*/
|
||||||
|
export interface CreateGraphRunnerApi {
|
||||||
|
(wasmLoaderScript?: string,
|
||||||
|
assetLoaderScript?: string,
|
||||||
|
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null,
|
||||||
|
fileLocator?: FileLocator): Promise<GraphRunnerApi>;
|
||||||
|
}
|
23
mediapipe/web/graph_runner/listener_types.d.ts
vendored
Normal file
23
mediapipe/web/graph_runner/listener_types.d.ts
vendored
Normal file
|
@ -0,0 +1,23 @@
|
||||||
|
/**
|
||||||
|
* A listener that receives the contents of a non-empty MediaPipe packet and
|
||||||
|
* its timestamp.
|
||||||
|
*/
|
||||||
|
export type SimpleListener<T> = (data: T, timestamp: number) => void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A listener that receives the current MediaPipe packet timestamp. This is
|
||||||
|
* invoked even for empty packet.
|
||||||
|
*/
|
||||||
|
export type EmptyPacketListener = (timestamp: number) => void;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A listener that receives a single element of vector-returning output packet.
|
||||||
|
* Receives one element at a time (in order). Once all elements are processed,
|
||||||
|
* the listener is invoked with `data` set to `unknown` and `done` set to true.
|
||||||
|
* Intended for internal usage.
|
||||||
|
*/
|
||||||
|
export type VectorListener<T> = (data: T, done: boolean, timestamp: number) =>
|
||||||
|
void;
|
||||||
|
|
||||||
|
/** A listener that will be invoked with an absl::StatusCode and message. */
|
||||||
|
export type ErrorListener = (code: number, message: string) => void;
|
111
mediapipe/web/graph_runner/wasm_module.d.ts
vendored
Normal file
111
mediapipe/web/graph_runner/wasm_module.d.ts
vendored
Normal file
|
@ -0,0 +1,111 @@
|
||||||
|
import {EmptyPacketListener, ErrorListener, SimpleListener, VectorListener} from './listener_types';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Declarations for Emscripten's WebAssembly Module behavior, so TS compiler
|
||||||
|
* doesn't break our various JS/C++ bridges. For internal usage.
|
||||||
|
*/
|
||||||
|
export declare interface WasmModule {
|
||||||
|
canvas: HTMLCanvasElement|OffscreenCanvas|null;
|
||||||
|
HEAPU8: Uint8Array;
|
||||||
|
HEAPU32: Uint32Array;
|
||||||
|
HEAPF32: Float32Array;
|
||||||
|
HEAPF64: Float64Array;
|
||||||
|
FS_createDataFile:
|
||||||
|
(parent: string, name: string, data: Uint8Array, canRead: boolean,
|
||||||
|
canWrite: boolean, canOwn: boolean) => void;
|
||||||
|
FS_createPath:
|
||||||
|
(parent: string, name: string, canRead: boolean,
|
||||||
|
canWrite: boolean) => void;
|
||||||
|
FS_unlink(path: string): void;
|
||||||
|
gpuOriginForWebTexturesIsBottomLeft?: boolean;
|
||||||
|
|
||||||
|
errorListener?: ErrorListener;
|
||||||
|
_bindTextureToCanvas: () => boolean;
|
||||||
|
_changeBinaryGraph: (size: number, dataPtr: number) => void;
|
||||||
|
_changeTextGraph: (size: number, dataPtr: number) => void;
|
||||||
|
_closeGraph: () => void;
|
||||||
|
_free: (ptr: number) => void;
|
||||||
|
_malloc: (size: number) => number;
|
||||||
|
_processFrame: (width: number, height: number, timestamp: number) => void;
|
||||||
|
_setAutoRenderToScreen: (enabled: boolean) => void;
|
||||||
|
_waitUntilIdle: () => void;
|
||||||
|
|
||||||
|
// Exposed so that clients of this lib can access this field
|
||||||
|
dataFileDownloads?: {[url: string]: {loaded: number, total: number}};
|
||||||
|
|
||||||
|
// Wasm Module multistream entrypoints. Require
|
||||||
|
// gl_graph_runner_internal_multi_input as a build dependency.
|
||||||
|
stringToNewUTF8: (data: string) => number;
|
||||||
|
_bindTextureToStream: (streamNamePtr: number) => void;
|
||||||
|
_addBoundTextureToStream:
|
||||||
|
(streamNamePtr: number, width: number, height: number,
|
||||||
|
timestamp: number) => void;
|
||||||
|
_addBoolToInputStream:
|
||||||
|
(data: boolean, streamNamePtr: number, timestamp: number) => void;
|
||||||
|
_addDoubleToInputStream:
|
||||||
|
(data: number, streamNamePtr: number, timestamp: number) => void;
|
||||||
|
_addFloatToInputStream:
|
||||||
|
(data: number, streamNamePtr: number, timestamp: number) => void;
|
||||||
|
_addIntToInputStream:
|
||||||
|
(data: number, streamNamePtr: number, timestamp: number) => void;
|
||||||
|
_addStringToInputStream:
|
||||||
|
(dataPtr: number, streamNamePtr: number, timestamp: number) => void;
|
||||||
|
_addFlatHashMapToInputStream:
|
||||||
|
(keysPtr: number, valuesPtr: number, count: number, streamNamePtr: number,
|
||||||
|
timestamp: number) => void;
|
||||||
|
_addProtoToInputStream:
|
||||||
|
(dataPtr: number, dataSize: number, protoNamePtr: number,
|
||||||
|
streamNamePtr: number, timestamp: number) => void;
|
||||||
|
_addEmptyPacketToInputStream:
|
||||||
|
(streamNamePtr: number, timestamp: number) => void;
|
||||||
|
|
||||||
|
// Input side packets
|
||||||
|
_addBoolToInputSidePacket: (data: boolean, streamNamePtr: number) => void;
|
||||||
|
_addDoubleToInputSidePacket: (data: number, streamNamePtr: number) => void;
|
||||||
|
_addFloatToInputSidePacket: (data: number, streamNamePtr: number) => void;
|
||||||
|
_addIntToInputSidePacket: (data: number, streamNamePtr: number) => void;
|
||||||
|
_addStringToInputSidePacket: (dataPtr: number, streamNamePtr: number) => void;
|
||||||
|
_addProtoToInputSidePacket:
|
||||||
|
(dataPtr: number, dataSize: number, protoNamePtr: number,
|
||||||
|
streamNamePtr: number) => void;
|
||||||
|
|
||||||
|
// Map of output streams to packet listeners. Also built as part of
|
||||||
|
// gl_graph_runner_internal_multi_input.
|
||||||
|
simpleListeners?:
|
||||||
|
Record<string, SimpleListener<unknown>|VectorListener<unknown>>;
|
||||||
|
// Map of output streams to empty packet listeners.
|
||||||
|
emptyPacketListeners?: Record<string, EmptyPacketListener>;
|
||||||
|
_attachBoolListener: (streamNamePtr: number) => void;
|
||||||
|
_attachBoolVectorListener: (streamNamePtr: number) => void;
|
||||||
|
_attachDoubleListener: (streamNamePtr: number) => void;
|
||||||
|
_attachDoubleVectorListener: (streamNamePtr: number) => void;
|
||||||
|
_attachFloatListener: (streamNamePtr: number) => void;
|
||||||
|
_attachFloatVectorListener: (streamNamePtr: number) => void;
|
||||||
|
_attachIntListener: (streamNamePtr: number) => void;
|
||||||
|
_attachIntVectorListener: (streamNamePtr: number) => void;
|
||||||
|
_attachStringListener: (streamNamePtr: number) => void;
|
||||||
|
_attachStringVectorListener: (streamNamePtr: number) => void;
|
||||||
|
_attachProtoListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
||||||
|
_attachProtoVectorListener:
|
||||||
|
(streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
||||||
|
|
||||||
|
// Require dependency ":gl_graph_runner_audio_out"
|
||||||
|
_attachAudioListener: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
||||||
|
|
||||||
|
// Require dependency ":gl_graph_runner_audio"
|
||||||
|
_addAudioToInputStream:
|
||||||
|
(dataPtr: number, numChannels: number, numSamples: number,
|
||||||
|
streamNamePtr: number, timestamp: number) => void;
|
||||||
|
_configureAudio:
|
||||||
|
(channels: number, samples: number, sampleRate: number,
|
||||||
|
streamNamePtr: number, headerNamePtr: number) => void;
|
||||||
|
|
||||||
|
// Get the graph configuration and invoke the listener configured under
|
||||||
|
// streamNamePtr
|
||||||
|
_getGraphConfig: (streamNamePtr: number, makeDeepCopy?: boolean) => void;
|
||||||
|
|
||||||
|
// TODO: Refactor to just use a few numbers (perhaps refactor away
|
||||||
|
// from gl_graph_runner_internal.cc entirely to use something a little more
|
||||||
|
// streamlined; new version is _processFrame above).
|
||||||
|
_processGl: (frameDataPtr: number) => number;
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user