From dd2966629600f98b27c67949bdc8f46e523b610e Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Wed, 11 Oct 2023 15:53:14 -0700 Subject: [PATCH] Adding vector versions of input calls to TS GraphRunner API PiperOrigin-RevId: 572711430 --- mediapipe/web/graph_runner/graph_runner.ts | 162 ++++++++++++++++++ .../web/graph_runner/graph_runner_api.d.ts | 94 ++++++++++ mediapipe/web/graph_runner/wasm_module.d.ts | 32 ++++ 3 files changed, 288 insertions(+) diff --git a/mediapipe/web/graph_runner/graph_runner.ts b/mediapipe/web/graph_runner/graph_runner.ts index 2ede0ae4f..06b615502 100644 --- a/mediapipe/web/graph_runner/graph_runner.ts +++ b/mediapipe/web/graph_runner/graph_runner.ts @@ -475,6 +475,88 @@ export class GraphRunner implements GraphRunnerApi { }); } + /** {@override GraphRunnerApi} */ + addBoolVectorToStream(data: boolean[], streamName: string, timestamp: number): + void { + this.wrapStringPtr(streamName, (streamNamePtr: number) => { + const vecPtr = this.wasmModule._allocateBoolVector(data.length); + if (!vecPtr) { + throw new Error('Unable to allocate new bool vector on heap.'); + } + for (const entry of data) { + this.wasmModule._addBoolVectorEntry(vecPtr, entry); + } + this.wasmModule._addBoolVectorToInputStream( + vecPtr, streamNamePtr, timestamp); + }); + } + + /** {@override GraphRunnerApi} */ + addDoubleVectorToStream( + data: number[], streamName: string, timestamp: number): void { + this.wrapStringPtr(streamName, (streamNamePtr: number) => { + const vecPtr = this.wasmModule._allocateDoubleVector(data.length); + if (!vecPtr) { + throw new Error('Unable to allocate new double vector on heap.'); + } + for (const entry of data) { + this.wasmModule._addDoubleVectorEntry(vecPtr, entry); + } + this.wasmModule._addDoubleVectorToInputStream( + vecPtr, streamNamePtr, timestamp); + }); + } + + /** {@override GraphRunnerApi} */ + addFloatVectorToStream(data: number[], streamName: string, timestamp: number): + void { + this.wrapStringPtr(streamName, (streamNamePtr: number) => { + const vecPtr = this.wasmModule._allocateFloatVector(data.length); + if (!vecPtr) { + throw new Error('Unable to allocate new float vector on heap.'); + } + for (const entry of data) { + this.wasmModule._addFloatVectorEntry(vecPtr, entry); + } + this.wasmModule._addFloatVectorToInputStream( + vecPtr, streamNamePtr, timestamp); + }); + } + + /** {@override GraphRunnerApi} */ + addIntVectorToStream(data: number[], streamName: string, timestamp: number): + void { + this.wrapStringPtr(streamName, (streamNamePtr: number) => { + const vecPtr = this.wasmModule._allocateIntVector(data.length); + if (!vecPtr) { + throw new Error('Unable to allocate new int vector on heap.'); + } + for (const entry of data) { + this.wasmModule._addIntVectorEntry(vecPtr, entry); + } + this.wasmModule._addIntVectorToInputStream( + vecPtr, streamNamePtr, timestamp); + }); + } + + /** {@override GraphRunnerApi} */ + addStringVectorToStream( + data: string[], streamName: string, timestamp: number): void { + this.wrapStringPtr(streamName, (streamNamePtr: number) => { + const vecPtr = this.wasmModule._allocateStringVector(data.length); + if (!vecPtr) { + throw new Error('Unable to allocate new string vector on heap.'); + } + for (const entry of data) { + this.wrapStringPtr(entry, (entryStringPtr: number) => { + this.wasmModule._addStringVectorEntry(vecPtr, entryStringPtr); + }); + } + this.wasmModule._addStringVectorToInputStream( + vecPtr, streamNamePtr, timestamp); + }); + } + /** {@override GraphRunnerApi} */ addBoolToInputSidePacket(data: boolean, sidePacketName: string): void { this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { @@ -528,6 +610,86 @@ export class GraphRunner implements GraphRunnerApi { }); } + /** {@override GraphRunnerApi} */ + addBoolVectorToInputSidePacket(data: boolean[], sidePacketName: string): + void { + this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { + const vecPtr = this.wasmModule._allocateBoolVector(data.length); + if (!vecPtr) { + throw new Error('Unable to allocate new bool vector on heap.'); + } + for (const entry of data) { + this.wasmModule._addBoolVectorEntry(vecPtr, entry); + } + this.wasmModule._addBoolVectorToInputSidePacket( + vecPtr, sidePacketNamePtr); + }); + } + + /** {@override GraphRunnerApi} */ + addDoubleVectorToInputSidePacket(data: number[], sidePacketName: string): + void { + this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { + const vecPtr = this.wasmModule._allocateDoubleVector(data.length); + if (!vecPtr) { + throw new Error('Unable to allocate new double vector on heap.'); + } + for (const entry of data) { + this.wasmModule._addDoubleVectorEntry(vecPtr, entry); + } + this.wasmModule._addDoubleVectorToInputSidePacket( + vecPtr, sidePacketNamePtr); + }); + } + + /** {@override GraphRunnerApi} */ + addFloatVectorToInputSidePacket(data: number[], sidePacketName: string): + void { + this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { + const vecPtr = this.wasmModule._allocateFloatVector(data.length); + if (!vecPtr) { + throw new Error('Unable to allocate new float vector on heap.'); + } + for (const entry of data) { + this.wasmModule._addFloatVectorEntry(vecPtr, entry); + } + this.wasmModule._addFloatVectorToInputSidePacket( + vecPtr, sidePacketNamePtr); + }); + } + + /** {@override GraphRunnerApi} */ + addIntVectorToInputSidePacket(data: number[], sidePacketName: string): void { + this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { + const vecPtr = this.wasmModule._allocateIntVector(data.length); + if (!vecPtr) { + throw new Error('Unable to allocate new int vector on heap.'); + } + for (const entry of data) { + this.wasmModule._addIntVectorEntry(vecPtr, entry); + } + this.wasmModule._addIntVectorToInputSidePacket(vecPtr, sidePacketNamePtr); + }); + } + + /** {@override GraphRunnerApi} */ + addStringVectorToInputSidePacket(data: string[], sidePacketName: string): + void { + this.wrapStringPtr(sidePacketName, (sidePacketNamePtr: number) => { + const vecPtr = this.wasmModule._allocateStringVector(data.length); + if (!vecPtr) { + throw new Error('Unable to allocate new string vector on heap.'); + } + for (const entry of data) { + this.wrapStringPtr(entry, (entryStringPtr: number) => { + this.wasmModule._addStringVectorEntry(vecPtr, entryStringPtr); + }); + } + this.wasmModule._addStringVectorToInputSidePacket( + vecPtr, sidePacketNamePtr); + }); + } + /** {@override GraphRunnerApi} */ attachBoolListener( outputStreamName: string, callbackFcn: SimpleListener): void { diff --git a/mediapipe/web/graph_runner/graph_runner_api.d.ts b/mediapipe/web/graph_runner/graph_runner_api.d.ts index cb5172491..f6318416f 100644 --- a/mediapipe/web/graph_runner/graph_runner_api.d.ts +++ b/mediapipe/web/graph_runner/graph_runner_api.d.ts @@ -320,6 +320,57 @@ export interface GraphRunnerApi { */ addEmptyPacketToStream(streamName: string, timestamp: number): void; + /** + * Sends a vector packet into the specified stream at the given + * timestamp. + * @param data The ordered array of boolean data to send as a vector. + * @param streamName The name of the graph input stream to send data into. + * @param timestamp The timestamp of the input data, in ms. + */ + addBoolVectorToStream(data: boolean[], streamName: string, timestamp: number): + void; + + /** + * Sends a vector packet into the specified stream at the given + * timestamp. + * @param data The ordered array of double-precision float data to send as a + * vector. + * @param streamName The name of the graph input stream to send data into. + * @param timestamp The timestamp of the input data, in ms. + */ + addDoubleVectorToStream( + data: number[], streamName: string, timestamp: number): void; + + /** + * Sends a vector packet into the specified stream at the given + * timestamp. + * @param data The ordered array of float data to send as a vector. + * @param streamName The name of the graph input stream to send data into. + * @param timestamp The timestamp of the input data, in ms. + */ + addFloatVectorToStream(data: number[], streamName: string, timestamp: number): + void; + + /** + * Sends a vector packet into the specified stream at the given + * timestamp. + * @param data The ordered array of integer data to send as a vector. + * @param streamName The name of the graph input stream to send data into. + * @param timestamp The timestamp of the input data, in ms. + */ + addIntVectorToStream(data: number[], streamName: string, timestamp: number): + void; + + /** + * Sends a vector packet into the specified stream at the given + * timestamp. + * @param data The ordered array of string data to send as a vector. + * @param streamName The name of the graph input stream to send data into. + * @param timestamp The timestamp of the input data, in ms. + */ + addStringVectorToStream( + data: string[], streamName: string, timestamp: number): void; + /** * Attaches a boolean packet to the specified input_side_packet. * @param data The boolean data to send. @@ -371,6 +422,49 @@ export interface GraphRunnerApi { addProtoToInputSidePacket( data: Uint8Array, protoType: string, sidePacketName: string): void; + /** + * Attaches a vector packet to the specified input_side_packet. + * @param data The ordered array of boolean data to send as a vector. + * @param sidePacketName The name of the graph input side packet to send data + * into. + */ + addBoolVectorToInputSidePacket(data: boolean[], sidePacketName: string): void; + + /** + * Attaches a vector packet to the specified input_side_packet. + * @param data The ordered array of double-precision float data to send as a + * vector. + * @param sidePacketName The name of the graph input side packet to send data + * into. + */ + addDoubleVectorToInputSidePacket(data: number[], sidePacketName: string): + void; + + /** + * Attaches a vector packet to the specified input_side_packet. + * @param data The ordered array of float data to send as a vector. + * @param sidePacketName The name of the graph input side packet to send data + * into. + */ + addFloatVectorToInputSidePacket(data: number[], sidePacketName: string): void; + + /** + * Attaches a vector packet to the specified input_side_packet. + * @param data The ordered array of integer data to send as a vector. + * @param sidePacketName The name of the graph input side packet to send data + * into. + */ + addIntVectorToInputSidePacket(data: number[], sidePacketName: string): void; + + /** + * Attaches a vector packet to the specified input_side_packet. + * @param data The ordered array of string data to send as a vector. + * @param sidePacketName The name of the graph input side packet to send data + * into. + */ + addStringVectorToInputSidePacket(data: string[], sidePacketName: string): + void; + /** * Attaches a boolean packet listener to the specified output_stream. * @param outputStreamName The name of the graph output stream to grab boolean diff --git a/mediapipe/web/graph_runner/wasm_module.d.ts b/mediapipe/web/graph_runner/wasm_module.d.ts index 1dad39a53..bc4c24c26 100644 --- a/mediapipe/web/graph_runner/wasm_module.d.ts +++ b/mediapipe/web/graph_runner/wasm_module.d.ts @@ -50,6 +50,16 @@ export declare interface WasmModule { (data: number, streamNamePtr: number, timestamp: number) => void; _addStringToInputStream: (dataPtr: number, streamNamePtr: number, timestamp: number) => void; + _addBoolVectorToInputStream: + (vecPtr: number, streamNamePtr: number, timestamp: number) => void; + _addDoubleVectorToInputStream: + (vecPtr: number, streamNamePtr: number, timestamp: number) => void; + _addFloatVectorToInputStream: + (vecPtr: number, streamNamePtr: number, timestamp: number) => void; + _addIntVectorToInputStream: + (vecPtr: number, streamNamePtr: number, timestamp: number) => void; + _addStringVectorToInputStream: + (vecPtr: number, streamNamePtr: number, timestamp: number) => void; _addFlatHashMapToInputStream: (keysPtr: number, valuesPtr: number, count: number, streamNamePtr: number, timestamp: number) => void; @@ -68,6 +78,28 @@ export declare interface WasmModule { _addProtoToInputSidePacket: (dataPtr: number, dataSize: number, protoNamePtr: number, streamNamePtr: number) => void; + _addBoolVectorToInputSidePacket: + (vecPtr: number, streamNamePtr: number) => void; + _addDoubleVectorToInputSidePacket: + (vecPtr: number, streamNamePtr: number) => void; + _addFloatVectorToInputSidePacket: + (vecPtr: number, streamNamePtr: number) => void; + _addIntVectorToInputSidePacket: + (vecPtr: number, streamNamePtr: number) => void; + _addStringVectorToInputSidePacket: + (vecPtr: number, streamNamePtr: number) => void; + + // Vector input creation + _allocateBoolVector: (size: number) => number; + _allocateDoubleVector: (size: number) => number; + _allocateFloatVector: (size: number) => number; + _allocateIntVector: (size: number) => number; + _allocateStringVector: (size: number) => number; + _addBoolVectorEntry: (vecPtr: number, entry: boolean) => void; + _addDoubleVectorEntry: (vecPtr: number, entry: number) => void; + _addFloatVectorEntry: (vecPtr: number, entry: number) => void; + _addIntVectorEntry: (vecPtr: number, entry: number) => void; + _addStringVectorEntry: (vecPtr: number, entryStringPtr: number) => void; // Map of output streams to packet listeners. Also built as part of // gl_graph_runner_internal_multi_input.