cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Open Source the remaining Web implementation files

Browse Source 
PiperOrigin-RevId: 486701932
This commit is contained in:
Sebastian Schmidt 2022-11-07 10:33:12 -08:00 committed by Copybara-Service
parent c02f3933d4
commit 2371051e17
27 changed files with 2931 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
mediapipe/tasks/web/audio/audio_classifier/BUILD Normal file
View File

@ -0,0 +1,33 @@
# This contains the MediaPipe Audio Classifier Task.
#
# This task takes audio data and outputs the classification result.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
mediapipe_ts_library(
name = "audio_classifier",
srcs = [
"audio_classifier.ts",
"audio_classifier_options.ts",
"audio_classifier_result.ts",
],
deps = [
"//mediapipe/framework:calculator_jspb_proto",
"//mediapipe/framework:calculator_options_jspb_proto",
"//mediapipe/tasks/cc/audio/audio_classifier/proto:audio_classifier_graph_options_jspb_proto",
"//mediapipe/tasks/cc/components/containers/proto:classifications_jspb_proto",
"//mediapipe/tasks/web/components/containers:category",
"//mediapipe/tasks/web/components/containers:classifications",
"//mediapipe/tasks/web/components/processors:base_options",
"//mediapipe/tasks/web/components/processors:classifier_options",
"//mediapipe/tasks/web/components/processors:classifier_result",
"//mediapipe/tasks/web/core",
"//mediapipe/tasks/web/core:classifier_options",
"//mediapipe/tasks/web/core:task_runner",
"//mediapipe/web/graph_runner:wasm_mediapipe_lib_ts",
],
)

215
mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts Normal file
View File

@ -0,0 +1,215 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {CalculatorGraphConfig} from '../../../../framework/calculator_pb';
import {CalculatorOptions} from '../../../../framework/calculator_options_pb';
import {AudioClassifierGraphOptions} from '../../../../tasks/cc/audio/audio_classifier/proto/audio_classifier_graph_options_pb';
import {ClassificationResult} from '../../../../tasks/cc/components/containers/proto/classifications_pb';
import {convertBaseOptionsToProto} from '../../../../tasks/web/components/processors/base_options';
import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options';
import {convertFromClassificationResultProto} from '../../../../tasks/web/components/processors/classifier_result';
import {TaskRunner} from '../../../../tasks/web/core/task_runner';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {createMediaPipeLib, FileLocator} from '../../../../web/graph_runner/wasm_mediapipe_lib';
// Placeholder for internal dependency on trusted resource url
import {AudioClassifierOptions} from './audio_classifier_options';
import {Classifications} from './audio_classifier_result';
const MEDIAPIPE_GRAPH =
'mediapipe.tasks.audio.audio_classifier.AudioClassifierGraph';
// Note: `input_audio` is hardcoded in 'gl_graph_runner_internal_audio' and
// cannot be changed
// TODO: Change this to `audio_in` to match the name in the CC
// implementation
const AUDIO_STREAM = 'input_audio';
const SAMPLE_RATE_STREAM = 'sample_rate';
const CLASSIFICATION_RESULT_STREAM = 'classification_result';
// The OSS JS API does not support the builder pattern.
// tslint:disable:jspb-use-builder-pattern
/** Performs audio classification. */
export class AudioClassifier extends TaskRunner {
private classifications: Classifications[] = [];
private defaultSampleRate = 48000;
private readonly options = new AudioClassifierGraphOptions();
/**
* Initializes the Wasm runtime and creates a new audio classifier from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param audioClassifierOptions The options for the audio classifier. Note
* that either a path to the model asset or a model buffer needs to be
* provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
audioClassifierOptions: AudioClassifierOptions):
Promise<AudioClassifier> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file loaded with this mechanism is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const classifier = await createMediaPipeLib(
AudioClassifier, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
await classifier.setOptions(audioClassifierOptions);
return classifier;
}
/**
* Initializes the Wasm runtime and creates a new audio classifier based on
* the provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
modelAssetBuffer: Uint8Array): Promise<AudioClassifier> {
return AudioClassifier.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new audio classifier based on
* the path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param modelAssetPath The path to the model asset.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
modelAssetPath: string): Promise<AudioClassifier> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return AudioClassifier.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
}
/**
* Sets new options for the audio classifier.
*
* Calling `setOptions()` with a subset of options only affects those options.
* You can reset an option back to its default value by explicitly setting it
* to `undefined`.
*
* @param options The options for the audio classifier.
*/
async setOptions(options: AudioClassifierOptions): Promise<void> {
if (options.baseOptions) {
const baseOptionsProto =
await convertBaseOptionsToProto(options.baseOptions);
this.options.setBaseOptions(baseOptionsProto);
}
this.options.setClassifierOptions(convertClassifierOptionsToProto(
options, this.options.getClassifierOptions()));
this.refreshGraph();
}
/**
* Sets the sample rate for all calls to `classify()` that omit an explicit
* sample rate. `48000` is used as a default if this method is not called.
*
* @param sampleRate A sample rate (e.g. `44100`).
*/
setDefaultSampleRate(sampleRate: number) {
this.defaultSampleRate = sampleRate;
}
/**
* Performs audio classification on the provided audio data and waits
* synchronously for the response.
*
* @param audioData An array of raw audio capture data, like
* from a call to getChannelData on an AudioBuffer.
* @param sampleRate The sample rate in Hz of the provided audio data. If not
* set, defaults to the sample rate set via `setDefaultSampleRate()` or
* `48000` if no custom default was set.
* @return The classification result of the audio datas
*/
classify(audioData: Float32Array, sampleRate?: number): Classifications[] {
sampleRate = sampleRate ?? this.defaultSampleRate;
// Configures the number of samples in the WASM layer. We re-configure the
// number of samples and the sample rate for every frame, but ignore other
// side effects of this function (such as sending the input side packet and
// the input stream header).
this.configureAudio(
/* numChannels= */ 1, /* numSamples= */ audioData.length, sampleRate);
const timestamp = performance.now();
this.addDoubleToStream(sampleRate, SAMPLE_RATE_STREAM, timestamp);
this.addAudioToStream(audioData, timestamp);
this.classifications = [];
this.finishProcessing();
return [...this.classifications];
}
/**
* Internal function for converting raw data into a classification, and
* adding it to our classfications list.
**/
private addJsAudioClassification(binaryProto: Uint8Array): void {
const classificationResult =
ClassificationResult.deserializeBinary(binaryProto);
this.classifications.push(
...convertFromClassificationResultProto(classificationResult));
}
/** Updates the MediaPipe graph configuration. */
private refreshGraph(): void {
const graphConfig = new CalculatorGraphConfig();
graphConfig.addInputStream(AUDIO_STREAM);
graphConfig.addInputStream(SAMPLE_RATE_STREAM);
graphConfig.addOutputStream(CLASSIFICATION_RESULT_STREAM);
const calculatorOptions = new CalculatorOptions();
calculatorOptions.setExtension(
AudioClassifierGraphOptions.ext, this.options);
// Perform audio classification. Pre-processing and results post-processing
// are built-in.
const classifierNode = new CalculatorGraphConfig.Node();
classifierNode.setCalculator(MEDIAPIPE_GRAPH);
classifierNode.addInputStream('AUDIO:' + AUDIO_STREAM);
classifierNode.addInputStream('SAMPLE_RATE:' + SAMPLE_RATE_STREAM);
classifierNode.addOutputStream(
'CLASSIFICATION_RESULT:' + CLASSIFICATION_RESULT_STREAM);
classifierNode.setOptions(calculatorOptions);
graphConfig.addNode(classifierNode);
this.attachProtoListener(CLASSIFICATION_RESULT_STREAM, binaryProto => {
this.addJsAudioClassification(binaryProto);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);
}
}

17
mediapipe/tasks/web/audio/audio_classifier/audio_classifier_options.ts Normal file
View File

@ -0,0 +1,17 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {ClassifierOptions as AudioClassifierOptions} from '../../../../tasks/web/core/classifier_options';

18
mediapipe/tasks/web/audio/audio_classifier/audio_classifier_result.ts Normal file
View File

@ -0,0 +1,18 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {Category} from '../../../../tasks/web/components/containers/category';
export {ClassificationEntry, Classifications} from '../../../../tasks/web/components/containers/classifications';

10
mediapipe/tasks/web/components/processors/classifier_options.ts
View File

@ -29,31 +29,31 @@ export function convertClassifierOptionsToProto(
baseOptions?: ClassifierOptionsProto): ClassifierOptionsProto { baseOptions?: ClassifierOptionsProto): ClassifierOptionsProto {
const classifierOptions = const classifierOptions =
baseOptions ? baseOptions.clone() : new ClassifierOptionsProto(); baseOptions ? baseOptions.clone() : new ClassifierOptionsProto();
if (options.displayNamesLocale) { if (options.displayNamesLocale !== undefined) {
classifierOptions.setDisplayNamesLocale(options.displayNamesLocale); classifierOptions.setDisplayNamesLocale(options.displayNamesLocale);
} else if (options.displayNamesLocale === undefined) { } else if (options.displayNamesLocale === undefined) {
classifierOptions.clearDisplayNamesLocale(); classifierOptions.clearDisplayNamesLocale();
} }
if (options.maxResults) { if (options.maxResults !== undefined) {
classifierOptions.setMaxResults(options.maxResults); classifierOptions.setMaxResults(options.maxResults);
} else if ('maxResults' in options) { // Check for undefined } else if ('maxResults' in options) { // Check for undefined
classifierOptions.clearMaxResults(); classifierOptions.clearMaxResults();
} }
if (options.scoreThreshold) { if (options.scoreThreshold !== undefined) {
classifierOptions.setScoreThreshold(options.scoreThreshold); classifierOptions.setScoreThreshold(options.scoreThreshold);
} else if ('scoreThreshold' in options) { // Check for undefined } else if ('scoreThreshold' in options) { // Check for undefined
classifierOptions.clearScoreThreshold(); classifierOptions.clearScoreThreshold();
} }
if (options.categoryAllowlist) { if (options.categoryAllowlist !== undefined) {
classifierOptions.setCategoryAllowlistList(options.categoryAllowlist); classifierOptions.setCategoryAllowlistList(options.categoryAllowlist);
} else if ('categoryAllowlist' in options) { // Check for undefined } else if ('categoryAllowlist' in options) { // Check for undefined
classifierOptions.clearCategoryAllowlistList(); classifierOptions.clearCategoryAllowlistList();
} }
if (options.categoryDenylist) { if (options.categoryDenylist !== undefined) {
classifierOptions.setCategoryDenylistList(options.categoryDenylist); classifierOptions.setCategoryDenylistList(options.categoryDenylist);
} else if ('categoryDenylist' in options) { // Check for undefined } else if ('categoryDenylist' in options) { // Check for undefined
classifierOptions.clearCategoryDenylistList(); classifierOptions.clearCategoryDenylistList();

12
mediapipe/tasks/web/core/BUILD
View File

@ -12,6 +12,18 @@ mediapipe_ts_library(
], ],
) )
mediapipe_ts_library(
name = "task_runner",
srcs = [
"task_runner.ts",
],
deps = [
"//mediapipe/web/graph_runner:register_model_resources_graph_service_ts",
"//mediapipe/web/graph_runner:wasm_mediapipe_image_lib_ts",
"//mediapipe/web/graph_runner:wasm_mediapipe_lib_ts",
],
)
mediapipe_ts_library( mediapipe_ts_library(
name = "classifier_options", name = "classifier_options",
srcs = [ srcs = [

83
mediapipe/tasks/web/core/task_runner.ts Normal file
View File

@ -0,0 +1,83 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {SupportModelResourcesGraphService} from '../../../web/graph_runner/register_model_resources_graph_service';
import {SupportImage} from '../../../web/graph_runner/wasm_mediapipe_image_lib';
import {WasmMediaPipeLib, WasmModule} from '../../../web/graph_runner/wasm_mediapipe_lib';
// tslint:disable-next-line:enforce-name-casing
const WasmMediaPipeImageLib =
SupportModelResourcesGraphService(SupportImage(WasmMediaPipeLib));
/** Base class for all MediaPipe Tasks. */
export abstract class TaskRunner extends WasmMediaPipeImageLib {
private processingErrors: Error[] = [];
constructor(wasmModule: WasmModule) {
super(wasmModule);
// Disables the automatic render-to-screen code, which allows for pure
// CPU processing.
this.setAutoRenderToScreen(false);
// Enables use of our model resource caching graph service.
this.registerModelResourcesGraphService();
}
/**
* Takes the raw data from a MediaPipe graph, and passes it to C++ to be run
* over the video stream. Will replace the previously running MediaPipe graph,
* if there is one.
* @param graphData The raw MediaPipe graph data, either in binary
* protobuffer format (.binarypb), or else in raw text format (.pbtxt or
* .textproto).
* @param isBinary This should be set to true if the graph is in
* binary format, and false if it is in human-readable text format.
*/
override setGraph(graphData: Uint8Array, isBinary: boolean): void {
this.attachErrorListener((code, message) => {
this.processingErrors.push(new Error(message));
});
super.setGraph(graphData, isBinary);
this.handleErrors();
}
/**
* Forces all queued-up packets to be pushed through the MediaPipe graph as
* far as possible, performing all processing until no more processing can be
* done.
*/
override finishProcessing(): void {
super.finishProcessing();
this.handleErrors();
}
/** Throws the error from the error listener if an error was raised. */
private handleErrors() {
const errorCount = this.processingErrors.length;
if (errorCount === 1) {
// Re-throw error to get a more meaningful stacktrace
throw new Error(this.processingErrors[0].message);
} else if (errorCount > 1) {
throw new Error(
'Encountered multiple errors: ' +
this.processingErrors.map(e => e.message).join(', '));
}
this.processingErrors = [];
}
}

34
mediapipe/tasks/web/text/text_classifier/BUILD Normal file
View File

@ -0,0 +1,34 @@
# This contains the MediaPipe Text Classifier Task.
#
# This task takes text input performs Natural Language classification (including
# BERT-based text classification).
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
mediapipe_ts_library(
name = "text_classifier",
srcs = [
"text_classifier.ts",
"text_classifier_options.d.ts",
"text_classifier_result.d.ts",
],
deps = [
"//mediapipe/framework:calculator_jspb_proto",
"//mediapipe/framework:calculator_options_jspb_proto",
"//mediapipe/tasks/cc/components/containers/proto:classifications_jspb_proto",
"//mediapipe/tasks/cc/text/text_classifier/proto:text_classifier_graph_options_jspb_proto",
"//mediapipe/tasks/web/components/containers:category",
"//mediapipe/tasks/web/components/containers:classifications",
"//mediapipe/tasks/web/components/processors:base_options",
"//mediapipe/tasks/web/components/processors:classifier_options",
"//mediapipe/tasks/web/components/processors:classifier_result",
"//mediapipe/tasks/web/core",
"//mediapipe/tasks/web/core:classifier_options",
"//mediapipe/tasks/web/core:task_runner",
"//mediapipe/web/graph_runner:wasm_mediapipe_lib_ts",
],
)

180
mediapipe/tasks/web/text/text_classifier/text_classifier.ts Normal file
View File

@ -0,0 +1,180 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {CalculatorGraphConfig} from '../../../../framework/calculator_pb';
import {CalculatorOptions} from '../../../../framework/calculator_options_pb';
import {ClassificationResult} from '../../../../tasks/cc/components/containers/proto/classifications_pb';
import {TextClassifierGraphOptions} from '../../../../tasks/cc/text/text_classifier/proto/text_classifier_graph_options_pb';
import {convertBaseOptionsToProto} from '../../../../tasks/web/components/processors/base_options';
import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options';
import {convertFromClassificationResultProto} from '../../../../tasks/web/components/processors/classifier_result';
import {TaskRunner} from '../../../../tasks/web/core/task_runner';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {createMediaPipeLib, FileLocator} from '../../../../web/graph_runner/wasm_mediapipe_lib';
// Placeholder for internal dependency on trusted resource url
import {TextClassifierOptions} from './text_classifier_options';
import {Classifications} from './text_classifier_result';
const INPUT_STREAM = 'text_in';
const CLASSIFICATION_RESULT_STREAM = 'classification_result_out';
const TEXT_CLASSIFIER_GRAPH =
'mediapipe.tasks.text.text_classifier.TextClassifierGraph';
// The OSS JS API does not support the builder pattern.
// tslint:disable:jspb-use-builder-pattern
/** Performs Natural Language classification. */
export class TextClassifier extends TaskRunner {
private classifications: Classifications[] = [];
private readonly options = new TextClassifierGraphOptions();
/**
* Initializes the Wasm runtime and creates a new text classifier from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param textClassifierOptions The options for the text classifier. Note that
* either a path to the TFLite model or the model itself needs to be
* provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
textClassifierOptions: TextClassifierOptions): Promise<TextClassifier> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const classifier = await createMediaPipeLib(
TextClassifier, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
await classifier.setOptions(textClassifierOptions);
return classifier;
}
/**
* Initializes the Wasm runtime and creates a new text classifier based on the
* provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
modelAssetBuffer: Uint8Array): Promise<TextClassifier> {
return TextClassifier.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new text classifier based on the
* path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param modelAssetPath The path to the model asset.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
modelAssetPath: string): Promise<TextClassifier> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return TextClassifier.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
}
/**
* Sets new options for the text classifier.
*
* Calling `setOptions()` with a subset of options only affects those options.
* You can reset an option back to its default value by explicitly setting it
* to `undefined`.
*
* @param options The options for the text classifier.
*/
async setOptions(options: TextClassifierOptions): Promise<void> {
if (options.baseOptions) {
const baseOptionsProto =
await convertBaseOptionsToProto(options.baseOptions);
this.options.setBaseOptions(baseOptionsProto);
}
this.options.setClassifierOptions(convertClassifierOptionsToProto(
options, this.options.getClassifierOptions()));
this.refreshGraph();
}
/**
* Performs Natural Language classification on the provided text and waits
* synchronously for the response.
*
* @param text The text to process.
* @return The classification result of the text
*/
classify(text: string): Classifications[] {
// Get classification classes by running our MediaPipe graph.
this.classifications = [];
this.addStringToStream(
text, INPUT_STREAM, /* timestamp= */ performance.now());
this.finishProcessing();
return [...this.classifications];
}
// Internal function for converting raw data into a classification, and
// adding it to our classifications list.
private addJsTextClassification(binaryProto: Uint8Array): void {
const classificationResult =
ClassificationResult.deserializeBinary(binaryProto);
console.log(classificationResult.toObject());
this.classifications.push(
...convertFromClassificationResultProto(classificationResult));
}
/** Updates the MediaPipe graph configuration. */
private refreshGraph(): void {
const graphConfig = new CalculatorGraphConfig();
graphConfig.addInputStream(INPUT_STREAM);
graphConfig.addOutputStream(CLASSIFICATION_RESULT_STREAM);
const calculatorOptions = new CalculatorOptions();
calculatorOptions.setExtension(
TextClassifierGraphOptions.ext, this.options);
const classifierNode = new CalculatorGraphConfig.Node();
classifierNode.setCalculator(TEXT_CLASSIFIER_GRAPH);
classifierNode.addInputStream('TEXT:' + INPUT_STREAM);
classifierNode.addOutputStream(
'CLASSIFICATION_RESULT:' + CLASSIFICATION_RESULT_STREAM);
classifierNode.setOptions(calculatorOptions);
graphConfig.addNode(classifierNode);
this.attachProtoListener(CLASSIFICATION_RESULT_STREAM, binaryProto => {
this.addJsTextClassification(binaryProto);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);
}
}

17
mediapipe/tasks/web/text/text_classifier/text_classifier_options.d.ts vendored Normal file
View File

@ -0,0 +1,17 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {ClassifierOptions as TextClassifierOptions} from '../../../../tasks/web/core/classifier_options';

18
mediapipe/tasks/web/text/text_classifier/text_classifier_result.d.ts vendored Normal file
View File

@ -0,0 +1,18 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {Category} from '../../../../tasks/web/components/containers/category';
export {ClassificationEntry, Classifications} from '../../../../tasks/web/components/containers/classifications';

40
mediapipe/tasks/web/vision/gesture_recognizer/BUILD Normal file
View File

@ -0,0 +1,40 @@
# This contains the MediaPipe Gesture Recognizer Task.
#
# This task takes video frames and outputs synchronized frames along with
# the detection results for one or more gesture categories, using Gesture Recognizer.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
mediapipe_ts_library(
name = "gesture_recognizer",
srcs = [
"gesture_recognizer.ts",
"gesture_recognizer_options.d.ts",
"gesture_recognizer_result.d.ts",
],
deps = [
"//mediapipe/framework:calculator_jspb_proto",
"//mediapipe/framework:calculator_options_jspb_proto",
"//mediapipe/framework/formats:classification_jspb_proto",
"//mediapipe/framework/formats:landmark_jspb_proto",
"//mediapipe/framework/formats:rect_jspb_proto",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_classifier_graph_options_jspb_proto",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:gesture_recognizer_graph_options_jspb_proto",
"//mediapipe/tasks/cc/vision/gesture_recognizer/proto:hand_gesture_recognizer_graph_options_jspb_proto",
"//mediapipe/tasks/cc/vision/hand_detector/proto:hand_detector_graph_options_jspb_proto",
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarker_graph_options_jspb_proto",
"//mediapipe/tasks/cc/vision/hand_landmarker/proto:hand_landmarks_detector_graph_options_jspb_proto",
"//mediapipe/tasks/web/components/containers:category",
"//mediapipe/tasks/web/components/containers:landmark",
"//mediapipe/tasks/web/components/processors:base_options",
"//mediapipe/tasks/web/components/processors:classifier_options",
"//mediapipe/tasks/web/core",
"//mediapipe/tasks/web/core:classifier_options",
"//mediapipe/tasks/web/core:task_runner",
"//mediapipe/web/graph_runner:wasm_mediapipe_lib_ts",
],
)

374
mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts Normal file
View File

@ -0,0 +1,374 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {CalculatorGraphConfig} from '../../../../framework/calculator_pb';
import {CalculatorOptions} from '../../../../framework/calculator_options_pb';
import {ClassificationList} from '../../../../framework/formats/classification_pb';
import {LandmarkList, NormalizedLandmarkList} from '../../../../framework/formats/landmark_pb';
import {NormalizedRect} from '../../../../framework/formats/rect_pb';
import {GestureClassifierGraphOptions} from '../../../../tasks/cc/vision/gesture_recognizer/proto/gesture_classifier_graph_options_pb';
import {GestureRecognizerGraphOptions} from '../../../../tasks/cc/vision/gesture_recognizer/proto/gesture_recognizer_graph_options_pb';
import {HandGestureRecognizerGraphOptions} from '../../../../tasks/cc/vision/gesture_recognizer/proto/hand_gesture_recognizer_graph_options_pb';
import {HandDetectorGraphOptions} from '../../../../tasks/cc/vision/hand_detector/proto/hand_detector_graph_options_pb';
import {HandLandmarkerGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarker_graph_options_pb';
import {HandLandmarksDetectorGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options_pb';
import {Category} from '../../../../tasks/web/components/containers/category';
import {Landmark} from '../../../../tasks/web/components/containers/landmark';
import {convertBaseOptionsToProto} from '../../../../tasks/web/components/processors/base_options';
import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options';
import {TaskRunner} from '../../../../tasks/web/core/task_runner';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {createMediaPipeLib, FileLocator, ImageSource, WasmModule} from '../../../../web/graph_runner/wasm_mediapipe_lib';
// Placeholder for internal dependency on trusted resource url
import {GestureRecognizerOptions} from './gesture_recognizer_options';
import {GestureRecognitionResult} from './gesture_recognizer_result';
export {ImageSource};
// The OSS JS API does not support the builder pattern.
// tslint:disable:jspb-use-builder-pattern
const IMAGE_STREAM = 'image_in';
const NORM_RECT_STREAM = 'norm_rect';
const HAND_GESTURES_STREAM = 'hand_gestures';
const LANDMARKS_STREAM = 'hand_landmarks';
const WORLD_LANDMARKS_STREAM = 'world_hand_landmarks';
const HANDEDNESS_STREAM = 'handedness';
const GESTURE_RECOGNIZER_GRAPH =
'mediapipe.tasks.vision.gesture_recognizer.GestureRecognizerGraph';
const DEFAULT_NUM_HANDS = 1;
const DEFAULT_SCORE_THRESHOLD = 0.5;
const DEFAULT_CATEGORY_INDEX = -1;
const FULL_IMAGE_RECT = new NormalizedRect();
FULL_IMAGE_RECT.setXCenter(0.5);
FULL_IMAGE_RECT.setYCenter(0.5);
FULL_IMAGE_RECT.setWidth(1);
FULL_IMAGE_RECT.setHeight(1);
/** Performs hand gesture recognition on images. */
export class GestureRecognizer extends TaskRunner {
private gestures: Category[][] = [];
private landmarks: Landmark[][] = [];
private worldLandmarks: Landmark[][] = [];
private handednesses: Category[][] = [];
private readonly options: GestureRecognizerGraphOptions;
private readonly handLandmarkerGraphOptions: HandLandmarkerGraphOptions;
private readonly handLandmarksDetectorGraphOptions:
HandLandmarksDetectorGraphOptions;
private readonly handDetectorGraphOptions: HandDetectorGraphOptions;
private readonly handGestureRecognizerGraphOptions:
HandGestureRecognizerGraphOptions;
/**
* Initializes the Wasm runtime and creates a new gesture recognizer from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param gestureRecognizerOptions The options for the gesture recognizer.
* Note that either a path to the model asset or a model buffer needs to
* be provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
gestureRecognizerOptions: GestureRecognizerOptions):
Promise<GestureRecognizer> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load via this mechanism is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const recognizer = await createMediaPipeLib(
GestureRecognizer, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
await recognizer.setOptions(gestureRecognizerOptions);
return recognizer;
}
/**
* Initializes the Wasm runtime and creates a new gesture recognizer based on
* the provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
modelAssetBuffer: Uint8Array): Promise<GestureRecognizer> {
return GestureRecognizer.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new gesture recognizer based on
* the path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param modelAssetPath The path to the model asset.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
modelAssetPath: string): Promise<GestureRecognizer> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return GestureRecognizer.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
}
constructor(wasmModule: WasmModule) {
super(wasmModule);
this.options = new GestureRecognizerGraphOptions();
this.handLandmarkerGraphOptions = new HandLandmarkerGraphOptions();
this.options.setHandLandmarkerGraphOptions(this.handLandmarkerGraphOptions);
this.handLandmarksDetectorGraphOptions =
new HandLandmarksDetectorGraphOptions();
this.handLandmarkerGraphOptions.setHandLandmarksDetectorGraphOptions(
this.handLandmarksDetectorGraphOptions);
this.handDetectorGraphOptions = new HandDetectorGraphOptions();
this.handLandmarkerGraphOptions.setHandDetectorGraphOptions(
this.handDetectorGraphOptions);
this.handGestureRecognizerGraphOptions =
new HandGestureRecognizerGraphOptions();
this.options.setHandGestureRecognizerGraphOptions(
this.handGestureRecognizerGraphOptions);
this.initDefaults();
// Disables the automatic render-to-screen code, which allows for pure
// CPU processing.
this.setAutoRenderToScreen(false);
}
/**
* Sets new options for the gesture recognizer.
*
* Calling `setOptions()` with a subset of options only affects those options.
* You can reset an option back to its default value by explicitly setting it
* to `undefined`.
*
* @param options The options for the gesture recognizer.
*/
async setOptions(options: GestureRecognizerOptions): Promise<void> {
if (options.baseOptions) {
const baseOptionsProto =
await convertBaseOptionsToProto(options.baseOptions);
this.options.setBaseOptions(baseOptionsProto);
}
if ('numHands' in options) {
this.handDetectorGraphOptions.setNumHands(
options.numHands ?? DEFAULT_NUM_HANDS);
}
if ('minHandDetectionConfidence' in options) {
this.handDetectorGraphOptions.setMinDetectionConfidence(
options.minHandDetectionConfidence ?? DEFAULT_SCORE_THRESHOLD);
}
if ('minHandPresenceConfidence' in options) {
this.handLandmarksDetectorGraphOptions.setMinDetectionConfidence(
options.minHandPresenceConfidence ?? DEFAULT_SCORE_THRESHOLD);
}
if ('minTrackingConfidence' in options) {
this.handLandmarkerGraphOptions.setMinTrackingConfidence(
options.minTrackingConfidence ?? DEFAULT_SCORE_THRESHOLD);
}
if (options.cannedGesturesClassifierOptions) {
// Note that we have to support both JSPB and ProtobufJS and cannot
// use JSPB's getMutableX() APIs.
const graphOptions = new GestureClassifierGraphOptions();
graphOptions.setClassifierOptions(convertClassifierOptionsToProto(
options.cannedGesturesClassifierOptions,
this.handGestureRecognizerGraphOptions
.getCannedGestureClassifierGraphOptions()
?.getClassifierOptions()));
this.handGestureRecognizerGraphOptions
.setCannedGestureClassifierGraphOptions(graphOptions);
} else if (options.cannedGesturesClassifierOptions === undefined) {
this.handGestureRecognizerGraphOptions
.getCannedGestureClassifierGraphOptions()
?.clearClassifierOptions();
}
if (options.customGesturesClassifierOptions) {
const graphOptions = new GestureClassifierGraphOptions();
graphOptions.setClassifierOptions(convertClassifierOptionsToProto(
options.customGesturesClassifierOptions,
this.handGestureRecognizerGraphOptions
.getCustomGestureClassifierGraphOptions()
?.getClassifierOptions()));
this.handGestureRecognizerGraphOptions
.setCustomGestureClassifierGraphOptions(graphOptions);
} else if (options.customGesturesClassifierOptions === undefined) {
this.handGestureRecognizerGraphOptions
.getCustomGestureClassifierGraphOptions()
?.clearClassifierOptions();
}
this.refreshGraph();
}
/**
* Performs gesture recognition on the provided single image and waits
* synchronously for the response.
* @param imageSource An image source to process.
* @param timestamp The timestamp of the current frame, in ms. If not
* provided, defaults to `performance.now()`.
* @return The detected gestures.
*/
recognize(imageSource: ImageSource, timestamp: number = performance.now()):
GestureRecognitionResult {
this.gestures = [];
this.landmarks = [];
this.worldLandmarks = [];
this.handednesses = [];
this.addGpuBufferAsImageToStream(imageSource, IMAGE_STREAM, timestamp);
this.addProtoToStream(
FULL_IMAGE_RECT.serializeBinary(), 'mediapipe.NormalizedRect',
NORM_RECT_STREAM, timestamp);
this.finishProcessing();
return {
gestures: this.gestures,
landmarks: this.landmarks,
worldLandmarks: this.worldLandmarks,
handednesses: this.handednesses
};
}
/** Sets the default values for the graph. */
private initDefaults(): void {
this.handDetectorGraphOptions.setNumHands(DEFAULT_NUM_HANDS);
this.handDetectorGraphOptions.setMinDetectionConfidence(
DEFAULT_SCORE_THRESHOLD);
this.handLandmarksDetectorGraphOptions.setMinDetectionConfidence(
DEFAULT_SCORE_THRESHOLD);
this.handLandmarkerGraphOptions.setMinTrackingConfidence(
DEFAULT_SCORE_THRESHOLD);
}
/** Converts the proto data to a Category[][] structure. */
private toJsCategories(data: Uint8Array[]): Category[][] {
const result: Category[][] = [];
for (const binaryProto of data) {
const inputList = ClassificationList.deserializeBinary(binaryProto);
const outputList: Category[] = [];
for (const classification of inputList.getClassificationList()) {
outputList.push({
score: classification.getScore() ?? 0,
index: classification.getIndex() ?? DEFAULT_CATEGORY_INDEX,
categoryName: classification.getLabel() ?? '',
displayName: classification.getDisplayName() ?? '',
});
}
result.push(outputList);
}
return result;
}
/** Converts raw data into a landmark, and adds it to our landmarks list. */
private addJsLandmarks(data: Uint8Array[]): void {
for (const binaryProto of data) {
const handLandmarksProto =
NormalizedLandmarkList.deserializeBinary(binaryProto);
const landmarks: Landmark[] = [];
for (const handLandmarkProto of handLandmarksProto.getLandmarkList()) {
landmarks.push({
x: handLandmarkProto.getX() ?? 0,
y: handLandmarkProto.getY() ?? 0,
z: handLandmarkProto.getZ() ?? 0,
normalized: true
});
}
this.landmarks.push(landmarks);
}
}
/**
* Converts raw data into a landmark, and adds it to our worldLandmarks
* list.
*/
private adddJsWorldLandmarks(data: Uint8Array[]): void {
for (const binaryProto of data) {
const handWorldLandmarksProto =
LandmarkList.deserializeBinary(binaryProto);
const worldLandmarks: Landmark[] = [];
for (const handWorldLandmarkProto of
handWorldLandmarksProto.getLandmarkList()) {
worldLandmarks.push({
x: handWorldLandmarkProto.getX() ?? 0,
y: handWorldLandmarkProto.getY() ?? 0,
z: handWorldLandmarkProto.getZ() ?? 0,
normalized: false
});
}
this.worldLandmarks.push(worldLandmarks);
}
}
/** Updates the MediaPipe graph configuration. */
private refreshGraph(): void {
const graphConfig = new CalculatorGraphConfig();
graphConfig.addInputStream(IMAGE_STREAM);
graphConfig.addInputStream(NORM_RECT_STREAM);
graphConfig.addOutputStream(HAND_GESTURES_STREAM);
graphConfig.addOutputStream(LANDMARKS_STREAM);
graphConfig.addOutputStream(WORLD_LANDMARKS_STREAM);
graphConfig.addOutputStream(HANDEDNESS_STREAM);
const calculatorOptions = new CalculatorOptions();
calculatorOptions.setExtension(
GestureRecognizerGraphOptions.ext, this.options);
const recognizerNode = new CalculatorGraphConfig.Node();
recognizerNode.setCalculator(GESTURE_RECOGNIZER_GRAPH);
recognizerNode.addInputStream('IMAGE:' + IMAGE_STREAM);
recognizerNode.addInputStream('NORM_RECT:' + NORM_RECT_STREAM);
recognizerNode.addOutputStream('HAND_GESTURES:' + HAND_GESTURES_STREAM);
recognizerNode.addOutputStream('LANDMARKS:' + LANDMARKS_STREAM);
recognizerNode.addOutputStream('WORLD_LANDMARKS:' + WORLD_LANDMARKS_STREAM);
recognizerNode.addOutputStream('HANDEDNESS:' + HANDEDNESS_STREAM);
recognizerNode.setOptions(calculatorOptions);
graphConfig.addNode(recognizerNode);
this.attachProtoVectorListener(LANDMARKS_STREAM, binaryProto => {
this.addJsLandmarks(binaryProto);
});
this.attachProtoVectorListener(WORLD_LANDMARKS_STREAM, binaryProto => {
this.adddJsWorldLandmarks(binaryProto);
});
this.attachProtoVectorListener(HAND_GESTURES_STREAM, binaryProto => {
this.gestures.push(...this.toJsCategories(binaryProto));
});
this.attachProtoVectorListener(HANDEDNESS_STREAM, binaryProto => {
this.handednesses.push(...this.toJsCategories(binaryProto));
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);
}
}

65
mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer_options.d.ts vendored Normal file
View File

@ -0,0 +1,65 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {BaseOptions} from '../../../../tasks/web/core/base_options';
import {ClassifierOptions} from '../../../../tasks/web/core/classifier_options';
/** Options to configure the MediaPipe Gesture Recognizer Task */
export interface GestureRecognizerOptions {
/** Options to configure the loading of the model assets. */
baseOptions?: BaseOptions;
/**
* The maximum number of hands can be detected by the GestureRecognizer.
* Defaults to 1.
*/
numHands?: number|undefined;
/**
* The minimum confidence score for the hand detection to be considered
* successful. Defaults to 0.5.
*/
minHandDetectionConfidence?: number|undefined;
/**
* The minimum confidence score of hand presence score in the hand landmark
* detection. Defaults to 0.5.
*/
minHandPresenceConfidence?: number|undefined;
/**
* The minimum confidence score for the hand tracking to be considered
* successful. Defaults to 0.5.
*/
minTrackingConfidence?: number|undefined;
/**
* Sets the optional `ClassifierOptions` controling the canned gestures
* classifier, such as score threshold, allow list and deny list of gestures.
* The categories for canned gesture
* classifiers are: ["None", "Closed_Fist", "Open_Palm", "Pointing_Up",
* "Thumb_Down", "Thumb_Up", "Victory", "ILoveYou"]
*/
// TODO: Note this option is subject to change
cannedGesturesClassifierOptions?: ClassifierOptions|undefined;
/**
* Options for configuring the custom gestures classifier, such as score
* threshold, allow list and deny list of gestures.
*/
// TODO b/251816640): Note this option is subject to change.
customGesturesClassifierOptions?: ClassifierOptions|undefined;
}

35
mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer_result.d.ts vendored Normal file
View File

@ -0,0 +1,35 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {Category} from '../../../../tasks/web/components/containers/category';
import {Landmark} from '../../../../tasks/web/components/containers/landmark';
/**
* Represents the gesture recognition results generated by `GestureRecognizer`.
*/
export interface GestureRecognitionResult {
/** Hand landmarks of detected hands. */
landmarks: Landmark[][];
/** Hand landmarks in world coordniates of detected hands. */
worldLandmarks: Landmark[][];
/** Handedness of detected hands. */
handednesses: Category[][];
/** Recognized hand gestures of detected hands */
gestures: Category[][];
}

33
mediapipe/tasks/web/vision/image_classifier/BUILD Normal file
View File

@ -0,0 +1,33 @@
# This contains the MediaPipe Image Classifier Task.
#
# This task takes video or image frames and outputs the classification result.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
mediapipe_ts_library(
name = "image_classifier",
srcs = [
"image_classifier.ts",
"image_classifier_options.ts",
"image_classifier_result.ts",
],
deps = [
"//mediapipe/framework:calculator_jspb_proto",
"//mediapipe/framework:calculator_options_jspb_proto",
"//mediapipe/tasks/cc/components/containers/proto:classifications_jspb_proto",
"//mediapipe/tasks/cc/vision/image_classifier/proto:image_classifier_graph_options_jspb_proto",
"//mediapipe/tasks/web/components/containers:category",
"//mediapipe/tasks/web/components/containers:classifications",
"//mediapipe/tasks/web/components/processors:base_options",
"//mediapipe/tasks/web/components/processors:classifier_options",
"//mediapipe/tasks/web/components/processors:classifier_result",
"//mediapipe/tasks/web/core",
"//mediapipe/tasks/web/core:classifier_options",
"//mediapipe/tasks/web/core:task_runner",
"//mediapipe/web/graph_runner:wasm_mediapipe_lib_ts",
],
)

186
mediapipe/tasks/web/vision/image_classifier/image_classifier.ts Normal file
View File

@ -0,0 +1,186 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {CalculatorGraphConfig} from '../../../../framework/calculator_pb';
import {CalculatorOptions} from '../../../../framework/calculator_options_pb';
import {ClassificationResult} from '../../../../tasks/cc/components/containers/proto/classifications_pb';
import {ImageClassifierGraphOptions} from '../../../../tasks/cc/vision/image_classifier/proto/image_classifier_graph_options_pb';
import {convertBaseOptionsToProto} from '../../../../tasks/web/components/processors/base_options';
import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options';
import {convertFromClassificationResultProto} from '../../../../tasks/web/components/processors/classifier_result';
import {TaskRunner} from '../../../../tasks/web/core/task_runner';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {createMediaPipeLib, FileLocator, ImageSource} from '../../../../web/graph_runner/wasm_mediapipe_lib';
// Placeholder for internal dependency on trusted resource url
import {ImageClassifierOptions} from './image_classifier_options';
import {Classifications} from './image_classifier_result';
const IMAGE_CLASSIFIER_GRAPH =
'mediapipe.tasks.vision.image_classifier.ImageClassifierGraph';
const INPUT_STREAM = 'input_image';
const CLASSIFICATION_RESULT_STREAM = 'classification_result';
export {ImageSource}; // Used in the public API
// The OSS JS API does not support the builder pattern.
// tslint:disable:jspb-use-builder-pattern
/** Performs classification on images. */
export class ImageClassifier extends TaskRunner {
private classifications: Classifications[] = [];
private readonly options = new ImageClassifierGraphOptions();
/**
* Initializes the Wasm runtime and creates a new image classifier from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param imageClassifierOptions The options for the image classifier. Note
* that either a path to the model asset or a model buffer needs to be
* provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
imageClassifierOptions: ImageClassifierOptions):
Promise<ImageClassifier> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const classifier = await createMediaPipeLib(
ImageClassifier, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
await classifier.setOptions(imageClassifierOptions);
return classifier;
}
/**
* Initializes the Wasm runtime and creates a new image classifier based on
* the provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
modelAssetBuffer: Uint8Array): Promise<ImageClassifier> {
return ImageClassifier.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new image classifier based on
* the path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param modelAssetPath The path to the model asset.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
modelAssetPath: string): Promise<ImageClassifier> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return ImageClassifier.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
}
/**
* Sets new options for the image classifier.
*
* Calling `setOptions()` with a subset of options only affects those options.
* You can reset an option back to its default value by explicitly setting it
* to `undefined`.
*
* @param options The options for the image classifier.
*/
async setOptions(options: ImageClassifierOptions): Promise<void> {
if (options.baseOptions) {
const baseOptionsProto =
await convertBaseOptionsToProto(options.baseOptions);
this.options.setBaseOptions(baseOptionsProto);
}
this.options.setClassifierOptions(convertClassifierOptionsToProto(
options, this.options.getClassifierOptions()));
this.refreshGraph();
}
/**
* Performs image classification on the provided image and waits synchronously
* for the response.
*
* @param imageSource An image source to process.
* @param timestamp The timestamp of the current frame, in ms. If not
* provided, defaults to `performance.now()`.
* @return The classification result of the image
*/
classify(imageSource: ImageSource, timestamp?: number): Classifications[] {
// Get classification classes by running our MediaPipe graph.
this.classifications = [];
this.addGpuBufferAsImageToStream(
imageSource, INPUT_STREAM, timestamp ?? performance.now());
this.finishProcessing();
return [...this.classifications];
}
/**
* Internal function for converting raw data into a classification, and
* adding it to our classfications list.
**/
private addJsImageClassification(binaryProto: Uint8Array): void {
const classificationResult =
ClassificationResult.deserializeBinary(binaryProto);
this.classifications.push(
...convertFromClassificationResultProto(classificationResult));
}
/** Updates the MediaPipe graph configuration. */
private refreshGraph(): void {
const graphConfig = new CalculatorGraphConfig();
graphConfig.addInputStream(INPUT_STREAM);
graphConfig.addOutputStream(CLASSIFICATION_RESULT_STREAM);
const calculatorOptions = new CalculatorOptions();
calculatorOptions.setExtension(
ImageClassifierGraphOptions.ext, this.options);
// Perform image classification. Pre-processing and results post-processing
// are built-in.
const classifierNode = new CalculatorGraphConfig.Node();
classifierNode.setCalculator(IMAGE_CLASSIFIER_GRAPH);
classifierNode.addInputStream('IMAGE:' + INPUT_STREAM);
classifierNode.addOutputStream(
'CLASSIFICATION_RESULT:' + CLASSIFICATION_RESULT_STREAM);
classifierNode.setOptions(calculatorOptions);
graphConfig.addNode(classifierNode);
this.attachProtoListener(CLASSIFICATION_RESULT_STREAM, binaryProto => {
this.addJsImageClassification(binaryProto);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);
}
}

17
mediapipe/tasks/web/vision/image_classifier/image_classifier_options.ts Normal file
View File

@ -0,0 +1,17 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {ClassifierOptions as ImageClassifierOptions} from '../../../../tasks/web/core/classifier_options';

18
mediapipe/tasks/web/vision/image_classifier/image_classifier_result.ts Normal file
View File

@ -0,0 +1,18 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export {Category} from '../../../../tasks/web/components/containers/category';
export {ClassificationEntry, Classifications} from '../../../../tasks/web/components/containers/classifications';

30
mediapipe/tasks/web/vision/object_detector/BUILD Normal file
View File

@ -0,0 +1,30 @@
# This contains the MediaPipe Object Detector Task.
#
# This task takes video frames and outputs synchronized frames along with
# the detection results for one or more object categories, using Object Detector.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
package(default_visibility = ["//mediapipe/tasks:internal"])
licenses(["notice"])
mediapipe_ts_library(
name = "object_detector",
srcs = [
"object_detector.ts",
"object_detector_options.d.ts",
"object_detector_result.d.ts",
],
deps = [
"//mediapipe/framework:calculator_jspb_proto",
"//mediapipe/framework:calculator_options_jspb_proto",
"//mediapipe/framework/formats:detection_jspb_proto",
"//mediapipe/tasks/cc/vision/object_detector/proto:object_detector_options_jspb_proto",
"//mediapipe/tasks/web/components/containers:category",
"//mediapipe/tasks/web/components/processors:base_options",
"//mediapipe/tasks/web/core",
"//mediapipe/tasks/web/core:task_runner",
"//mediapipe/web/graph_runner:wasm_mediapipe_lib_ts",
],
)

233
mediapipe/tasks/web/vision/object_detector/object_detector.ts Normal file
View File

@ -0,0 +1,233 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {CalculatorGraphConfig} from '../../../../framework/calculator_pb';
import {CalculatorOptions} from '../../../../framework/calculator_options_pb';
import {Detection as DetectionProto} from '../../../../framework/formats/detection_pb';
import {ObjectDetectorOptions as ObjectDetectorOptionsProto} from '../../../../tasks/cc/vision/object_detector/proto/object_detector_options_pb';
import {convertBaseOptionsToProto} from '../../../../tasks/web/components/processors/base_options';
import {TaskRunner} from '../../../../tasks/web/core/task_runner';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {createMediaPipeLib, FileLocator, ImageSource} from '../../../../web/graph_runner/wasm_mediapipe_lib';
// Placeholder for internal dependency on trusted resource url
import {ObjectDetectorOptions} from './object_detector_options';
import {Detection} from './object_detector_result';
const INPUT_STREAM = 'input_frame_gpu';
const DETECTIONS_STREAM = 'detections';
const OBJECT_DETECTOR_GRAPH = 'mediapipe.tasks.vision.ObjectDetectorGraph';
const DEFAULT_CATEGORY_INDEX = -1;
export {ImageSource}; // Used in the public API
// The OSS JS API does not support the builder pattern.
// tslint:disable:jspb-use-builder-pattern
/** Performs object detection on images. */
export class ObjectDetector extends TaskRunner {
private detections: Detection[] = [];
private readonly options = new ObjectDetectorOptionsProto();
/**
* Initializes the Wasm runtime and creates a new object detector from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param objectDetectorOptions The options for the Object Detector. Note that
* either a path to the model asset or a model buffer needs to be
* provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
objectDetectorOptions: ObjectDetectorOptions): Promise<ObjectDetector> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const detector = await createMediaPipeLib(
ObjectDetector, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
await detector.setOptions(objectDetectorOptions);
return detector;
}
/**
* Initializes the Wasm runtime and creates a new object detector based on the
* provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
modelAssetBuffer: Uint8Array): Promise<ObjectDetector> {
return ObjectDetector.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new object detector based on the
* path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param modelAssetPath The path to the model asset.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
modelAssetPath: string): Promise<ObjectDetector> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return ObjectDetector.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
}
/**
* Sets new options for the object detector.
*
* Calling `setOptions()` with a subset of options only affects those options.
* You can reset an option back to its default value by explicitly setting it
* to `undefined`.
*
* @param options The options for the object detector.
*/
async setOptions(options: ObjectDetectorOptions): Promise<void> {
if (options.baseOptions) {
const baseOptionsProto =
await convertBaseOptionsToProto(options.baseOptions);
this.options.setBaseOptions(baseOptionsProto);
}
// Note that we have to support both JSPB and ProtobufJS, hence we
// have to expliclity clear the values instead of setting them to
// `undefined`.
if (options.displayNamesLocale !== undefined) {
this.options.setDisplayNamesLocale(options.displayNamesLocale);
} else if ('displayNamesLocale' in options) { // Check for undefined
this.options.clearDisplayNamesLocale();
}
if (options.maxResults !== undefined) {
this.options.setMaxResults(options.maxResults);
} else if ('maxResults' in options) { // Check for undefined
this.options.clearMaxResults();
}
if (options.scoreThreshold !== undefined) {
this.options.setScoreThreshold(options.scoreThreshold);
} else if ('scoreThreshold' in options) { // Check for undefined
this.options.clearScoreThreshold();
}
if (options.categoryAllowlist !== undefined) {
this.options.setCategoryAllowlistList(options.categoryAllowlist);
} else if ('categoryAllowlist' in options) { // Check for undefined
this.options.clearCategoryAllowlistList();
}
if (options.categoryDenylist !== undefined) {
this.options.setCategoryDenylistList(options.categoryDenylist);
} else if ('categoryDenylist' in options) { // Check for undefined
this.options.clearCategoryDenylistList();
}
this.refreshGraph();
}
/**
* Performs object detection on the provided single image and waits
* synchronously for the response.
* @param imageSource An image source to process.
* @param timestamp The timestamp of the current frame, in ms. If not
* provided, defaults to `performance.now()`.
* @return The list of detected objects
*/
detect(imageSource: ImageSource, timestamp?: number): Detection[] {
// Get detections by running our MediaPipe graph.
this.detections = [];
this.addGpuBufferAsImageToStream(
imageSource, INPUT_STREAM, timestamp ?? performance.now());
this.finishProcessing();
return [...this.detections];
}
/** Converts raw data into a Detection, and adds it to our detection list. */
private addJsObjectDetections(data: Uint8Array[]): void {
for (const binaryProto of data) {
const detectionProto = DetectionProto.deserializeBinary(binaryProto);
const scores = detectionProto.getScoreList();
const indexes = detectionProto.getLabelIdList();
const labels = detectionProto.getLabelList();
const displayNames = detectionProto.getDisplayNameList();
const detection: Detection = {categories: []};
for (let i = 0; i < scores.length; i++) {
detection.categories.push({
score: scores[i],
index: indexes[i] ?? DEFAULT_CATEGORY_INDEX,
categoryName: labels[i] ?? '',
displayName: displayNames[i] ?? '',
});
}
const boundingBox = detectionProto.getLocationData()?.getBoundingBox();
if (boundingBox) {
detection.boundingBox = {
originX: boundingBox.getXmin() ?? 0,
originY: boundingBox.getYmin() ?? 0,
width: boundingBox.getWidth() ?? 0,
height: boundingBox.getHeight() ?? 0
};
}
this.detections.push(detection);
}
}
/** Updates the MediaPipe graph configuration. */
private refreshGraph(): void {
const graphConfig = new CalculatorGraphConfig();
graphConfig.addInputStream(INPUT_STREAM);
graphConfig.addOutputStream(DETECTIONS_STREAM);
const calculatorOptions = new CalculatorOptions();
calculatorOptions.setExtension(
ObjectDetectorOptionsProto.ext, this.options);
const detectorNode = new CalculatorGraphConfig.Node();
detectorNode.setCalculator(OBJECT_DETECTOR_GRAPH);
detectorNode.addInputStream('IMAGE:' + INPUT_STREAM);
detectorNode.addOutputStream('DETECTIONS:' + DETECTIONS_STREAM);
detectorNode.setOptions(calculatorOptions);
graphConfig.addNode(detectorNode);
this.attachProtoVectorListener(DETECTIONS_STREAM, binaryProto => {
this.addJsObjectDetections(binaryProto);
});
const binaryGraph = graphConfig.serializeBinary();
this.setGraph(new Uint8Array(binaryGraph), /* isBinary= */ true);
}
}

52
mediapipe/tasks/web/vision/object_detector/object_detector_options.d.ts vendored Normal file
View File

@ -0,0 +1,52 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {BaseOptions} from '../../../../tasks/web/core/base_options';
/** Options to configure the MediaPipe Object Detector Task */
export interface ObjectDetectorOptions {
/** Options to configure the loading of the model assets. */
baseOptions?: BaseOptions;
/**
* The locale to use for display names specified through the TFLite Model
* Metadata, if any. Defaults to English.
*/
displayNamesLocale?: string|undefined;
/** The maximum number of top-scored detection results to return. */
maxResults?: number|undefined;
/**
* Overrides the value provided in the model metadata. Results below this
* value are rejected.
*/
scoreThreshold?: number|undefined;
/**
* Allowlist of category names. If non-empty, detection results whose category
* name is not in this set will be filtered out. Duplicate or unknown category
* names are ignored. Mutually exclusive with `categoryDenylist`.
*/
categoryAllowlist?: string[]|undefined;
/**
* Denylist of category names. If non-empty, detection results whose category
* name is in this set will be filtered out. Duplicate or unknown category
* names are ignored. Mutually exclusive with `categoryAllowlist`.
*/
categoryDenylist?: string[]|undefined;
}

38
mediapipe/tasks/web/vision/object_detector/object_detector_result.d.ts vendored Normal file
View File

@ -0,0 +1,38 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {Category} from '../../../../tasks/web/components/containers/category';
/** An integer bounding box, axis aligned. */
export interface BoundingBox {
/** The X coordinate of the top-left corner, in pixels. */
originX: number;
/** The Y coordinate of the top-left corner, in pixels. */
originY: number;
/** The width of the bounding box, in pixels. */
width: number;
/** The height of the bounding box, in pixels. */
height: number;
}
/** Represents one object detected by the `ObjectDetector`. */
export interface Detection {
/** A list of `Category` objects. */
categories: Category[];
/** The bounding box of the detected objects. */
boundingBox?: BoundingBox;
}

41
mediapipe/web/graph_runner/BUILD Normal file
View File

@ -0,0 +1,41 @@
# The TypeScript graph runner used by all MediaPipe Web tasks.
load("//mediapipe/framework/port:build_config.bzl", "mediapipe_ts_library")
package(default_visibility = [
":internal",
"//mediapipe/tasks:internal",
])
package_group(
name = "internal",
packages = [
"//mediapipe/app/pursuit/wasm/web_ml_cpu/typescript/...",
],
)
mediapipe_ts_library(
name = "wasm_mediapipe_lib_ts",
srcs = [
":wasm_mediapipe_lib.ts",
],
allow_unoptimized_namespaces = True,
)
mediapipe_ts_library(
name = "wasm_mediapipe_image_lib_ts",
srcs = [
":wasm_mediapipe_image_lib.ts",
],
allow_unoptimized_namespaces = True,
deps = [":wasm_mediapipe_lib_ts"],
)
mediapipe_ts_library(
name = "register_model_resources_graph_service_ts",
srcs = [
":register_model_resources_graph_service.ts",
],
allow_unoptimized_namespaces = True,
deps = [":wasm_mediapipe_lib_ts"],
)

41
mediapipe/web/graph_runner/register_model_resources_graph_service.ts Normal file
View File

@ -0,0 +1,41 @@
import {WasmMediaPipeLib} from './wasm_mediapipe_lib';
/**
* We extend from a WasmMediaPipeLib constructor. This ensures our mixin has
* access to the wasmModule, among other things. The `any` type is required for
* mixin constructors.
*/
// tslint:disable-next-line:no-any
type LibConstructor = new (...args: any[]) => WasmMediaPipeLib;
/**
* Declarations for Emscripten's WebAssembly Module behavior, so TS compiler
* doesn't break our JS/C++ bridge.
*/
export declare interface WasmModuleRegisterModelResources {
_registerModelResourcesGraphService: () => void;
}
/**
* An implementation of WasmMediaPipeLib that supports registering model
* resources to a cache, in the form of a GraphService C++-side. We implement as
* a proper TS mixin, to allow for effective multiple inheritance. Sample usage:
* `const WasmMediaPipeImageLib = SupportModelResourcesGraphService(
* WasmMediaPipeLib);`
*/
// tslint:disable:enforce-name-casing
export function SupportModelResourcesGraphService<TBase extends LibConstructor>(
Base: TBase) {
return class extends Base {
// tslint:enable:enforce-name-casing
/**
* Instructs the graph runner to use the model resource caching graph
* service for both graph expansion/inintialization, as well as for graph
* run.
*/
registerModelResourcesGraphService(): void {
(this.wasmModule as unknown as WasmModuleRegisterModelResources)
._registerModelResourcesGraphService();
}
};
}

52
mediapipe/web/graph_runner/wasm_mediapipe_image_lib.ts Normal file
View File

@ -0,0 +1,52 @@
import {ImageSource, WasmMediaPipeLib} from './wasm_mediapipe_lib';
/**
* We extend from a WasmMediaPipeLib constructor. This ensures our mixin has
* access to the wasmModule, among other things. The `any` type is required for
* mixin constructors.
*/
// tslint:disable-next-line:no-any
type LibConstructor = new (...args: any[]) => WasmMediaPipeLib;
/**
* Declarations for Emscripten's WebAssembly Module behavior, so TS compiler
* doesn't break our JS/C++ bridge.
*/
export declare interface WasmImageModule {
_addBoundTextureAsImageToStream:
(streamNamePtr: number, width: number, height: number,
timestamp: number) => void;
}
/**
* An implementation of WasmMediaPipeLib that supports binding GPU image data as
* `mediapipe::Image` instances. We implement as a proper TS mixin, to allow for
* effective multiple inheritance. Example usage:
* `const WasmMediaPipeImageLib = SupportImage(WasmMediaPipeLib);`
*/
// tslint:disable-next-line:enforce-name-casing
export function SupportImage<TBase extends LibConstructor>(Base: TBase) {
return class extends Base {
/**
* Takes the relevant information from the HTML video or image element, and
* passes it into the WebGL-based graph for processing on the given stream
* at the given timestamp as a MediaPipe image. Processing will not occur
* until a blocking call (like processVideoGl or finishProcessing) is made.
* @param imageSource Reference to the video frame we wish to add into our
* graph.
* @param streamName The name of the MediaPipe graph stream to add the frame
* to.
* @param timestamp The timestamp of the input frame, in ms.
*/
addGpuBufferAsImageToStream(
imageSource: ImageSource, streamName: string, timestamp: number): void {
this.wrapStringPtr(streamName, (streamNamePtr: number) => {
const [width, height] =
this.bindTextureToStream(imageSource, streamNamePtr);
(this.wasmModule as unknown as WasmImageModule)
._addBoundTextureAsImageToStream(
streamNamePtr, width, height, timestamp);
});
}
};
}

1044
mediapipe/web/graph_runner/wasm_mediapipe_lib.ts Normal file
View File

File diff suppressed because it is too large Load Diff