cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for browsers without SIMD

Browse Source 
PiperOrigin-RevId: 491371277
This commit is contained in:
Sebastian Schmidt 2022-11-28 09:52:40 -08:00 committed by Copybara-Service
parent 395d9d8ea2
commit 153edc59a1
28 changed files with 410 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
mediapipe/tasks/web/BUILD
View File

@ -13,10 +13,16 @@ package(default_visibility = ["//mediapipe/tasks:internal"])
mediapipe_files(srcs = [
"wasm/audio_wasm_internal.js",
"wasm/audio_wasm_internal.wasm",
"wasm/audio_wasm_nosimd_internal.js",
"wasm/audio_wasm_nosimd_internal.wasm",
"wasm/text_wasm_internal.js",
"wasm/text_wasm_internal.wasm",
"wasm/text_wasm_nosimd_internal.js",
"wasm/text_wasm_nosimd_internal.wasm",
"wasm/vision_wasm_internal.js",
"wasm/vision_wasm_internal.wasm",
"wasm/vision_wasm_nosimd_internal.js",
"wasm/vision_wasm_nosimd_internal.wasm",
])
# Audio
@ -57,6 +63,8 @@ pkg_npm(
deps = [
"wasm/audio_wasm_internal.js",
"wasm/audio_wasm_internal.wasm",
"wasm/audio_wasm_nosimd_internal.js",
"wasm/audio_wasm_nosimd_internal.wasm",
":audio_bundle",
],
)
@ -99,6 +107,8 @@ pkg_npm(
deps = [
"wasm/text_wasm_internal.js",
"wasm/text_wasm_internal.wasm",
"wasm/text_wasm_nosimd_internal.js",
"wasm/text_wasm_nosimd_internal.wasm",
":text_bundle",
],
)
@ -141,6 +151,8 @@ pkg_npm(
deps = [
"wasm/vision_wasm_internal.js",
"wasm/vision_wasm_internal.wasm",
"wasm/vision_wasm_nosimd_internal.js",
"wasm/vision_wasm_nosimd_internal.wasm",
":vision_bundle",
],
)

5
mediapipe/tasks/web/audio.ts
View File

@ -14,11 +14,12 @@
* limitations under the License.
*/
import {AudioClassifier as AudioClassifierImpl, AudioEmbedder as AudioEmbedderImpl} from '../../tasks/web/audio/index';
import {AudioClassifier as AudioClassifierImpl, AudioEmbedder as AudioEmbedderImpl, FilesetResolver as FilesetResolverImpl} from '../../tasks/web/audio/index';
// Declare the variables locally so that Rollup in OSS includes them explcilty
// as exports.
const AudioClassifier = AudioClassifierImpl;
const AudioEmbedder = AudioEmbedderImpl;
const FilesetResolver = FilesetResolverImpl;
export {AudioClassifier, AudioEmbedder};
export {AudioClassifier, AudioEmbedder, FilesetResolver};

1
mediapipe/tasks/web/audio/BUILD
View File

@ -10,5 +10,6 @@ mediapipe_ts_library(
deps = [
"//mediapipe/tasks/web/audio/audio_classifier",
"//mediapipe/tasks/web/audio/audio_embedder",
"//mediapipe/tasks/web/core:fileset_resolver",
],
)

2
mediapipe/tasks/web/audio/audio_classifier/BUILD
View File

@ -25,7 +25,7 @@ mediapipe_ts_library(
"//mediapipe/tasks/web/components/processors:classifier_result",
"//mediapipe/tasks/web/core",
"//mediapipe/tasks/web/core:classifier_options",
"//mediapipe/web/graph_runner:graph_runner_ts",
"//mediapipe/tasks/web/core:task_runner",
],
)

41
mediapipe/tasks/web/audio/audio_classifier/audio_classifier.ts
View File

@ -22,8 +22,8 @@ import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/b
import {AudioTaskRunner} from '../../../../tasks/web/audio/core/audio_task_runner';
import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options';
import {convertFromClassificationResultProto} from '../../../../tasks/web/components/processors/classifier_result';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {createMediaPipeLib, FileLocator} from '../../../../web/graph_runner/graph_runner';
import {TaskRunner} from '../../../../tasks/web/core/task_runner';
import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
// Placeholder for internal dependency on trusted resource url
import {AudioClassifierOptions} from './audio_classifier_options';
@ -50,28 +50,17 @@ export class AudioClassifier extends AudioTaskRunner<AudioClassifierResult[]> {
/**
* Initializes the Wasm runtime and creates a new audio classifier from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param audioClassifierOptions The options for the audio classifier. Note
* that either a path to the model asset or a model buffer needs to be
* provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
audioClassifierOptions: AudioClassifierOptions):
wasmFileset: WasmFileset, audioClassifierOptions: AudioClassifierOptions):
Promise<AudioClassifier> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file loaded with this mechanism is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const classifier = await createMediaPipeLib(
AudioClassifier, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
const classifier = await TaskRunner.createInstance(
AudioClassifier, /* initializeCanvas= */ false, wasmFileset);
await classifier.setOptions(audioClassifierOptions);
return classifier;
}
@ -79,31 +68,31 @@ export class AudioClassifier extends AudioTaskRunner<AudioClassifierResult[]> {
/**
* Initializes the Wasm runtime and creates a new audio classifier based on
* the provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetBuffer: Uint8Array): Promise<AudioClassifier> {
return AudioClassifier.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
wasmFileset, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new audio classifier based on
* the path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetPath The path to the model asset.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetPath: string): Promise<AudioClassifier> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return AudioClassifier.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
wasmFileset, new Uint8Array(graphData));
}
protected override get baseOptions(): BaseOptionsProto|undefined {

28
mediapipe/tasks/web/audio/audio_embedder/audio_embedder.ts
View File

@ -24,7 +24,7 @@ import {Embedding} from '../../../../tasks/web/components/containers/embedding_r
import {convertEmbedderOptionsToProto} from '../../../../tasks/web/components/processors/embedder_options';
import {convertFromEmbeddingResultProto} from '../../../../tasks/web/components/processors/embedder_result';
import {computeCosineSimilarity} from '../../../../tasks/web/components/utils/cosine_similarity';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
import {createMediaPipeLib, FileLocator} from '../../../../web/graph_runner/graph_runner';
// Placeholder for internal dependency on trusted resource url
@ -52,25 +52,25 @@ export class AudioEmbedder extends AudioTaskRunner<AudioEmbedderResult[]> {
/**
* Initializes the Wasm runtime and creates a new audio embedder from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param audioEmbedderOptions The options for the audio embedder. Note that
* either a path to the TFLite model or the model itself needs to be
* provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
audioEmbedderOptions: AudioEmbedderOptions): Promise<AudioEmbedder> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
return wasmFileset.wasmBinaryPath.toString();
}
};
const embedder = await createMediaPipeLib(
AudioEmbedder, wasmLoaderOptions.wasmLoaderPath,
AudioEmbedder, wasmFileset.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
await embedder.setOptions(audioEmbedderOptions);
@ -80,31 +80,31 @@ export class AudioEmbedder extends AudioTaskRunner<AudioEmbedderResult[]> {
/**
* Initializes the Wasm runtime and creates a new audio embedder based on the
* provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the TFLite model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetBuffer: Uint8Array): Promise<AudioEmbedder> {
return AudioEmbedder.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
wasmFileset, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new audio embedder based on the
* path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetPath The path to the TFLite model.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetPath: string): Promise<AudioEmbedder> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return AudioEmbedder.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
wasmFileset, new Uint8Array(graphData));
}
protected override get baseOptions(): BaseOptionsProto|undefined {

1
mediapipe/tasks/web/audio/index.ts
View File

@ -16,3 +16,4 @@
export * from '../../../tasks/web/audio/audio_classifier/audio_classifier';
export * from '../../../tasks/web/audio/audio_embedder/audio_embedder';
export * from '../../../tasks/web/core/fileset_resolver';

9
mediapipe/tasks/web/core/BUILD
View File

@ -8,7 +8,7 @@ mediapipe_ts_declaration(
name = "core",
srcs = [
"base_options.d.ts",
"wasm_loader_options.d.ts",
"wasm_fileset.d.ts",
],
)
@ -18,12 +18,19 @@ mediapipe_ts_library(
"task_runner.ts",
],
deps = [
":core",
"//mediapipe/web/graph_runner:graph_runner_image_lib_ts",
"//mediapipe/web/graph_runner:graph_runner_ts",
"//mediapipe/web/graph_runner:register_model_resources_graph_service_ts",
],
)
mediapipe_ts_library(
name = "fileset_resolver",
srcs = ["fileset_resolver.ts"],
deps = [":core"],
)
mediapipe_ts_declaration(
name = "classifier_options",
srcs = ["classifier_options.d.ts"],

130
mediapipe/tasks/web/core/fileset_resolver.ts Normal file
View File

@ -0,0 +1,130 @@
/**
* Copyright 2022 The MediaPipe Authors. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// Placeholder for internal dependency on trusted resource URL builder
import {WasmFileset} from './wasm_fileset';
let supportsSimd: boolean|undefined;
/**
* Simple WASM program to test compatibility with the M91 instruction set.
* Compiled from
* https://github.com/GoogleChromeLabs/wasm-feature-detect/blob/main/src/detectors/simd/module.wat
*/
const WASM_SIMD_CHECK = new Uint8Array([
0, 97, 115, 109, 1, 0, 0, 0, 1, 5, 1, 96, 0, 1, 123, 3,
2, 1, 0, 10, 10, 1, 8, 0, 65, 0, 253, 15, 253, 98, 11
]);
async function isSimdSupported(): Promise<boolean> {
if (supportsSimd === undefined) {
try {
await WebAssembly.instantiate(WASM_SIMD_CHECK);
supportsSimd = true;
} catch {
supportsSimd = false;
}
}
return supportsSimd;
}
async function createFileset(
taskName: string, basePath: string = '.'): Promise<WasmFileset> {
if (await isSimdSupported()) {
return {
wasmLoaderPath:
`/${basePath}/${taskName}_wasm_internal.js`,
wasmBinaryPath:
`/${basePath}/${taskName}_wasm_internal.wasm`,
};
} else {
return {
wasmLoaderPath:
`/${basePath}/${taskName}_wasm_nosimd_internal.js`,
wasmBinaryPath: `/${basePath}/${
taskName}_wasm_nosimd_internal.wasm`,
};
}
}
// tslint:disable:class-as-namespace
/**
* Resolves the files required for the MediaPipe Task APIs.
*
* This class verifies whether SIMD is supported in the current environment and
* loads the SIMD files only if support is detected. The returned filesets
* require that the Wasm files are published without renaming. If this is not
* possible, you can invoke the MediaPipe Tasks APIs using a manually created
* `WasmFileset`.
*/
export class FilesetResolver {
/**
* Returns whether SIMD is supported in the current environment.
*
* If your environment requires custom locations for the MediaPipe Wasm files,
* you can use `isSimdSupported()` to decide whether to load the SIMD-based
* assets.
*
* @return Whether SIMD support was detected in the current environment.
*/
static isSimdSupported(): Promise<boolean> {
return isSimdSupported();
}
/**
* Creates a fileset for the MediaPipe Audio tasks.
*
* @param basePath An optional base path to specify the directory the Wasm
* files should be loaded from. If not specified, the Wasm files are
* loaded from the host's root directory.
* @return A `WasmFileset` that can be used to initialize MediaPipe Audio
* tasks.
*/
static forAudioTasks(basePath?: string): Promise<WasmFileset> {
return createFileset('audio', basePath);
}
/**
* Creates a fileset for the MediaPipe Text tasks.
*
* @param basePath An optional base path to specify the directory the Wasm
* files should be loaded from. If not specified, the Wasm files are
* loaded from the host's root directory.
* @return A `WasmFileset` that can be used to initialize MediaPipe Text
* tasks.
*/
static forTextTasks(basePath?: string): Promise<WasmFileset> {
return createFileset('text', basePath);
}
/**
* Creates a fileset for the MediaPipe Vision tasks.
*
* @param basePath An optional base path to specify the directory the Wasm
* files should be loaded from. If not specified, the Wasm files are
* loaded from the host's root directory.
* @return A `WasmFileset` that can be used to initialize MediaPipe Vision
* tasks.
*/
static forVisionTasks(basePath?: string): Promise<WasmFileset> {
return createFileset('vision', basePath);
}
}

45
mediapipe/tasks/web/core/task_runner.ts
View File

@ -14,9 +14,14 @@
* limitations under the License.
*/
import {SupportModelResourcesGraphService} from '../../../web/graph_runner/register_model_resources_graph_service';
import {createMediaPipeLib, FileLocator, GraphRunner, WasmMediaPipeConstructor, WasmModule} from '../../../web/graph_runner/graph_runner';
import {SupportImage} from '../../../web/graph_runner/graph_runner_image_lib';
import {GraphRunner, WasmModule} from '../../../web/graph_runner/graph_runner';
import {SupportModelResourcesGraphService} from '../../../web/graph_runner/register_model_resources_graph_service';
import {WasmFileset} from './wasm_fileset';
// None of the MP Tasks ship bundle assets.
const NO_ASSETS = undefined;
// tslint:disable-next-line:enforce-name-casing
const WasmMediaPipeImageLib =
@ -26,8 +31,40 @@ const WasmMediaPipeImageLib =
export abstract class TaskRunner extends WasmMediaPipeImageLib {
private processingErrors: Error[] = [];
constructor(wasmModule: WasmModule) {
super(wasmModule);
/**
* Creates a new instance of a Mediapipe Task. Determines if SIMD is
* supported and loads the relevant WASM binary.
* @return A fully instantiated instance of `T`.
*/
protected static async createInstance<T extends TaskRunner>(
type: WasmMediaPipeConstructor<T>, initializeCanvas: boolean,
fileset: WasmFileset): Promise<T> {
const fileLocator: FileLocator = {
locateFile() {
// The only file loaded with this mechanism is the Wasm binary
return fileset.wasmBinaryPath.toString();
}
};
if (initializeCanvas) {
// Fall back to an OffscreenCanvas created by the GraphRunner if
// OffscreenCanvas is available
const canvas = typeof OffscreenCanvas === 'undefined' ?
document.createElement('canvas') :
undefined;
return createMediaPipeLib(
type, fileset.wasmLoaderPath, NO_ASSETS, canvas, fileLocator);
} else {
return createMediaPipeLib(
type, fileset.wasmLoaderPath, NO_ASSETS, /* glCanvas= */ null,
fileLocator);
}
}
constructor(
wasmModule: WasmModule,
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null) {
super(wasmModule, glCanvas);
// Disables the automatic render-to-screen code, which allows for pure
// CPU processing.

4
mediapipe/tasks/web/core/wasm_loader_options.d.ts → mediapipe/tasks/web/core/wasm_fileset.d.ts vendored
View File

@ -16,8 +16,8 @@
// Placeholder for internal dependency on trusted resource url
/** An object containing the locations of all Wasm assets */
export declare interface WasmLoaderOptions {
/** An object containing the locations of the Wasm assets */
export declare interface WasmFileset {
/** The path to the Wasm loader script. */
wasmLoaderPath: string;
/** The path to the Wasm binary. */

5
mediapipe/tasks/web/text.ts
View File

@ -14,11 +14,12 @@
* limitations under the License.
*/
import {TextClassifier as TextClassifierImpl, TextEmbedder as TextEmbedderImpl} from '../../tasks/web/text/index';
import {FilesetResolver as FilesetResolverImpl, TextClassifier as TextClassifierImpl, TextEmbedder as TextEmbedderImpl} from '../../tasks/web/text/index';
// Declare the variables locally so that Rollup in OSS includes them explcilty
// as exports.
const FilesetResolver = FilesetResolverImpl;
const TextClassifier = TextClassifierImpl;
const TextEmbedder = TextEmbedderImpl;
export {TextClassifier, TextEmbedder};
export {FilesetResolver, TextClassifier, TextEmbedder};

1
mediapipe/tasks/web/text/BUILD
View File

@ -8,6 +8,7 @@ mediapipe_ts_library(
name = "text_lib",
srcs = ["index.ts"],
deps = [
"//mediapipe/tasks/web/core:fileset_resolver",
"//mediapipe/tasks/web/text/text_classifier",
"//mediapipe/tasks/web/text/text_embedder",
],

1
mediapipe/tasks/web/text/index.ts
View File

@ -16,3 +16,4 @@
export * from '../../../tasks/web/text/text_classifier/text_classifier';
export * from '../../../tasks/web/text/text_embedder/text_embedder';
export * from '../../../tasks/web/core/fileset_resolver';

1
mediapipe/tasks/web/text/text_classifier/BUILD
View File

@ -26,7 +26,6 @@ mediapipe_ts_library(
"//mediapipe/tasks/web/core",
"//mediapipe/tasks/web/core:classifier_options",
"//mediapipe/tasks/web/core:task_runner",
"//mediapipe/web/graph_runner:graph_runner_ts",
],
)

39
mediapipe/tasks/web/text/text_classifier/text_classifier.ts
View File

@ -22,8 +22,7 @@ import {convertBaseOptionsToProto} from '../../../../tasks/web/components/proces
import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options';
import {convertFromClassificationResultProto} from '../../../../tasks/web/components/processors/classifier_result';
import {TaskRunner} from '../../../../tasks/web/core/task_runner';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {createMediaPipeLib, FileLocator} from '../../../../web/graph_runner/graph_runner';
import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
// Placeholder for internal dependency on trusted resource url
import {TextClassifierOptions} from './text_classifier_options';
@ -48,27 +47,17 @@ export class TextClassifier extends TaskRunner {
/**
* Initializes the Wasm runtime and creates a new text classifier from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param textClassifierOptions The options for the text classifier. Note that
* either a path to the TFLite model or the model itself needs to be
* provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
textClassifierOptions: TextClassifierOptions): Promise<TextClassifier> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const classifier = await createMediaPipeLib(
TextClassifier, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
const classifier = await TaskRunner.createInstance(
TextClassifier, /* initializeCanvas= */ false, wasmFileset);
await classifier.setOptions(textClassifierOptions);
return classifier;
}
@ -76,31 +65,31 @@ export class TextClassifier extends TaskRunner {
/**
* Initializes the Wasm runtime and creates a new text classifier based on the
* provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetBuffer: Uint8Array): Promise<TextClassifier> {
return TextClassifier.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
wasmFileset, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new text classifier based on the
* path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetPath The path to the model asset.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetPath: string): Promise<TextClassifier> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return TextClassifier.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
wasmFileset, new Uint8Array(graphData));
}
/**

1
mediapipe/tasks/web/text/text_embedder/BUILD
View File

@ -26,7 +26,6 @@ mediapipe_ts_library(
"//mediapipe/tasks/web/core",
"//mediapipe/tasks/web/core:embedder_options",
"//mediapipe/tasks/web/core:task_runner",
"//mediapipe/web/graph_runner:graph_runner_ts",
],
)

42
mediapipe/tasks/web/text/text_embedder/text_embedder.ts
View File

@ -24,8 +24,7 @@ import {convertEmbedderOptionsToProto} from '../../../../tasks/web/components/pr
import {convertFromEmbeddingResultProto} from '../../../../tasks/web/components/processors/embedder_result';
import {computeCosineSimilarity} from '../../../../tasks/web/components/utils/cosine_similarity';
import {TaskRunner} from '../../../../tasks/web/core/task_runner';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {createMediaPipeLib, FileLocator} from '../../../../web/graph_runner/graph_runner';
import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
// Placeholder for internal dependency on trusted resource url
import {TextEmbedderOptions} from './text_embedder_options';
@ -52,27 +51,17 @@ export class TextEmbedder extends TaskRunner {
/**
* Initializes the Wasm runtime and creates a new text embedder from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param textEmbedderOptions The options for the text embedder. Note that
* either a path to the TFLite model or the model itself needs to be
* provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
textEmbedderOptions: TextEmbedderOptions): Promise<TextEmbedder> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const embedder = await createMediaPipeLib(
TextEmbedder, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
const embedder = await TaskRunner.createInstance(
TextEmbedder, /* initializeCanvas= */ false, wasmFileset);
await embedder.setOptions(textEmbedderOptions);
return embedder;
}
@ -80,31 +69,31 @@ export class TextEmbedder extends TaskRunner {
/**
* Initializes the Wasm runtime and creates a new text embedder based on the
* provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the TFLite model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetBuffer: Uint8Array): Promise<TextEmbedder> {
return TextEmbedder.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
wasmFileset, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new text embedder based on the
* path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetPath The path to the TFLite model.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetPath: string): Promise<TextEmbedder> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return TextEmbedder.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
wasmFileset, new Uint8Array(graphData));
}
/**
@ -122,14 +111,11 @@ export class TextEmbedder extends TaskRunner {
options.baseOptions, this.options.getBaseOptions());
this.options.setBaseOptions(baseOptionsProto);
}
this.options.setEmbedderOptions(convertEmbedderOptionsToProto(
options, this.options.getEmbedderOptions()));
this.refreshGraph();
}
/**
* Performs embeding extraction on the provided text and waits synchronously
* for the response.

4
mediapipe/tasks/web/vision.ts
View File

@ -14,10 +14,11 @@
* limitations under the License.
*/
import {GestureRecognizer as GestureRecognizerImpl, HandLandmarker as HandLandmarkerImpl, ImageClassifier as ImageClassifierImpl, ImageEmbedder as ImageEmbedderImpl, ObjectDetector as ObjectDetectorImpl} from '../../tasks/web/vision/index';
import {FilesetResolver as FilesetResolverImpl, GestureRecognizer as GestureRecognizerImpl, HandLandmarker as HandLandmarkerImpl, ImageClassifier as ImageClassifierImpl, ImageEmbedder as ImageEmbedderImpl, ObjectDetector as ObjectDetectorImpl} from '../../tasks/web/vision/index';
// Declare the variables locally so that Rollup in OSS includes them explcilty
// as exports.
const FilesetResolver = FilesetResolverImpl;
const GestureRecognizer = GestureRecognizerImpl;
const HandLandmarker = HandLandmarkerImpl;
const ImageClassifier = ImageClassifierImpl;
@ -25,6 +26,7 @@ const ImageEmbedder = ImageEmbedderImpl;
const ObjectDetector = ObjectDetectorImpl;
export {
FilesetResolver,
GestureRecognizer,
HandLandmarker,
ImageClassifier,

1
mediapipe/tasks/web/vision/BUILD
View File

@ -8,6 +8,7 @@ mediapipe_ts_library(
name = "vision_lib",
srcs = ["index.ts"],
deps = [
"//mediapipe/tasks/web/core:fileset_resolver",
"//mediapipe/tasks/web/vision/gesture_recognizer",
"//mediapipe/tasks/web/vision/hand_landmarker",
"//mediapipe/tasks/web/vision/image_classifier",

46
mediapipe/tasks/web/vision/gesture_recognizer/gesture_recognizer.ts
View File

@ -29,9 +29,9 @@ import {HandLandmarksDetectorGraphOptions} from '../../../../tasks/cc/vision/han
import {Category} from '../../../../tasks/web/components/containers/category';
import {Landmark} from '../../../../tasks/web/components/containers/landmark';
import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner';
import {createMediaPipeLib, FileLocator, ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner';
import {ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner';
// Placeholder for internal dependency on trusted resource url
import {GestureRecognizerOptions} from './gesture_recognizer_options';
@ -82,28 +82,18 @@ export class GestureRecognizer extends
/**
* Initializes the Wasm runtime and creates a new gesture recognizer from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param gestureRecognizerOptions The options for the gesture recognizer.
* Note that either a path to the model asset or a model buffer needs to
* be provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
gestureRecognizerOptions: GestureRecognizerOptions):
Promise<GestureRecognizer> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load via this mechanism is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const recognizer = await createMediaPipeLib(
GestureRecognizer, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
const recognizer = await VisionTaskRunner.createInstance(
GestureRecognizer, /* initializeCanvas= */ true, wasmFileset);
await recognizer.setOptions(gestureRecognizerOptions);
return recognizer;
}
@ -111,35 +101,37 @@ export class GestureRecognizer extends
/**
* Initializes the Wasm runtime and creates a new gesture recognizer based on
* the provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetBuffer: Uint8Array): Promise<GestureRecognizer> {
return GestureRecognizer.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
wasmFileset, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new gesture recognizer based on
* the path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetPath The path to the model asset.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetPath: string): Promise<GestureRecognizer> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return GestureRecognizer.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
wasmFileset, new Uint8Array(graphData));
}
constructor(wasmModule: WasmModule) {
super(wasmModule);
constructor(
wasmModule: WasmModule,
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null) {
super(wasmModule, glCanvas);
this.options = new GestureRecognizerGraphOptions();
this.handLandmarkerGraphOptions = new HandLandmarkerGraphOptions();

46
mediapipe/tasks/web/vision/hand_landmarker/hand_landmarker.ts
View File

@ -25,9 +25,9 @@ import {HandLandmarkerGraphOptions} from '../../../../tasks/cc/vision/hand_landm
import {HandLandmarksDetectorGraphOptions} from '../../../../tasks/cc/vision/hand_landmarker/proto/hand_landmarks_detector_graph_options_pb';
import {Category} from '../../../../tasks/web/components/containers/category';
import {Landmark} from '../../../../tasks/web/components/containers/landmark';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner';
import {createMediaPipeLib, FileLocator, ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner';
import {ImageSource, WasmModule} from '../../../../web/graph_runner/graph_runner';
// Placeholder for internal dependency on trusted resource url
import {HandLandmarkerOptions} from './hand_landmarker_options';
@ -71,27 +71,17 @@ export class HandLandmarker extends VisionTaskRunner<HandLandmarkerResult> {
/**
* Initializes the Wasm runtime and creates a new `HandLandmarker` from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param handLandmarkerOptions The options for the HandLandmarker.
* Note that either a path to the model asset or a model buffer needs to
* be provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
handLandmarkerOptions: HandLandmarkerOptions): Promise<HandLandmarker> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load via this mechanism is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const landmarker = await createMediaPipeLib(
HandLandmarker, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
const landmarker = await VisionTaskRunner.createInstance(
HandLandmarker, /* initializeCanvas= */ true, wasmFileset);
await landmarker.setOptions(handLandmarkerOptions);
return landmarker;
}
@ -99,35 +89,37 @@ export class HandLandmarker extends VisionTaskRunner<HandLandmarkerResult> {
/**
* Initializes the Wasm runtime and creates a new `HandLandmarker` based on
* the provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetBuffer: Uint8Array): Promise<HandLandmarker> {
return HandLandmarker.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
wasmFileset, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new `HandLandmarker` based on
* the path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetPath The path to the model asset.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetPath: string): Promise<HandLandmarker> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return HandLandmarker.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
wasmFileset, new Uint8Array(graphData));
}
constructor(wasmModule: WasmModule) {
super(wasmModule);
constructor(
wasmModule: WasmModule,
glCanvas?: HTMLCanvasElement|OffscreenCanvas|null) {
super(wasmModule, glCanvas);
this.options = new HandLandmarkerGraphOptions();
this.handLandmarksDetectorGraphOptions =

41
mediapipe/tasks/web/vision/image_classifier/image_classifier.ts
View File

@ -21,9 +21,9 @@ import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/b
import {ImageClassifierGraphOptions} from '../../../../tasks/cc/vision/image_classifier/proto/image_classifier_graph_options_pb';
import {convertClassifierOptionsToProto} from '../../../../tasks/web/components/processors/classifier_options';
import {convertFromClassificationResultProto} from '../../../../tasks/web/components/processors/classifier_result';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner';
import {createMediaPipeLib, FileLocator, ImageSource} from '../../../../web/graph_runner/graph_runner';
import {ImageSource} from '../../../../web/graph_runner/graph_runner';
// Placeholder for internal dependency on trusted resource url
import {ImageClassifierOptions} from './image_classifier_options';
@ -49,28 +49,17 @@ export class ImageClassifier extends VisionTaskRunner<ImageClassifierResult> {
/**
* Initializes the Wasm runtime and creates a new image classifier from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location
* Wasm binary and its loader.
* @param imageClassifierOptions The options for the image classifier. Note
* that either a path to the model asset or a model buffer needs to be
* provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
imageClassifierOptions: ImageClassifierOptions):
wasmFileset: WasmFileset, imageClassifierOptions: ImageClassifierOptions):
Promise<ImageClassifier> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const classifier = await createMediaPipeLib(
ImageClassifier, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
const classifier = await VisionTaskRunner.createInstance(
ImageClassifier, /* initializeCanvas= */ true, wasmFileset);
await classifier.setOptions(imageClassifierOptions);
return classifier;
}
@ -78,31 +67,31 @@ export class ImageClassifier extends VisionTaskRunner<ImageClassifierResult> {
/**
* Initializes the Wasm runtime and creates a new image classifier based on
* the provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetBuffer: Uint8Array): Promise<ImageClassifier> {
return ImageClassifier.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
wasmFileset, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new image classifier based on
* the path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetPath The path to the model asset.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetPath: string): Promise<ImageClassifier> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return ImageClassifier.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
wasmFileset, new Uint8Array(graphData));
}
protected override get baseOptions(): BaseOptionsProto|undefined {

40
mediapipe/tasks/web/vision/image_embedder/image_embedder.ts
View File

@ -23,9 +23,9 @@ import {Embedding} from '../../../../tasks/web/components/containers/embedding_r
import {convertEmbedderOptionsToProto} from '../../../../tasks/web/components/processors/embedder_options';
import {convertFromEmbeddingResultProto} from '../../../../tasks/web/components/processors/embedder_result';
import {computeCosineSimilarity} from '../../../../tasks/web/components/utils/cosine_similarity';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner';
import {createMediaPipeLib, FileLocator, ImageSource} from '../../../../web/graph_runner/graph_runner';
import {ImageSource} from '../../../../web/graph_runner/graph_runner';
// Placeholder for internal dependency on trusted resource url
import {ImageEmbedderOptions} from './image_embedder_options';
@ -51,27 +51,17 @@ export class ImageEmbedder extends VisionTaskRunner<ImageEmbedderResult> {
/**
* Initializes the Wasm runtime and creates a new image embedder from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param imageEmbedderOptions The options for the image embedder. Note that
* either a path to the TFLite model or the model itself needs to be
* provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
imageEmbedderOptions: ImageEmbedderOptions): Promise<ImageEmbedder> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const embedder = await createMediaPipeLib(
ImageEmbedder, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
const embedder = await VisionTaskRunner.createInstance(
ImageEmbedder, /* initializeCanvas= */ true, wasmFileset);
await embedder.setOptions(imageEmbedderOptions);
return embedder;
}
@ -79,31 +69,31 @@ export class ImageEmbedder extends VisionTaskRunner<ImageEmbedderResult> {
/**
* Initializes the Wasm runtime and creates a new image embedder based on the
* provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the TFLite model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetBuffer: Uint8Array): Promise<ImageEmbedder> {
return ImageEmbedder.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
wasmFileset, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new image embedder based on the
* path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetPath The path to the TFLite model.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetPath: string): Promise<ImageEmbedder> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return ImageEmbedder.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
wasmFileset, new Uint8Array(graphData));
}
protected override get baseOptions(): BaseOptionsProto|undefined {

1
mediapipe/tasks/web/vision/index.ts
View File

@ -19,3 +19,4 @@ export * from '../../../tasks/web/vision/image_embedder/image_embedder';
export * from '../../../tasks/web/vision/gesture_recognizer/gesture_recognizer';
export * from '../../../tasks/web/vision/hand_landmarker/hand_landmarker';
export * from '../../../tasks/web/vision/object_detector/object_detector';
export * from '../../../tasks/web/core/fileset_resolver';

40
mediapipe/tasks/web/vision/object_detector/object_detector.ts
View File

@ -19,9 +19,9 @@ import {CalculatorOptions} from '../../../../framework/calculator_options_pb';
import {Detection as DetectionProto} from '../../../../framework/formats/detection_pb';
import {BaseOptions as BaseOptionsProto} from '../../../../tasks/cc/core/proto/base_options_pb';
import {ObjectDetectorOptions as ObjectDetectorOptionsProto} from '../../../../tasks/cc/vision/object_detector/proto/object_detector_options_pb';
import {WasmLoaderOptions} from '../../../../tasks/web/core/wasm_loader_options';
import {WasmFileset} from '../../../../tasks/web/core/wasm_fileset';
import {VisionTaskRunner} from '../../../../tasks/web/vision/core/vision_task_runner';
import {createMediaPipeLib, FileLocator, ImageSource} from '../../../../web/graph_runner/graph_runner';
import {ImageSource} from '../../../../web/graph_runner/graph_runner';
// Placeholder for internal dependency on trusted resource url
import {ObjectDetectorOptions} from './object_detector_options';
@ -48,27 +48,17 @@ export class ObjectDetector extends VisionTaskRunner<Detection[]> {
/**
* Initializes the Wasm runtime and creates a new object detector from the
* provided options.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param objectDetectorOptions The options for the Object Detector. Note that
* either a path to the model asset or a model buffer needs to be
* provided (via `baseOptions`).
*/
static async createFromOptions(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
objectDetectorOptions: ObjectDetectorOptions): Promise<ObjectDetector> {
// Create a file locator based on the loader options
const fileLocator: FileLocator = {
locateFile() {
// The only file we load is the Wasm binary
return wasmLoaderOptions.wasmBinaryPath.toString();
}
};
const detector = await createMediaPipeLib(
ObjectDetector, wasmLoaderOptions.wasmLoaderPath,
/* assetLoaderScript= */ undefined,
/* glCanvas= */ undefined, fileLocator);
const detector = await VisionTaskRunner.createInstance(
ObjectDetector, /* initializeCanvas= */ true, wasmFileset);
await detector.setOptions(objectDetectorOptions);
return detector;
}
@ -76,31 +66,31 @@ export class ObjectDetector extends VisionTaskRunner<Detection[]> {
/**
* Initializes the Wasm runtime and creates a new object detector based on the
* provided model asset buffer.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetBuffer A binary representation of the model.
*/
static createFromModelBuffer(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetBuffer: Uint8Array): Promise<ObjectDetector> {
return ObjectDetector.createFromOptions(
wasmLoaderOptions, {baseOptions: {modelAssetBuffer}});
wasmFileset, {baseOptions: {modelAssetBuffer}});
}
/**
* Initializes the Wasm runtime and creates a new object detector based on the
* path to the model asset.
* @param wasmLoaderOptions A configuration object that provides the location
* of the Wasm binary and its loader.
* @param wasmFileset A configuration object that provides the location of the
* Wasm binary and its loader.
* @param modelAssetPath The path to the model asset.
*/
static async createFromModelPath(
wasmLoaderOptions: WasmLoaderOptions,
wasmFileset: WasmFileset,
modelAssetPath: string): Promise<ObjectDetector> {
const response = await fetch(modelAssetPath.toString());
const graphData = await response.arrayBuffer();
return ObjectDetector.createFromModelBuffer(
wasmLoaderOptions, new Uint8Array(graphData));
wasmFileset, new Uint8Array(graphData));
}
protected override get baseOptions(): BaseOptionsProto|undefined {

8
mediapipe/web/graph_runner/graph_runner.ts
View File

@ -133,9 +133,11 @@ export type ImageSource =
/** A listener that will be invoked with an absl::StatusCode and message. */
export type ErrorListener = (code: number, message: string) => void;
// Internal type of constructors used for initializing GraphRunner and
// subclasses.
type WasmMediaPipeConstructor<LibType> =
/**
* Internal type of constructors used for initializing GraphRunner and
* subclasses.
*/
export type WasmMediaPipeConstructor<LibType> =
(new (
module: WasmModule, canvas?: HTMLCanvasElement|OffscreenCanvas|null) =>
LibType);

76
third_party/wasm_files.bzl vendored
View File

@ -12,36 +12,72 @@ def wasm_files():
http_file(
name = "com_google_mediapipe_wasm_audio_wasm_internal_js",
sha256 = "9419766229f24790388805d891af907cf11fe8e2cdacabcf016feb054b720c82",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.js?generation=1667934266184984"],
)
http_file(
name = "com_google_mediapipe_wasm_text_wasm_internal_js",
sha256 = "39d9445ab3b90f625a3332251fe82e59b40cd0501a5657475f3b115b7c6122c8",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.js?generation=1667934268229056"],
)
http_file(
name = "com_google_mediapipe_wasm_vision_wasm_internal_js",
sha256 = "b43c7078fe5da72990394af4fefd798bd844b4ac47849a49067bd68c3c910a3d",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.js?generation=1667934270239845"],
sha256 = "42d2d0ade6e2e8b81425b23686be93eb1423b7777f043eb8f18ad671e2ca803f",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.js?generation=1669173769507080"],
)
http_file(
name = "com_google_mediapipe_wasm_audio_wasm_internal_wasm",
sha256 = "9f2abe2a51d1ebc854859f620759cec1cc643773f3748d0d19e0868578c3d746",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.wasm?generation=1667934272818542"],
sha256 = "20200ee9b0866d5176f633a9b375e8a44e53204c01ea2e159e2f9245afb00e80",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_internal.wasm?generation=1669173772528997"],
)
http_file(
name = "com_google_mediapipe_wasm_audio_wasm_nosimd_internal_js",
sha256 = "11bbf73d48723b19a5a6a13ec296ecdb2aa178cdc3db9d7bc54265a7d4b94c6a",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.js?generation=1669173774625527"],
)
http_file(
name = "com_google_mediapipe_wasm_audio_wasm_nosimd_internal_wasm",
sha256 = "d4528972219033996a83a62798952b6ee8b6b396bcffd96fd5bda5458d57d3a3",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/audio_wasm_nosimd_internal.wasm?generation=1669173777474822"],
)
http_file(
name = "com_google_mediapipe_wasm_text_wasm_internal_js",
sha256 = "29e72e177122f92bda6a3ecd463ebacf30b920559b06c97068112a22eeea4d0e",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.js?generation=1669173779706893"],
)
http_file(
name = "com_google_mediapipe_wasm_text_wasm_internal_wasm",
sha256 = "8334caec5fb10cd1f936f6ee41f8853771c7bf3a421f5c15c39ee41aa503ca54",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.wasm?generation=1667934275451198"],
sha256 = "84e5f5ac70f7718baeaa09a89b155abbea67386e7d50663301b3af7ef0941e74",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_internal.wasm?generation=1669173782728605"],
)
http_file(
name = "com_google_mediapipe_wasm_text_wasm_nosimd_internal_js",
sha256 = "36f247673124e32535f217265b96508c1badee8fe2458c11c1efa95b6bec5daa",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.js?generation=1669173785027190"],
)
http_file(
name = "com_google_mediapipe_wasm_text_wasm_nosimd_internal_wasm",
sha256 = "cc74d90a8aaf6d006ec24048cc80c33f96baeeb0075a6c6739f30d41da54e450",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/text_wasm_nosimd_internal.wasm?generation=1669173787903754"],
)
http_file(
name = "com_google_mediapipe_wasm_vision_wasm_internal_js",
sha256 = "c3451423186766b08008e07ef6d52f628fcc0aca75beedd9bb4d87d380f29edd",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.js?generation=1669173790070986"],
)
http_file(
name = "com_google_mediapipe_wasm_vision_wasm_internal_wasm",
sha256 = "b996eaa324da151359ad8e16edad27d9768505f1fd073625bc50dbb0f252e098",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.wasm?generation=1667934277855507"],
sha256 = "d1e8ad748913e3f190bfd3f72e0e8a4a308f78b918d54c79cec60a2cf30a49f0",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_internal.wasm?generation=1669173792993881"],
)
http_file(
name = "com_google_mediapipe_wasm_vision_wasm_nosimd_internal_js",
sha256 = "e5f1b5e8264ff9a90371653cb0fdbf9ce3b30b712acbd72068af18ebca2293ac",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.js?generation=1669173794969702"],
)
http_file(
name = "com_google_mediapipe_wasm_vision_wasm_nosimd_internal_wasm",
sha256 = "24351fe580e88f2065b1978b8b3c0f3ad7b90f1c95805aafa07971ce422b5854",
urls = ["https://storage.googleapis.com/mediapipe-assets/wasm/vision_wasm_nosimd_internal.wasm?generation=1669173797596874"],
)