cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Shorten MPImage API

Browse Source 
PiperOrigin-RevId: 528039371
This commit is contained in:
Sebastian Schmidt 2023-04-28 18:58:26 -07:00 committed by Copybara-Service
parent b1f93b3b27
commit e15add2475
5 changed files with 105 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

80
mediapipe/tasks/web/vision/core/image.test.ts
View File

@ -16,7 +16,7 @@
import 'jasmine'; import 'jasmine';
import {MPImage, MPImageShaderContext, MPImageStorageType} from './image'; import {MPImage, MPImageShaderContext, MPImageType} from './image';
const WIDTH = 2; const WIDTH = 2;
const HEIGHT = 2; const HEIGHT = 2;
@ -130,20 +130,20 @@ class MPImageTestContext {
function assertEquality(image: MPImage, expected: ImageType): void { function assertEquality(image: MPImage, expected: ImageType): void {
if (expected instanceof Uint8ClampedArray) { if (expected instanceof Uint8ClampedArray) {
const result = image.getImage(MPImageStorageType.UINT8_CLAMPED_ARRAY); const result = image.get(MPImageType.UINT8_CLAMPED_ARRAY);
expect(result).toEqual(expected); expect(result).toEqual(expected);
} else if (expected instanceof Float32Array) { } else if (expected instanceof Float32Array) {
const result = image.getImage(MPImageStorageType.FLOAT32_ARRAY); const result = image.get(MPImageType.FLOAT32_ARRAY);
expect(result).toEqual(expected); expect(result).toEqual(expected);
} else if (expected instanceof ImageData) { } else if (expected instanceof ImageData) {
const result = image.getImage(MPImageStorageType.IMAGE_DATA); const result = image.get(MPImageType.IMAGE_DATA);
expect(result).toEqual(expected); expect(result).toEqual(expected);
} else if (expected instanceof ImageBitmap) { } else if (expected instanceof ImageBitmap) {
const result = image.getImage(MPImageStorageType.IMAGE_BITMAP); const result = image.get(MPImageType.IMAGE_BITMAP);
expect(readPixelsFromImageBitmap(result)) expect(readPixelsFromImageBitmap(result))
.toEqual(readPixelsFromImageBitmap(expected)); .toEqual(readPixelsFromImageBitmap(expected));
} else { // WebGLTexture } else { // WebGLTexture
const result = image.getImage(MPImageStorageType.WEBGL_TEXTURE); const result = image.get(MPImageType.WEBGL_TEXTURE);
expect(readPixelsFromWebGLTexture(result)) expect(readPixelsFromWebGLTexture(result))
.toEqual(readPixelsFromWebGLTexture(expected)); .toEqual(readPixelsFromWebGLTexture(expected));
} }
@ -206,7 +206,7 @@ class MPImageTestContext {
/* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false, /* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false,
context.canvas, shaderContext, WIDTH, HEIGHT); context.canvas, shaderContext, WIDTH, HEIGHT);
const result = image.clone().getImage(MPImageStorageType.IMAGE_DATA); const result = image.clone().get(MPImageType.IMAGE_DATA);
expect(result).toEqual(context.imageData); expect(result).toEqual(context.imageData);
shaderContext.close(); shaderContext.close();
@ -223,61 +223,61 @@ class MPImageTestContext {
// Verify that we can mix the different shader modes by running them out of // Verify that we can mix the different shader modes by running them out of
// order. // order.
let result = image.getImage(MPImageStorageType.IMAGE_DATA); let result = image.get(MPImageType.IMAGE_DATA);
expect(result).toEqual(context.imageData); expect(result).toEqual(context.imageData);
result = image.clone().getImage(MPImageStorageType.IMAGE_DATA); result = image.clone().get(MPImageType.IMAGE_DATA);
expect(result).toEqual(context.imageData); expect(result).toEqual(context.imageData);
result = image.getImage(MPImageStorageType.IMAGE_DATA); result = image.get(MPImageType.IMAGE_DATA);
expect(result).toEqual(context.imageData); expect(result).toEqual(context.imageData);
shaderContext.close(); shaderContext.close();
}); });
it('supports hasType()', async () => { it('supports has()', async () => {
await context.init(); await context.init();
const shaderContext = new MPImageShaderContext(); const shaderContext = new MPImageShaderContext();
const image = createImage(shaderContext, context.imageData, WIDTH, HEIGHT); const image = createImage(shaderContext, context.imageData, WIDTH, HEIGHT);
expect(image.hasType(MPImageStorageType.IMAGE_DATA)).toBe(true); expect(image.has(MPImageType.IMAGE_DATA)).toBe(true);
expect(image.hasType(MPImageStorageType.UINT8_CLAMPED_ARRAY)).toBe(false); expect(image.has(MPImageType.UINT8_CLAMPED_ARRAY)).toBe(false);
expect(image.hasType(MPImageStorageType.FLOAT32_ARRAY)).toBe(false); expect(image.has(MPImageType.FLOAT32_ARRAY)).toBe(false);
expect(image.hasType(MPImageStorageType.WEBGL_TEXTURE)).toBe(false); expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(false);
expect(image.hasType(MPImageStorageType.IMAGE_BITMAP)).toBe(false); expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(false);
image.getImage(MPImageStorageType.UINT8_CLAMPED_ARRAY); image.get(MPImageType.UINT8_CLAMPED_ARRAY);
expect(image.hasType(MPImageStorageType.IMAGE_DATA)).toBe(true); expect(image.has(MPImageType.IMAGE_DATA)).toBe(true);
expect(image.hasType(MPImageStorageType.UINT8_CLAMPED_ARRAY)).toBe(true); expect(image.has(MPImageType.UINT8_CLAMPED_ARRAY)).toBe(true);
expect(image.hasType(MPImageStorageType.FLOAT32_ARRAY)).toBe(false); expect(image.has(MPImageType.FLOAT32_ARRAY)).toBe(false);
expect(image.hasType(MPImageStorageType.WEBGL_TEXTURE)).toBe(false); expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(false);
expect(image.hasType(MPImageStorageType.IMAGE_BITMAP)).toBe(false); expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(false);
image.getImage(MPImageStorageType.FLOAT32_ARRAY); image.get(MPImageType.FLOAT32_ARRAY);
expect(image.hasType(MPImageStorageType.IMAGE_DATA)).toBe(true); expect(image.has(MPImageType.IMAGE_DATA)).toBe(true);
expect(image.hasType(MPImageStorageType.UINT8_CLAMPED_ARRAY)).toBe(true); expect(image.has(MPImageType.UINT8_CLAMPED_ARRAY)).toBe(true);
expect(image.hasType(MPImageStorageType.FLOAT32_ARRAY)).toBe(true); expect(image.has(MPImageType.FLOAT32_ARRAY)).toBe(true);
expect(image.hasType(MPImageStorageType.WEBGL_TEXTURE)).toBe(false); expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(false);
expect(image.hasType(MPImageStorageType.IMAGE_BITMAP)).toBe(false); expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(false);
image.getImage(MPImageStorageType.WEBGL_TEXTURE); image.get(MPImageType.WEBGL_TEXTURE);
expect(image.hasType(MPImageStorageType.IMAGE_DATA)).toBe(true); expect(image.has(MPImageType.IMAGE_DATA)).toBe(true);
expect(image.hasType(MPImageStorageType.UINT8_CLAMPED_ARRAY)).toBe(true); expect(image.has(MPImageType.UINT8_CLAMPED_ARRAY)).toBe(true);
expect(image.hasType(MPImageStorageType.FLOAT32_ARRAY)).toBe(true); expect(image.has(MPImageType.FLOAT32_ARRAY)).toBe(true);
expect(image.hasType(MPImageStorageType.WEBGL_TEXTURE)).toBe(true); expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(true);
expect(image.hasType(MPImageStorageType.IMAGE_BITMAP)).toBe(false); expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(false);
image.getImage(MPImageStorageType.IMAGE_BITMAP); image.get(MPImageType.IMAGE_BITMAP);
expect(image.hasType(MPImageStorageType.IMAGE_DATA)).toBe(true); expect(image.has(MPImageType.IMAGE_DATA)).toBe(true);
expect(image.hasType(MPImageStorageType.UINT8_CLAMPED_ARRAY)).toBe(true); expect(image.has(MPImageType.UINT8_CLAMPED_ARRAY)).toBe(true);
expect(image.hasType(MPImageStorageType.FLOAT32_ARRAY)).toBe(true); expect(image.has(MPImageType.FLOAT32_ARRAY)).toBe(true);
expect(image.hasType(MPImageStorageType.WEBGL_TEXTURE)).toBe(true); expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(true);
expect(image.hasType(MPImageStorageType.IMAGE_BITMAP)).toBe(true); expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(true);
image.close(); image.close();
shaderContext.close(); shaderContext.close();

129
mediapipe/tasks/web/vision/core/image.ts
View File

@ -15,7 +15,7 @@
*/ */
/** The underlying type of the image. */ /** The underlying type of the image. */
export enum MPImageStorageType { export enum MPImageType {
/** Represents the native `UInt8ClampedArray` type. */ /** Represents the native `UInt8ClampedArray` type. */
UINT8_CLAMPED_ARRAY, UINT8_CLAMPED_ARRAY,
/** /**
@ -31,7 +31,7 @@ export enum MPImageStorageType {
} }
/** The supported image formats. For internal usage. */ /** The supported image formats. For internal usage. */
export type MPImageNativeContainer = export type MPImageContainer =
Uint8ClampedArray|Float32Array|ImageData|ImageBitmap|WebGLTexture; Uint8ClampedArray|Float32Array|ImageData|ImageBitmap|WebGLTexture;
const VERTEX_SHADER = ` const VERTEX_SHADER = `
@ -359,9 +359,9 @@ const DEFAULT_CONVERTER: Required<MPImageChannelConverter> = {
* *
* Images are stored as `ImageData`, `ImageBitmap` or `WebGLTexture` objects. * Images are stored as `ImageData`, `ImageBitmap` or `WebGLTexture` objects.
* You can convert the underlying type to any other type by passing the * You can convert the underlying type to any other type by passing the
* desired type to `getImage()`. As type conversions can be expensive, it is * desired type to `get()`. As type conversions can be expensive, it is
* recommended to limit these conversions. You can verify what underlying * recommended to limit these conversions. You can verify what underlying
* types are already available by invoking `hasType()`. * types are already available by invoking `has()`.
* *
* Images that are returned from a MediaPipe Tasks are owned by by the * Images that are returned from a MediaPipe Tasks are owned by by the
* underlying C++ Task. If you need to extend the lifetime of these objects, * underlying C++ Task. If you need to extend the lifetime of these objects,
@ -379,15 +379,18 @@ const DEFAULT_CONVERTER: Required<MPImageChannelConverter> = {
* single-channel arrays). To convert these type to other formats a conversion * single-channel arrays). To convert these type to other formats a conversion
* function is invoked to convert pixel values between single channel and four * function is invoked to convert pixel values between single channel and four
* channel RGBA values. To customize this conversion, you can specify these * channel RGBA values. To customize this conversion, you can specify these
* conversion functions when you invoke `getImage()`. If you use the default * conversion functions when you invoke `get()`. If you use the default
* conversion function a warning will be logged to the console. * conversion function a warning will be logged to the console.
*/ */
export class MPImage { export class MPImage {
private gl?: WebGL2RenderingContext; private gl?: WebGL2RenderingContext;
/** The underlying type of the image. */
static TYPE = MPImageType;
/** @hideconstructor */ /** @hideconstructor */
constructor( constructor(
private readonly containers: MPImageNativeContainer[], private readonly containers: MPImageContainer[],
private ownsImageBitmap: boolean, private ownsImageBitmap: boolean,
private ownsWebGLTexture: boolean, private ownsWebGLTexture: boolean,
/** Returns the canvas element that the image is bound to. */ /** Returns the canvas element that the image is bound to. */
@ -402,9 +405,9 @@ export class MPImage {
/** /**
* Returns whether this `MPImage` stores the image in the desired format. * Returns whether this `MPImage` stores the image in the desired format.
* This method can be called to reduce expensive conversion before invoking * This method can be called to reduce expensive conversion before invoking
* `getType()`. * `get()`.
*/ */
hasType(type: MPImageStorageType): boolean { has(type: MPImageType): boolean {
return !!this.getContainer(type); return !!this.getContainer(type);
} }
@ -421,8 +424,7 @@ export class MPImage {
* function if the requested conversion does not change the pixel format. * function if the requested conversion does not change the pixel format.
* @return The current data as a Uint8ClampedArray. * @return The current data as a Uint8ClampedArray.
*/ */
getImage( get(type: MPImageType.UINT8_CLAMPED_ARRAY,
type: MPImageStorageType.UINT8_CLAMPED_ARRAY,
converter?: MPImageChannelConverter): Uint8ClampedArray; converter?: MPImageChannelConverter): Uint8ClampedArray;
/** /**
* Returns the underlying image as a single channel `Float32Array`. Note * Returns the underlying image as a single channel `Float32Array`. Note
@ -437,8 +439,7 @@ export class MPImage {
* function if the requested conversion does not change the pixel format. * function if the requested conversion does not change the pixel format.
* @return The current image as a Float32Array. * @return The current image as a Float32Array.
*/ */
getImage( get(type: MPImageType.FLOAT32_ARRAY,
type: MPImageStorageType.FLOAT32_ARRAY,
converter?: MPImageChannelConverter): Float32Array; converter?: MPImageChannelConverter): Float32Array;
/** /**
* Returns the underlying image as an `ImageData` object. Note that this * Returns the underlying image as an `ImageData` object. Note that this
@ -449,8 +450,7 @@ export class MPImage {
* *
* @return The current image as an ImageData object. * @return The current image as an ImageData object.
*/ */
getImage( get(type: MPImageType.IMAGE_DATA,
type: MPImageStorageType.IMAGE_DATA,
converter?: MPImageChannelConverter): ImageData; converter?: MPImageChannelConverter): ImageData;
/** /**
* Returns the underlying image as an `ImageBitmap`. Note that * Returns the underlying image as an `ImageBitmap`. Note that
@ -470,8 +470,7 @@ export class MPImage {
* function if the requested conversion does not change the pixel format. * function if the requested conversion does not change the pixel format.
* @return The current image as an ImageBitmap object. * @return The current image as an ImageBitmap object.
*/ */
getImage( get(type: MPImageType.IMAGE_BITMAP,
type: MPImageStorageType.IMAGE_BITMAP,
converter?: MPImageChannelConverter): ImageBitmap; converter?: MPImageChannelConverter): ImageBitmap;
/** /**
* Returns the underlying image as a `WebGLTexture` object. Note that this * Returns the underlying image as a `WebGLTexture` object. Note that this
@ -485,22 +484,21 @@ export class MPImage {
* function if the requested conversion does not change the pixel format. * function if the requested conversion does not change the pixel format.
* @return The current image as a WebGLTexture. * @return The current image as a WebGLTexture.
*/ */
getImage( get(type: MPImageType.WEBGL_TEXTURE,
type: MPImageStorageType.WEBGL_TEXTURE,
converter?: MPImageChannelConverter): WebGLTexture; converter?: MPImageChannelConverter): WebGLTexture;
getImage(type?: MPImageStorageType, converter?: MPImageChannelConverter): get(type?: MPImageType,
MPImageNativeContainer { converter?: MPImageChannelConverter): MPImageContainer {
const internalConverter = {...DEFAULT_CONVERTER, ...converter}; const internalConverter = {...DEFAULT_CONVERTER, ...converter};
switch (type) { switch (type) {
case MPImageStorageType.UINT8_CLAMPED_ARRAY: case MPImageType.UINT8_CLAMPED_ARRAY:
return this.convertToUint8ClampedArray(internalConverter); return this.convertToUint8ClampedArray(internalConverter);
case MPImageStorageType.FLOAT32_ARRAY: case MPImageType.FLOAT32_ARRAY:
return this.convertToFloat32Array(internalConverter); return this.convertToFloat32Array(internalConverter);
case MPImageStorageType.IMAGE_DATA: case MPImageType.IMAGE_DATA:
return this.convertToImageData(internalConverter); return this.convertToImageData(internalConverter);
case MPImageStorageType.IMAGE_BITMAP: case MPImageType.IMAGE_BITMAP:
return this.convertToImageBitmap(internalConverter); return this.convertToImageBitmap(internalConverter);
case MPImageStorageType.WEBGL_TEXTURE: case MPImageType.WEBGL_TEXTURE:
return this.convertToWebGLTexture(internalConverter); return this.convertToWebGLTexture(internalConverter);
default: default:
throw new Error(`Type is not supported: ${type}`); throw new Error(`Type is not supported: ${type}`);
@ -508,31 +506,25 @@ export class MPImage {
} }
private getContainer(type: MPImageStorageType.UINT8_CLAMPED_ARRAY): private getContainer(type: MPImageType.UINT8_CLAMPED_ARRAY): Uint8ClampedArray
Uint8ClampedArray|undefined;
private getContainer(type: MPImageStorageType.FLOAT32_ARRAY): Float32Array
|undefined;
private getContainer(type: MPImageStorageType.IMAGE_DATA): ImageData
|undefined;
private getContainer(type: MPImageStorageType.IMAGE_BITMAP): ImageBitmap
|undefined;
private getContainer(type: MPImageStorageType.WEBGL_TEXTURE): WebGLTexture
|undefined;
private getContainer(type: MPImageStorageType): MPImageNativeContainer
|undefined; |undefined;
private getContainer(type: MPImageType.FLOAT32_ARRAY): Float32Array|undefined;
private getContainer(type: MPImageType.IMAGE_DATA): ImageData|undefined;
private getContainer(type: MPImageType.IMAGE_BITMAP): ImageBitmap|undefined;
private getContainer(type: MPImageType.WEBGL_TEXTURE): WebGLTexture|undefined;
private getContainer(type: MPImageType): MPImageContainer|undefined;
/** Returns the container for the requested storage type iff it exists. */ /** Returns the container for the requested storage type iff it exists. */
private getContainer(type: MPImageStorageType): MPImageNativeContainer private getContainer(type: MPImageType): MPImageContainer|undefined {
|undefined {
switch (type) { switch (type) {
case MPImageStorageType.UINT8_CLAMPED_ARRAY: case MPImageType.UINT8_CLAMPED_ARRAY:
return this.containers.find(img => img instanceof Uint8ClampedArray); return this.containers.find(img => img instanceof Uint8ClampedArray);
case MPImageStorageType.FLOAT32_ARRAY: case MPImageType.FLOAT32_ARRAY:
return this.containers.find(img => img instanceof Float32Array); return this.containers.find(img => img instanceof Float32Array);
case MPImageStorageType.IMAGE_DATA: case MPImageType.IMAGE_DATA:
return this.containers.find(img => img instanceof ImageData); return this.containers.find(img => img instanceof ImageData);
case MPImageStorageType.IMAGE_BITMAP: case MPImageType.IMAGE_BITMAP:
return this.containers.find(img => img instanceof ImageBitmap); return this.containers.find(img => img instanceof ImageBitmap);
case MPImageStorageType.WEBGL_TEXTURE: case MPImageType.WEBGL_TEXTURE:
return this.containers.find(img => img instanceof WebGLTexture); return this.containers.find(img => img instanceof WebGLTexture);
default: default:
throw new Error(`Type is not supported: ${type}`); throw new Error(`Type is not supported: ${type}`);
@ -547,12 +539,12 @@ export class MPImage {
* avoided. * avoided.
*/ */
clone(): MPImage { clone(): MPImage {
const destinationContainers: MPImageNativeContainer[] = []; const destinationContainers: MPImageContainer[] = [];
// TODO: We might only want to clone one backing datastructure // TODO: We might only want to clone one backing datastructure
// even if multiple are defined; // even if multiple are defined;
for (const container of this.containers) { for (const container of this.containers) {
let destinationContainer: MPImageNativeContainer; let destinationContainer: MPImageContainer;
if (container instanceof Uint8ClampedArray) { if (container instanceof Uint8ClampedArray) {
destinationContainer = new Uint8ClampedArray(container); destinationContainer = new Uint8ClampedArray(container);
@ -599,9 +591,9 @@ export class MPImage {
} }
return new MPImage( return new MPImage(
destinationContainers, this.hasType(MPImageStorageType.IMAGE_BITMAP), destinationContainers, this.has(MPImageType.IMAGE_BITMAP),
this.hasType(MPImageStorageType.WEBGL_TEXTURE), this.canvas, this.has(MPImageType.WEBGL_TEXTURE), this.canvas, this.shaderContext,
this.shaderContext, this.width, this.height); this.width, this.height);
} }
private getOffscreenCanvas(): OffscreenCanvas { private getOffscreenCanvas(): OffscreenCanvas {
@ -637,7 +629,7 @@ export class MPImage {
private convertToImageBitmap(converter: Required<MPImageChannelConverter>): private convertToImageBitmap(converter: Required<MPImageChannelConverter>):
ImageBitmap { ImageBitmap {
let imageBitmap = this.getContainer(MPImageStorageType.IMAGE_BITMAP); let imageBitmap = this.getContainer(MPImageType.IMAGE_BITMAP);
if (!imageBitmap) { if (!imageBitmap) {
this.convertToWebGLTexture(converter); this.convertToWebGLTexture(converter);
imageBitmap = this.convertWebGLTextureToImageBitmap(); imageBitmap = this.convertWebGLTextureToImageBitmap();
@ -650,11 +642,10 @@ export class MPImage {
private convertToImageData(converter: Required<MPImageChannelConverter>): private convertToImageData(converter: Required<MPImageChannelConverter>):
ImageData { ImageData {
let imageData = this.getContainer(MPImageStorageType.IMAGE_DATA); let imageData = this.getContainer(MPImageType.IMAGE_DATA);
if (!imageData) { if (!imageData) {
if (this.hasType(MPImageStorageType.UINT8_CLAMPED_ARRAY)) { if (this.has(MPImageType.UINT8_CLAMPED_ARRAY)) {
const source = const source = this.getContainer(MPImageType.UINT8_CLAMPED_ARRAY)!;
this.getContainer(MPImageStorageType.UINT8_CLAMPED_ARRAY)!;
const destination = new Uint8ClampedArray(this.width * this.height * 4); const destination = new Uint8ClampedArray(this.width * this.height * 4);
for (let i = 0; i < this.width * this.height; i++) { for (let i = 0; i < this.width * this.height; i++) {
const rgba = converter.uint8ToRGBAConverter(source[i]); const rgba = converter.uint8ToRGBAConverter(source[i]);
@ -665,8 +656,8 @@ export class MPImage {
} }
imageData = new ImageData(destination, this.width, this.height); imageData = new ImageData(destination, this.width, this.height);
this.containers.push(imageData); this.containers.push(imageData);
} else if (this.hasType(MPImageStorageType.FLOAT32_ARRAY)) { } else if (this.has(MPImageType.FLOAT32_ARRAY)) {
const source = this.getContainer(MPImageStorageType.FLOAT32_ARRAY)!; const source = this.getContainer(MPImageType.FLOAT32_ARRAY)!;
const destination = new Uint8ClampedArray(this.width * this.height * 4); const destination = new Uint8ClampedArray(this.width * this.height * 4);
for (let i = 0; i < this.width * this.height; i++) { for (let i = 0; i < this.width * this.height; i++) {
const rgba = converter.floatToRGBAConverter(source[i]); const rgba = converter.floatToRGBAConverter(source[i]);
@ -678,8 +669,8 @@ export class MPImage {
imageData = new ImageData(destination, this.width, this.height); imageData = new ImageData(destination, this.width, this.height);
this.containers.push(imageData); this.containers.push(imageData);
} else if ( } else if (
this.hasType(MPImageStorageType.IMAGE_BITMAP) || this.has(MPImageType.IMAGE_BITMAP) ||
this.hasType(MPImageStorageType.WEBGL_TEXTURE)) { this.has(MPImageType.WEBGL_TEXTURE)) {
const gl = this.getGL(); const gl = this.getGL();
const shaderContext = this.getShaderContext(); const shaderContext = this.getShaderContext();
const pixels = new Uint8Array(this.width * this.height * 4); const pixels = new Uint8Array(this.width * this.height * 4);
@ -706,11 +697,10 @@ export class MPImage {
private convertToUint8ClampedArray( private convertToUint8ClampedArray(
converter: Required<MPImageChannelConverter>): Uint8ClampedArray { converter: Required<MPImageChannelConverter>): Uint8ClampedArray {
let uint8ClampedArray = let uint8ClampedArray = this.getContainer(MPImageType.UINT8_CLAMPED_ARRAY);
this.getContainer(MPImageStorageType.UINT8_CLAMPED_ARRAY);
if (!uint8ClampedArray) { if (!uint8ClampedArray) {
if (this.hasType(MPImageStorageType.FLOAT32_ARRAY)) { if (this.has(MPImageType.FLOAT32_ARRAY)) {
const source = this.getContainer(MPImageStorageType.FLOAT32_ARRAY)!; const source = this.getContainer(MPImageType.FLOAT32_ARRAY)!;
uint8ClampedArray = new Uint8ClampedArray( uint8ClampedArray = new Uint8ClampedArray(
source.map(v => converter.floatToUint8Converter(v))); source.map(v => converter.floatToUint8Converter(v)));
} else { } else {
@ -730,11 +720,10 @@ export class MPImage {
private convertToFloat32Array(converter: Required<MPImageChannelConverter>): private convertToFloat32Array(converter: Required<MPImageChannelConverter>):
Float32Array { Float32Array {
let float32Array = this.getContainer(MPImageStorageType.FLOAT32_ARRAY); let float32Array = this.getContainer(MPImageType.FLOAT32_ARRAY);
if (!float32Array) { if (!float32Array) {
if (this.hasType(MPImageStorageType.UINT8_CLAMPED_ARRAY)) { if (this.has(MPImageType.UINT8_CLAMPED_ARRAY)) {
const source = const source = this.getContainer(MPImageType.UINT8_CLAMPED_ARRAY)!;
this.getContainer(MPImageStorageType.UINT8_CLAMPED_ARRAY)!;
float32Array = new Float32Array(source).map( float32Array = new Float32Array(source).map(
v => converter.uint8ToFloatConverter(v)); v => converter.uint8ToFloatConverter(v));
} else { } else {
@ -754,11 +743,11 @@ export class MPImage {
private convertToWebGLTexture(converter: Required<MPImageChannelConverter>): private convertToWebGLTexture(converter: Required<MPImageChannelConverter>):
WebGLTexture { WebGLTexture {
let webGLTexture = this.getContainer(MPImageStorageType.WEBGL_TEXTURE); let webGLTexture = this.getContainer(MPImageType.WEBGL_TEXTURE);
if (!webGLTexture) { if (!webGLTexture) {
const gl = this.getGL(); const gl = this.getGL();
webGLTexture = this.bindTexture(); webGLTexture = this.bindTexture();
const source = this.getContainer(MPImageStorageType.IMAGE_BITMAP) || const source = this.getContainer(MPImageType.IMAGE_BITMAP) ||
this.convertToImageData(converter); this.convertToImageData(converter);
gl.texImage2D( gl.texImage2D(
gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source); gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source);
@ -778,7 +767,7 @@ export class MPImage {
gl.viewport(0, 0, this.width, this.height); gl.viewport(0, 0, this.width, this.height);
gl.activeTexture(gl.TEXTURE0); gl.activeTexture(gl.TEXTURE0);
let webGLTexture = this.getContainer(MPImageStorageType.WEBGL_TEXTURE); let webGLTexture = this.getContainer(MPImageType.WEBGL_TEXTURE);
if (!webGLTexture) { if (!webGLTexture) {
webGLTexture = webGLTexture =
assertNotNull(gl.createTexture(), 'Failed to create texture'); assertNotNull(gl.createTexture(), 'Failed to create texture');
@ -868,12 +857,12 @@ export class MPImage {
*/ */
close(): void { close(): void {
if (this.ownsImageBitmap) { if (this.ownsImageBitmap) {
this.getContainer(MPImageStorageType.IMAGE_BITMAP)!.close(); this.getContainer(MPImageType.IMAGE_BITMAP)!.close();
} }
if (this.ownsWebGLTexture) { if (this.ownsWebGLTexture) {
const gl = this.getGL(); const gl = this.getGL();
gl.deleteTexture(this.getContainer(MPImageStorageType.WEBGL_TEXTURE)!); gl.deleteTexture(this.getContainer(MPImageType.WEBGL_TEXTURE)!);
} }
} }
} }

4
mediapipe/tasks/web/vision/face_stylizer/face_stylizer_test.ts
View File

@ -19,7 +19,7 @@ import 'jasmine';
// Placeholder for internal dependency on encodeByteArray // Placeholder for internal dependency on encodeByteArray
import {CalculatorGraphConfig} from '../../../../framework/calculator_pb'; import {CalculatorGraphConfig} from '../../../../framework/calculator_pb';
import {addJasmineCustomFloatEqualityTester, createSpyWasmModule, MediapipeTasksFake, SpyWasmModule, verifyGraph, verifyListenersRegistered} from '../../../../tasks/web/core/task_runner_test_utils'; import {addJasmineCustomFloatEqualityTester, createSpyWasmModule, MediapipeTasksFake, SpyWasmModule, verifyGraph, verifyListenersRegistered} from '../../../../tasks/web/core/task_runner_test_utils';
import {MPImageStorageType} from '../../../../tasks/web/vision/core/image'; import {MPImage} from '../../../../tasks/web/vision/core/image';
import {WasmImage} from '../../../../web/graph_runner/graph_runner_image_lib'; import {WasmImage} from '../../../../web/graph_runner/graph_runner_image_lib';
import {FaceStylizer} from './face_stylizer'; import {FaceStylizer} from './face_stylizer';
@ -118,7 +118,7 @@ describe('FaceStylizer', () => {
faceStylizer.stylize({} as HTMLImageElement, image => { faceStylizer.stylize({} as HTMLImageElement, image => {
expect(faceStylizer.fakeWasmModule._waitUntilIdle).toHaveBeenCalled(); expect(faceStylizer.fakeWasmModule._waitUntilIdle).toHaveBeenCalled();
expect(image).not.toBeNull(); expect(image).not.toBeNull();
expect(image!.hasType(MPImageStorageType.IMAGE_DATA)).toBeTrue(); expect(image!.has(MPImage.TYPE.IMAGE_DATA)).toBeTrue();
expect(image!.width).toEqual(1); expect(image!.width).toEqual(1);
expect(image!.height).toEqual(1); expect(image!.height).toEqual(1);
done(); done();

6
mediapipe/tasks/web/vision/index.ts
View File

@ -16,7 +16,7 @@
import {FilesetResolver as FilesetResolverImpl} from '../../../tasks/web/core/fileset_resolver'; import {FilesetResolver as FilesetResolverImpl} from '../../../tasks/web/core/fileset_resolver';
import {DrawingUtils as DrawingUtilsImpl} from '../../../tasks/web/vision/core/drawing_utils'; import {DrawingUtils as DrawingUtilsImpl} from '../../../tasks/web/vision/core/drawing_utils';
import {MPImage as MPImageImpl, MPImageStorageType as MPImageStorageTypeImpl} from '../../../tasks/web/vision/core/image'; import {MPImage as MPImageImpl, MPImageType as MPImageTypeImpl} from '../../../tasks/web/vision/core/image';
import {FaceDetector as FaceDetectorImpl} from '../../../tasks/web/vision/face_detector/face_detector'; import {FaceDetector as FaceDetectorImpl} from '../../../tasks/web/vision/face_detector/face_detector';
import {FaceLandmarker as FaceLandmarkerImpl, FaceLandmarksConnections as FaceLandmarksConnectionsImpl} from '../../../tasks/web/vision/face_landmarker/face_landmarker'; import {FaceLandmarker as FaceLandmarkerImpl, FaceLandmarksConnections as FaceLandmarksConnectionsImpl} from '../../../tasks/web/vision/face_landmarker/face_landmarker';
import {FaceStylizer as FaceStylizerImpl} from '../../../tasks/web/vision/face_stylizer/face_stylizer'; import {FaceStylizer as FaceStylizerImpl} from '../../../tasks/web/vision/face_stylizer/face_stylizer';
@ -34,7 +34,7 @@ import {PoseLandmarker as PoseLandmarkerImpl} from '../../../tasks/web/vision/po
const DrawingUtils = DrawingUtilsImpl; const DrawingUtils = DrawingUtilsImpl;
const FilesetResolver = FilesetResolverImpl; const FilesetResolver = FilesetResolverImpl;
const MPImage = MPImageImpl; const MPImage = MPImageImpl;
const MPImageStorageType = MPImageStorageTypeImpl; const MPImageType = MPImageTypeImpl;
const FaceDetector = FaceDetectorImpl; const FaceDetector = FaceDetectorImpl;
const FaceLandmarker = FaceLandmarkerImpl; const FaceLandmarker = FaceLandmarkerImpl;
const FaceLandmarksConnections = FaceLandmarksConnectionsImpl; const FaceLandmarksConnections = FaceLandmarksConnectionsImpl;
@ -52,7 +52,7 @@ export {
DrawingUtils, DrawingUtils,
FilesetResolver, FilesetResolver,
MPImage, MPImage,
MPImageStorageType, MPImageType,
FaceDetector, FaceDetector,
FaceLandmarker, FaceLandmarker,
FaceLandmarksConnections, FaceLandmarksConnections,

2
mediapipe/tasks/web/vision/types.ts
View File

@ -16,7 +16,7 @@
export * from '../../../tasks/web/core/fileset_resolver'; export * from '../../../tasks/web/core/fileset_resolver';
export * from '../../../tasks/web/vision/core/drawing_utils'; export * from '../../../tasks/web/vision/core/drawing_utils';
export {MPImage, MPImageChannelConverter, MPImageStorageType} from '../../../tasks/web/vision/core/image'; export {MPImage, MPImageChannelConverter, MPImageType} from '../../../tasks/web/vision/core/image';
export * from '../../../tasks/web/vision/face_detector/face_detector'; export * from '../../../tasks/web/vision/face_detector/face_detector';
export * from '../../../tasks/web/vision/face_landmarker/face_landmarker'; export * from '../../../tasks/web/vision/face_landmarker/face_landmarker';
export * from '../../../tasks/web/vision/face_stylizer/face_stylizer'; export * from '../../../tasks/web/vision/face_stylizer/face_stylizer';