cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Generify tests for MPImage

Browse Source 
PiperOrigin-RevId: 527611864
This commit is contained in:
Sebastian Schmidt 2023-04-27 10:10:03 -07:00 committed by Copybara-Service
parent bc3434108e
commit b457060c3a
1 changed files with 104 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

217
mediapipe/tasks/web/vision/core/image.test.ts
View File

@ -29,43 +29,64 @@ if (skip) {
/** The image types supported by MPImage. */ /** The image types supported by MPImage. */
type ImageType = ImageData|ImageBitmap|WebGLTexture; type ImageType = ImageData|ImageBitmap|WebGLTexture;
async function createTestData( const IMAGE_2_2 = [1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, 4, 0, 0, 255];
gl: WebGL2RenderingContext, data: number[], width: number, const IMAGE_2_1 = [1, 0, 0, 255, 2, 0, 0, 255];
height: number): Promise<[ImageData, ImageBitmap, WebGLTexture]> { const IMAGE_2_3 = [
const imageData = new ImageData(new Uint8ClampedArray(data), width, height); 1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255,
const imageBitmap = await createImageBitmap(imageData); 4, 0, 0, 255, 5, 0, 0, 255, 6, 0, 0, 255
const webGlTexture = gl.createTexture()!; ];
gl.bindTexture(gl.TEXTURE_2D, webGlTexture); /** The test images and data to use for the unit tests below. */
class MPImageTestContext {
canvas!: OffscreenCanvas;
gl!: WebGL2RenderingContext;
imageData!: ImageData;
imageBitmap!: ImageBitmap;
webGLTexture!: WebGLTexture;
async init(pixels = IMAGE_2_2, width = WIDTH, height = HEIGHT):
Promise<void> {
// Initialize a canvas with default dimensions. Note that the canvas size
// can be different from the image size.
this.canvas = new OffscreenCanvas(WIDTH, HEIGHT);
this.gl = this.canvas.getContext('webgl2') as WebGL2RenderingContext;
const gl = this.gl;
this.imageData =
new ImageData(new Uint8ClampedArray(pixels), width, height);
this.imageBitmap = await createImageBitmap(this.imageData);
this.webGLTexture = gl.createTexture()!;
gl.bindTexture(gl.TEXTURE_2D, this.webGLTexture);
gl.texImage2D( gl.texImage2D(
gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, imageBitmap); gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, this.imageBitmap);
gl.bindTexture(gl.TEXTURE_2D, null); gl.bindTexture(gl.TEXTURE_2D, null);
}
return [imageData, imageBitmap, webGlTexture]; get(type: unknown) {
switch (type) {
case ImageData:
return this.imageData;
case ImageBitmap:
return this.imageBitmap;
case WebGLTexture:
return this.webGLTexture;
default:
throw new Error(`Unsupported type: ${type}`);
}
}
close(): void {
this.gl.deleteTexture(this.webGLTexture);
this.imageBitmap.close();
}
} }
(skip ? xdescribe : describe)('MPImage', () => { (skip ? xdescribe : describe)('MPImage', () => {
let canvas: OffscreenCanvas; const context = new MPImageTestContext();
let gl: WebGL2RenderingContext;
let imageData: ImageData;
let imageBitmap: ImageBitmap;
let webGlTexture: WebGLTexture;
beforeEach(async () => {
canvas = new OffscreenCanvas(WIDTH, HEIGHT);
gl = canvas.getContext('webgl2') as WebGL2RenderingContext;
const images = await createTestData(
gl, [1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, 4, 0, 0, 255], WIDTH,
HEIGHT);
imageData = images[0];
imageBitmap = images[1];
webGlTexture = images[2];
});
afterEach(() => { afterEach(() => {
gl.deleteTexture(webGlTexture); context.close();
imageBitmap.close();
}); });
function readPixelsFromImageBitmap(imageBitmap: ImageBitmap): ImageData { function readPixelsFromImageBitmap(imageBitmap: ImageBitmap): ImageData {
@ -78,6 +99,7 @@ async function createTestData(
function readPixelsFromWebGLTexture(texture: WebGLTexture): Uint8Array { function readPixelsFromWebGLTexture(texture: WebGLTexture): Uint8Array {
const pixels = new Uint8Array(WIDTH * WIDTH * 4); const pixels = new Uint8Array(WIDTH * WIDTH * 4);
const gl = context.gl;
gl.bindTexture(gl.TEXTURE_2D, texture); gl.bindTexture(gl.TEXTURE_2D, texture);
const framebuffer = gl.createFramebuffer()!; const framebuffer = gl.createFramebuffer()!;
@ -113,8 +135,8 @@ async function createTestData(
height: number): MPImage { height: number): MPImage {
return new MPImage( return new MPImage(
[input], [input],
/* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false, canvas, /* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false,
shaderContext, width, height); context.canvas, shaderContext, width, height);
} }
function runConversionTest( function runConversionTest(
@ -136,105 +158,67 @@ async function createTestData(
shaderContext.close(); shaderContext.close();
} }
it(`converts from ImageData to ImageData`, () => { const sources = skip ? [] : [ImageData, ImageBitmap, WebGLTexture];
runConversionTest(imageData, imageData);
});
it(`converts from ImageData to ImageBitmap`, () => { for (let i = 0; i < sources.length; i++) {
runConversionTest(imageData, imageBitmap); for (let j = 0; j < sources.length; j++) {
it(`converts from ${sources[i].name} to ${sources[j].name}`, async () => {
await context.init();
runConversionTest(context.get(sources[i]), context.get(sources[j]));
}); });
}
}
it(`converts from ImageData to WebGLTexture`, () => { for (let i = 0; i < sources.length; i++) {
runConversionTest(imageData, webGlTexture); it(`clones ${sources[i].name}`, async () => {
}); await context.init();
runCloneTest(context.get(sources[i]));
it(`converts from ImageBitmap to ImageData`, () => {
runConversionTest(imageBitmap, imageData);
});
it(`converts from ImageBitmap to ImageBitmap`, () => {
runConversionTest(imageBitmap, imageBitmap);
});
it(`converts from ImageBitmap to WebGLTexture`, () => {
runConversionTest(imageBitmap, webGlTexture);
});
it(`converts from WebGLTexture to ImageData`, () => {
runConversionTest(webGlTexture, imageData);
});
it(`converts from WebGLTexture to ImageBitmap`, () => {
runConversionTest(webGlTexture, imageBitmap);
});
it(`converts from WebGLTexture to WebGLTexture`, () => {
runConversionTest(webGlTexture, webGlTexture);
});
it(`clones ImageData`, () => {
runCloneTest(imageData);
});
it(`clones ImageBitmap`, () => {
runCloneTest(imageBitmap);
});
it(`clones WebGLTextures`, () => {
runCloneTest(webGlTexture);
}); });
}
it(`does not flip textures twice`, async () => { it(`does not flip textures twice`, async () => {
const [imageData, , webGlTexture] = await createTestData( await context.init();
gl, [1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, 4, 0, 0, 255], WIDTH,
HEIGHT);
const shaderContext = new MPImageShaderContext(); const shaderContext = new MPImageShaderContext();
const image = new MPImage( const image = new MPImage(
[webGlTexture], [context.webGLTexture],
/* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false, canvas, /* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false,
shaderContext, WIDTH, HEIGHT); context.canvas, shaderContext, WIDTH, HEIGHT);
const result = image.clone().getImage(MPImageStorageType.IMAGE_DATA); const result = image.clone().getImage(MPImageStorageType.IMAGE_DATA);
expect(result).toEqual(imageData); expect(result).toEqual(context.imageData);
gl.deleteTexture(webGlTexture);
shaderContext.close(); shaderContext.close();
}); });
it(`can clone and get image`, async () => { it(`can clone and get image`, async () => {
const [imageData, , webGlTexture] = await createTestData( await context.init();
gl, [1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255, 4, 0, 0, 255], WIDTH,
HEIGHT);
const shaderContext = new MPImageShaderContext(); const shaderContext = new MPImageShaderContext();
const image = new MPImage( const image = new MPImage(
[webGlTexture], [context.webGLTexture],
/* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false, canvas, /* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false,
shaderContext, WIDTH, HEIGHT); context.canvas, shaderContext, WIDTH, HEIGHT);
// Verify that we can mix the different shader modes by running them out of // Verify that we can mix the different shader modes by running them out of
// order. // order.
let result = image.getImage(MPImageStorageType.IMAGE_DATA); let result = image.getImage(MPImageStorageType.IMAGE_DATA);
expect(result).toEqual(imageData); expect(result).toEqual(context.imageData);
result = image.clone().getImage(MPImageStorageType.IMAGE_DATA); result = image.clone().getImage(MPImageStorageType.IMAGE_DATA);
expect(result).toEqual(imageData); expect(result).toEqual(context.imageData);
result = image.getImage(MPImageStorageType.IMAGE_DATA); result = image.getImage(MPImageStorageType.IMAGE_DATA);
expect(result).toEqual(imageData); expect(result).toEqual(context.imageData);
gl.deleteTexture(webGlTexture);
shaderContext.close(); shaderContext.close();
}); });
it('supports hasType()', async () => { it('supports hasType()', async () => {
const shaderContext = new MPImageShaderContext(); await context.init();
const image = createImage(shaderContext, imageData, WIDTH, HEIGHT);
expect(image.hasType(MPImageStorageType.IMAGE_DATA)).toBe(true); const shaderContext = new MPImageShaderContext();
expect(image.hasType(MPImageStorageType.WEBGL_TEXTURE)).toBe(false); const image = createImage(shaderContext, context.imageData, WIDTH, HEIGHT);
expect(image.hasType(MPImageStorageType.IMAGE_BITMAP)).toBe(false);
image.getImage(MPImageStorageType.WEBGL_TEXTURE); image.getImage(MPImageStorageType.WEBGL_TEXTURE);
@ -242,7 +226,7 @@ async function createTestData(
expect(image.hasType(MPImageStorageType.WEBGL_TEXTURE)).toBe(true); expect(image.hasType(MPImageStorageType.WEBGL_TEXTURE)).toBe(true);
expect(image.hasType(MPImageStorageType.IMAGE_BITMAP)).toBe(false); expect(image.hasType(MPImageStorageType.IMAGE_BITMAP)).toBe(false);
await image.getImage(MPImageStorageType.IMAGE_BITMAP); image.getImage(MPImageStorageType.IMAGE_BITMAP);
expect(image.hasType(MPImageStorageType.IMAGE_DATA)).toBe(true); expect(image.hasType(MPImageStorageType.IMAGE_DATA)).toBe(true);
expect(image.hasType(MPImageStorageType.WEBGL_TEXTURE)).toBe(true); expect(image.hasType(MPImageStorageType.WEBGL_TEXTURE)).toBe(true);
@ -253,33 +237,32 @@ async function createTestData(
}); });
it('supports image that is smaller than the canvas', async () => { it('supports image that is smaller than the canvas', async () => {
const [imageData, imageBitmap, webGlTexture] = await createTestData( await context.init(IMAGE_2_1, /* width= */ 2, /* height= */ 1);
gl, [1, 0, 0, 255, 2, 0, 0, 255], /* width= */ 2, /* height= */ 1);
runConversionTest(imageData, webGlTexture, /* width= */ 2, /* height= */ 1);
runConversionTest( runConversionTest(
webGlTexture, imageBitmap, /* width= */ 2, /* height= */ 1); context.imageData, context.webGLTexture, /* width= */ 2,
runConversionTest(imageBitmap, imageData, /* width= */ 2, /* height= */ 1); /* height= */ 1);
runConversionTest(
context.webGLTexture, context.imageBitmap, /* width= */ 2,
/* height= */ 1);
runConversionTest(
context.imageBitmap, context.imageData, /* width= */ 2,
/* height= */ 1);
gl.deleteTexture(webGlTexture); context.close();
imageBitmap.close();
}); });
it('supports image that is larger than the canvas', async () => { it('supports image that is larger than the canvas', async () => {
const [imageData, imageBitmap, webGlTexture] = await createTestData( await context.init(IMAGE_2_3, /* width= */ 2, /* height= */ 3);
gl,
[
1, 0, 0, 255, 2, 0, 0, 255, 3, 0, 0, 255,
4, 0, 0, 255, 5, 0, 0, 255, 6, 0, 0, 255
],
/* width= */ 2, /* height= */ 3);
runConversionTest(imageData, webGlTexture, /* width= */ 2, /* height= */ 3);
runConversionTest( runConversionTest(
webGlTexture, imageBitmap, /* width= */ 2, /* height= */ 3); context.imageData, context.webGLTexture, /* width= */ 2,
runConversionTest(imageBitmap, imageData, /* width= */ 2, /* height= */ 3); /* height= */ 3);
runConversionTest(
gl.deleteTexture(webGlTexture); context.webGLTexture, context.imageBitmap, /* width= */ 2,
imageBitmap.close(); /* height= */ 3);
runConversionTest(
context.imageBitmap, context.imageData, /* width= */ 2,
/* height= */ 3);
}); });
}); });