Remove single-channel types from MPImage

PiperOrigin-RevId: 529956549
This commit is contained in:
Sebastian Schmidt 2023-05-06 07:02:35 -07:00 committed by Copybara-Service
parent 800a7b4a27
commit 8a6fe90759
5 changed files with 33 additions and 377 deletions

View File

@ -43,7 +43,6 @@ mediapipe_ts_library(
name = "image", name = "image",
srcs = [ srcs = [
"image.ts", "image.ts",
"image_converter.ts",
"image_shader_context.ts", "image_shader_context.ts",
], ],
) )
@ -117,7 +116,6 @@ mediapipe_ts_library(
mediapipe_ts_library( mediapipe_ts_library(
name = "render_utils", name = "render_utils",
srcs = ["render_utils.ts"], srcs = ["render_utils.ts"],
deps = [":image"],
) )
jasmine_node_test( jasmine_node_test(

View File

@ -41,8 +41,6 @@ const IMAGE_2_3 = [
class MPImageTestContext { class MPImageTestContext {
canvas!: OffscreenCanvas; canvas!: OffscreenCanvas;
gl!: WebGL2RenderingContext; gl!: WebGL2RenderingContext;
uint8ClampedArray!: Uint8ClampedArray;
float32Array!: Float32Array;
imageData!: ImageData; imageData!: ImageData;
imageBitmap!: ImageBitmap; imageBitmap!: ImageBitmap;
webGLTexture!: WebGLTexture; webGLTexture!: WebGLTexture;
@ -56,17 +54,11 @@ class MPImageTestContext {
const gl = this.gl; const gl = this.gl;
this.uint8ClampedArray = new Uint8ClampedArray(pixels.length / 4);
this.float32Array = new Float32Array(pixels.length / 4);
for (let i = 0; i < this.uint8ClampedArray.length; ++i) {
this.uint8ClampedArray[i] = pixels[i * 4];
this.float32Array[i] = pixels[i * 4] / 255;
}
this.imageData = this.imageData =
new ImageData(new Uint8ClampedArray(pixels), width, height); new ImageData(new Uint8ClampedArray(pixels), width, height);
this.imageBitmap = await createImageBitmap(this.imageData); this.imageBitmap = await createImageBitmap(this.imageData);
this.webGLTexture = gl.createTexture()!;
this.webGLTexture = gl.createTexture()!;
gl.bindTexture(gl.TEXTURE_2D, this.webGLTexture); gl.bindTexture(gl.TEXTURE_2D, this.webGLTexture);
gl.texImage2D( gl.texImage2D(
gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, this.imageBitmap); gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, this.imageBitmap);
@ -75,10 +67,6 @@ class MPImageTestContext {
get(type: unknown) { get(type: unknown) {
switch (type) { switch (type) {
case Uint8ClampedArray:
return this.uint8ClampedArray;
case Float32Array:
return this.float32Array;
case ImageData: case ImageData:
return this.imageData; return this.imageData;
case ImageBitmap: case ImageBitmap:
@ -126,17 +114,14 @@ class MPImageTestContext {
gl.bindTexture(gl.TEXTURE_2D, null); gl.bindTexture(gl.TEXTURE_2D, null);
// Sanity check
expect(pixels.find(v => !!v)).toBeDefined();
return pixels; return pixels;
} }
function assertEquality(image: MPImage, expected: ImageType): void { function assertEquality(image: MPImage, expected: ImageType): void {
if (expected instanceof Uint8ClampedArray) { if (expected instanceof ImageData) {
const result = image.get(MPImageType.UINT8_CLAMPED_ARRAY);
expect(result).toEqual(expected);
} else if (expected instanceof Float32Array) {
const result = image.get(MPImageType.FLOAT32_ARRAY);
expect(result).toEqual(expected);
} else if (expected instanceof ImageData) {
const result = image.get(MPImageType.IMAGE_DATA); const result = image.get(MPImageType.IMAGE_DATA);
expect(result).toEqual(expected); expect(result).toEqual(expected);
} else if (expected instanceof ImageBitmap) { } else if (expected instanceof ImageBitmap) {
@ -154,8 +139,7 @@ class MPImageTestContext {
shaderContext: MPImageShaderContext, input: ImageType, width: number, shaderContext: MPImageShaderContext, input: ImageType, width: number,
height: number): MPImage { height: number): MPImage {
return new MPImage( return new MPImage(
[input], [input], /* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false,
/* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false,
context.canvas, shaderContext, width, height); context.canvas, shaderContext, width, height);
} }
@ -178,9 +162,7 @@ class MPImageTestContext {
shaderContext.close(); shaderContext.close();
} }
const sources = skip ? const sources = skip ? [] : [ImageData, ImageBitmap, WebGLTexture];
[] :
[Uint8ClampedArray, Float32Array, ImageData, ImageBitmap, WebGLTexture];
for (let i = 0; i < sources.length; i++) { for (let i = 0; i < sources.length; i++) {
for (let j = 0; j < sources.length; j++) { for (let j = 0; j < sources.length; j++) {
@ -203,9 +185,9 @@ class MPImageTestContext {
const shaderContext = new MPImageShaderContext(); const shaderContext = new MPImageShaderContext();
const image = new MPImage( const image = new MPImage(
[context.webGLTexture], [context.webGLTexture], /* ownsImageBitmap= */ false,
/* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false, /* ownsWebGLTexture= */ false, context.canvas, shaderContext, WIDTH,
context.canvas, shaderContext, WIDTH, HEIGHT); HEIGHT);
const result = image.clone().get(MPImageType.IMAGE_DATA); const result = image.clone().get(MPImageType.IMAGE_DATA);
expect(result).toEqual(context.imageData); expect(result).toEqual(context.imageData);
@ -218,9 +200,9 @@ class MPImageTestContext {
const shaderContext = new MPImageShaderContext(); const shaderContext = new MPImageShaderContext();
const image = new MPImage( const image = new MPImage(
[context.webGLTexture], [context.webGLTexture], /* ownsImageBitmap= */ false,
/* ownsImageBitmap= */ false, /* ownsWebGLTexture= */ false, /* ownsWebGLTexture= */ false, context.canvas, shaderContext, WIDTH,
context.canvas, shaderContext, WIDTH, HEIGHT); HEIGHT);
// Verify that we can mix the different shader modes by running them out of // Verify that we can mix the different shader modes by running them out of
// order. // order.
@ -243,40 +225,18 @@ class MPImageTestContext {
const image = createImage(shaderContext, context.imageData, WIDTH, HEIGHT); const image = createImage(shaderContext, context.imageData, WIDTH, HEIGHT);
expect(image.has(MPImageType.IMAGE_DATA)).toBe(true); expect(image.has(MPImageType.IMAGE_DATA)).toBe(true);
expect(image.has(MPImageType.UINT8_CLAMPED_ARRAY)).toBe(false);
expect(image.has(MPImageType.FLOAT32_ARRAY)).toBe(false);
expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(false);
expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(false);
image.get(MPImageType.UINT8_CLAMPED_ARRAY);
expect(image.has(MPImageType.IMAGE_DATA)).toBe(true);
expect(image.has(MPImageType.UINT8_CLAMPED_ARRAY)).toBe(true);
expect(image.has(MPImageType.FLOAT32_ARRAY)).toBe(false);
expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(false);
expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(false);
image.get(MPImageType.FLOAT32_ARRAY);
expect(image.has(MPImageType.IMAGE_DATA)).toBe(true);
expect(image.has(MPImageType.UINT8_CLAMPED_ARRAY)).toBe(true);
expect(image.has(MPImageType.FLOAT32_ARRAY)).toBe(true);
expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(false); expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(false);
expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(false); expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(false);
image.get(MPImageType.WEBGL_TEXTURE); image.get(MPImageType.WEBGL_TEXTURE);
expect(image.has(MPImageType.IMAGE_DATA)).toBe(true); expect(image.has(MPImageType.IMAGE_DATA)).toBe(true);
expect(image.has(MPImageType.UINT8_CLAMPED_ARRAY)).toBe(true);
expect(image.has(MPImageType.FLOAT32_ARRAY)).toBe(true);
expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(true); expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(true);
expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(false); expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(false);
image.get(MPImageType.IMAGE_BITMAP); image.get(MPImageType.IMAGE_BITMAP);
expect(image.has(MPImageType.IMAGE_DATA)).toBe(true); expect(image.has(MPImageType.IMAGE_DATA)).toBe(true);
expect(image.has(MPImageType.UINT8_CLAMPED_ARRAY)).toBe(true);
expect(image.has(MPImageType.FLOAT32_ARRAY)).toBe(true);
expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(true); expect(image.has(MPImageType.WEBGL_TEXTURE)).toBe(true);
expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(true); expect(image.has(MPImageType.IMAGE_BITMAP)).toBe(true);

View File

@ -14,17 +14,10 @@
* limitations under the License. * limitations under the License.
*/ */
import {DefaultColorConverter} from '../../../../tasks/web/vision/core/image_converter';
import {assertNotNull, MPImageShaderContext} from '../../../../tasks/web/vision/core/image_shader_context'; import {assertNotNull, MPImageShaderContext} from '../../../../tasks/web/vision/core/image_shader_context';
/** The underlying type of the image. */ /** The underlying type of the image. */
export enum MPImageType { export enum MPImageType {
/** Represents the native `UInt8ClampedArray` type. */
UINT8_CLAMPED_ARRAY,
/**
* Represents the native `Float32Array` type. Values range from [0.0, 1.0].
*/
FLOAT32_ARRAY,
/** Represents the native `ImageData` type. */ /** Represents the native `ImageData` type. */
IMAGE_DATA, IMAGE_DATA,
/** Represents the native `ImageBitmap` type. */ /** Represents the native `ImageBitmap` type. */
@ -34,75 +27,7 @@ export enum MPImageType {
} }
/** The supported image formats. For internal usage. */ /** The supported image formats. For internal usage. */
export type MPImageContainer = export type MPImageContainer = ImageData|ImageBitmap|WebGLTexture;
Uint8ClampedArray|Float32Array|ImageData|ImageBitmap|WebGLTexture;
/** A four channel color with a red, green, blue and alpha values. */
export type RGBAColor = [number, number, number, number];
/**
* An interface that can be used to provide custom conversion functions. These
* functions are invoked to convert pixel values between different channel
* counts and value ranges. Any conversion function that is not specified will
* result in a default conversion.
*/
export interface MPImageChannelConverter {
/**
* A conversion function to convert a number in the [0.0, 1.0] range to RGBA.
* The output is an array with four elemeents whose values range from 0 to 255
* inclusive.
*
* The default conversion function is `[v * 255, v * 255, v * 255, 255]`
* and will log a warning if invoked.
*/
floatToRGBAConverter?: (value: number) => RGBAColor;
/*
* A conversion function to convert a number in the [0, 255] range to RGBA.
* The output is an array with four elemeents whose values range from 0 to 255
* inclusive.
*
* The default conversion function is `[v, v , v , 255]` and will log a
* warning if invoked.
*/
uint8ToRGBAConverter?: (value: number) => RGBAColor;
/**
* A conversion function to convert an RGBA value in the range of 0 to 255 to
* a single value in the [0.0, 1.0] range.
*
* The default conversion function is `(r / 3 + g / 3 + b / 3) / 255` and will
* log a warning if invoked.
*/
rgbaToFloatConverter?: (r: number, g: number, b: number, a: number) => number;
/**
* A conversion function to convert an RGBA value in the range of 0 to 255 to
* a single value in the [0, 255] range.
*
* The default conversion function is `r / 3 + g / 3 + b / 3` and will log a
* warning if invoked.
*/
rgbaToUint8Converter?: (r: number, g: number, b: number, a: number) => number;
/**
* A conversion function to convert a single value in the 0.0 to 1.0 range to
* [0, 255].
*
* The default conversion function is `r * 255` and will log a warning if
* invoked.
*/
floatToUint8Converter?: (value: number) => number;
/**
* A conversion function to convert a single value in the 0 to 255 range to
* [0.0, 1.0] .
*
* The default conversion function is `r / 255` and will log a warning if
* invoked.
*/
uint8ToFloatConverter?: (value: number) => number;
}
/** /**
* The wrapper class for MediaPipe Image objects. * The wrapper class for MediaPipe Image objects.
@ -123,14 +48,6 @@ export interface MPImageChannelConverter {
* initialized with an `OffscreenCanvas`. As we require WebGL2 support, this * initialized with an `OffscreenCanvas`. As we require WebGL2 support, this
* places some limitations on Browser support as outlined here: * places some limitations on Browser support as outlined here:
* https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas/getContext * https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas/getContext
*
* Some MediaPipe tasks return single channel masks. These masks are stored
* using an underlying `Uint8ClampedArray` an `Float32Array` (represented as
* single-channel arrays). To convert these type to other formats a conversion
* function is invoked to convert pixel values between single channel and four
* channel RGBA values. To customize this conversion, you can specify these
* conversion functions when you invoke `get()`. If you use the default
* conversion function a warning will be logged to the console.
*/ */
export class MPImage { export class MPImage {
private gl?: WebGL2RenderingContext; private gl?: WebGL2RenderingContext;
@ -161,53 +78,18 @@ export class MPImage {
return !!this.getContainer(type); return !!this.getContainer(type);
} }
/**
* Returns the underlying image as a single channel `Uint8ClampedArray`. Note
* that this involves an expensive GPU to CPU transfer if the current image is
* only available as an `ImageBitmap` or `WebGLTexture`. If necessary, this
* function converts RGBA data pixel-by-pixel to a single channel value by
* invoking a conversion function (see class comment for detail).
*
* @param type The type of image to return.
* @param converter A set of conversion functions that will be invoked to
* convert the underlying pixel data if necessary. You may omit this
* function if the requested conversion does not change the pixel format.
* @return The current data as a Uint8ClampedArray.
*/
get(type: MPImageType.UINT8_CLAMPED_ARRAY,
converter?: MPImageChannelConverter): Uint8ClampedArray;
/**
* Returns the underlying image as a single channel `Float32Array`. Note
* that this involves an expensive GPU to CPU transfer if the current image is
* only available as an `ImageBitmap` or `WebGLTexture`. If necessary, this
* function converts RGBA data pixel-by-pixel to a single channel value by
* invoking a conversion function (see class comment for detail).
*
* @param type The type of image to return.
* @param converter A set of conversion functions that will be invoked to
* convert the underlying pixel data if necessary. You may omit this
* function if the requested conversion does not change the pixel format.
* @return The current image as a Float32Array.
*/
get(type: MPImageType.FLOAT32_ARRAY,
converter?: MPImageChannelConverter): Float32Array;
/** /**
* Returns the underlying image as an `ImageData` object. Note that this * Returns the underlying image as an `ImageData` object. Note that this
* involves an expensive GPU to CPU transfer if the current image is only * involves an expensive GPU to CPU transfer if the current image is only
* available as an `ImageBitmap` or `WebGLTexture`. If necessary, this * available as an `ImageBitmap` or `WebGLTexture`.
* function converts single channel pixel values to RGBA by invoking a
* conversion function (see class comment for detail).
* *
* @return The current image as an ImageData object. * @return The current image as an ImageData object.
*/ */
get(type: MPImageType.IMAGE_DATA, get(type: MPImageType.IMAGE_DATA): ImageData;
converter?: MPImageChannelConverter): ImageData;
/** /**
* Returns the underlying image as an `ImageBitmap`. Note that * Returns the underlying image as an `ImageBitmap`. Note that
* conversions to `ImageBitmap` are expensive, especially if the data * conversions to `ImageBitmap` are expensive, especially if the data
* currently resides on CPU. If necessary, this function first converts single * currently resides on CPU.
* channel pixel values to RGBA by invoking a conversion function (see class
* comment for detail).
* *
* Processing with `ImageBitmap`s requires that the MediaPipe Task was * Processing with `ImageBitmap`s requires that the MediaPipe Task was
* initialized with an `OffscreenCanvas` with WebGL2 support. See * initialized with an `OffscreenCanvas` with WebGL2 support. See
@ -215,13 +97,9 @@ export class MPImage {
* for a list of supported platforms. * for a list of supported platforms.
* *
* @param type The type of image to return. * @param type The type of image to return.
* @param converter A set of conversion functions that will be invoked to
* convert the underlying pixel data if necessary. You may omit this
* function if the requested conversion does not change the pixel format.
* @return The current image as an ImageBitmap object. * @return The current image as an ImageBitmap object.
*/ */
get(type: MPImageType.IMAGE_BITMAP, get(type: MPImageType.IMAGE_BITMAP): ImageBitmap;
converter?: MPImageChannelConverter): ImageBitmap;
/** /**
* Returns the underlying image as a `WebGLTexture` object. Note that this * Returns the underlying image as a `WebGLTexture` object. Note that this
* involves a CPU to GPU transfer if the current image is only available as * involves a CPU to GPU transfer if the current image is only available as
@ -229,36 +107,21 @@ export class MPImage {
* canvas (see `.canvas`). * canvas (see `.canvas`).
* *
* @param type The type of image to return. * @param type The type of image to return.
* @param converter A set of conversion functions that will be invoked to
* convert the underlying pixel data if necessary. You may omit this
* function if the requested conversion does not change the pixel format.
* @return The current image as a WebGLTexture. * @return The current image as a WebGLTexture.
*/ */
get(type: MPImageType.WEBGL_TEXTURE, get(type: MPImageType.WEBGL_TEXTURE): WebGLTexture;
converter?: MPImageChannelConverter): WebGLTexture; get(type?: MPImageType): MPImageContainer {
get(type?: MPImageType,
converter?: MPImageChannelConverter): MPImageContainer {
const internalConverter = new DefaultColorConverter(converter ?? {});
switch (type) { switch (type) {
case MPImageType.UINT8_CLAMPED_ARRAY:
return this.convertToUint8ClampedArray(internalConverter);
case MPImageType.FLOAT32_ARRAY:
return this.convertToFloat32Array(internalConverter);
case MPImageType.IMAGE_DATA: case MPImageType.IMAGE_DATA:
return this.convertToImageData(internalConverter); return this.convertToImageData();
case MPImageType.IMAGE_BITMAP: case MPImageType.IMAGE_BITMAP:
return this.convertToImageBitmap(internalConverter); return this.convertToImageBitmap();
case MPImageType.WEBGL_TEXTURE: case MPImageType.WEBGL_TEXTURE:
return this.convertToWebGLTexture(internalConverter); return this.convertToWebGLTexture();
default: default:
throw new Error(`Type is not supported: ${type}`); throw new Error(`Type is not supported: ${type}`);
} }
} }
private getContainer(type: MPImageType.UINT8_CLAMPED_ARRAY): Uint8ClampedArray
|undefined;
private getContainer(type: MPImageType.FLOAT32_ARRAY): Float32Array|undefined;
private getContainer(type: MPImageType.IMAGE_DATA): ImageData|undefined; private getContainer(type: MPImageType.IMAGE_DATA): ImageData|undefined;
private getContainer(type: MPImageType.IMAGE_BITMAP): ImageBitmap|undefined; private getContainer(type: MPImageType.IMAGE_BITMAP): ImageBitmap|undefined;
private getContainer(type: MPImageType.WEBGL_TEXTURE): WebGLTexture|undefined; private getContainer(type: MPImageType.WEBGL_TEXTURE): WebGLTexture|undefined;
@ -266,10 +129,6 @@ export class MPImage {
/** Returns the container for the requested storage type iff it exists. */ /** Returns the container for the requested storage type iff it exists. */
private getContainer(type: MPImageType): MPImageContainer|undefined { private getContainer(type: MPImageType): MPImageContainer|undefined {
switch (type) { switch (type) {
case MPImageType.UINT8_CLAMPED_ARRAY:
return this.containers.find(img => img instanceof Uint8ClampedArray);
case MPImageType.FLOAT32_ARRAY:
return this.containers.find(img => img instanceof Float32Array);
case MPImageType.IMAGE_DATA: case MPImageType.IMAGE_DATA:
return this.containers.find(img => img instanceof ImageData); return this.containers.find(img => img instanceof ImageData);
case MPImageType.IMAGE_BITMAP: case MPImageType.IMAGE_BITMAP:
@ -300,11 +159,7 @@ export class MPImage {
for (const container of this.containers) { for (const container of this.containers) {
let destinationContainer: MPImageContainer; let destinationContainer: MPImageContainer;
if (container instanceof Uint8ClampedArray) { if (container instanceof ImageData) {
destinationContainer = new Uint8ClampedArray(container);
} else if (container instanceof Float32Array) {
destinationContainer = new Float32Array(container);
} else if (container instanceof ImageData) {
destinationContainer = destinationContainer =
new ImageData(container.data, this.width, this.height); new ImageData(container.data, this.width, this.height);
} else if (container instanceof WebGLTexture) { } else if (container instanceof WebGLTexture) {
@ -333,7 +188,7 @@ export class MPImage {
this.unbindTexture(); this.unbindTexture();
} else if (container instanceof ImageBitmap) { } else if (container instanceof ImageBitmap) {
this.convertToWebGLTexture(new DefaultColorConverter({})); this.convertToWebGLTexture();
this.bindTexture(); this.bindTexture();
destinationContainer = this.copyTextureToBitmap(); destinationContainer = this.copyTextureToBitmap();
this.unbindTexture(); this.unbindTexture();
@ -381,11 +236,10 @@ export class MPImage {
return this.shaderContext; return this.shaderContext;
} }
private convertToImageBitmap(converter: Required<MPImageChannelConverter>): private convertToImageBitmap(): ImageBitmap {
ImageBitmap {
let imageBitmap = this.getContainer(MPImageType.IMAGE_BITMAP); let imageBitmap = this.getContainer(MPImageType.IMAGE_BITMAP);
if (!imageBitmap) { if (!imageBitmap) {
this.convertToWebGLTexture(converter); this.convertToWebGLTexture();
imageBitmap = this.convertWebGLTextureToImageBitmap(); imageBitmap = this.convertWebGLTextureToImageBitmap();
this.containers.push(imageBitmap); this.containers.push(imageBitmap);
this.ownsImageBitmap = true; this.ownsImageBitmap = true;
@ -394,43 +248,17 @@ export class MPImage {
return imageBitmap; return imageBitmap;
} }
private convertToImageData(converter: Required<MPImageChannelConverter>): private convertToImageData(): ImageData {
ImageData {
let imageData = this.getContainer(MPImageType.IMAGE_DATA); let imageData = this.getContainer(MPImageType.IMAGE_DATA);
if (!imageData) { if (!imageData) {
if (this.has(MPImageType.UINT8_CLAMPED_ARRAY)) { if (this.has(MPImageType.IMAGE_BITMAP) ||
const source = this.getContainer(MPImageType.UINT8_CLAMPED_ARRAY)!;
const destination = new Uint8ClampedArray(this.width * this.height * 4);
for (let i = 0; i < this.width * this.height; i++) {
const rgba = converter.uint8ToRGBAConverter(source[i]);
destination[i * 4] = rgba[0];
destination[i * 4 + 1] = rgba[1];
destination[i * 4 + 2] = rgba[2];
destination[i * 4 + 3] = rgba[3];
}
imageData = new ImageData(destination, this.width, this.height);
this.containers.push(imageData);
} else if (this.has(MPImageType.FLOAT32_ARRAY)) {
const source = this.getContainer(MPImageType.FLOAT32_ARRAY)!;
const destination = new Uint8ClampedArray(this.width * this.height * 4);
for (let i = 0; i < this.width * this.height; i++) {
const rgba = converter.floatToRGBAConverter(source[i]);
destination[i * 4] = rgba[0];
destination[i * 4 + 1] = rgba[1];
destination[i * 4 + 2] = rgba[2];
destination[i * 4 + 3] = rgba[3];
}
imageData = new ImageData(destination, this.width, this.height);
this.containers.push(imageData);
} else if (
this.has(MPImageType.IMAGE_BITMAP) ||
this.has(MPImageType.WEBGL_TEXTURE)) { this.has(MPImageType.WEBGL_TEXTURE)) {
const gl = this.getGL(); const gl = this.getGL();
const shaderContext = this.getShaderContext(); const shaderContext = this.getShaderContext();
const pixels = new Uint8Array(this.width * this.height * 4); const pixels = new Uint8Array(this.width * this.height * 4);
// Create texture if needed // Create texture if needed
const webGlTexture = this.convertToWebGLTexture(converter); const webGlTexture = this.convertToWebGLTexture();
// Create a framebuffer from the texture and read back pixels // Create a framebuffer from the texture and read back pixels
shaderContext.bindFramebuffer(gl, webGlTexture); shaderContext.bindFramebuffer(gl, webGlTexture);
@ -449,60 +277,13 @@ export class MPImage {
return imageData; return imageData;
} }
private convertToUint8ClampedArray( private convertToWebGLTexture(): WebGLTexture {
converter: Required<MPImageChannelConverter>): Uint8ClampedArray {
let uint8ClampedArray = this.getContainer(MPImageType.UINT8_CLAMPED_ARRAY);
if (!uint8ClampedArray) {
if (this.has(MPImageType.FLOAT32_ARRAY)) {
const source = this.getContainer(MPImageType.FLOAT32_ARRAY)!;
uint8ClampedArray = new Uint8ClampedArray(
source.map(v => converter.floatToUint8Converter(v)));
} else {
const source = this.convertToImageData(converter).data;
uint8ClampedArray = new Uint8ClampedArray(this.width * this.height);
for (let i = 0; i < this.width * this.height; i++) {
uint8ClampedArray[i] = converter.rgbaToUint8Converter(
source[i * 4], source[i * 4 + 1], source[i * 4 + 2],
source[i * 4 + 3]);
}
}
this.containers.push(uint8ClampedArray);
}
return uint8ClampedArray;
}
private convertToFloat32Array(converter: Required<MPImageChannelConverter>):
Float32Array {
let float32Array = this.getContainer(MPImageType.FLOAT32_ARRAY);
if (!float32Array) {
if (this.has(MPImageType.UINT8_CLAMPED_ARRAY)) {
const source = this.getContainer(MPImageType.UINT8_CLAMPED_ARRAY)!;
float32Array = new Float32Array(source).map(
v => converter.uint8ToFloatConverter(v));
} else {
const source = this.convertToImageData(converter).data;
float32Array = new Float32Array(this.width * this.height);
for (let i = 0; i < this.width * this.height; i++) {
float32Array[i] = converter.rgbaToFloatConverter(
source[i * 4], source[i * 4 + 1], source[i * 4 + 2],
source[i * 4 + 3]);
}
}
this.containers.push(float32Array);
}
return float32Array;
}
private convertToWebGLTexture(converter: Required<MPImageChannelConverter>):
WebGLTexture {
let webGLTexture = this.getContainer(MPImageType.WEBGL_TEXTURE); let webGLTexture = this.getContainer(MPImageType.WEBGL_TEXTURE);
if (!webGLTexture) { if (!webGLTexture) {
const gl = this.getGL(); const gl = this.getGL();
webGLTexture = this.bindTexture(); webGLTexture = this.bindTexture();
const source = this.getContainer(MPImageType.IMAGE_BITMAP) || const source = this.getContainer(MPImageType.IMAGE_BITMAP) ||
this.convertToImageData(converter); this.convertToImageData();
gl.texImage2D( gl.texImage2D(
gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source); gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, source);
this.unbindTexture(); this.unbindTexture();

View File

@ -1,83 +0,0 @@
/**
* Copyright 2023 The MediaPipe Authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import {MPImageChannelConverter, RGBAColor} from '../../../../tasks/web/vision/core/image';
/**
* Color converter that falls back to a default implementation if the
* user-provided converter does not specify a conversion.
*/
export class DefaultColorConverter implements
Required<MPImageChannelConverter> {
private static readonly WARNINGS_LOGGED = new Set<string>();
constructor(private readonly customConverter: MPImageChannelConverter) {}
floatToRGBAConverter(v: number): RGBAColor {
if (this.customConverter.floatToRGBAConverter) {
return this.customConverter.floatToRGBAConverter(v);
}
this.logWarningOnce('floatToRGBAConverter');
return [v * 255, v * 255, v * 255, 255];
}
uint8ToRGBAConverter(v: number): RGBAColor {
if (this.customConverter.uint8ToRGBAConverter) {
return this.customConverter.uint8ToRGBAConverter(v);
}
this.logWarningOnce('uint8ToRGBAConverter');
return [v, v, v, 255];
}
rgbaToFloatConverter(r: number, g: number, b: number, a: number): number {
if (this.customConverter.rgbaToFloatConverter) {
return this.customConverter.rgbaToFloatConverter(r, g, b, a);
}
this.logWarningOnce('rgbaToFloatConverter');
return (r / 3 + g / 3 + b / 3) / 255;
}
rgbaToUint8Converter(r: number, g: number, b: number, a: number): number {
if (this.customConverter.rgbaToUint8Converter) {
return this.customConverter.rgbaToUint8Converter(r, g, b, a);
}
this.logWarningOnce('rgbaToUint8Converter');
return r / 3 + g / 3 + b / 3;
}
floatToUint8Converter(v: number): number {
if (this.customConverter.floatToUint8Converter) {
return this.customConverter.floatToUint8Converter(v);
}
this.logWarningOnce('floatToUint8Converter');
return v * 255;
}
uint8ToFloatConverter(v: number): number {
if (this.customConverter.uint8ToFloatConverter) {
return this.customConverter.uint8ToFloatConverter(v);
}
this.logWarningOnce('uint8ToFloatConverter');
return v / 255;
}
private logWarningOnce(methodName: string): void {
if (!DefaultColorConverter.WARNINGS_LOGGED.has(methodName)) {
console.log(`Using default ${methodName}`);
DefaultColorConverter.WARNINGS_LOGGED.add(methodName);
}
}
}

View File

@ -16,7 +16,7 @@
export * from '../../../tasks/web/core/fileset_resolver'; export * from '../../../tasks/web/core/fileset_resolver';
export * from '../../../tasks/web/vision/core/drawing_utils'; export * from '../../../tasks/web/vision/core/drawing_utils';
export {MPImage, MPImageChannelConverter, MPImageType} from '../../../tasks/web/vision/core/image'; export {MPImage, MPImageType} from '../../../tasks/web/vision/core/image';
export {MPMask, MPMaskType} from '../../../tasks/web/vision/core/mask'; export {MPMask, MPMaskType} from '../../../tasks/web/vision/core/mask';
export * from '../../../tasks/web/vision/face_detector/face_detector'; export * from '../../../tasks/web/vision/face_detector/face_detector';
export * from '../../../tasks/web/vision/face_landmarker/face_landmarker'; export * from '../../../tasks/web/vision/face_landmarker/face_landmarker';