cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix rendering of MPMask and MPImage clone

Browse Source 
PiperOrigin-RevId: 533551170
This commit is contained in:
Sebastian Schmidt 2023-05-19 14:24:19 -07:00 committed by Copybara-Service
parent a6457d87b2
commit 7c28c5d58f
4 changed files with 48 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mediapipe/tasks/web/vision/core/image.test.ts
View File

@ -60,6 +60,10 @@ class MPImageTestContext {
this.webGLTexture = gl.createTexture()!;
gl.bindTexture(gl.TEXTURE_2D, this.webGLTexture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
gl.texImage2D(
gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, this.imageBitmap);
gl.bindTexture(gl.TEXTURE_2D, null);

31
mediapipe/tasks/web/vision/core/image.ts
View File

@ -187,10 +187,11 @@ export class MPImage {
destinationContainer =
assertNotNull(gl.createTexture(), 'Failed to create texture');
gl.bindTexture(gl.TEXTURE_2D, destinationContainer);
this.configureTextureParams();
gl.texImage2D(
gl.TEXTURE_2D, 0, gl.RGBA, this.width, this.height, 0, gl.RGBA,
gl.UNSIGNED_BYTE, null);
gl.bindTexture(gl.TEXTURE_2D, null);
shaderContext.bindFramebuffer(gl, destinationContainer);
shaderContext.run(gl, /* flipVertically= */ false, () => {
@ -302,6 +303,20 @@ export class MPImage {
return webGLTexture;
}
/** Sets texture params for the currently bound texture. */
private configureTextureParams() {
const gl = this.getGL();
// `gl.LINEAR` might break rendering for some textures, but it allows us to
// do smooth resizing. Ideally, this would be user-configurable, but for now
// we hard-code the value here to `gl.LINEAR` (versus `gl.NEAREST` for
// `MPMask` where we do not want to interpolate mask values, especially for
// category masks).
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
}
/**
* Binds the backing texture to the canvas. If the texture does not yet
* exist, creates it first.
@ -318,16 +333,12 @@ export class MPImage {
assertNotNull(gl.createTexture(), 'Failed to create texture');
this.containers.push(webGLTexture);
this.ownsWebGLTexture = true;
gl.bindTexture(gl.TEXTURE_2D, webGLTexture);
this.configureTextureParams();
} else {
gl.bindTexture(gl.TEXTURE_2D, webGLTexture);
}
gl.bindTexture(gl.TEXTURE_2D, webGLTexture);
// TODO: Ideally, we would only set these once per texture and
// not once every frame.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
return webGLTexture;
}

5
mediapipe/tasks/web/vision/core/mask.test.ts
View File

@ -60,8 +60,11 @@ class MPMaskTestContext {
}
this.webGLTexture = gl.createTexture()!;
gl.bindTexture(gl.TEXTURE_2D, this.webGLTexture);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.texImage2D(
gl.TEXTURE_2D, 0, gl.R32F, width, height, 0, gl.RED, gl.FLOAT,
new Float32Array(pixels).map(v => v / 255));

27
mediapipe/tasks/web/vision/core/mask.ts
View File

@ -175,6 +175,7 @@ export class MPMask {
destinationContainer =
assertNotNull(gl.createTexture(), 'Failed to create texture');
gl.bindTexture(gl.TEXTURE_2D, destinationContainer);
this.configureTextureParams();
gl.texImage2D(
gl.TEXTURE_2D, 0, gl.R32F, this.width, this.height, 0, gl.RED,
gl.FLOAT, null);
@ -283,6 +284,19 @@ export class MPMask {
return webGLTexture;
}
/** Sets texture params for the currently bound texture. */
private configureTextureParams() {
const gl = this.getGL();
// `gl.NEAREST` ensures that we do not get interpolated values for
// masks. In some cases, the user might want interpolation (e.g. for
// confidence masks), so we might want to make this user-configurable.
// Note that `MPImage` uses `gl.LINEAR`.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
}
/**
* Binds the backing texture to the canvas. If the texture does not yet
* exist, creates it first.
@ -299,15 +313,12 @@ export class MPMask {
assertNotNull(gl.createTexture(), 'Failed to create texture');
this.containers.push(webGLTexture);
this.ownsWebGLTexture = true;
}
gl.bindTexture(gl.TEXTURE_2D, webGLTexture);
// TODO: Ideally, we would only set these once per texture and
// not once every frame.
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
gl.bindTexture(gl.TEXTURE_2D, webGLTexture);
this.configureTextureParams();
} else {
gl.bindTexture(gl.TEXTURE_2D, webGLTexture);
}
return webGLTexture;
}