diff --git a/mediapipe/tasks/web/vision/core/image.test.ts b/mediapipe/tasks/web/vision/core/image.test.ts index e92debc2e..3c30c7293 100644 --- a/mediapipe/tasks/web/vision/core/image.test.ts +++ b/mediapipe/tasks/web/vision/core/image.test.ts @@ -60,6 +60,10 @@ class MPImageTestContext { this.webGLTexture = gl.createTexture()!; gl.bindTexture(gl.TEXTURE_2D, this.webGLTexture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texImage2D( gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, this.imageBitmap); gl.bindTexture(gl.TEXTURE_2D, null); diff --git a/mediapipe/tasks/web/vision/core/image.ts b/mediapipe/tasks/web/vision/core/image.ts index 3b067bd78..9a5d0de86 100644 --- a/mediapipe/tasks/web/vision/core/image.ts +++ b/mediapipe/tasks/web/vision/core/image.ts @@ -187,10 +187,11 @@ export class MPImage { destinationContainer = assertNotNull(gl.createTexture(), 'Failed to create texture'); gl.bindTexture(gl.TEXTURE_2D, destinationContainer); - + this.configureTextureParams(); gl.texImage2D( gl.TEXTURE_2D, 0, gl.RGBA, this.width, this.height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null); + gl.bindTexture(gl.TEXTURE_2D, null); shaderContext.bindFramebuffer(gl, destinationContainer); shaderContext.run(gl, /* flipVertically= */ false, () => { @@ -302,6 +303,20 @@ export class MPImage { return webGLTexture; } + /** Sets texture params for the currently bound texture. */ + private configureTextureParams() { + const gl = this.getGL(); + // `gl.LINEAR` might break rendering for some textures, but it allows us to + // do smooth resizing. Ideally, this would be user-configurable, but for now + // we hard-code the value here to `gl.LINEAR` (versus `gl.NEAREST` for + // `MPMask` where we do not want to interpolate mask values, especially for + // category masks). + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + } + /** * Binds the backing texture to the canvas. If the texture does not yet * exist, creates it first. @@ -318,16 +333,12 @@ export class MPImage { assertNotNull(gl.createTexture(), 'Failed to create texture'); this.containers.push(webGLTexture); this.ownsWebGLTexture = true; + + gl.bindTexture(gl.TEXTURE_2D, webGLTexture); + this.configureTextureParams(); + } else { + gl.bindTexture(gl.TEXTURE_2D, webGLTexture); } - - gl.bindTexture(gl.TEXTURE_2D, webGLTexture); - // TODO: Ideally, we would only set these once per texture and - // not once every frame. - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); - return webGLTexture; } diff --git a/mediapipe/tasks/web/vision/core/mask.test.ts b/mediapipe/tasks/web/vision/core/mask.test.ts index b632f2dc5..d2f5ddb09 100644 --- a/mediapipe/tasks/web/vision/core/mask.test.ts +++ b/mediapipe/tasks/web/vision/core/mask.test.ts @@ -60,8 +60,11 @@ class MPMaskTestContext { } this.webGLTexture = gl.createTexture()!; - gl.bindTexture(gl.TEXTURE_2D, this.webGLTexture); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); gl.texImage2D( gl.TEXTURE_2D, 0, gl.R32F, width, height, 0, gl.RED, gl.FLOAT, new Float32Array(pixels).map(v => v / 255)); diff --git a/mediapipe/tasks/web/vision/core/mask.ts b/mediapipe/tasks/web/vision/core/mask.ts index d7cf59e5f..3f37e804f 100644 --- a/mediapipe/tasks/web/vision/core/mask.ts +++ b/mediapipe/tasks/web/vision/core/mask.ts @@ -175,6 +175,7 @@ export class MPMask { destinationContainer = assertNotNull(gl.createTexture(), 'Failed to create texture'); gl.bindTexture(gl.TEXTURE_2D, destinationContainer); + this.configureTextureParams(); gl.texImage2D( gl.TEXTURE_2D, 0, gl.R32F, this.width, this.height, 0, gl.RED, gl.FLOAT, null); @@ -283,6 +284,19 @@ export class MPMask { return webGLTexture; } + /** Sets texture params for the currently bound texture. */ + private configureTextureParams() { + const gl = this.getGL(); + // `gl.NEAREST` ensures that we do not get interpolated values for + // masks. In some cases, the user might want interpolation (e.g. for + // confidence masks), so we might want to make this user-configurable. + // Note that `MPImage` uses `gl.LINEAR`. + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + } + /** * Binds the backing texture to the canvas. If the texture does not yet * exist, creates it first. @@ -299,15 +313,12 @@ export class MPMask { assertNotNull(gl.createTexture(), 'Failed to create texture'); this.containers.push(webGLTexture); this.ownsWebGLTexture = true; - } - gl.bindTexture(gl.TEXTURE_2D, webGLTexture); - // TODO: Ideally, we would only set these once per texture and - // not once every frame. - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); - gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.bindTexture(gl.TEXTURE_2D, webGLTexture); + this.configureTextureParams(); + } else { + gl.bindTexture(gl.TEXTURE_2D, webGLTexture); + } return webGLTexture; }