Fix rendering of MPMask and MPImage clone

PiperOrigin-RevId: 533551170

mediapipe/tasks/web/vision/core/image.test.ts

@@ -60,6 +60,10 @@ class MPImageTestContext {
 
     this.webGLTexture = gl.createTexture()!;
     gl.bindTexture(gl.TEXTURE_2D, this.webGLTexture);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
     gl.texImage2D(
         gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, this.imageBitmap);
     gl.bindTexture(gl.TEXTURE_2D, null);

mediapipe/tasks/web/vision/core/image.ts

@@ -187,10 +187,11 @@ export class MPImage {
         destinationContainer =
             assertNotNull(gl.createTexture(), 'Failed to create texture');
         gl.bindTexture(gl.TEXTURE_2D, destinationContainer);
-
+        this.configureTextureParams();
         gl.texImage2D(
             gl.TEXTURE_2D, 0, gl.RGBA, this.width, this.height, 0, gl.RGBA,
             gl.UNSIGNED_BYTE, null);
+        gl.bindTexture(gl.TEXTURE_2D, null);
 
         shaderContext.bindFramebuffer(gl, destinationContainer);
         shaderContext.run(gl, /* flipVertically= */ false, () => {
@@ -302,6 +303,20 @@ export class MPImage {
     return webGLTexture;
   }
 
+  /** Sets texture params for the currently bound texture. */
+  private configureTextureParams() {
+    const gl = this.getGL();
+    // `gl.LINEAR` might break rendering for some textures, but it allows us to
+    // do smooth resizing. Ideally, this would be user-configurable, but for now
+    // we hard-code the value here to `gl.LINEAR` (versus `gl.NEAREST` for
+    // `MPMask` where we do not want to interpolate mask values, especially for
+    // category masks).
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+  }
+
   /**
    * Binds the backing texture to the canvas. If the texture does not yet
    * exist, creates it first.
@@ -318,16 +333,12 @@ export class MPImage {
           assertNotNull(gl.createTexture(), 'Failed to create texture');
       this.containers.push(webGLTexture);
       this.ownsWebGLTexture = true;
-    }
 
       gl.bindTexture(gl.TEXTURE_2D, webGLTexture);
-    // TODO: Ideally, we would only set these once per texture and
+      this.configureTextureParams();
-    // not once every frame.
+    } else {
-    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+      gl.bindTexture(gl.TEXTURE_2D, webGLTexture);
-    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    }
-    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
-    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
-
     return webGLTexture;
   }
 

mediapipe/tasks/web/vision/core/mask.test.ts

@@ -60,8 +60,11 @@ class MPMaskTestContext {
     }
 
     this.webGLTexture = gl.createTexture()!;
-
     gl.bindTexture(gl.TEXTURE_2D, this.webGLTexture);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
     gl.texImage2D(
         gl.TEXTURE_2D, 0, gl.R32F, width, height, 0, gl.RED, gl.FLOAT,
         new Float32Array(pixels).map(v => v / 255));

mediapipe/tasks/web/vision/core/mask.ts

@@ -175,6 +175,7 @@ export class MPMask {
         destinationContainer =
             assertNotNull(gl.createTexture(), 'Failed to create texture');
         gl.bindTexture(gl.TEXTURE_2D, destinationContainer);
+        this.configureTextureParams();
         gl.texImage2D(
             gl.TEXTURE_2D, 0, gl.R32F, this.width, this.height, 0, gl.RED,
             gl.FLOAT, null);
@@ -283,6 +284,19 @@ export class MPMask {
     return webGLTexture;
   }
 
+  /** Sets texture params for the currently bound texture. */
+  private configureTextureParams() {
+    const gl = this.getGL();
+    // `gl.NEAREST` ensures that we do not get interpolated values for
+    // masks. In some cases, the user might want interpolation (e.g. for
+    // confidence masks), so we might want to make this user-configurable.
+    // Note that `MPImage` uses `gl.LINEAR`.
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
+  }
+
   /**
    * Binds the backing texture to the canvas. If the texture does not yet
    * exist, creates it first.
@@ -299,15 +313,12 @@ export class MPMask {
           assertNotNull(gl.createTexture(), 'Failed to create texture');
       this.containers.push(webGLTexture);
       this.ownsWebGLTexture = true;
-    }
 
       gl.bindTexture(gl.TEXTURE_2D, webGLTexture);
-    // TODO: Ideally, we would only set these once per texture and
+      this.configureTextureParams();
-    // not once every frame.
+    } else {
-    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+      gl.bindTexture(gl.TEXTURE_2D, webGLTexture);
-    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+    }
-    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST);
-    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST);
 
     return webGLTexture;
   }