From b6a19ea9e82b0256cca6ce42d01b9af8a260319c Mon Sep 17 00:00:00 2001
From: MediaPipe Team <mediapipe-team@google.com>
Date: Tue, 11 Apr 2023 20:08:20 -0700
Subject: [PATCH] Add shaders that support better landscape rendering with
 GlSurfaceViewRenderer.

PiperOrigin-RevId: 523579769
---
 .../components/GlSurfaceViewRenderer.java     | 28 +++++++++++++++--
 .../mediapipe/glutil/CommonShaders.java       | 31 +++++++++++++++++++
 2 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java
index 23178b265..4e659d2a8 100644
--- a/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java
+++ b/mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java
@@ -77,6 +77,8 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
   private final AtomicReference<TextureFrame> nextFrame = new AtomicReference<>();
   private final AtomicBoolean captureNextFrameBitmap = new AtomicBoolean();
   private BitmapCaptureListener bitmapCaptureListener;
+  // Specifies whether a black CLAMP_TO_BORDER effect should be used.
+  private boolean shouldClampToBorder = false;
 
   /**
    * Sets the {@link BitmapCaptureListener}.
@@ -104,12 +106,20 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
     attributeLocations.put("position", ATTRIB_POSITION);
     attributeLocations.put("texture_coordinate", ATTRIB_TEXTURE_COORDINATE);
     Log.d(TAG, "external texture: " + isExternalTexture());
+    String fragmentShader;
+    if (shouldClampToBorder) {
+      fragmentShader = isExternalTexture()
+          ? CommonShaders.FRAGMENT_SHADER_EXTERNAL_CLAMP_TO_BORDER
+          : CommonShaders.FRAGMENT_SHADER_CLAMP_TO_BORDER;
+    } else {
+      fragmentShader = isExternalTexture()
+          ? CommonShaders.FRAGMENT_SHADER_EXTERNAL
+          : CommonShaders.FRAGMENT_SHADER;
+    }
     program =
         ShaderUtil.createProgram(
             CommonShaders.VERTEX_SHADER,
-            isExternalTexture()
-                ? CommonShaders.FRAGMENT_SHADER_EXTERNAL
-                : CommonShaders.FRAGMENT_SHADER,
+            fragmentShader,
             attributeLocations);
     frameUniform = GLES20.glGetUniformLocation(program, "video_frame");
     textureTransformUniform = GLES20.glGetUniformLocation(program, "texture_transform");
@@ -308,6 +318,18 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
     alignmentVertical = vertical;
   }
 
+  /**
+   * Whether to use GL_CLAMP_TO_BORDER-like mode. This is useful when rendering landscape or
+   * different aspect ratio frames. The remaining area will be rendered black.
+   */
+  public void setClampToBorder(boolean shouldClampToBorder) {
+    if (program != 0) {
+      throw new IllegalStateException(
+          "setClampToBorder must be called before the surface is created");
+    }
+    this.shouldClampToBorder = shouldClampToBorder;
+  }
+
   private boolean isExternalTexture() {
     return textureTarget == GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
   }
diff --git a/mediapipe/java/com/google/mediapipe/glutil/CommonShaders.java b/mediapipe/java/com/google/mediapipe/glutil/CommonShaders.java
index 1e601bc9e..408ca32d8 100644
--- a/mediapipe/java/com/google/mediapipe/glutil/CommonShaders.java
+++ b/mediapipe/java/com/google/mediapipe/glutil/CommonShaders.java
@@ -56,6 +56,37 @@ public class CommonShaders {
           + "void main() {\n"
           + "  gl_FragColor = texture2D(video_frame, sample_coordinate);\n"
           + "}";
+  /**
+   * Fragment shader that renders a 2D texture with a black CLAMP_TO_BORDER effect for
+   * out-of-bounds co-ordinates.
+   */
+  public static final String FRAGMENT_SHADER_CLAMP_TO_BORDER =
+      "varying mediump vec2 sample_coordinate;\n"
+          + "uniform sampler2D video_frame;\n"
+          + "\n"
+          + "void main() {\n"
+          +  " gl_FragColor = vec4(0.0, 0.0, 0.0, 0.0);\n"
+          + "  if (all(greaterThan(sample_coordinate, vec2(0.0))) \n"
+          + "      && all(lessThan(sample_coordinate, vec2(1.0)))) {\n"
+          + "    gl_FragColor = texture2D(video_frame, sample_coordinate);\n"
+          + "  }\n"
+          + "}";
+  /** Simple fragment shader that renders a texture bound to the {@link
+   * android.opengl.GLES11Ext#GL_TEXTURE_EXTERNAL_OES} target with a black CLAMP_TO_BORDER effect
+   * for out-of-bounds co-ordinates. See {@link android.graphics.SurfaceTexture}.
+   */
+  public static final String FRAGMENT_SHADER_EXTERNAL_CLAMP_TO_BORDER =
+      "#extension GL_OES_EGL_image_external : require\n"
+          + "varying mediump vec2 sample_coordinate;\n"
+          + "uniform sampler2D video_frame;\n"
+          + "\n"
+          + "void main() {\n"
+          +  " gl_FragColor = vec4(0.0, 0.0, 0.0, 0.0);\n"
+          + "  if (all(greaterThan(sample_coordinate, vec2(0.0))) \n"
+          + "      && all(lessThan(sample_coordinate, vec2(1.0)))) {\n"
+          + "    gl_FragColor = texture2D(video_frame, sample_coordinate);\n"
+          + "  }\n"
+          + "}";
   /**
    * Vertices for a quad that fills the drawing area.
    *