cassdlcm/mediapipe
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add shaders that support better landscape rendering with GlSurfaceViewRenderer.

Browse Source 
PiperOrigin-RevId: 523579769
This commit is contained in:
MediaPipe Team 2023-04-11 20:08:20 -07:00 committed by Copybara-Service
parent 1b947df0c2
commit b6a19ea9e8
2 changed files with 56 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
mediapipe/java/com/google/mediapipe/components/GlSurfaceViewRenderer.java
View File

@ -77,6 +77,8 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
private final AtomicReference<TextureFrame> nextFrame = new AtomicReference<>();
private final AtomicBoolean captureNextFrameBitmap = new AtomicBoolean();
private BitmapCaptureListener bitmapCaptureListener;
// Specifies whether a black CLAMP_TO_BORDER effect should be used.
private boolean shouldClampToBorder = false;
/**
* Sets the {@link BitmapCaptureListener}.
@ -104,12 +106,20 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
attributeLocations.put("position", ATTRIB_POSITION);
attributeLocations.put("texture_coordinate", ATTRIB_TEXTURE_COORDINATE);
Log.d(TAG, "external texture: " + isExternalTexture());
String fragmentShader;
if (shouldClampToBorder) {
fragmentShader = isExternalTexture()
? CommonShaders.FRAGMENT_SHADER_EXTERNAL_CLAMP_TO_BORDER
: CommonShaders.FRAGMENT_SHADER_CLAMP_TO_BORDER;
} else {
fragmentShader = isExternalTexture()
? CommonShaders.FRAGMENT_SHADER_EXTERNAL
: CommonShaders.FRAGMENT_SHADER;
}
program =
ShaderUtil.createProgram(
CommonShaders.VERTEX_SHADER,
isExternalTexture()
? CommonShaders.FRAGMENT_SHADER_EXTERNAL
: CommonShaders.FRAGMENT_SHADER,
fragmentShader,
attributeLocations);
frameUniform = GLES20.glGetUniformLocation(program, "video_frame");
textureTransformUniform = GLES20.glGetUniformLocation(program, "texture_transform");
@ -308,6 +318,18 @@ public class GlSurfaceViewRenderer implements GLSurfaceView.Renderer {
alignmentVertical = vertical;
}
/**
* Whether to use GL_CLAMP_TO_BORDER-like mode. This is useful when rendering landscape or
* different aspect ratio frames. The remaining area will be rendered black.
*/
public void setClampToBorder(boolean shouldClampToBorder) {
if (program != 0) {
throw new IllegalStateException(
"setClampToBorder must be called before the surface is created");
}
this.shouldClampToBorder = shouldClampToBorder;
}
private boolean isExternalTexture() {
return textureTarget == GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
}

31
mediapipe/java/com/google/mediapipe/glutil/CommonShaders.java
View File

@ -56,6 +56,37 @@ public class CommonShaders {
+ "void main() {\n"
+ " gl_FragColor = texture2D(video_frame, sample_coordinate);\n"
+ "}";
/**
* Fragment shader that renders a 2D texture with a black CLAMP_TO_BORDER effect for
* out-of-bounds co-ordinates.
*/
public static final String FRAGMENT_SHADER_CLAMP_TO_BORDER =
"varying mediump vec2 sample_coordinate;\n"
+ "uniform sampler2D video_frame;\n"
+ "\n"
+ "void main() {\n"
+ " gl_FragColor = vec4(0.0, 0.0, 0.0, 0.0);\n"
+ " if (all(greaterThan(sample_coordinate, vec2(0.0))) \n"
+ " && all(lessThan(sample_coordinate, vec2(1.0)))) {\n"
+ " gl_FragColor = texture2D(video_frame, sample_coordinate);\n"
+ " }\n"
+ "}";
/** Simple fragment shader that renders a texture bound to the {@link
* android.opengl.GLES11Ext#GL_TEXTURE_EXTERNAL_OES} target with a black CLAMP_TO_BORDER effect
* for out-of-bounds co-ordinates. See {@link android.graphics.SurfaceTexture}.
*/
public static final String FRAGMENT_SHADER_EXTERNAL_CLAMP_TO_BORDER =
"#extension GL_OES_EGL_image_external : require\n"
+ "varying mediump vec2 sample_coordinate;\n"
+ "uniform sampler2D video_frame;\n"
+ "\n"
+ "void main() {\n"
+ " gl_FragColor = vec4(0.0, 0.0, 0.0, 0.0);\n"
+ " if (all(greaterThan(sample_coordinate, vec2(0.0))) \n"
+ " && all(lessThan(sample_coordinate, vec2(1.0)))) {\n"
+ " gl_FragColor = texture2D(video_frame, sample_coordinate);\n"
+ " }\n"
+ "}";
/**
* Vertices for a quad that fills the drawing area.
*