/* * Copyright 2018 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ package org.webrtc; import android.opengl.GLES11Ext; import android.opengl.GLES20; import androidx.annotation.Nullable; import java.nio.FloatBuffer; import org.webrtc.GlShader; import org.webrtc.GlUtil; import org.webrtc.RendererCommon; /** * Helper class to implement an instance of RendererCommon.GlDrawer that can accept multiple input * sources (OES, RGB, or YUV) using a generic fragment shader as input. The generic fragment shader * should sample pixel values from the function "sample" that will be provided by this class and * provides an abstraction for the input source type (OES, RGB, or YUV). The texture coordinate * variable name will be "tc" and the texture matrix in the vertex shader will be "tex_mat". The * simplest possible generic shader that just draws pixel from the frame unmodified looks like: * void main() { * gl_FragColor = sample(tc); * } * This class covers the cases for most simple shaders and generates the necessary boiler plate. * Advanced shaders can always implement RendererCommon.GlDrawer directly. */ class GlGenericDrawer implements RendererCommon.GlDrawer { /** * The different shader types representing different input sources. YUV here represents three * separate Y, U, V textures. */ public static enum ShaderType { OES, RGB, YUV } /** * The shader callbacks is used to customize behavior for a GlDrawer. It provides a hook to set * uniform variables in the shader before a frame is drawn. */ public static interface ShaderCallbacks { /** * This callback is called when a new shader has been compiled and created. It will be called * for the first frame as well as when the shader type is changed. This callback can be used to * do custom initialization of the shader that only needs to happen once. */ void onNewShader(GlShader shader); /** * This callback is called before rendering a frame. It can be used to do custom preparation of * the shader that needs to happen every frame. */ void onPrepareShader(GlShader shader, float[] texMatrix, int frameWidth, int frameHeight, int viewportWidth, int viewportHeight); } private static final String INPUT_VERTEX_COORDINATE_NAME = "in_pos"; private static final String INPUT_TEXTURE_COORDINATE_NAME = "in_tc"; private static final String TEXTURE_MATRIX_NAME = "tex_mat"; private static final String DEFAULT_VERTEX_SHADER_STRING = "varying vec2 tc;\n" + "attribute vec4 in_pos;\n" + "attribute vec4 in_tc;\n" + "uniform mat4 tex_mat;\n" + "void main() {\n" + " gl_Position = in_pos;\n" + " tc = (tex_mat * in_tc).xy;\n" + "}\n"; // Vertex coordinates in Normalized Device Coordinates, i.e. (-1, -1) is bottom-left and (1, 1) // is top-right. private static final FloatBuffer FULL_RECTANGLE_BUFFER = GlUtil.createFloatBuffer(new float[] { -1.0f, -1.0f, // Bottom left. 1.0f, -1.0f, // Bottom right. -1.0f, 1.0f, // Top left. 1.0f, 1.0f, // Top right. }); // Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right. private static final FloatBuffer FULL_RECTANGLE_TEXTURE_BUFFER = GlUtil.createFloatBuffer(new float[] { 0.0f, 0.0f, // Bottom left. 1.0f, 0.0f, // Bottom right. 0.0f, 1.0f, // Top left. 1.0f, 1.0f, // Top right. }); static String createFragmentShaderString(String genericFragmentSource, ShaderType shaderType) { final StringBuilder stringBuilder = new StringBuilder(); if (shaderType == ShaderType.OES) { stringBuilder.append("#extension GL_OES_EGL_image_external : require\n"); } stringBuilder.append("precision mediump float;\n"); stringBuilder.append("varying vec2 tc;\n"); if (shaderType == ShaderType.YUV) { stringBuilder.append("uniform sampler2D y_tex;\n"); stringBuilder.append("uniform sampler2D u_tex;\n"); stringBuilder.append("uniform sampler2D v_tex;\n"); // Add separate function for sampling texture. // yuv_to_rgb_mat is inverse of the matrix defined in YuvConverter. stringBuilder.append("vec4 sample(vec2 p) {\n"); stringBuilder.append(" float y = texture2D(y_tex, p).r * 1.16438;\n"); stringBuilder.append(" float u = texture2D(u_tex, p).r;\n"); stringBuilder.append(" float v = texture2D(v_tex, p).r;\n"); stringBuilder.append(" return vec4(y + 1.59603 * v - 0.874202,\n"); stringBuilder.append(" y - 0.391762 * u - 0.812968 * v + 0.531668,\n"); stringBuilder.append(" y + 2.01723 * u - 1.08563, 1);\n"); stringBuilder.append("}\n"); stringBuilder.append(genericFragmentSource); } else { final String samplerName = shaderType == ShaderType.OES ? "samplerExternalOES" : "sampler2D"; stringBuilder.append("uniform ").append(samplerName).append(" tex;\n"); // Update the sampling function in-place. stringBuilder.append(genericFragmentSource.replace("sample(", "texture2D(tex, ")); } return stringBuilder.toString(); } private final String genericFragmentSource; private final String vertexShader; private final ShaderCallbacks shaderCallbacks; @Nullable private ShaderType currentShaderType; @Nullable private GlShader currentShader; private int inPosLocation; private int inTcLocation; private int texMatrixLocation; public GlGenericDrawer(String genericFragmentSource, ShaderCallbacks shaderCallbacks) { this(DEFAULT_VERTEX_SHADER_STRING, genericFragmentSource, shaderCallbacks); } public GlGenericDrawer( String vertexShader, String genericFragmentSource, ShaderCallbacks shaderCallbacks) { this.vertexShader = vertexShader; this.genericFragmentSource = genericFragmentSource; this.shaderCallbacks = shaderCallbacks; } // Visible for testing. GlShader createShader(ShaderType shaderType) { return new GlShader( vertexShader, createFragmentShaderString(genericFragmentSource, shaderType)); } /** * Draw an OES texture frame with specified texture transformation matrix. Required resources are * allocated at the first call to this function. */ @Override public void drawOes(int oesTextureId, float[] texMatrix, int frameWidth, int frameHeight, int viewportX, int viewportY, int viewportWidth, int viewportHeight) { prepareShader( ShaderType.OES, texMatrix, frameWidth, frameHeight, viewportWidth, viewportHeight); // Bind the texture. GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, oesTextureId); // Draw the texture. GLES20.glViewport(viewportX, viewportY, viewportWidth, viewportHeight); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); // Unbind the texture as a precaution. GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0); } /** * Draw a RGB(A) texture frame with specified texture transformation matrix. Required resources * are allocated at the first call to this function. */ @Override public void drawRgb(int textureId, float[] texMatrix, int frameWidth, int frameHeight, int viewportX, int viewportY, int viewportWidth, int viewportHeight) { prepareShader( ShaderType.RGB, texMatrix, frameWidth, frameHeight, viewportWidth, viewportHeight); // Bind the texture. GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId); // Draw the texture. GLES20.glViewport(viewportX, viewportY, viewportWidth, viewportHeight); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); // Unbind the texture as a precaution. GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); } /** * Draw a YUV frame with specified texture transformation matrix. Required resources are allocated * at the first call to this function. */ @Override public void drawYuv(int[] yuvTextures, float[] texMatrix, int frameWidth, int frameHeight, int viewportX, int viewportY, int viewportWidth, int viewportHeight) { prepareShader( ShaderType.YUV, texMatrix, frameWidth, frameHeight, viewportWidth, viewportHeight); // Bind the textures. for (int i = 0; i < 3; ++i) { GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); } // Draw the textures. GLES20.glViewport(viewportX, viewportY, viewportWidth, viewportHeight); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); // Unbind the textures as a precaution. for (int i = 0; i < 3; ++i) { GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); } } private void prepareShader(ShaderType shaderType, float[] texMatrix, int frameWidth, int frameHeight, int viewportWidth, int viewportHeight) { final GlShader shader; if (shaderType.equals(currentShaderType)) { // Same shader type as before, reuse exising shader. shader = currentShader; } else { // Allocate new shader. currentShaderType = shaderType; if (currentShader != null) { currentShader.release(); } shader = createShader(shaderType); currentShader = shader; shader.useProgram(); // Set input texture units. if (shaderType == ShaderType.YUV) { GLES20.glUniform1i(shader.getUniformLocation("y_tex"), 0); GLES20.glUniform1i(shader.getUniformLocation("u_tex"), 1); GLES20.glUniform1i(shader.getUniformLocation("v_tex"), 2); } else { GLES20.glUniform1i(shader.getUniformLocation("tex"), 0); } GlUtil.checkNoGLES2Error("Create shader"); shaderCallbacks.onNewShader(shader); texMatrixLocation = shader.getUniformLocation(TEXTURE_MATRIX_NAME); inPosLocation = shader.getAttribLocation(INPUT_VERTEX_COORDINATE_NAME); inTcLocation = shader.getAttribLocation(INPUT_TEXTURE_COORDINATE_NAME); } shader.useProgram(); // Upload the vertex coordinates. GLES20.glEnableVertexAttribArray(inPosLocation); GLES20.glVertexAttribPointer(inPosLocation, /* size= */ 2, /* type= */ GLES20.GL_FLOAT, /* normalized= */ false, /* stride= */ 0, FULL_RECTANGLE_BUFFER); // Upload the texture coordinates. GLES20.glEnableVertexAttribArray(inTcLocation); GLES20.glVertexAttribPointer(inTcLocation, /* size= */ 2, /* type= */ GLES20.GL_FLOAT, /* normalized= */ false, /* stride= */ 0, FULL_RECTANGLE_TEXTURE_BUFFER); // Upload the texture transformation matrix. GLES20.glUniformMatrix4fv( texMatrixLocation, 1 /* count= */, false /* transpose= */, texMatrix, 0 /* offset= */); // Do custom per-frame shader preparation. shaderCallbacks.onPrepareShader( shader, texMatrix, frameWidth, frameHeight, viewportWidth, viewportHeight); GlUtil.checkNoGLES2Error("Prepare shader"); } /** * Release all GLES resources. This needs to be done manually, otherwise the resources are leaked. */ @Override public void release() { if (currentShader != null) { currentShader.release(); currentShader = null; currentShaderType = null; } } }