1、渲染YUV数据原理
1.1、为什么用OpenGL来处理YUVP颜色格式视频？
OpenGL中是不能直接渲染YUV数据的，但是我们可以用3个纹理来
分别获取Y、U和V的值，然后根据公式：
r = y + 1.403 * v;
g = y - 0.344 * u - 0.714 * v;
b = y + 1.770 * u;
转为rgb颜色格式显示出来。这个转换过程是在GPU中完成的，计算
效率比在CPU中计算高很多倍！
1.2、Shader编写
vertex_shader.glsl
attribute vec4 av_Position;
attribute vec2 af_Position;
varying vec2 v_texPosition;
void main() {
v_texPosition = af_Position;
gl_Position = av_Position;
}
注： attribute 只能在vertex中使用
varying 用于vertex和fragment之间传递值
fragment_shader.glsl
precision mediump float;
varying vec2 v_texPosition;
uniform sampler2D sampler_y;
uniform sampler2D sampler_u;
uniform sampler2D sampler_v;
void main() {
float y,u,v;
y = texture2D(sampler_y,v_texPosition).r;
u = texture2D(sampler_u,v_texPosition).r- 0.5;
v = texture2D(sampler_v,v_texPosition).r- 0.5;
vec3 rgb;
rgb.r = y + 1.403 * v;
rgb.g = y - 0.344 * u - 0.714 * v;
rgb.b = y + 1.770 * u;
gl_FragColor = vec4(rgb,1);
}
注： uniform 用于在application中向vertex和fragment中传递值。
2、OpenGL ES渲染YUV数据
2.1、实现OpenGL ES
实现OpenGL ES的工具类-JfShaderUtil.java
package com.example.myplayer.opengl;
import android.content.Context;
import android.opengl.GLES20;
import com.example.myplayer.log.JfLog;
import java.io.BufferedReader;
import java.io.InputStream;
import java.io.InputStreamReader;
public class JfShaderUtil {
public static String readRawTxt(Context context, int rawId) {
InputStream inputStream = context.getResources().openRawResource(rawId);
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
StringBuffer sb = new StringBuffer();
String line;
try
{
while((line = reader.readLine()) != null)
{
sb.append(line).append("\n");
}
reader.close();
}
catch (Exception e)
{
e.printStackTrace();
}
return sb.toString();
}
public static int loadShader(int shaderType, String source)
{
int shader = GLES20.glCreateShader(shaderType);
if(shader != 0)
{
GLES20.glShaderSource(shader, source);
GLES20.glCompileShader(shader);
int[] compile = new int[1];
GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compile, 0);
if(compile[0] != GLES20.GL_TRUE)
{
JfLog.e("shader compile error");
GLES20.glDeleteShader(shader);
shader = 0;
}
}
return shader;
}
public static int createProgram(String vertexSource, String fragmentSource)
{
int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
if(vertexShader == 0)
{
return 0;
}
int fragmentShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
if(fragmentShader == 0)
{
return 0;
}
int program = GLES20.glCreateProgram();
if(program != 0)
{
GLES20.glAttachShader(program, vertexShader);
GLES20.glAttachShader(program, fragmentShader);
GLES20.glLinkProgram(program);
int[] linsStatus = new int[1];
GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linsStatus, 0);
if(linsStatus[0] != GLES20.GL_TRUE)
{
JfLog.e( "link program error");
GLES20.glDeleteProgram(program);
program = 0;
}
}
return program;
}
}
实现JfRender
package com.example.myplayer.opengl;
import android.content.Context;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
import com.example.myplayer.R;
import com.example.myplayer.log.JfLog;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
public class JfRender implements GLSurfaceView.Renderer {
private Context context;
private final float[] vertexData ={//顶点坐标
-1f, -1f,
1f, -1f,
-1f, 1f,
1f, 1f
};
private final float[] textureData ={//纹理坐标
0f,1f,
1f, 1f,
0f, 0f,
1f, 0f
};
private FloatBuffer vertexBuffer;
private FloatBuffer textureBuffer;
private int program_yuv;
private int avPosition_yuv;
private int afPosition_yuv;
private int sampler_y;
private int sampler_u;
private int sampler_v;
private int[] textureId_yuv;
//渲染用
private int width_yuv;
private int height_yuv;
private ByteBuffer y;
private ByteBuffer u;
private ByteBuffer v;
public JfRender(Context context){
this.context = context;
//存储顶点坐标数据
vertexBuffer = ByteBuffer.allocateDirect(vertexData.length * 4)
.order(ByteOrder.nativeOrder())
.asFloatBuffer()
.put(vertexData);
vertexBuffer.position(0);
//存储纹理坐标
textureBuffer = ByteBuffer.allocateDirect(textureData.length * 4)
.order(ByteOrder.nativeOrder())
.asFloatBuffer()
.put(textureData);
textureBuffer.position(0);
}
@Override
public void onSurfaceCreated(GL10 gl, EGLConfig config) {
initRenderYUV();
}
@Override
public void onSurfaceChanged(GL10 gl, int width, int height) {
GLES20.glViewport(0, 0, width, height);
}
@Override
public void onDrawFrame(GL10 gl) {
//用黑色清屏
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
renderYUV();
}
/**
* 初始化
*/
private void initRenderYUV(){
String vertexSource = JfShaderUtil.readRawTxt(context, R.raw.vertex_shader);
String fragmentSource = JfShaderUtil.readRawTxt(context,R.raw.fragment_shader);
//创建一个渲染程序
program_yuv = JfShaderUtil.createProgram(vertexSource,fragmentSource);
//得到着色器中的属性
avPosition_yuv = GLES20.glGetAttribLocation(program_yuv,"av_Position");
afPosition_yuv = GLES20.glGetAttribLocation(program_yuv,"af_Position");
sampler_y = GLES20.glGetUniformLocation(program_yuv, "sampler_y");
sampler_u = GLES20.glGetUniformLocation(program_yuv, "sampler_u");
sampler_v = GLES20.glGetUniformLocation(program_yuv, "sampler_v");
//创建纹理
textureId_yuv = new int[3];
GLES20.glGenTextures(3, textureId_yuv, 0);
for(int i = 0; i < 3; i++)
{
//绑定纹理
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId_yuv[i]);
//设置环绕和过滤方式
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_REPEAT);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_REPEAT);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
}
JfLog.d("initRenderYUV");
}
public void setYUVRenderData(int width, int height, byte[] y, byte[] u, byte[] v)
{
this.width_yuv = width;
this.height_yuv = height;
this.y = ByteBuffer.wrap(y);
this.u = ByteBuffer.wrap(u);
this.v = ByteBuffer.wrap(v);
}
/**
* 渲染
*/
private void renderYUV(){
JfLog.d("渲染中");
if(width_yuv > 0 && height_yuv > 0 && y != null && u != null && v != null){
GLES20.glUseProgram(program_yuv);//使用源程序
GLES20.glEnableVertexAttribArray(avPosition_yuv);//使顶点属性数组有效
GLES20.glVertexAttribPointer(avPosition_yuv, 2, GLES20.GL_FLOAT, false, 8, vertexBuffer);//为顶点属性赋值
GLES20.glEnableVertexAttribArray(afPosition_yuv);
GLES20.glVertexAttribPointer(afPosition_yuv, 2, GLES20.GL_FLOAT, false, 8, textureBuffer);
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);//激活纹理
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId_yuv[0]);//绑定纹理
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, width_yuv, height_yuv, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, y);//
GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId_yuv[1]);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, width_yuv / 2, height_yuv / 2, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, u);
GLES20.glActiveTexture(GLES20.GL_TEXTURE2);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId_yuv[2]);
GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, width_yuv / 2, height_yuv / 2, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, v);
GLES20.glUniform1i(sampler_y, 0);
GLES20.glUniform1i(sampler_u, 1);
GLES20.glUniform1i(sampler_v, 2);
y.clear();
u.clear();
v.clear();
y = null;
u = null;
v = null;
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
}
}
}
实现JfGLSurfaceView
package com.example.myplayer.opengl;
import android.content.Context;
import android.opengl.GLSurfaceView;
import android.util.AttributeSet;
public class JfGLSurfaceView extends GLSurfaceView {
private JfRender jfRender;
public JfGLSurfaceView(Context context) {
this(context,null);
}
public JfGLSurfaceView(Context context, AttributeSet attrs) {
super(context, attrs);
setEGLContextClientVersion(2);
jfRender = new JfRender(context);
setRenderer(jfRender);
setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY);//requestRender()时不会重复渲染
}
public void setYUVData(int width,int height,byte[] y,byte[] u,byte[] v){
if (jfRender != null) {
jfRender.setYUVRenderData(width, height, y, u, v);
requestRender();
}
}
}
JfPlayer.java中调用
private JfGLSurfaceView jfGLSurfaceView;
public void setJfGLSurfaceView(JfGLSurfaceView jfGLSurfaceView) {
this.jfGLSurfaceView = jfGLSurfaceView;
}
/**
*
* @param width
* @param height
* @param y
* @param u
* @param v
*/
public void onCallRenderYUV(int width,int height,byte[] y,byte[] u,byte[] v){
JfLog.d("获取到视频的数据");
if (jfGLSurfaceView != null) {
jfGLSurfaceView.setYUVData(width, height, y, u, v);
}
}
MainActivity中设置WlGLSurfaceView：
private WlGLSurfaceView wlGLSurfaceView;
jfGLSurfaceView = (JfGLSurfaceView) findViewById(R.id.jfglsfv);
jfPlayer.setJfGLSurfaceView(jfGLSurfaceView);
问题
播放的时候会闪黑屏！
因为在JfRender.java中的private void renderYUV()方法中有一个判断语句,
if(width_yuv > 0 && height_yuv > 0 && y != null && u != null && v != null){}
有可能视频中的某些AVFrame可能不满足这些条件，比如y、u、v数据某一个为空，
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
导致这句不执行，所以没有画出来矩形；
@Override
public void onDrawFrame(GL10 gl) {
//用黑色清屏
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
renderYUV();
}
在onDrawFrame回调方法中就用黑色清了一次屏，所以会闪黑屏！
如果改为
@Override
public void onDrawFrame(GL10 gl) {
//用黑色清屏
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glClearColor(1.0f, 0.0f, 0.0f, 1.0f);
renderYUV();
}
会很明显的闪红屏！
解决方法：
不管有没有渲染都把矩形画出来！
@Override
public void onDrawFrame(GL10 gl) {
//用黑色清屏
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
renderYUV();
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
}
