FFmpeg In Android - H264碼流解碼/OpenGL ES渲染
阿新 • • 發佈:2018-11-19
主要思路是FFmpeg解碼H264得到一張yuv420p圖片後,傳遞給opengl es在著色器內部做圖片轉換yuv->rgb,然後通過紋理貼圖的方式渲染出來.這種方式的效率更高.核心程式碼如下:
#include "common.h" #include "gl_util.h" #ifdef __cplusplus extern "C" { #endif #include <libavformat/avformat.h> #include <libavutil/imgutils.h> #include <libavutil/pixfmt.h> #ifdef __cplusplus }; #endif auto gVertexShader = "attribute vec4 aPosition;\n" "attribute vec2 a_TexCoord;\n" "varying vec2 v_TexCoord;\n" "void main() {\n" " gl_Position = aPosition;\n" " v_TexCoord = a_TexCoord;\n" "}\n"; auto gFragmentShader = "precision mediump float;\n" "uniform sampler2D samplerY;\n" "uniform sampler2D samplerU;\n" "uniform sampler2D samplerV;\n" "varying vec2 v_TexCoord;\n" "void main() {\n" " vec3 yuv;\n" " vec3 rgb;\n" " yuv.x = texture2D(samplerY, v_TexCoord).r;\n" " yuv.y = texture2D(samplerU, v_TexCoord).r - 0.5;\n" " yuv.z = texture2D(samplerV, v_TexCoord).r - 0.5;\n" " rgb = mat3(1, 1, 1, 0, -0.39465, 2.03211, 1.13983, -0.58060, 0) * yuv;\n" " gl_FragColor = vec4(rgb, 1);\n" "}\n"; AVPacket avpkt; AVCodecContext *avctx; const AVCodec *codec; AVFrame *frame; uint8_t *video_dst_data[4] = {NULL}; int video_dst_linesize[4]; int video_dst_bufsize; const GLfloat gTriangleVertices[] = { -0.5f, 0.5f, 0.0f, 1 - 1.0f, -0.5f, -0.5f, 0.0f, 1 - 0.0f, 0.5f, 0.5f, 1.0f, 1 - 1.0f, 0.5f, -0.5f, 1.0f, 1 - 0.0f}; GLuint gProgram; GLuint gaPositionHandle; GLuint gaTexCoordHandle; GLuint uTextureSamplerYHandle; GLuint uTextureSamplerUHandle; GLuint uTextureSamplerVHandle; void ffmpeg_log_callback(void *ptr, int level, const char *fmt, va_list vl) { va_list vl2; char line[1024]; static int print_prefix = 1; va_copy(vl2, vl); // av_log_default_callback(ptr, level, fmt, vl); av_log_format_line(ptr, level, fmt, vl2, line, sizeof(line), &print_prefix); va_end(vl2); LOGD("%s", line); } extern "C" JNIEXPORT void JNICALL Java_com_example_king_ffmpegexample_GLESRenderUtil_init( JNIEnv* env, jobject obj) { av_register_all(); av_log_set_callback(ffmpeg_log_callback); av_init_packet(&avpkt); codec = avcodec_find_decoder(AV_CODEC_ID_H264); if (!codec) { LOGE("Codec not found"); } avctx = avcodec_alloc_context3(codec); if (!avctx) { LOGE("Could not allocate video codec context\n"); } if (codec->capabilities & AV_CODEC_CAP_TRUNCATED) avctx->flags |= AV_CODEC_FLAG_TRUNCATED; // we do not send complete frames avctx->width = 240; avctx->height = 320; avctx->time_base.num = 1; avctx->frame_number = 1; avctx->codec_type = AVMEDIA_TYPE_VIDEO; avctx->bit_rate = 0; /* open it */ if (avcodec_open2(avctx, codec, NULL) < 0) { LOGE("Could not open codec\n"); } frame = av_frame_alloc(); if (!frame) { LOGE("Could not allocate video frame\n"); } int ret = av_image_alloc(video_dst_data, video_dst_linesize, avctx->width, avctx->height, AV_PIX_FMT_YUV420P, 1); if (ret < 0) { LOGE("Could not allocate raw video buffer\n"); return; } video_dst_bufsize = ret; } extern "C" JNIEXPORT void JNICALL Java_com_example_king_ffmpegexample_GLESRenderUtil_resize( JNIEnv* env, jobject obj, jint width, jint height) { glViewport(0, 0, width, height); gProgram = linkProgram(gVertexShader, gFragmentShader); if (!gProgram) { LOGE("Could not create program."); return; } glUseProgram(gProgram); gaPositionHandle = glGetAttribLocation(gProgram, "aPosition"); glVertexAttribPointer(gaPositionHandle, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), gTriangleVertices); glEnableVertexAttribArray(gaPositionHandle); gaTexCoordHandle = glGetAttribLocation(gProgram, "a_TexCoord"); glVertexAttribPointer(gaTexCoordHandle, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(GLfloat), gTriangleVertices + 2); glEnableVertexAttribArray(gaTexCoordHandle); uTextureSamplerYHandle = glGetUniformLocation(gProgram, "samplerY"); uTextureSamplerUHandle = glGetUniformLocation(gProgram, "samplerU"); uTextureSamplerVHandle = glGetUniformLocation(gProgram, "samplerV"); } void videoFrameRender(char * yuvDataBuf, int yuv_width, int yuv_height) { GLuint textureYID; GLuint textureUID; GLuint textureVID; glGenTextures(1, &textureYID); glGenTextures(1, &textureUID); glGenTextures(1, &textureVID); glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, textureYID); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, yuv_width, yuv_height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, yuvDataBuf); glGenerateMipmap(GL_TEXTURE_2D); glUniform1i(uTextureSamplerYHandle, 0); glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, textureUID); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, yuv_width/2, yuv_height/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, yuvDataBuf + yuv_width * yuv_height); glGenerateMipmap(GL_TEXTURE_2D); glUniform1i(uTextureSamplerUHandle, 1); glActiveTexture(GL_TEXTURE2); glBindTexture(GL_TEXTURE_2D, textureVID); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, yuv_width/2, yuv_height/2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, yuvDataBuf + yuv_width * yuv_height * 5 / 4); glGenerateMipmap(GL_TEXTURE_2D); glUniform1i(uTextureSamplerVHandle, 2); glClearColor(0.0f, 0.0f, 0.0f, 1.0f); glClear(GL_COLOR_BUFFER_BIT); glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); glBindTexture(GL_TEXTURE_2D, 0); glDeleteTextures(1, &textureYID); glDeleteTextures(1, &textureUID); glDeleteTextures(1, &textureVID); } extern "C" JNIEXPORT void JNICALL Java_com_example_king_ffmpegexample_GLESRenderUtil_step( JNIEnv* env, jobject obj, jbyteArray in, jint len) { if (len > 0) { jbyte *buf = env->GetByteArrayElements(in, NULL); if (buf) { avpkt.size = len; avpkt.data = (uint8_t *) buf; int got_frame = 0; int decodedLen = avcodec_decode_video2(avctx, frame, &got_frame, &avpkt); if (decodedLen < 0) { LOGE("Error while decoding frame"); } if (got_frame) { LOGD("got_frame----------------------step"); av_image_copy(video_dst_data, video_dst_linesize, (const uint8_t **) (frame->data), frame->linesize, AV_PIX_FMT_YUV420P, avctx->width, avctx->height); videoFrameRender((char*)video_dst_data[0], avctx->width, avctx->height); } env->ReleaseByteArrayElements(in, buf, 0); } } } extern "C" JNIEXPORT void JNICALL Java_com_example_king_ffmpegexample_GLESRenderUtil_release( JNIEnv* env, jobject obj) { avcodec_free_context(&avctx); av_frame_free(&frame); av_free(video_dst_data[0]); }
著色器用到的yuv轉換rgb公式如下,用矩陣乘法實現:
R = Y + 1.13983V
G = Y - 0.39465U - 0.58060V
B = Y + 2.03211U
本例子完整原始碼H264AndroidGLActivity,gl_render.cpp
轉:OpenGL播放yuv視訊
轉:最簡單的視音訊播放示例6:OpenGL播放YUV420P(通過Texture,使用Shader)