MediaCodec、OpenGL、OpenSL/AudioTrack 實現一款簡單的視訊播放器
功能很簡單,大致流程為:
- MediaCodec 解碼視訊檔案得到 YUV、PCM 資料
- OpenGL 將 YUV 轉為 RGB,並渲染到 Surface 上
- OpenSL/AudoTrack 獲取 PCM 資料並播放
需要的前置知識有:
- YUV、PCM 等基礎音視訊知識,如 YUV 轉 RGB
- MediaCodec 的使用
- OpenGL,包括 EGL、紋理等
- OpenSL 或 AudioTrack 的使用
MediaCodec 解碼
大致流程和普通的解碼類似,在編寫視訊播放器這個功能時,需要注意的地方有兩個:
- 監聽解碼流程
public interface OnDecodeListener { void onImageDecoded(byte[] data); void onSampleDecoded(byte[] data); void onDecodeEnded(); } 複製程式碼
也可以加一個 onDecodeError() 的介面,看需要擴充套件即可。
- 播放和解碼同步
因為視訊資料量很大,不可能把解碼後的 YUV 資料儲存在一個佇列裡,再慢慢拿出來使用 OpenGL 渲染(很容易就 OOM 了),因此,必須控制解碼的速率,最簡單的控制方式是和播放同步,如下所示:
ByteBuffer outputBuffer = outputBuffers[outIndex]; outputBuffer.position(bufferInfo.offset); outputBuffer.limit(bufferInfo.offset + bufferInfo.size); byte[] data = new byte[bufferInfo.size]; outputBuffer.get(data); if (mIsDecodeWithPts) { if (startTime == 0) { startTime = System.nanoTime(); } else { passTime = (System.nanoTime() - startTime) / 1000; if (passTime < bufferInfo.presentationTimeUs) { TimeUnit.MICROSECONDS.sleep(bufferInfo.presentationTimeUs - passTime); } } } if (mediaType == HWCodec.MEDIA_TYPE_VIDEO && listener != null) { listener.onImageDecoded(data); } else if (listener != null) { listener.onSampleDecoded(data); } 複製程式碼
OpenGL 渲染 YUV 資料
和渲染紋理的流程類似,不同的地方在於需要轉換 YUV 資料為 RGB,而 YUV 資料又有 YUV420P、YUV420SP 等多種格式,因此在轉換 RGB 之前,需要統一 YUV 資料的格式,這裡使用的是 YUV420P。
YUV 資料格式之間的轉換可以自己寫,比如 YUV420SP 轉換為 YUV420P,只需要把最後的 U、V 資料分別逐個放入到一個數組裡即可,但考慮到視訊裁剪、旋轉,以及之後可能用到的各種 YUV 資料處理功能,因此這裡引入了一個 libyuv 的庫,使用非常簡單:
Yuv* convertToI420(AVModel *model) { ... Yuv *yuv = new Yuv(model->width, model->height); ConvertToI420(model->image, (size_t) model->imageLen, yuv->bufY, yuv->strideY, yuv->bufU, yuv->strideU, yuv->bufV, yuv->strideV, 0, 0, model->width, model->height, model->width, model->height, kRotate0, getFourCC(model->pixelFormat)); return yuv; } 複製程式碼
AVModel、Yuv 是我自定義的兩個類,分別用於儲存音視訊資料及相關資訊、YUV 資料及相關資訊,原始碼可見GitHub 。
YUV 轉 RGB 的相關係數在可上網查詢,fragment shader 示例如下:
#version 300 es precision highp float; uniform sampler2D yTexture; uniform sampler2D uTexture; uniform sampler2D vTexture; in vec2 vTexCoord; layout(location=0) out vec4 fragColor; void main() { highp float y = texture(yTexture, vTexCoord).r; highp float u = texture(uTexture, vTexCoord).r - 0.5; highp float v = texture(vTexture, vTexCoord).r - 0.5; highp float r = y + 1.402 * v; highp float g = y - 0.344 * u - 0.714 * v; highp float b = y + 1.772 * u; fragColor = vec4(r, g, b, 1.0); } 複製程式碼
OpenGL 關鍵程式碼如下:
bool YuvRenderer::doInit() { std::string *vShader = readShaderFromAsset(mAssetManager, "yuv_renderer.vert"); std::string *fShader = readShaderFromAsset(mAssetManager, "yuv_renderer.frag"); mProgram = loadProgram(vShader->c_str(), fShader->c_str()); mMatrixLoc = glGetUniformLocation(mProgram, "mMatrix"); mSamplerY = glGetUniformLocation(mProgram, "yTexture"); mSamplerU = glGetUniformLocation(mProgram, "uTexture"); mSamplerV = glGetUniformLocation(mProgram, "vTexture"); glPixelStorei(GL_UNPACK_ALIGNMENT, 1); // 生成三個紋理,分別用於裝載 Y、U、V 資料 glGenTextures(3, mTextures); glBindTexture(GL_TEXTURE_2D, mTextures[0]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth, mTexHeight, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glBindTexture(GL_TEXTURE_2D, mTextures[1]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth / 2, mTexHeight / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); glBindTexture(GL_TEXTURE_2D, mTextures[2]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth / 2, mTexHeight / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, 0); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); // 快取頂點座標、紋理座標、索引資料到 VBO 中 glGenBuffers(3, mVboIds); glBindBuffer(GL_ARRAY_BUFFER, mVboIds[0]); glBufferData(GL_ARRAY_BUFFER, sizeof(VERTICES), VERTICES, GL_STATIC_DRAW); glBindBuffer(GL_ARRAY_BUFFER, mVboIds[1]); glBufferData(GL_ARRAY_BUFFER, sizeof(TEX_COORDS), TEX_COORDS, GL_STATIC_DRAW); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mVboIds[2]); glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(INDICES), INDICES, GL_STATIC_DRAW); // 快取 VBO 到 VAO 中 glGenVertexArrays(1, &mVaoId); glBindVertexArray(mVaoId); glBindBuffer(GL_ARRAY_BUFFER, mVboIds[0]); glEnableVertexAttribArray(ATTRIB_POSITION); glVertexAttribPointer(ATTRIB_POSITION, VERTEX_POS_SIZE, GL_FLOAT, GL_FALSE, sizeof(GLfloat) * VERTEX_POS_SIZE, 0); glBindBuffer(GL_ARRAY_BUFFER, mVboIds[1]); glEnableVertexAttribArray(ATTRIB_TEX_COORD); glVertexAttribPointer(ATTRIB_TEX_COORD, TEX_COORD_SIZE, GL_FLOAT, GL_FALSE, sizeof(GLfloat) * TEX_COORD_SIZE, 0); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, mVboIds[2]); glBindVertexArray(0); glBindBuffer(GL_ARRAY_BUFFER, 0); glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, 0); glClearColor(1.0f, 1.0f, 1.0f, 1.0f); delete vShader; delete fShader; return true; } void YuvRenderer::doDraw() { glViewport(0, 0, mWidth, mHeight); glClear(GL_COLOR_BUFFER_BIT); glUseProgram(mProgram); glUniformMatrix4fv(mMatrixLoc, 1, GL_FALSE, mMatrix); if (!mYuv) { LOGW("YuvRenderer doDraw failed: yuv data have not assigned"); return; } // 分別載入 Y、U、V 資料到對應的紋理中 glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_2D, mTextures[0]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth, mTexHeight, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, mYuv->bufY); glUniform1i(mSamplerY, 0); glActiveTexture(GL_TEXTURE1); glBindTexture(GL_TEXTURE_2D, mTextures[1]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth / 2, mTexHeight / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, mYuv->bufU); glUniform1i(mSamplerU, 1); glActiveTexture(GL_TEXTURE2); glBindTexture(GL_TEXTURE_2D, mTextures[2]); glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, mTexWidth / 2, mTexHeight / 2, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE, mYuv->bufV); glUniform1i(mSamplerV, 2); // 使用 VAO 快取的座標資料繪製圖像 glBindVertexArray(mVaoId); glDrawElements(GL_TRIANGLES, INDEX_NUMBER, GL_UNSIGNED_SHORT, 0); glBindVertexArray(0); glBindTexture(GL_TEXTURE_2D, 0); } 複製程式碼
OpenSL 播放 PCM 資料
初始化播放器:
bool BQAudioPlayer::init() { SLresult result; SLDataLocator_AndroidSimpleBufferQueue locBufq = {SL_DATALOCATOR_ANDROIDSIMPLEBUFFERQUEUE, 2}; // channelMask: 位數和 channel 相等,0 代表 SL_SPEAKER_FRONT_LEFT | SL_SPEAKER_FRONT_RIGHT SLDataFormat_PCM formatPcm = {SL_DATAFORMAT_PCM, (SLuint32) mChannels, mSampleRate, (SLuint32) mSampleFormat, (SLuint32) mSampleFormat, mChannels == 2 ? 0 : SL_SPEAKER_FRONT_CENTER, SL_BYTEORDER_LITTLEENDIAN}; if (mSampleRate) { formatPcm.samplesPerSec = mSampleRate; } SLDataSource audioSrc = {&locBufq, &formatPcm}; SLDataLocator_OutputMix locOutpuMix = {SL_DATALOCATOR_OUTPUTMIX, mAudioEngine->outputMixObj}; SLDataSink audioSink = {&locOutpuMix, nullptr}; const SLInterfaceID ids[3] = {SL_IID_BUFFERQUEUE, SL_IID_VOLUME, SL_IID_EFFECTSEND}; const SLboolean req[3] = {SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE, SL_BOOLEAN_TRUE}; result = (*mAudioEngine->engine)->CreateAudioPlayer(mAudioEngine->engine, &mPlayerObj, &audioSrc, &audioSink, mSampleRate ? 2 : 3, ids, req); if (result != SL_RESULT_SUCCESS) { LOGE("CreateAudioPlayer failed: %d", result); return false; } result = (*mPlayerObj)->Realize(mPlayerObj, SL_BOOLEAN_FALSE); if (result != SL_RESULT_SUCCESS) { LOGE("mPlayerObj Realize failed: %d", result); return false; } result = (*mPlayerObj)->GetInterface(mPlayerObj, SL_IID_PLAY, &mPlayer); if (result != SL_RESULT_SUCCESS) { LOGE("mPlayerObj GetInterface failed: %d", result); return false; } result = (*mPlayerObj)->GetInterface(mPlayerObj, SL_IID_BUFFERQUEUE, &mBufferQueue); if (result != SL_RESULT_SUCCESS) { LOGE("mPlayerObj GetInterface failed: %d", result); return false; } result = (*mBufferQueue)->RegisterCallback(mBufferQueue, playerCallback, this); if (result != SL_RESULT_SUCCESS) { LOGE("mPlayerObj RegisterCallback failed: %d", result); return false; } mEffectSend = nullptr; if (mSampleRate == 0) { result = (*mPlayerObj)->GetInterface(mPlayerObj, SL_IID_EFFECTSEND, &mEffectSend); if (result != SL_RESULT_SUCCESS) { LOGE("mPlayerObj GetInterface failed: %d", result); return false; } } result = (*mPlayerObj)->GetInterface(mPlayerObj, SL_IID_VOLUME, &mVolume); if (result != SL_RESULT_SUCCESS) { LOGE("mPlayerObj GetInterface failed: %d", result); return false; } result = (*mPlayer)->SetPlayState(mPlayer, SL_PLAYSTATE_PLAYING); if (result != SL_RESULT_SUCCESS) { LOGE("mPlayerObj SetPlayState failed: %d", result); return false; } return true; } 複製程式碼
之後只需要把 PCM 入隊即可:
// 一幀音訊播放完畢後就會回撥這個函式 void playerCallback(SLAndroidSimpleBufferQueueItf bq, void *context) { BQAudioPlayer *player = (BQAudioPlayer *) context; assert(bq == player->mBufferQueue); pthread_mutex_unlock(&player->mMutex); } void BQAudioPlayer::enqueueSample(void *data, size_t length) { // 必須等待一幀音訊播放完畢後才可以 Enqueue 第二幀音訊 pthread_mutex_lock(&mMutex); if (mBufSize < length) { mBufSize = length; if (mBuffers[0]) { delete[] mBuffers[0]; } if (mBuffers[1]) { delete[] mBuffers[1]; } mBuffers[0] = new uint8_t[mBufSize]; mBuffers[1] = new uint8_t[mBufSize]; } memcpy(mBuffers[mIndex], data, length); (*mBufferQueue)->Enqueue(mBufferQueue, mBuffers[mIndex], length); mIndex = 1 - mIndex; } 複製程式碼
結束播放:
void BQAudioPlayer::release() { pthread_mutex_lock(&mMutex); if (mPlayerObj) { (*mPlayerObj)->Destroy(mPlayerObj); mPlayerObj = nullptr; mPlayer = nullptr; mBufferQueue = nullptr; mEffectSend = nullptr; mVolume = nullptr; } if (mAudioEngine) { delete mAudioEngine; mAudioEngine = nullptr; } if (mBuffers[0]) { delete[] mBuffers[0]; mBuffers[0] = nullptr; } if (mBuffers[1]) { delete[] mBuffers[1]; mBuffers[1] = nullptr; } pthread_mutex_unlock(&mMutex); pthread_mutex_destroy(&mMutex); } 複製程式碼
AudioTrack 播放 PCM 資料
相對 OpenSL,AudioTrack 程式碼量少很多,設定 AudioTrack:
private void setupAudioTrack() { int channelConfig = mChannels == 1 ? AudioFormat.CHANNEL_OUT_MONO : AudioFormat.CHANNEL_OUT_STEREO; // 獲取 sample format 的 API 要求高,這裡預設使用 ENCODING_PCM_16BIT int bufferSize = AudioTrack.getMinBufferSize(mSampleRate, channelConfig, AudioFormat.ENCODING_PCM_16BIT); mAudioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, mSampleRate, channelConfig, AudioFormat.ENCODING_PCM_16BIT, bufferSize, AudioTrack.MODE_STREAM); } 複製程式碼
播放 PCM 資料:
@Override public void onSampleDecoded(byte[] data) { if (mIsPlaying) { mAudioTrack.write(data, 0, data.length); mAudioTrack.play(); } } 複製程式碼
結束播放:
private void releaseAudioTrack() { if (mAudioTrack != null) { mAudioTrack.stop(); mAudioTrack.release(); mAudioTrack = null; } } 複製程式碼
以上,一款簡單的視訊播放器就完成了,當然還有很多細節沒有處理,有興趣的可以參考 ijkplayer 自行完善。
原始碼已上傳到GitHub 。