基於iOS的網路音視訊實時傳輸系統（三）- VideoToolbox編碼音視訊資料為H264、AAC
阿新 • • 發佈：2019-02-17
server端 -- 編碼音視訊資料為H264、AAC
這部分花了好多時間，本身就不具備這方面的相關知識，查閱了不少資料，不過關於VideoToolbox和AudioToolbox方面的編碼資料寥寥無幾，雖然網上搜索結果看似特別多，其實一看 內容也大同小異，建議還是看看官方的文件。
下載
VideoToolbox編碼視訊資料為H264
初始化--建立session
// ----- 1. 建立session ----- int width = 640, height = 480; OSStatus status = VTCompressionSessionCreate(NULL, width, height, kCMVideoCodecType_H264, NULL, NULL, NULL, didCompressH264, (__bridge void *)(self), &EncodingSession); NSLog(@"H264: VTCompressionSessionCreate %d", (int)status); if (status != 0) { NSLog(@"H264: session 建立失敗"); return ; } // ----- 2. 設定session屬性 ----- // 設定實時編碼輸出（避免延遲） VTSessionSetProperty(EncodingSession, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue); VTSessionSetProperty(EncodingSession, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Baseline_AutoLevel); // 設定關鍵幀（GOPsize)間隔 int frameInterval = 10; CFNumberRef frameIntervalRef = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &frameInterval); VTSessionSetProperty(EncodingSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, frameIntervalRef); // 設定期望幀率 int fps = 10; CFNumberRef fpsRef = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &fps); VTSessionSetProperty(EncodingSession, kVTCompressionPropertyKey_ExpectedFrameRate, fpsRef); //設定位元速率，上限，單位是bps int bitRate = width * height * 3 * 4 * 8; CFNumberRef bitRateRef = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &bitRate); VTSessionSetProperty(EncodingSession, kVTCompressionPropertyKey_AverageBitRate, bitRateRef); //設定位元速率，均值，單位是byte int bitRateLimit = width * height * 3 * 4; CFNumberRef bitRateLimitRef = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &bitRateLimit); VTSessionSetProperty(EncodingSession, kVTCompressionPropertyKey_DataRateLimits, bitRateLimitRef); // Tell the encoder to start encoding VTCompressionSessionPrepareToEncodeFrames(EncodingSession);
編碼完成回撥
將來通過這個回撥獲取H264資料
void didCompressH264(void *outputCallbackRefCon, void *sourceFrameRefCon, OSStatus status, VTEncodeInfoFlags infoFlags, CMSampleBufferRef sampleBuffer) { // NSLog(@"didCompressH264 called with status %d infoFlags %d", (int)status, (int)infoFlags); // 0 1 if (status != 0) { return; } if (!CMSampleBufferDataIsReady(sampleBuffer)) { NSLog(@"didCompressH264 data is not ready "); return; } // ViewController* encoder = (__bridge ViewController*)outputCallbackRefCon; HJH264Encoder *encoder = (__bridge HJH264Encoder*)(outputCallbackRefCon); // ----- 關鍵幀獲取SPS和PPS ------ bool keyframe = !CFDictionaryContainsKey( (CFArrayGetValueAtIndex(CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true), 0)), kCMSampleAttachmentKey_NotSync); // 判斷當前幀是否為關鍵幀 // 獲取sps & pps資料 if (keyframe) { CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer); size_t sparameterSetSize, sparameterSetCount; const uint8_t *sparameterSet; OSStatus statusCode = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 0, &sparameterSet, &sparameterSetSize, &sparameterSetCount, 0 ); if (statusCode == noErr) { // Found sps and now check for pps size_t pparameterSetSize, pparameterSetCount; const uint8_t *pparameterSet; OSStatus statusCode = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 1, &pparameterSet, &pparameterSetSize, &pparameterSetCount, 0 ); if (statusCode == noErr) { // Found pps NSData *sps = [NSData dataWithBytes:sparameterSet length:sparameterSetSize]; NSData *pps = [NSData dataWithBytes:pparameterSet length:pparameterSetSize]; if (encoder) { [encoder gotSpsPps:sps pps:pps]; // 獲取sps & pps資料 } } } } // --------- 寫入資料 ---------- CMBlockBufferRef dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer); size_t length, totalLength; char *dataPointer; OSStatus statusCodeRet = CMBlockBufferGetDataPointer(dataBuffer, 0, &length, &totalLength, &dataPointer); if (statusCodeRet == noErr) { size_t bufferOffset = 0; static const int AVCCHeaderLength = 4; // 返回的nalu資料前四個位元組不是0001的startcode，而是大端模式的幀長度length // 迴圈獲取nalu資料 while (bufferOffset < totalLength - AVCCHeaderLength) { uint32_t NALUnitLength = 0; // Read the NAL unit length memcpy(&NALUnitLength, dataPointer + bufferOffset, AVCCHeaderLength); // 從大端轉系統端 NALUnitLength = CFSwapInt32BigToHost(NALUnitLength); NSData* data = [[NSData alloc] initWithBytes:(dataPointer + bufferOffset + AVCCHeaderLength) length:NALUnitLength]; [encoder gotEncodedData:data isKeyFrame:keyframe]; // Move to the next NAL unit in the block buffer bufferOffset += AVCCHeaderLength + NALUnitLength; } } }
傳入需要編碼的幀
- (void) encode:(CMSampleBufferRef )sampleBuffer { CVImageBufferRef imageBuffer = (CVImageBufferRef)CMSampleBufferGetImageBuffer(sampleBuffer); // 幀時間，如果不設定會導致時間軸過長。 CMTime presentationTimeStamp = CMTimeMake(frameID++, 1000); // CMTimeMake(分子，分母)；分子/分母 = 時間(秒) VTEncodeInfoFlags flags; OSStatus statusCode = VTCompressionSessionEncodeFrame(EncodingSession, imageBuffer, presentationTimeStamp, kCMTimeInvalid, NULL, NULL, &flags); if (statusCode != noErr) { NSLog(@"H264: VTCompressionSessionEncodeFrame failed with %d", (int)statusCode); VTCompressionSessionInvalidate(EncodingSession); CFRelease(EncodingSession); EncodingSession = NULL; return; } }
然後就可以在上面的回撥裡取得編碼後的資料，再把資料通過socket發給客戶端即可。
在每個階段都要記得測試、列印日誌，不然以後找bug會很辛苦的。
這裡可以把編碼後的資料寫入本地檔案，然後用VLC工具開啟，檢測編碼是否有問題。
最後不要忘記關閉編碼器
- (void)EndVideoToolBox
{
VTCompressionSessionCompleteFrames(EncodingSession, kCMTimeInvalid);
VTCompressionSessionInvalidate(EncodingSession);
CFRelease(EncodingSession);
EncodingSession = NULL;
}另：在macOS環境下使用VideoToolbox編碼的過程在這個部落格裡：
AudioToolbox編碼音訊資料為AAC
設定編碼引數
- (void) setupEncoderFromSampleBuffer:(CMSampleBufferRef)sampleBuffer {
AudioStreamBasicDescription inAudioStreamBasicDescription = *CMAudioFormatDescriptionGetStreamBasicDescription((CMAudioFormatDescriptionRef)CMSampleBufferGetFormatDescription(sampleBuffer));
AudioStreamBasicDescription outAudioStreamBasicDescription = {0}; // 初始化輸出流的結構體描述為0. 很重要。
outAudioStreamBasicDescription.mSampleRate = inAudioStreamBasicDescription.mSampleRate; // 音訊流，在正常播放情況下的幀率。如果是壓縮的格式，這個屬性表示解壓縮後的幀率。幀率不能為0。
outAudioStreamBasicDescription.mFormatID = kAudioFormatMPEG4AAC; // 設定編碼格式
outAudioStreamBasicDescription.mFormatFlags = kMPEG4Object_AAC_LC; // 無損編碼 ，0表示沒有
outAudioStreamBasicDescription.mBytesPerPacket = 0; // 每一個packet的音訊資料大小。如果的動態大小，設定為0。動態大小的格式，需要用AudioStreamPacketDescription 來確定每個packet的大小。
outAudioStreamBasicDescription.mFramesPerPacket = 1024; // 每個packet的幀數。如果是未壓縮的音訊資料，值是1。動態位元速率格式，這個值是一個較大的固定數字，比如說AAC的1024。如果是動態大小幀數（比如Ogg格式）設定為0。
outAudioStreamBasicDescription.mBytesPerFrame = 0; // 每幀的大小。每一幀的起始點到下一幀的起始點。如果是壓縮格式，設定為0 。
outAudioStreamBasicDescription.mChannelsPerFrame = 1; // 聲道數
outAudioStreamBasicDescription.mBitsPerChannel = 0; // 壓縮格式設定為0
outAudioStreamBasicDescription.mReserved = 0; // 8位元組對齊，填0.
AudioClassDescription *description = [self
getAudioClassDescriptionWithType:kAudioFormatMPEG4AAC
fromManufacturer:kAppleSoftwareAudioCodecManufacturer]; //軟編
OSStatus status = AudioConverterNewSpecific(&inAudioStreamBasicDescription, &outAudioStreamBasicDescription, 1, description, &_audioConverter); // 建立轉換器
if (status != 0) {
NSLog(@"setup converter: %d", (int)status);
}
}獲取編解碼器
- (AudioClassDescription *)getAudioClassDescriptionWithType:(UInt32)type
fromManufacturer:(UInt32)manufacturer
{
static AudioClassDescription desc;
UInt32 encoderSpecifier = type;
OSStatus st;
UInt32 size;
st = AudioFormatGetPropertyInfo(kAudioFormatProperty_Encoders,
sizeof(encoderSpecifier),
&encoderSpecifier,
&size);
if (st) {
NSLog(@"error getting audio format propery info: %d", (int)(st));
return nil;
}
unsigned int count = size / sizeof(AudioClassDescription);
AudioClassDescription descriptions[count];
st = AudioFormatGetProperty(kAudioFormatProperty_Encoders,
sizeof(encoderSpecifier),
&encoderSpecifier,
&size,
descriptions);
if (st) {
NSLog(@"error getting audio format propery: %d", (int)(st));
return nil;
}
for (unsigned int i = 0; i < count; i++) {
if ((type == descriptions[i].mSubType) &&
(manufacturer == descriptions[i].mManufacturer)) {
memcpy(&desc, &(descriptions[i]), sizeof(desc));
return &desc;
}
}
return nil;
}將裝置捕獲到的音訊資料傳給編碼器
- (void) encodeSampleBuffer:(CMSampleBufferRef)sampleBuffer completionBlock:(void (^)(NSData * encodedData, NSError* error))completionBlock {
CFRetain(sampleBuffer);
dispatch_async(_encoderQueue, ^{
if (!_audioConverter) {
[self setupEncoderFromSampleBuffer:sampleBuffer];
}
CMBlockBufferRef blockBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
CFRetain(blockBuffer);
// --------- 通過CMBlockBufferGetDataPointer獲取到_pcmBufferSize和_pcmBuffer --------
OSStatus status = CMBlockBufferGetDataPointer(blockBuffer, 0, NULL, &_pcmBufferSize, &_pcmBuffer);
NSError *error = nil;
if (status != kCMBlockBufferNoErr) {
error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil];
}
memset(_aacBuffer, 0, _aacBufferSize);
AudioBufferList outAudioBufferList = {0};
outAudioBufferList.mNumberBuffers = 1;
outAudioBufferList.mBuffers[0].mNumberChannels = 1;
outAudioBufferList.mBuffers[0].mDataByteSize = (int)_aacBufferSize;
outAudioBufferList.mBuffers[0].mData = _aacBuffer;
AudioStreamPacketDescription *outPacketDescription = NULL;
UInt32 ioOutputDataPacketSize = 1;
// Converts data supplied by an input callback function, supporting non-interleaved and packetized formats.
// Produces a buffer list of output data from an AudioConverter. The supplied input callback function is called whenever necessary.
status = AudioConverterFillComplexBuffer(_audioConverter, inInputDataProc, (__bridge void *)(self), &ioOutputDataPacketSize, &outAudioBufferList, outPacketDescription);
NSData *data = nil;
if (status == 0) {
NSData *rawAAC = [NSData dataWithBytes:outAudioBufferList.mBuffers[0].mData length:outAudioBufferList.mBuffers[0].mDataByteSize];
NSData *adtsHeader = [self adtsDataForPacketLength:rawAAC.length];
NSMutableData *fullData = [NSMutableData dataWithData:adtsHeader];
[fullData appendData:rawAAC];
data = fullData;
} else {
error = [NSError errorWithDomain:NSOSStatusErrorDomain code:status userInfo:nil];
}
if (completionBlock) {
dispatch_async(_callbackQueue, ^{
// printf("----- audio data len = %d ----\n",(int)[data length]);
completionBlock(data, error);
});
}
CFRelease(sampleBuffer);
CFRelease(blockBuffer);
});
}回撥函式
OSStatus inInputDataProc(AudioConverterRef inAudioConverter, UInt32 *ioNumberDataPackets, AudioBufferList *ioData, AudioStreamPacketDescription **outDataPacketDescription, void *inUserData)
{
AACEncoder *encoder = (__bridge AACEncoder *)(inUserData);
UInt32 requestedPackets = *ioNumberDataPackets;
size_t copiedSamples = [encoder copyPCMSamplesIntoBuffer:ioData];
if (copiedSamples < requestedPackets) {
//PCM 緩衝區還沒滿
*ioNumberDataPackets = 0;
return -1;
}
*ioNumberDataPackets = 1;
return noErr;
}
/**
* 填充PCM到緩衝區
*/
- (size_t) copyPCMSamplesIntoBuffer:(AudioBufferList*)ioData {
size_t originalBufferSize = _pcmBufferSize;
if (!originalBufferSize) {
return 0;
}
ioData->mBuffers[0].mData = _pcmBuffer;
ioData->mBuffers[0].mDataByteSize = (int)_pcmBufferSize;
_pcmBuffer = NULL;
_pcmBufferSize = 0;
return originalBufferSize;
}最後在需要的地方釋放編碼器
- (void) dealloc {
AudioConverterDispose(_audioConverter);
free(_aacBuffer);
}