webrtc中的video rtp packet 接收組包過程
當前版本為webrtc58；
一：webrtc中，接收視訊packet的基本過程是（這裡說的的H264，vp8和vp9是一樣的過程，在解析後會判斷型別，構造RtpDepacketizer）：
//接收 video rtp_packet
通過 class PhysicalSocket : public AsyncSocket, public sigslot::has_slots<> ，真正實現網路UDP資料傳送，接收；
// rtp packet 解析 為 RTPHeader + ParsedPayload -> 構造 WebRtcRTPHeader -> 構造 VCMPacket -> packet_buffer_->InsertPacket(&packet)
bool RtpReceiverImpl::IncomingRtpPacket -> int32_t RTPReceiverVideo::ParseRtpPacket -> int32_t RtpStreamReceiver::OnReceivedPayloadData
// video rtp packet -> h264 Frame
recv_video_rtp_packet -> tracker_.CopyAndFixBitstream(&packet) （h264 rtp packet 解析） -> packet_buffer_->InsertPacket(&packet) -> 獲取完整H264Frame (VideoReceiveStream::OnCompleteFrame )
，OnReceivedFrameCallback::OnReceivedFrame -> class FrameBuffer (webrtc58\src\webrtc\modules\video_coding\frame_buffer2.h) ->解碼顯示；
再看一個函式：接收端解碼前的的資料，其實這個資料更符合實際顯示資料：
VideoReceiveStream::OnEncodedImage（...）
{
}
// Called for each incoming video frame, i.e. in encoded state. E.g. used
// when
// saving the stream to a file. 'nullptr' disables the callback.
EncodedFrameObserver* pre_decode_callback
二：webrtc中的視訊快取佇列
這裡有兩個佇列：
1：video_coding::PacketBuffer；
將video rtp解析後，新增start code，然後packet_buffer_->InsertPacket(&packet) ，當判斷有完整的H264 Frame的時候，將響應：OnReceivedFrameCallback::OnReceivedFrame(std::unique_ptr<RtpFrameObject> frame) ，可以獲取完成H264 Frame;
clas PacketBuffer中的主要思路：
獲取解析後的video packet，根據 first_packet_in_frame，rtp_maker（或者根據single，或者FU-A的E==1），rtp_sqNum，每次新增排序查詢完整的Frame；
當有完整的Frame時，在查詢這個真的參考相關幀存在的完整性，將完整的Frame回調出去（放到Frame buffer 中），同時將之前的資料清空；
2： frame_buffer(frame_buffer2.h)，將完整的h264 Frame排序，然後Decoder獲取解碼；
這裡說了視訊的rtp pakcet接收後的解析組包流程；
如果不用這個視訊緩衝的話，也可以向我之前的文章，通過 webrtc58\src\webrtc\modules\rtp_rtcp\source\rtp_format_h264.h 的 class RtpDepacketizerH264 : public RtpDepacketizer解析，然後自己組包；
#ifndef test_rtp_receiver_impl_h__
#define tes_trtp_receiver_impl_h__
#include "webrtc\modules\rtp_rtcp\source\rtp_receiver_impl.h"
#include "webrtc\modules\rtp_rtcp\include\rtp_receiver.h"
#include "webrtc/base/rate_limiter.h"
#include "webrtc/common_types.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_payload_registry.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "webrtc/modules/rtp_rtcp/source/byte_io.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_receiver_video.h"
//#include "webrtc/modules/rtp_rtcp/test/testAPI/test_api.h"
#include "webrtc\system_wrappers\include\clock.h"
#include "webrtc/base/array_view.h"
#include "webrtc/common_video/h264/h264_common.h"
#include "webrtc/modules/include/module_common_types.h"
//#include "webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h"
#include "webrtc/modules/rtp_rtcp/source/byte_io.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_format.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h"
#include "webrtc/base/buffer.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_format_h264.h"
#include "webrtc/base/random.h"
#include "webrtc/modules/video_coding/frame_object.h"
#include "webrtc/modules/video_coding/packet_buffer.h"
#include "webrtc/system_wrappers/include/clock.h"
#include "webrtc\modules\video_coding\h264_sps_pps_tracker.h"
#include "webrtc\modules\rtp_rtcp\include\remote_ntp_time_estimator.h"
using namespace webrtc;
constexpr int kPacketBufferStartSize = 32;
constexpr int kPacketBufferMaxSixe = 2048;
namespace webrtc {
namespace video_coding {
class CVideoRtpRecv : public OnReceivedFrameCallback
{
public:
void OnReceivedFrame(std::unique_ptr<RtpFrameObject> frame) override
{
static FILE * f = fopen("E:\\video_packet_buffer.h264", "w+b");
fwrite(frame->Buffer(), 1, frame->Length(), f);
}
public:
CVideoRtpRecv() :
depacketizer_(RtpDepacketizer::Create(kRtpVideoH264)),
clock_(Clock::GetRealTimeClock()),
ntp_estimator_(clock_),
last_received_timestamp_(0),
last_received_frame_time_ms_(-1),
last_received_sequence_number_(0)
{
packet_buffer_ = video_coding::PacketBuffer::Create(clock_, kPacketBufferStartSize, kPacketBufferMaxSixe, this);
}
public:
bool HaveReceivedFrame()
{
return last_received_frame_time_ms_ >= 0;
}
bool get_is_first_packet_in_frame(const RTPHeader& rtp_header)
{
bool in_order = true;
//
bool is_first_packet_in_frame = false;
{
if (HaveReceivedFrame())
{
is_first_packet_in_frame =
last_received_sequence_number_ + 1 == rtp_header.sequenceNumber &&
last_received_timestamp_ != rtp_header.timestamp;
}
else
{
is_first_packet_in_frame = true;
}
}
{
if (in_order) {
if (last_received_timestamp_ != rtp_header.timestamp) {
last_received_timestamp_ = rtp_header.timestamp;
last_received_frame_time_ms_ = clock_->TimeInMilliseconds();
}
last_received_sequence_number_ = rtp_header.sequenceNumber;
}
}
return is_first_packet_in_frame;
}
void insert_VCMPacket(uint8_t * rtp_buffer, size_t rtp_buffer_lenght)
{
RTPHeader rtp_header;
std::unique_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
bool bIsRtcp = parser->IsRtcp(rtp_buffer, rtp_buffer_lenght);
if (bIsRtcp)
{
return;
}
bool bParser = parser->Parse(rtp_buffer, rtp_buffer_lenght, &rtp_header);
if (!bParser)
{
return;
}
//---------------------------------------------------------------
size_t nRtpHeaderLen = rtp_header.headerLength;
uint8_t * pbufPayload = (uint8_t *)(rtp_buffer + nRtpHeaderLen);
size_t nLenPayload = rtp_buffer_lenght - nRtpHeaderLen;
//
std::unique_ptr<RtpDepacketizer> depacketizer(RtpDepacketizer::Create(kRtpVideoH264));
RtpDepacketizer::ParsedPayload parsed_payload;
depacketizer->Parse(&parsed_payload, pbufPayload, nLenPayload);
//EXPECT_EQ(kVideoFrameKey, payload.frame_type);
//EXPECT_EQ(kRtpVideoH264, payload.type.Video.codec);
//EXPECT_TRUE(payload.type.Video.is_first_packet_in_frame);
//EXPECT_EQ(kH264SingleNalu, payload.type.Video.codecHeader.H264.packetization_type);
//EXPECT_EQ(kIdr, payload.type.Video.codecHeader.H264.nalu_type);
//parsed_payload.type.Video;
//const RTPVideoHeaderH264& h264 = parsed_payload.type.Video.codecHeader.H264;
//-----------------------------------------------------------------------------
#if 0
//bool RtpReceiverImpl::IncomingRtpPacket
WebRtcRTPHeader webrtc_rtp_header;
memset(&webrtc_rtp_header, 0, sizeof(webrtc_rtp_header));
webrtc_rtp_header.header = rtp_header;
//bool is_first_packet_in_frame = get_is_first_packet_in_frame(rtp_header);
if (0)
{
bool is_first_packet_in_frame = false;
{
if (HaveReceivedFrame()) {
is_first_packet_in_frame =
last_received_sequence_number_ + 1 == rtp_header.sequenceNumber &&
last_received_timestamp_ != rtp_header.timestamp;
}
else {
is_first_packet_in_frame = true;
//last_received_timestamp_ = rtp_header.timestamp; //y;
}
}
}
//int32_t RTPReceiverVideo::ParseRtpPacket
webrtc_rtp_header.type.Video.codec = parsed_payload.type.Video.codec;
webrtc_rtp_header.type.Video.is_first_packet_in_frame = is_first_packet_in_frame;
webrtc_rtp_header.frameType = parsed_payload.frame_type;
webrtc_rtp_header.type = parsed_payload.type;
webrtc_rtp_header.type.Video.rotation = kVideoRotation_0;
// Retrieve the video rotation information.
if (webrtc_rtp_header.header.extension.hasVideoRotation) {
webrtc_rtp_header.type.Video.rotation =
webrtc_rtp_header.header.extension.videoRotation;
}
webrtc_rtp_header.type.Video.playout_delay = webrtc_rtp_header.header.extension.playout_delay;
//int32_t RtpStreamReceiver::OnReceivedPayloadData(
WebRtcRTPHeader rtp_header_with_ntp = webrtc_rtp_header;
rtp_header_with_ntp.ntp_time_ms = ntp_estimator_.Estimate(webrtc_rtp_header.header.timestamp);
VCMPacket packet(parsed_payload.payload, parsed_payload.payload_length, rtp_header_with_ntp);
#else
VCMPacket packet;
packet.frameType = parsed_payload.frame_type;
packet.timestamp = rtp_header.timestamp;
packet.seqNum = rtp_header.sequenceNumber;
packet.dataPtr = parsed_payload.payload;
packet.sizeBytes = parsed_payload.payload_length;
packet.markerBit = rtp_header.markerBit;
packet.codec = kVideoCodecH264;
packet.video_header = parsed_payload.type.Video;
//
packet.width = parsed_payload.type.Video.width;
packet.height = parsed_payload.type.Video.height;
packet.is_first_packet_in_frame = packet.video_header.is_first_packet_in_frame;
packet.isFirstPacket = packet.video_header.isFirstPacket;
#endif
if (packet.codec == kVideoCodecH264)
{
switch (tracker_.CopyAndFixBitstream(&packet))
{
case video_coding::H264SpsPpsTracker::kRequestKeyframe:
//keyframe_request_sender_->RequestKeyFrame();
FALLTHROUGH();
case video_coding::H264SpsPpsTracker::kDrop:
return;
case video_coding::H264SpsPpsTracker::kInsert:
break;
}
}
packet_buffer_->InsertPacket(&packet);
}
private:
rtc::scoped_refptr<video_coding::PacketBuffer> packet_buffer_;
uint32_t last_received_timestamp_;
int64_t last_received_frame_time_ms_;
uint16_t last_received_sequence_number_;
video_coding::H264SpsPpsTracker tracker_;
Clock* clock_;
RemoteNtpTimeEstimator ntp_estimator_;
int16_t last_payload_type_ = -1;
std::unique_ptr<RtpDepacketizer> depacketizer_;
};
}
}
static int file_size(char* filename)
{
FILE *fp = fopen(filename, "r");
if (!fp) return -1;
fseek(fp, 0L, SEEK_END);
int size = ftell(fp);
fclose(fp);
return size;
}
static void test_rtp_recv()
{
video_coding::CVideoRtpRecv video_rtp_pakcet_recv;
uint8_t * pRtpbuf = new uint8_t[1024 * 10];
for (int n = 0; n < 2700; n++)
{
//std::string strRtpFile = std::string("E:\\h264_rtp\\") + std::to_string(n + 1) + std::string("_h264.rtp");
std::string strRtpFile = std::string("E:\\h264_rtp_2\\") + std::to_string(n + 1) + std::string("_h264.rtp");
int nFileSize = file_size((char *)strRtpFile.c_str()); if (nFileSize <= 0) { return; }
{
FILE * f = fopen(strRtpFile.c_str(), "r+b");
fread(pRtpbuf, 1, nFileSize, f);
fclose(f);
}
//video_rtp_pakcet_recv.test(pRtpbuf, nFileSize);
video_rtp_pakcet_recv.insert_VCMPacket(pRtpbuf, nFileSize);
Sleep(5);
}
}
// webrtc58\src\webrtc\modules\video_coding\video_packet_buffer_unittest.cc(32): PacketBuffer::Create(clock_.get(), kStartSize, kMaxSize, this)) {}
// webrtc58\src\webrtc\video\rtp_stream_receiver.cc(195): packet_buffer_ = video_coding::PacketBuffer::Create(
#endif
在寫一個示例，根據 video packet 獲取完整Frame；再判斷這個完整frame的相關參考幀和獨立性；
如果可以，那麼將觸發： void OnCompleteFrame(std::unique_ptr<video_coding::FrameObject> frame)
#ifndef test_rtp_receiver_Framebuffer2_impl_h__
#define test_rtp_receiver_impl_h__
#include "webrtc\modules\rtp_rtcp\source\rtp_receiver_impl.h"
#include "webrtc\modules\rtp_rtcp\include\rtp_receiver.h"
#include "webrtc/base/rate_limiter.h"
#include "webrtc/common_types.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_payload_registry.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "webrtc/modules/rtp_rtcp/source/byte_io.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_receiver_video.h"
//#include "webrtc/modules/rtp_rtcp/test/testAPI/test_api.h"
#include "webrtc\system_wrappers\include\clock.h"
#include "webrtc/base/array_view.h"
#include "webrtc/common_video/h264/h264_common.h"
#include "webrtc/modules/include/module_common_types.h"
//#include "webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h"
#include "webrtc/modules/rtp_rtcp/source/byte_io.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_format.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h"
#include "webrtc/base/buffer.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_format_h264.h"
#include "webrtc/base/random.h"
#include "webrtc/modules/video_coding/frame_object.h"
#include "webrtc/modules/video_coding/packet_buffer.h"
#include "webrtc/modules/video_coding/rtp_frame_reference_finder.h"
#include "webrtc/modules/video_coding/sequence_number_util.h"
#include "webrtc/system_wrappers/include/clock.h"
#include "webrtc\modules\video_coding\h264_sps_pps_tracker.h"
#include "webrtc\modules\rtp_rtcp\include\remote_ntp_time_estimator.h"
#include "webrtc/modules/video_coding/timing.h"
using namespace webrtc;
constexpr int kPacketBufferStartSize = 32;
constexpr int kPacketBufferMaxSixe = 2048;
namespace webrtc {
namespace video_coding {
class CVideoRtpRecv : public OnReceivedFrameCallback,
public video_coding::OnCompleteFrameCallback
{
public:
void OnReceivedFrame(std::unique_ptr<RtpFrameObject> frame) override
{
static FILE * f = fopen("E:\\video_packet_buffer.h264", "w+b");
fwrite(frame->Buffer(), 1, frame->Length(), f);
if (!frame->delayed_by_retransmission())
timing_->IncomingTimestamp(frame->timestamp, clock_->TimeInMilliseconds());
reference_finder_->ManageFrame(std::move(frame));
}
void /*RtpStreamReceiver::*/OnCompleteFrame(std::unique_ptr<video_coding::FrameObject> frame)
{
static FILE * f = fopen("E:\\video_packet_buffer_OnCompleteFrame.h264", "w+b");
fwrite(frame->Buffer(), 1, frame->Length(), f);
{
rtc::CritScope lock(&last_seq_num_cs_);
video_coding::RtpFrameObject* rtp_frame =
static_cast<video_coding::RtpFrameObject*>(frame.get());
last_seq_num_for_pic_id_[rtp_frame->picture_id] = rtp_frame->last_seq_num();
}
//complete_frame_callback_->OnCompleteFrame(std::move(frame)); //another class need completeFrame; now, we not need this one;
}
public:
CVideoRtpRecv() :
depacketizer_(RtpDepacketizer::Create(kRtpVideoH264)),
clock_(Clock::GetRealTimeClock()),
ntp_estimator_(clock_),
timing_from_VideoReceiveStream_(new VCMTiming(clock_)),
timing_(timing_from_VideoReceiveStream_.get()),
last_received_timestamp_(0),
last_received_frame_time_ms_(-1),
last_received_sequence_number_(0)
{
packet_buffer_ = video_coding::PacketBuffer::Create(clock_, kPacketBufferStartSize, kPacketBufferMaxSixe, this);
reference_finder_.reset(new video_coding::RtpFrameReferenceFinder(this));
}
public:
bool HaveReceivedFrame()
{
return last_received_frame_time_ms_ >= 0;
}
bool get_is_first_packet_in_frame(const RTPHeader& rtp_header)
{
bool in_order = true;
//
bool is_first_packet_in_frame = false;
{
if (HaveReceivedFrame())
{
is_first_packet_in_frame =
last_received_sequence_number_ + 1 == rtp_header.sequenceNumber &&
last_received_timestamp_ != rtp_header.timestamp;
}
else
{
is_first_packet_in_frame = true;
}
}
{
if (in_order) {
if (last_received_timestamp_ != rtp_header.timestamp) {
last_received_timestamp_ = rtp_header.timestamp;
last_received_frame_time_ms_ = clock_->TimeInMilliseconds();
}
last_received_sequence_number_ = rtp_header.sequenceNumber;
}
}
return is_first_packet_in_frame;
}
void insert_VCMPacket(uint8_t * rtp_buffer, size_t rtp_buffer_lenght)
{
RTPHeader rtp_header;
std::unique_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
bool bIsRtcp = parser->IsRtcp(rtp_buffer, rtp_buffer_lenght);
if (bIsRtcp)
{
return;
}
bool bParser = parser->Parse(rtp_buffer, rtp_buffer_lenght, &rtp_header);
if (!bParser)
{
return;
}
//---------------------------------------------------------------
size_t nRtpHeaderLen = rtp_header.headerLength;
uint8_t * pbufPayload = (uint8_t *)(rtp_buffer + nRtpHeaderLen);
size_t nLenPayload = rtp_buffer_lenght - nRtpHeaderLen;
//
std::unique_ptr<RtpDepacketizer> depacketizer(RtpDepacketizer::Create(kRtpVideoH264));
RtpDepacketizer::ParsedPayload parsed_payload;
depacketizer->Parse(&parsed_payload, pbufPayload, nLenPayload);
//EXPECT_EQ(kVideoFrameKey, payload.frame_type);
//EXPECT_EQ(kRtpVideoH264, payload.type.Video.codec);
//EXPECT_TRUE(payload.type.Video.is_first_packet_in_frame);
//EXPECT_EQ(kH264SingleNalu, payload.type.Video.codecHeader.H264.packetization_type);
//EXPECT_EQ(kIdr, payload.type.Video.codecHeader.H264.nalu_type);
//parsed_payload.type.Video;
//const RTPVideoHeaderH264& h264 = parsed_payload.type.Video.codecHeader.H264;
//-----------------------------------------------------------------------------
VCMPacket packet;
packet.frameType = parsed_payload.frame_type;
packet.timestamp = rtp_header.timestamp;
packet.seqNum = rtp_header.sequenceNumber;
packet.dataPtr = parsed_payload.payload;
packet.sizeBytes = parsed_payload.payload_length;
packet.markerBit = rtp_header.markerBit;
packet.codec = kVideoCodecH264;
packet.video_header = parsed_payload.type.Video;
//
packet.width = parsed_payload.type.Video.width;
packet.height = parsed_payload.type.Video.height;
packet.is_first_packet_in_frame = packet.video_header.is_first_packet_in_frame;
packet.isFirstPacket = packet.video_header.isFirstPacket;
if (packet.codec == kVideoCodecH264)
{
switch (tracker_.CopyAndFixBitstream(&packet))
{
case video_coding::H264SpsPpsTracker::kRequestKeyframe:
//keyframe_request_sender_->RequestKeyFrame();
FALLTHROUGH();
case video_coding::H264SpsPpsTracker::kDrop:
return;
case video_coding::H264SpsPpsTracker::kInsert:
break;
}
}
packet_buffer_->InsertPacket(&packet);
}
private:
rtc::scoped_refptr<video_coding::PacketBuffer> packet_buffer_;
uint32_t last_received_timestamp_;
int64_t last_received_frame_time_ms_;
uint16_t last_received_sequence_number_;
video_coding::H264SpsPpsTracker tracker_;
Clock* clock_;
//from: VideoReceiveStream; VCMTiming* timing_ use the VideoReceiveStream std::unique_ptr<VCMTiming> timing_; created, and input to rtp_stream_receiver.cc;
//the same to the webrtc code;
std::unique_ptr<VCMTiming> timing_from_VideoReceiveStream_; // Jitter buffer experiment.
RemoteNtpTimeEstimator ntp_estimator_;
int16_t last_payload_type_ = -1;
std::unique_ptr<RtpDepacketizer> depacketizer_;
// Members for the new jitter buffer experiment.
video_coding::OnCompleteFrameCallback* complete_frame_callback_;
VCMTiming* timing_;
rtc::CriticalSection last_seq_num_cs_;
std::map<uint16_t, uint16_t, DescendingSeqNumComp<uint16_t>>
last_seq_num_for_pic_id_ GUARDED_BY(last_seq_num_cs_);
std::unique_ptr<video_coding::RtpFrameReferenceFinder> reference_finder_;
};
}
}
static int file_size(char* filename)
{
FILE *fp = fopen(filename, "r");
if (!fp) return -1;
fseek(fp, 0L, SEEK_END);
int size = ftell(fp);
fclose(fp);
return size;
}
static void test_rtp_recv()
{
video_coding::CVideoRtpRecv video_rtp_pakcet_recv;
uint8_t * pRtpbuf = new uint8_t[1024 * 10];
for (int n = 0; n < 2700; n++)
{
std::string strRtpFile = std::string("E:\\h264_rtp\\") + std::to_string(n + 1) + std::string("_h264.rtp");
//std::string strRtpFile = std::string("E:\\h264_rtp_2\\") + std::to_string(n + 1) + std::string("_h264.rtp");
int nFileSize = file_size((char *)strRtpFile.c_str()); if (nFileSize <= 0) { return; }
{
FILE * f = fopen(strRtpFile.c_str(), "r+b");
fread(pRtpbuf, 1, nFileSize, f);
fclose(f);
}
//video_rtp_pakcet_recv.test(pRtpbuf, nFileSize);
video_rtp_pakcet_recv.insert_VCMPacket(pRtpbuf, nFileSize);
Sleep(5);
}
}
#endif // test_rtp_receiver_impl_h__
示例：
這裡是一個完整的示例，從video_packet -> bufferframe2;
bufferframe2主要是解決抖動問題；
通常情況，上述的第二個示例就可以了；
#ifndef test_rtp_receiver_Framebuffer2_impl_h__
#define test_rtp_receiver_Framebuffer2_impl_h__
#include "webrtc\modules\rtp_rtcp\source\rtp_receiver_impl.h"
#include "webrtc\modules\rtp_rtcp\include\rtp_receiver.h"
#include "webrtc/base/rate_limiter.h"
#include "webrtc/common_types.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_payload_registry.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "webrtc/modules/rtp_rtcp/source/byte_io.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_receiver_video.h"
//#include "webrtc/modules/rtp_rtcp/test/testAPI/test_api.h"
#include "webrtc\system_wrappers\include\clock.h"
#include "webrtc/base/array_view.h"
#include "webrtc/common_video/h264/h264_common.h"
#include "webrtc/modules/include/module_common_types.h"
//#include "webrtc/modules/rtp_rtcp/mocks/mock_rtp_rtcp.h"
#include "webrtc/modules/rtp_rtcp/source/byte_io.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_format.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_packet_to_send.h"
#include "webrtc/base/buffer.h"
#include "webrtc/modules/rtp_rtcp/include/rtp_header_parser.h"
#include "webrtc/modules/rtp_rtcp/source/rtp_format_h264.h"
#include "webrtc/base/random.h"
#include "webrtc/modules/video_coding/frame_object.h"
#include "webrtc/modules/video_coding/packet_buffer.h"
#include "webrtc/modules/video_coding/rtp_frame_reference_finder.h"
#include "webrtc/modules/video_coding/sequence_number_util.h"
#include "webrtc/system_wrappers/include/clock.h"
#include "webrtc\modules\video_coding\h264_sps_pps_tracker.h"
#include "webrtc\modules\rtp_rtcp\include\remote_ntp_time_estimator.h"
#include "webrtc/modules/video_coding/timing.h"
//
#include "webrtc/modules/video_coding/frame_buffer2.h"
#include "webrtc/modules/video_coding/jitter_estimator.h"
#include "webrtc/video/rtp_stream_receiver.h"
#include "webrtc/base/platform_thread.h"
using namespace webrtc;
constexpr int kPacketBufferStartSize = 32;
constexpr int kPacketBufferMaxSixe = 2048;
class Interface_VideoFrameToDecod
{
public:
virtual void frame_to_decode(const webrtc::VCMEncodedFrame* frame) = 0; //decoder to decode video frame in this function;
};
namespace webrtc {
namespace video_coding {
class CVideoRtpRecv : public OnReceivedFrameCallback,
public video_coding::OnCompleteFrameCallback
{
public:
void OnReceivedFrame(std::unique_ptr<RtpFrameObject> frame) override
{
static FILE * f = fopen("E:\\video_packet_buffer.h264", "w+b");
fwrite(frame->Buffer(), 1, frame->Length(), f);
if (!frame->delayed_by_retransmission())
timing_->IncomingTimestamp(frame->timestamp, clock_->TimeInMilliseconds());
reference_finder_->ManageFrame(std::move(frame));
}
void /*RtpStreamReceiver::*/OnCompleteFrame(std::unique_ptr<video_coding::FrameObject> frame)
{
static FILE * f = fopen("E:\\video_packet_buffer_OnCompleteFrame.h264", "w+b");
fwrite(frame->Buffer(), 1, frame->Length(), f);
{
rtc::CritScope lock(&last_seq_num_cs_);
video_coding::RtpFrameObject* rtp_frame =
static_cast<video_coding::RtpFrameObject*>(frame.get());
last_seq_num_for_pic_id_[rtp_frame->picture_id] = rtp_frame->last_seq_num();
}
//complete_frame_callback_->OnCompleteFrame(std::move(frame)); //callback VideoReceiveStream::OnCompleteFrame;
this->VideoReceiveStream_OnCompleteFrame(std::move(frame));
}
//
void /*VideoReceiveStream::*/VideoReceiveStream_OnCompleteFrame(std::unique_ptr<video_coding::FrameObject> frame)
{
int last_continuous_pid = frame_buffer_->InsertFrame(std::move(frame));
if (last_continuous_pid != -1)
{
//rtp_stream_receiver_.FrameContinuous(last_continuous_pid);
}
}
public:
CVideoRtpRecv(Interface_VideoFrameToDecod * pInterface_VideoFrameToDecod) :
depacketizer_(RtpDepacketizer::Create(kRtpVideoH264)),
clock_(Clock::GetRealTimeClock()),
ntp_estimator_(clock_),
timing_from_VideoReceiveStream_(new VCMTiming(clock_)),
timing_(timing_from_VideoReceiveStream_.get()),
last_received_timestamp_(0),
last_received_frame_time_ms_(-1),
last_received_sequence_number_(0),
decode_thread_(DecodeThreadFunction, this, "DecodingThread"),
m_pInterface_VideoFrameToDecod(pInterface_VideoFrameToDecod)
{
if (m_pInterface_VideoFrameToDecod == NULL)
{
LOG(LS_ERROR) << "Interface_VideoFrameToDecod must be set, and Interface_VideoFrameToDecod must be implement;";
//exit(0);
}
packet_buffer_ = video_coding::PacketBuffer::Create(clock_, kPacketBufferStartSize, kPacketBufferMaxSixe, this);
reference_finder_.reset(new video_coding::RtpFrameReferenceFinder(this));
jitter_estimator_.reset(new VCMJitterEstimator(clock_));
frame_buffer_.reset(new video_coding::FrameBuffer(
clock_, jitter_estimator_.get(), /*timing_.get()*/timing_, /*&stats_proxy_*/NULL));
Start(); //FrameBuffre2 get Frame for decode;
}
~CVideoRtpRecv()
{
Stop();
}
public:
bool HaveReceivedFrame()
{
return last_received_frame_time_ms_ >= 0;
}
bool get_is_first_packet_in_frame(const RTPHeader& rtp_header)
{
bool in_order = true;
//
bool is_first_packet_in_frame = false;
{
if (HaveReceivedFrame())
{
is_first_packet_in_frame =
last_received_sequence_number_ + 1 == rtp_header.sequenceNumber &&
last_received_timestamp_ != rtp_header.timestamp;
}
else
{
is_first_packet_in_frame = true;
}
}
{
if (in_order) {
if (last_received_timestamp_ != rtp_header.timestamp) {
last_received_timestamp_ = rtp_header.timestamp;
last_received_frame_time_ms_ = clock_->TimeInMilliseconds();
}
last_received_sequence_number_ = rtp_header.sequenceNumber;
}
}
return is_first_packet_in_frame;
}
void insert_VCMPacket(uint8_t * rtp_buffer, size_t rtp_buffer_lenght)
{
RTPHeader rtp_header;
std::unique_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
bool bIsRtcp = parser->IsRtcp(rtp_buffer, rtp_buffer_lenght);
if (bIsRtcp)
{
return;
}
bool bParser = parser->Parse(rtp_buffer, rtp_buffer_lenght, &rtp_header);
if (!bParser)
{
return;
}
//---------------------------------------------------------------
size_t nRtpHeaderLen = rtp_header.headerLength;
uint8_t * pbufPayload = (uint8_t *)(rtp_buffer + nRtpHeaderLen);
size_t nLenPayload = rtp_buffer_lenght - nRtpHeaderLen;
//
std::unique_ptr<RtpDepacketizer> depacketizer(RtpDepacketizer::Create(kRtpVideoH264));
RtpDepacketizer::ParsedPayload parsed_payload;
depacketizer->Parse(&parsed_payload, pbufPayload, nLenPayload);
//EXPECT_EQ(kVideoFrameKey, payload.frame_type);
//EXPECT_EQ(kRtpVideoH264, payload.type.Video.codec);
//EXPECT_TRUE(payload.type.Video.is_first_packet_in_frame);
//EXPECT_EQ(kH264SingleNalu, payload.type.Video.codecHeader.H264.packetization_type);
//EXPECT_EQ(kIdr, payload.type.Video.codecHeader.H264.nalu_type);
//parsed_payload.type.Video;
//const RTPVideoHeaderH264& h264 = parsed_payload.type.Video.codecHeader.H264;
//-----------------------------------------------------------------------------
VCMPacket packet;
packet.frameType = parsed_payload.frame_type;
packet.timestamp = rtp_header.timestamp;
packet.seqNum = rtp_header.sequenceNumber;
packet.dataPtr = parsed_payload.payload;
packet.sizeBytes = parsed_payload.payload_length;
packet.markerBit = rtp_header.markerBit;
packet.codec = kVideoCodecH264;
packet.video_header = parsed_payload.type.Video;
//
packet.width = parsed_payload.type.Video.width;
packet.height = parsed_payload.type.Video.height;
packet.is_first_packet_in_frame = packet.video_header.is_first_packet_in_frame;
packet.isFirstPacket = packet.video_header.isFirstPacket;
if (packet.codec == kVideoCodecH264)
{
switch (tracker_.CopyAndFixBitstream(&packet))
{
case video_coding::H264SpsPpsTracker::kRequestKeyframe:
//keyframe_request_sender_->RequestKeyFrame();
FALLTHROUGH();
case video_coding::H264SpsPpsTracker::kDrop:
return;
case video_coding::H264SpsPpsTracker::kInsert:
break;
}
}
packet_buffer_->InsertPacket(&packet);
}
private:
void /*VideoReceiveStream::*/Start()
{
if (decode_thread_.IsRunning())
return;
frame_buffer_->Start();
// Start the decode thread
decode_thread_.Start();
decode_thread_.SetPriority(rtc::kHighestPriority);
}
void /*VideoReceiveStream::*/Stop()
{
if (decode_thread_.IsRunning()) {
decode_thread_.Stop();
}
frame_buffer_->Stop();
}
public://private:
static bool /*VideoReceiveStream::*/DecodeThreadFunction(void* ptr)
{
//return static_cast<VideoReceiveStream*>(ptr)->Decode();
return static_cast<CVideoRtpRecv*>(ptr)->Decode();
((CVideoRtpRecv*)(ptr))->Decode();
}
bool /*VideoReceiveStream::*/Decode()
{
static const int kMaxWaitForFrameMs = 3000;
std::unique_ptr<video_coding::FrameObject> frame;
video_coding::FrameBuffer::ReturnReason res =
frame_buffer_->NextFrame(kMaxWaitForFrameMs, &frame);
if (res == video_coding::FrameBuffer::ReturnReason::kStopped)
return false;
if (frame) {
//here, wait for decode video frame, and then clear the buffer and calculate the some time; so, must to decoded , then, do next process;
//if (video_receiver_.Decode(frame.get()) == VCM_OK)
// rtp_stream_receiver_.FrameDecoded(frame->picture_id);
if (m_pInterface_VideoFrameToDecod)
{
//rtc::CritScope lock(&receive_crit_);
m_pInterface_VideoFrameToDecod->frame_to_decode(frame.get());
}
this->FrameDecoded(frame->picture_id);
}
else {
LOG(LS_WARNING) << "No decodable frame in " << kMaxWaitForFrameMs
<< " ms, requesting keyframe.";
//RequestKeyFrame();
}
return true;
}
void /*RtpStreamReceiver::*/FrameDecoded(uint16_t picture_id) {
int seq_num = -1;
{
rtc::CritScope lock(&last_seq_num_cs_);
auto seq_num_it = last_seq_num_for_pic_id_.find(picture_id);
if (seq_num_it != last_seq_num_for_pic_id_.end()) {
seq_num = seq_num_it->second;
last_seq_num_for_pic_id_.erase(last_seq_num_for_pic_id_.begin(),
++seq_num_it);
}
}
if (seq_num != -1) {
packet_buffer_->ClearTo(seq_num);
reference_finder_->ClearTo(seq_num);
}
}
private:
Interface_VideoFrameToDecod * m_pInterface_VideoFrameToDecod;
private:
rtc::scoped_refptr<video_coding::PacketBuffer> packet_buffer_;
uint32_t last_received_timestamp_;
int64_t last_received_frame_time_ms_;
uint16_t last_received_sequence_number_;
video_coding::H264SpsPpsTracker tracker_;
Clock* clock_;
//from: VideoReceiveStream; VCMTiming* timing_ use the VideoReceiveStream std::unique_ptr<VCMTiming> timing_; created, and input to rtp_stream_receiver.cc;
//the same to the webrtc code;
std::unique_ptr<VCMTiming> timing_from_VideoReceiveStream_; // Jitter buffer experiment.
RemoteNtpTimeEstimator ntp_estimator_;
int16_t last_payload_type_ = -1;
std::unique_ptr<RtpDepacketizer> depacketizer_;
// Members for the new jitter buffer experiment.
video_coding::OnCompleteFrameCallback* complete_frame_callback_;
VCMTiming* timing_;
rtc::CriticalSection last_seq_num_cs_;
std::map<uint16_t, uint16_t, DescendingSeqNumComp<uint16_t>>
last_seq_num_for_pic_id_ GUARDED_BY(last_seq_num_cs_);
std::unique_ptr<video_coding::RtpFrameReferenceFinder> reference_finder_;
//VideoReceiveStream member
rtc::PlatformThread decode_thread_;
// Members for the new jitter buffer experiment.
std::unique_ptr<VCMJitterEstimator> jitter_estimator_;
std::unique_ptr<video_coding::FrameBuffer> frame_buffer_;
//RtpStreamReceiver rtp_stream_receiver_;
};
}
}
static int file_size(char* filename)
{
FILE *fp = fopen(filename, "r");
if (!fp) return -1;
fseek(fp, 0L, SEEK_END);
int size = ftell(fp);
fclose(fp);
return size;
}
static void test_rtp_recv()
{
video_coding::CVideoRtpRecv video_rtp_pakcet_recv(NULL);
uint8_t * pRtpbuf = new uint8_t[1024 * 10];
for (int n = 0; n < 2700; n++)
{
std::string strRtpFile = std::string("E:\\h264_rtp\\") + std::to_string(n + 1) + std::string("_h264.rtp");
//std::string strRtpFile = std::string("E:\\h264_rtp_2\\") + std::to_string(n + 1) + std::string("_h264.rtp");
int nFileSize = file_size((char *)strRtpFile.c_str()); if (nFileSize <= 0) { return; }
{
FILE * f = fopen(strRtpFile.c_str(), "r+b");
fread(pRtpbuf, 1, nFileSize, f);
fclose(f);
}
//video_rtp_pakcet_recv.test(pRtpbuf, nFileSize);
video_rtp_pakcet_recv.insert_VCMPacket(pRtpbuf, nFileSize);
Sleep(5);
}
}
//----------------------------------------------framebuffer2
class CVideoRecvFramebuffer2 : public video_coding::CVideoRtpRecv, public Interface_VideoFrameToDecod
{
public:
CVideoRecvFramebuffer2() :video_coding::CVideoRtpRecv(this)
{
}
public:
virtual void frame_to_decode(const webrtc::VCMEncodedFrame* frame) override
{
// if (pre_decode_image_callback_) {
// EncodedImage encoded_image(frame->EncodedImage());
// int qp = -1;
// if (qp_parser_.GetQp(*frame, &qp)) {
// encoded_image.qp_ = qp;
// }
static FILE * f = fopen("E:\\video_packet_buffer_framebuffer2.h264", "w+b");
fwrite(frame->Buffer(), 1, frame->Length(), f);
}
};
static void test_rtp_recv_framebuffer2()
{
CVideoRecvFramebuffer2 video_rtp_pakcet_recv_framebuffer2;
uint8_t * pRtpbuf = new uint8_t[1024 * 10];
for (int n = 0; n < 2700; n++)
{
std::string strRtpFile = std::string("E:\\h264_rtp\\") + std::to_string(n + 1) + std::string("_h264.rtp");
//std::string strRtpFile = std::string("E:\\h264_rtp_2\\") + std::to_string(n + 1) + std::string("_h264.rtp");
int nFileSize = file_size((char *)strRtpFile.c_str()); if (nFileSize <= 0) { return; }
{
FILE * f = fopen(strRtpFile.c_str(), "r+b");
fread(pRtpbuf, 1, nFileSize, f);
fclose(f);
}
video_rtp_pakcet_recv_framebuffer2.insert_VCMPacket(pRtpbuf, nFileSize);
}
}
#endif // test_rtp_receiver_Framebuffer2_impl_h__
寫在最後：
重要函式1：
這有一個很重要的發現，記得之前驗證過，現在有些忘了；
1.1webrtc 單片幀？？？
webrtc的frame判斷，一個frame中的任何一個FU-A分包丟失都會將整個frame drop；
所以按照這個論點也沒有問題；
1.2:
packet資料到 PacketBuffer::InsertPacket(VCMPacket* packet) 中的時候，會儲存到PacketBuffer類的sequence_buffer_；
sequence_buffer_[index].frame_begin = packet->is_first_packet_in_frame; //frame的第一個packet，webrtc後面的註釋說通過s為true判斷第一個packet是謊言；（如果不是fu-a，非fu-a沒有S和E判斷；或者，如果多slice，有可能，但是webrtc應該不是）
sequence_buffer_[index].frame_end = packet->markerBit; //很直接，通過M位；fu-a可以通過E==true；
sequence_buffer_[index].seq_num = packet->seqNum;
sequence_buffer_[index].continuous = false;
sequence_buffer_[index].frame_created = false;
sequence_buffer_[index].used = true;
data_buffer_[index] = *packet;
簡單說一下sequence_buffer_，通過packet sequenceNum % sequence buffer size，將packet放到響應的位置；
sequence_buffer_就是 std::vector<ContinuityInfo> sequence_buffer_
1.3：
這個函式是在packet_buffer中查詢完整的frame；
基本邏輯：
接收到當前packet，新增到packet_buffer相應位置，通過（ size_t index = seq_num % size_）；
然後從當前位置的向後開始查詢frame_end；
如果找到frame_end，然後向前查詢，sequence_buffer_中的packet的sequence_num必須是連續的；直到找到時間戳不等的兩個連續的packet，那麼就認為找到first_packet了；
如果frame查詢完成，那麼 當前frame在packet_buffer索引之前的所有packet會被清理；
std::vector<std::unique_ptr<RtpFrameObject>> PacketBuffer::FindFrames(
uint16_t seq_num) {
std::vector<std::unique_ptr<RtpFrameObject>> found_frames;
for (size_t i = 0; i < size_ && PotentialNewFrame(seq_num); ++i) {
size_t index = seq_num % size_;
sequence_buffer_[index].continuous = true;
// If all packets of the frame is continuous, find the first packet of the
// frame and create an RtpFrameObject.
if (sequence_buffer_[index].frame_end) {
size_t frame_size = 0;
int max_nack_count = -1;
uint16_t start_seq_num = seq_num;
// Find the start index by searching backward until the packet with
// the |frame_begin| flag is set.
int start_index = index;
size_t tested_packets = 0;
int64_t frame_timestamp = data_buffer_[start_index].timestamp;
// Identify H.264 keyframes by means of SPS, PPS, and IDR.
bool is_h264 = data_buffer_[start_index].codec == kVideoCodecH264;
bool has_h264_sps = false;
bool has_h264_pps = false;
bool has_h264_idr = false;
bool is_h264_keyframe = false;
while (true) {
++tested_packets;
frame_size += data_buffer_[start_index].sizeBytes;
max_nack_count =
std::max(max_nack_count, data_buffer_[start_index].timesNacked);
sequence_buffer_[start_index].frame_created = true;
if (!is_h264 && sequence_buffer_[start_index].frame_begin)
break;
if (is_h264 && !is_h264_keyframe) {
const RTPVideoHeaderH264& header =
data_buffer_[start_index].video_header.codecHeader.H264;
for (size_t j = 0; j < header.nalus_length; ++j) {
if (header.nalus[j].type == H264::NaluType::kSps) {
has_h264_sps = true;
} else if (header.nalus[j].type == H264::NaluType::kPps) {
has_h264_pps = true;
} else if (header.nalus[j].type == H264::NaluType::kIdr) {
has_h264_idr = true;
}
}
if ((sps_pps_idr_is_h264_keyframe_ && has_h264_idr && has_h264_sps &&
has_h264_pps) ||
(!sps_pps_idr_is_h264_keyframe_ && has_h264_idr)) {
is_h264_keyframe = true;
}
}
if (tested_packets == size_)
break;
start_index = start_index > 0 ? start_index - 1 : size_ - 1;
// In the case of H264 we don't have a frame_begin bit (yes,
// |frame_begin| might be set to true but that is a lie).（為什麼這麼說？因為多slice，不合適；但是webrtc不是；！？
所以說RtpDepacketizerH264::ParseFuaNalu中的判斷不嚴謹，但是符合目前webrtc的H264結構要求；bool first_fragment = (payload_data[1] & kSBit) > 0;parsed_payload->type.Video.is_first_packet_in_frame = first_fragment; 可能是兩個人寫的程式碼；）So instead
// we traverese backwards as long as we have a previous packet and
// the timestamp of that packet is the same as this one. This may cause
// the PacketBuffer to hand out incomplete frames.
// See: https://bugs.chromium.org/p/webrtc/issues/detail?id=7106
if (is_h264 &&
(!sequence_buffer_[start_index].used ||
data_buffer_[start_index].timestamp != frame_timestamp)) {
break;
}
--start_seq_num;
}
if (is_h264) {
// Warn if this is an unsafe frame.
if (has_h264_idr && (!has_h264_sps || !has_h264_pps)) {
std::stringstream ss;
ss << "Received H.264-IDR frame "
<< "(SPS: " << has_h264_sps << ", PPS: " << has_h264_pps << "). ";
if (sps_pps_idr_is_h264_keyframe_) {
ss << "Treating as delta frame since "
"WebRTC-SpsPpsIdrIsH264Keyframe is enabled.";
} else {
ss << "Treating as key frame since "
"WebRTC-SpsPpsIdrIsH264Keyframe is disabled.";
}
RTC_LOG(LS_WARNING) << ss.str();
}
// Now that we have decided whether to treat this frame as a key frame
// or delta frame in the frame buffer, we update the field that
// determines if the RtpFrameObject is a key frame or delta frame.
const size_t first_packet_index = start_seq_num % size_;
RTC_CHECK_LT(first_packet_index, size_);
if (is_h264_keyframe) {
data_buffer_[first_packet_index].frameType = kVideoFrameKey;
} else {
data_buffer_[first_packet_index].frameType = kVideoFrameDelta;
}
// If this is not a keyframe, make sure there are no gaps in the
// packet sequence numbers up until this point.
if (!is_h264_keyframe && missing_packets_.upper_bound(start_seq_num) !=
missing_packets_.begin()) {
uint16_t stop_index = (index + 1) % size_;
while (start_index != stop_index) {
sequence_buffer_[start_index].frame_created = false;
start_index = (start_index + 1) % size_;
}
return found_frames;
}
}
missing_packets_.erase(missing_packets_.begin(),
missing_packets_.upper_bound(seq_num));
found_frames.emplace_back(
new RtpFrameObject(this, start_seq_num, seq_num, frame_size,
max_nack_count, clock_->TimeInMilliseconds()));
}
++seq_num;
}
return found_frames;
}
重要的函式2：
當packet_buffer_找到了完整frame，新增到 FrameBuffer中，還要做很多判斷：
1：如果當前frame前面的參考frame丟棄，那麼當前frame也丟棄；
2：如果framebuffer超過kMaxFramesBuffered（600），那麼新到達的frame也丟棄；
3：如果當前的frame的id是早期的，但是時間戳是新的，那麼清空framebuffer：ClearFramesAndHistory();
從當前frame開始從新解碼；
4：根據framebuffer中的最後一個frame解碼需要，可能會丟棄當前frame；
5： << "A jump in picture id was detected, clearing buffer.";
ClearFramesAndHistory();
6： RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< key.picture_id << ":"
<< static_cast<int>(key.spatial_layer)
<< ") already inserted, dropping frame.";
7：
// Check how many dependencies that have already been fulfilled.
for (size_t i = 0; i < frame.num_references; ++i) {
FrameKey ref_key(frame.references[i], frame.spatial_layer);
auto ref_info = frames_.find(ref_key);
// Does |frame| depend on a frame earlier than the last decoded frame?
if (last_decoded_frame_it_ != frames_.end() &&
ref_key <= last_decoded_frame_it_->first) {
if (ref_info == frames_.end()) {
int64_t now_ms = clock_->TimeInMilliseconds();
if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) {
RTC_LOG(LS_WARNING)
<< "Frame with (picture_id:spatial_id) (" << key.picture_id << ":"
<< static_cast<int>(key.spatial_layer)
<< ") depends on a non-decoded frame more previous than"
<< " the last decoded frame, dropping frame.";
last_log_non_decoded_ms_ = now_ms;
}
return false;
}
8：如果上述frame判斷都ok，那麼，可以將當前frame新增到framebuffer中：
info->second.frame = std::move(frame);
++num_frames_buffered_;
所以，要想獲取完整不花屏參考不相關的h264frame，那麼就要從：上述第8點獲取；
通常從： 回撥函式中獲取（可用從一開始說的decode之前獲取）；
bool VideoReceiveStream::Decode() {
TRACE_EVENT0("webrtc", "VideoReceiveStream::Decode");
static const int kMaxWaitForFrameMs = 3000;
static const int kMaxWaitForKeyFrameMs = 200;
int wait_ms = keyframe_required_ ? kMaxWaitForKeyFrameMs : kMaxWaitForFrameMs;
std::unique_ptr<video_coding::FrameObject> frame;
// TODO(philipel): Call NextFrame with |keyframe_required| argument when
// downstream project has been fixed.
video_coding::FrameBuffer::ReturnReason res =
frame_buffer_->NextFrame(wait_ms, &frame); //解碼前，從frame_buffer_獲取視訊frame；
if (res == video_coding::FrameBuffer::ReturnReason::kStopped) {
video_receiver_.DecodingStopped();
return false;
}
if (frame)
{
//解碼：。。。
}
else
{
}
//原始碼：更多細節參考webrtc原始碼：
int64_t FrameBuffer::InsertFrame(std::unique_ptr<FrameObject> frame) {
TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame");
RTC_DCHECK(frame);
if (stats_callback_)
stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(),
frame->contentType());
FrameKey key(frame->picture_id, frame->spatial_layer);
rtc::CritScope lock(&crit_);
int64_t last_continuous_picture_id =
last_continuous_frame_it_ == frames_.end()
? -1
: last_continuous_frame_it_->first.picture_id;
if (!ValidReferences(*frame)) {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< key.picture_id << ":"
<< static_cast<int>(key.spatial_layer)
<< ") has invalid frame references, dropping frame.";
return last_continuous_picture_id;
}
if (num_frames_buffered_ >= kMaxFramesBuffered) {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< key.picture_id << ":"
<< static_cast<int>(key.spatial_layer)
<< ") could not be inserted due to the frame "
<< "buffer being full, dropping frame.";
return last_continuous_picture_id;
}
if (last_decoded_frame_it_ != frames_.end() &&
key <= last_decoded_frame_it_->first) {
if (AheadOf(frame->timestamp, last_decoded_frame_timestamp_) &&
frame->is_keyframe()) {
// If this frame has a newer timestamp but an earlier picture id then we
// assume there has been a jump in the picture id due to some encoder
// reconfiguration or some other reason. Even though this is not according
// to spec we can still continue to decode from this frame if it is a
// keyframe.
RTC_LOG(LS_WARNING)
<< "A jump in picture id was detected, clearing buffer.";
ClearFramesAndHistory();
last_continuous_picture_id = -1;
} else {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< key.picture_id << ":"
<< static_cast<int>(key.spatial_layer)
<< ") inserted after frame ("
<< last_decoded_frame_it_->first.picture_id << ":"
<< static_cast<int>(
last_decoded_frame_it_->first.spatial_layer)
<< ") was handed off for decoding, dropping frame.";
return last_continuous_picture_id;
}
}
// Test if inserting this frame would cause the order of the frames to become
// ambiguous (covering more than half the interval of 2^16). This can happen
// when the picture id make large jumps mid stream.
if (!frames_.empty() &&
key < frames_.begin()->first &&
frames_.rbegin()->first < key) {
RTC_LOG(LS_WARNING)
<< "A jump in picture id was detected, clearing buffer.";
ClearFramesAndHistory();
last_continuous_picture_id = -1;
}
auto info = frames_.insert(std::make_pair(key, FrameInfo())).first;
if (info->second.frame) {
RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
<< key.picture_id << ":"
<< static_cast<int>(key.spatial_layer)
<< ") already inserted, dropping frame.";
return last_continuous_picture_id;
}
if (!UpdateFrameInfoWithIncomingFrame(*frame, info))
return last_continuous_picture_id;
UpdatePlayoutDelays(*frame);
info->second.frame = std::move(frame);
++num_frames_buffered_;
if (info->second.num_missing_continuous == 0) {
info->second.continuous = true;
PropagateContinuity(info);
last_continuous_picture_id = last_continuous_frame_it_->first.picture_id;
// Since we now have new continuous frames there might be a better frame
// to return from NextFrame. Signal that thread so that it again can choose
// which frame to return.
new_continuous_frame_event_.Set();
}
return last_continuous_picture_id;
}