------------------------------------------------------------
author: hjjdebug
date: 2024年 08月 06日 星期二 14:51:29 CST
description: ffmpeg 解包流程
------------------------------------------------------------
ffmpeg 解包, 即从avcodec_send_packet 交出数据包,
到avcodec_receive_frame 的流程, 关键是前者, avcodec_receive_frame 只是个外壳.
0 in alloc_picture of libavcodec/h264_slice.c:194
1 in h264_frame_start of libavcodec/h264_slice.c:510
2 in h264_field_start of libavcodec/h264_slice.c:1663
3 in ff_h264_queue_decode_slice of libavcodec/h264_slice.c:2186
4 in decode_nal_units of libavcodec/h264dec.c:622
5 in h264_decode_frame of libavcodec/h264dec.c:991
6 in decode_simple_internal of libavcodec/decode.c:327
7 in decode_simple_receive_frame of libavcodec/decode.c:526
8 in decode_receive_frame_internal of libavcodec/decode.c:546
9 in avcodec_send_packet of libavcodec/decode.c:608
10 in decode of decode_video.c:57 //测试程序
11 in main of decode_video.c:173 //测试程序
下面倒着分析,即自底向上:
----------------------------------------
第0层 alloc_picture
----------------------------------------
为H264Picture 进行各种内存分配活动，分配的大小要看缓冲池的定义
static int alloc_picture(H264Context *h, H264Picture *pic)
{
av_assert0(!pic->f->data[0]);
pic->tf.f = pic->f;
#define AV_GET_BUFFER_FLAG_REF (1 << 0), 该项作为flags传入，查无用途！
//该函数在另一篇博客, 内存分配中分析过了, 传进的是 ThreadFrame*
int ret = ff_thread_get_buffer(h->avctx, &pic->tf, pic->reference ? AV_GET_BUFFER_FLAG_REF : 0);
if (!h->qscale_table_pool) {
ret = init_table_pools(h); //初始化4个内存池
}
//为pic 进行内存分配,从内存池获取, 更底层的内存操作，可参考另一片博客
pic->qscale_table_buf = av_buffer_pool_get(h->qscale_table_pool);
pic->mb_type_buf = av_buffer_pool_get(h->mb_type_pool);
pic->mb_type = (uint32_t*)pic->mb_type_buf->data + 2 * h->mb_stride + 1;
pic->qscale_table = pic->qscale_table_buf->data + 2 * h->mb_stride + 1;
//为pic 进行内存分配
for (i = 0; i < 2; i++) {
pic->motion_val_buf[i] = av_buffer_pool_get(h->motion_val_pool);
pic->ref_index_buf[i] = av_buffer_pool_get(h->ref_index_pool);
pic->motion_val[i] = (int16_t (*)[2])pic->motion_val_buf[i]->data + 4;
pic->ref_index[i] = pic->ref_index_buf[i]->data;
}
pic->pps_buf = av_buffer_ref(h->ps.pps_ref);
pic->pps = (const PPS*)pic->pps_buf->data;
//对pic 其它属性进行设置,从H264Context 中
pic->mb_width = h->mb_width;
pic->mb_height = h->mb_height;
pic->mb_stride = h->mb_stride;
return 0;
}
//初始化4个pool,
static int init_table_pools(H264Context *h)
{ //名字猜测着来吧
const int big_mb_num = h->mb_stride * (h->mb_height + 1) + 1; //宏块
const int mb_array_size = h->mb_stride * h->mb_height; //宏块数组
const int b4_stride = h->mb_width * 4 + 1; // 步长
const int b4_array_size = b4_stride * h->mb_height * 4; //b4数组
//传进的内存分配函数是av_buffer_allocz, 否则默认是av_buffer_alloc 函数
h->qscale_table_pool = av_buffer_pool_init(big_mb_num + h->mb_stride, av_buffer_allocz);
h->mb_type_pool = av_buffer_pool_init((big_mb_num + h->mb_stride) * sizeof(uint32_t), av_buffer_allocz);
h->motion_val_pool = av_buffer_pool_init(2 * (b4_array_size + 4) * sizeof(int16_t), av_buffer_allocz);
h->ref_index_pool = av_buffer_pool_init(4 * mb_array_size, av_buffer_allocz);
return 0;
}
----------------------------------------
第1层 h264_frame_start
----------------------------------------
一堆代码看过去，其是就是给pic 付初值,也是初始化h中的一些值
static int h264_frame_start(H264Context *h)
{
if (!ff_thread_can_start_frame(h->avctx)) {
av_log(h->avctx, AV_LOG_ERROR, "Attempt to start a frame outside SETUP state\n");
return -1;
}
release_unused_pictures(h, 1);
h->cur_pic_ptr = NULL;
i = find_unused_picture(h);
if (i < 0) {
av_log(h->avctx, AV_LOG_ERROR, "no frame buffer available\n");
return i;
}
//赋值pic 一些基本属性
H264Picture *pic = &h->DPB[i];
pic->reference = h->droppable ? 0 : h->picture_structure;
pic->f->coded_picture_number = h->coded_picture_number++;
pic->field_picture = h->picture_structure != PICT_FRAME;
pic->frame_num = h->poc.frame_num;
pic->f->key_frame = 0;
pic->mmco_reset = 0;
pic->recovered = 0;
pic->invalid_gap = 0;
pic->sei_recovery_frame_cnt = h->sei.recovery_point.recovery_frame_cnt;
pic->f->pict_type = h->slice_ctx[0].slice_type;
pic->f->crop_left = h->crop_left;
pic->f->crop_right = h->crop_right;
pic->f->crop_top = h->crop_top;
pic->f->crop_bottom = h->crop_bottom;
//为pic分配内存， 就是前面的分析
if ((ret = alloc_picture(h, pic)) < 0) return ret;
h->cur_pic_ptr = pic;
ff_h264_unref_picture(h, &h->cur_pic);
if (CONFIG_ERROR_RESILIENCE) {
ff_h264_set_erpic(&h->slice_ctx[0].er.cur_pic, NULL);
}
if ((ret = ff_h264_ref_picture(h, &h->cur_pic, h->cur_pic_ptr)) < 0) return ret;
for (i = 0; i < h->nb_slice_ctx; i++) {
h->slice_ctx[i].linesize = h->cur_pic_ptr->f->linesize[0];
h->slice_ctx[i].uvlinesize = h->cur_pic_ptr->f->linesize[1];
}
if (CONFIG_ERROR_RESILIENCE && h->enable_er) {
ff_er_frame_start(&h->slice_ctx[0].er);
ff_h264_set_erpic(&h->slice_ctx[0].er.last_pic, NULL);
ff_h264_set_erpic(&h->slice_ctx[0].er.next_pic, NULL);
}
const int pixel_shift = h->pixel_shift;
//scan8 就是一些全局常量,可见还是付给初始值
for (i = 0; i < 16; i++) {
h->block_offset[i] = (4 * ((scan8[i] - scan8[0]) & 7) << pixel_shift) + 4 * pic->f->linesize[0] * ((scan8[i] - scan8[0]) >> 3);
h->block_offset[48 + i] = (4 * ((scan8[i] - scan8[0]) & 7) << pixel_shift) + 8 * pic->f->linesize[0] * ((scan8[i] - scan8[0]) >> 3);
}
for (i = 0; i < 16; i++) {
h->block_offset[16 + i] =
h->block_offset[32 + i] = (4 * ((scan8[i] - scan8[0]) & 7) << pixel_shift) + 4 * pic->f->linesize[1] * ((scan8[i] - scan8[0]) >> 3);
h->block_offset[48 + 16 + i] =
h->block_offset[48 + 32 + i] = (4 * ((scan8[i] - scan8[0]) & 7) << pixel_shift) + 8 * pic->f->linesize[1] * ((scan8[i] - scan8[0]) >> 3);
}
/* We mark the current picture as non-reference after allocating it, so
* that if we break out due to an error it can be released automatically
* in the next ff_mpv_frame_start().
*/
h->cur_pic_ptr->reference = 0;
h->cur_pic_ptr->field_poc[0] = h->cur_pic_ptr->field_poc[1] = INT_MAX;
h->next_output_pic = NULL;
h->postpone_filter = 0;
h->mb_aff_frame = h->ps.sps->mb_aff && (h->picture_structure == PICT_FRAME);
if (h->sei.unregistered.x264_build >= 0)
h->x264_build = h->sei.unregistered.x264_build;
assert(h->cur_pic_ptr->long_ref == 0);
return 0;
}
----------------------------------------
第2层 h264_field_start
----------------------------------------
static int h264_field_start(H264Context *h, const H264SliceContext *sl,
const H2645NAL *nal, int first_slice)
暂时不分析
----------------------------------------
第3层 ff_h264_queue_decode_slice (pause)
----------------------------------------
暂时不分析,真正的解码算法.
----------------------------------------
第4层 decode_nal_unit
----------------------------------------
将buf,buf_size中的数据， 分析到H264Context中, 解码的核心控制函数
它由两部分主要工作,一个是分析ff_h2645_packet_split, 一个是处理for循环
代码有简化！
static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size)
{
AVCodecContext *const avctx = h->avctx;
int nals_needed = 0;
int idr_cleared=0;
int i, ret = 0;
h->has_slice = 0;
h->nal_unit_type= 0;
if (!(avctx->flags2 & AV_CODEC_FLAG2_CHUNKS)) {
h->current_slice = 0;
if (!h->first_field) {
h->cur_pic_ptr = NULL;
ff_h264_sei_uninit(&h->sei);
}
}
if (h->nal_length_size == 4) {
if (buf_size > 8 && AV_RB32(buf) == 1 && AV_RB32(buf+5) > (unsigned)buf_size) {
h->is_avc = 0;
}else if(buf_size > 3 && AV_RB32(buf) > 1 && AV_RB32(buf) <= (unsigned)buf_size)
h->is_avc = 1;
}
//将输入数据分割为NAL单元, 经此数据都进入内存，并变成了计算机方便处理的内存分布形式.
//在这里把整体分割成一个个部分, 我才突然对nal, 网络抽象单元有了认识, 怎么起这样一个名字呢？
//其实叫分析单元或数据单元可能更通俗吧! 就是说整个文件是由一块一块的NAL单元来构成的.
//通过该函数把它们都分割完毕，都到了计算机内存中，所以解码时就可以一个循环处理完毕
ret = ff_h2645_packet_split(&h->pkt, buf, buf_size, avctx, h->is_avc, h->nal_length_size,
avctx->codec_id, 0, 0);
//以下的循环对分割的nal 依次进行处理，可以方便的知道nal的类型，拿到nal数据，完成解码过程
//例如解析pps,sps,idr_slice, slice, ... 等等
for (i = 0; i < h->pkt.nb_nals; i++) {
H2645NAL *nal = &h->pkt.nals[i]; //数据就在nal* 中
h->nal_ref_idc = nal->ref_idc;
h->nal_unit_type = nal->type;
int err = 0;
switch (nal->type) {
case H264_NAL_IDR_SLICE: //IDR 帧, 自动滑落到 slice 处理
if(!idr_cleared) {
idr(h); // FIXME ensure we don't lose some frames if there is reordering
}
idr_cleared = 1;
h->has_recovery_point = 1;
case H264_NAL_SLICE:
h->has_slice = 1;
err = ff_h264_queue_decode_slice(h, nal;
int max_slice_ctx = avctx->hwaccel ? 1 : h->nb_slice_ctx;
if (h->nb_slice_ctx_queued == max_slice_ctx) {
ret = ff_h264_execute_decode_slices(h);
}
break;
case H264_NAL_DPA:
case H264_NAL_DPB:
case H264_NAL_DPC:
avpriv_request_sample(avctx, "data partitioning");
break;
case H264_NAL_SEI:
ret = ff_h264_sei_decode(&h->sei, &nal->gb, &h->ps, avctx);
h->has_recovery_point = h->has_recovery_point || h->sei.recovery_point.recovery_frame_cnt != -1;
break;
case H264_NAL_SPS: {
GetBitContext tmp_gb = nal->gb;
if (ff_h264_decode_seq_parameter_set(&tmp_gb, avctx, &h->ps, 0) >= 0) break;
ff_h264_decode_seq_parameter_set(&nal->gb, avctx, &h->ps, 1);
break;
}
case H264_NAL_PPS:
ret = ff_h264_decode_picture_parameter_set(&nal->gb, avctx, &h->ps, nal->size_bits);
break;
case H264_NAL_AUD:
case H264_NAL_END_SEQUENCE:
case H264_NAL_END_STREAM:
case H264_NAL_FILLER_DATA:
case H264_NAL_SPS_EXT:
case H264_NAL_AUXILIARY_SLICE:
break;
default:
av_log(avctx, AV_LOG_DEBUG, "Unknown NAL code: %d (%d bits)\n",
nal->type, nal->size_bits);
}
}
ret = ff_h264_execute_decode_slices(h); //真正slice 解码
// set decode_error_flags to allow users to detect concealed decoding errors
if ((ret < 0 || h->slice_ctx->er.error_occurred) && h->cur_pic_ptr) {
h->cur_pic_ptr->f->decode_error_flags |= FF_DECODE_ERROR_DECODE_SLICES;
}
ret = 0;
end:
/* clean up */
if (h->cur_pic_ptr && !h->droppable && h->has_slice) {
ff_thread_report_progress(&h->cur_pic_ptr->tf, INT_MAX,
h->picture_structure == PICT_BOTTOM_FIELD);
}
return (ret < 0) ? ret : buf_size;
}
----------------------------------------
第5层 h264_decode_frame
----------------------------------------
实际的解frame函数, 数据从avpkt 到 pict或说data
static int h264_decode_frame(AVCodecContext *avctx, void *data,
int *got_frame, AVPacket *avpkt)
{
H264Context *h = avctx->priv_data;
AVFrame *pict = data;
int buf_index;
int ret;
h->flags = avctx->flags;
h->setup_finished = 0;
h->nb_slice_ctx_queued = 0;
ff_h264_unref_picture(h, &h->last_pic_for_ec); //先释放
const uint8_t *buf = avpkt->data;
int buf_size = avpkt->size;
if (buf_size == 0) // pkt 中无数据，继续发包并返回
return send_next_delayed_frame(h, pict, got_frame, 0);
//pkt 中有额外数据，那就先解额外数据
if (av_packet_get_side_data(avpkt, AV_PKT_DATA_NEW_EXTRADATA, NULL)) {
buffer_size_t side_size;
uint8_t *side = av_packet_get_side_data(avpkt, AV_PKT_DATA_NEW_EXTRADATA, &side_size);
ff_h264_decode_extradata(side, side_size,
&h->ps, &h->is_avc, &h->nal_length_size,
avctx->err_recognition, avctx);
}
if (h->is_avc && buf_size >= 9 && buf[0]==1 && buf[2]==0 && (buf[4]&0xFC)==0xFC) {
if (is_avcc_extradata(buf, buf_size))
return ff_h264_decode_extradata(buf, buf_size,
&h->ps, &h->is_avc, &h->nal_length_size,
avctx->err_recognition, avctx);
}
//主角登场,解码nal单元, buf 是输入数据
buf_index = decode_nal_units(h, buf, buf_size);
//还需要数据，那就再发送
if (!h->cur_pic_ptr && h->nal_unit_type == H264_NAL_END_SEQUENCE) {
return send_next_delayed_frame(h, pict, got_frame, buf_index);
}
if (!(avctx->flags2 & AV_CODEC_FLAG2_CHUNKS) && (!h->cur_pic_ptr || !h->has_slice)) {
if (avctx->skip_frame >= AVDISCARD_NONREF || buf_size >= 4 && !memcmp("Q264", buf, 4))
return buf_size;
av_log(avctx, AV_LOG_ERROR, "no frame!\n");
return AVERROR_INVALIDDATA;
}
if (!(avctx->flags2 & AV_CODEC_FLAG2_CHUNKS) || (h->mb_y >= h->mb_height && h->mb_height))
{
if ((ret = ff_h264_field_end(h, &h->slice_ctx[0], 0)) < 0) //终止h264解码,返回
return ret;
/* Wait for second field. */
if (h->next_output_pic) {
ret = finalize_frame(h, pict, h->next_output_pic, got_frame);
if (ret < 0)
return ret;
}
}
av_assert0(pict->buf[0] || !*got_frame);
ff_h264_unref_picture(h, &h->last_pic_for_ec);
return get_consumed_bytes(buf_index, buf_size);
}
----------------------------------------
第6层 decode_simple_internal
----------------------------------------
最主要的封装函数.
一个包装函数，为什么也写这么多代码? 它就是让具体的解码函数去解码
static inline int decode_simple_internal(AVCodecContext *avctx, AVFrame *frame, int64_t *discarded_samples)
{
AVCodecInternal *avci = avctx->internal;
DecodeSimpleContext *ds = &avci->ds;
AVPacket *pkt = ds->in_pkt;
int got_frame, actual_got_frame;
int ret;
//处理数据合法性判别
if (!pkt->data && !avci->draining) { //pkt 中无数据，那就从avctx中再取一个
av_packet_unref(pkt);
ret = ff_decode_get_packet(avctx, pkt);
if (ret < 0 && ret != AVERROR_EOF)
return ret;
}
if (avci->draining_done) return AVERROR_EOF;
got_frame = 0;
if (HAVE_THREADS && avctx->active_thread_type & FF_THREAD_FRAME) {
ret = ff_thread_decode_frame(avctx, frame, &got_frame, pkt);
} else {
ret = avctx->codec->decode(avctx, frame, &got_frame, pkt); //向下调用真正的实现
if (!(avctx->codec->caps_internal & FF_CODEC_CAP_SETS_PKT_DTS))
frame->pkt_dts = pkt->dts;
if (avctx->codec->type == AVMEDIA_TYPE_VIDEO) {
if(!avctx->has_b_frames)
frame->pkt_pos = pkt->pos;
//FIXME these should be under if(!avctx->has_b_frames)
/* get_buffer is supposed to set frame parameters */
if (!(avctx->codec->capabilities & AV_CODEC_CAP_DR1)) {
if (!frame->sample_aspect_ratio.num) frame->sample_aspect_ratio = avctx->sample_aspect_ratio;
if (!frame->width) frame->width = avctx->width;
if (!frame->height) frame->height = avctx->height;
if (frame->format == AV_PIX_FMT_NONE) frame->format = avctx->pix_fmt;
}
}
}
emms_c();
actual_got_frame = got_frame;
if (avctx->codec->type == AVMEDIA_TYPE_VIDEO) {
if (frame->flags & AV_FRAME_FLAG_DISCARD)
got_frame = 0;
} else if (avctx->codec->type == AVMEDIA_TYPE_AUDIO)
{
// 一大堆音频处理，用途不大干扰视线忽略之
...
}
if (!got_frame)
av_frame_unref(frame);
if (ret >= 0 && avctx->codec->type == AVMEDIA_TYPE_VIDEO && !(avctx->flags & AV_CODEC_FLAG_TRUNCATED))
ret = pkt->size;
if (avctx->framerate.num > 0 && avctx->framerate.den > 0)
avctx->time_base = av_inv_q(av_mul_q(avctx->framerate, (AVRational){avctx->ticks_per_frame, 1}));
/* do not stop draining when actual_got_frame != 0 or ret < 0 */
/* got_frame == 0 but actual_got_frame != 0 when frame is discarded */
if (avci->draining && !actual_got_frame) {
if (ret < 0) {
/* prevent infinite loop if a decoder wrongly always return error on draining */
/* reasonable nb_errors_max = maximum b frames + thread count */
int nb_errors_max = 20 + (HAVE_THREADS && avctx->active_thread_type & FF_THREAD_FRAME ?
avctx->thread_count : 1);
if (avci->nb_draining_errors++ >= nb_errors_max) {
av_log(avctx, AV_LOG_ERROR, "Too many errors when draining, this is a bug. "
"Stop draining and force EOF.\n");
avci->draining_done = 1;
ret = AVERROR_BUG;
}
} else {
avci->draining_done = 1;
}
}
avci->compat_decode_consumed += ret;
if (ret >= pkt->size || ret < 0) {
av_packet_unref(pkt);
av_packet_unref(avci->last_pkt_props);
} else {
int consumed = ret;
pkt->data += consumed;
pkt->size -= consumed;
avci->last_pkt_props->size -= consumed; // See extract_packet_props() comment.
pkt->pts = AV_NOPTS_VALUE;
pkt->dts = AV_NOPTS_VALUE;
avci->last_pkt_props->pts = AV_NOPTS_VALUE;
avci->last_pkt_props->dts = AV_NOPTS_VALUE;
}
if (got_frame)
av_assert0(frame->buf[0]);
return ret < 0 ? ret : 0;
}
----------------------------------------
第7层 decode_simple_receive_frame
----------------------------------------
不停的调用decode_simple_internal,直到返回error_again,error_EOF或者取到frame数据
static int decode_simple_receive_frame(AVCodecContext *avctx, AVFrame *frame)
{
int ret;
int64_t discarded_samples = 0;
while (!frame->buf[0]) {
if (discarded_samples > avctx->max_samples) return AVERROR(EAGAIN);
ret = decode_simple_internal(avctx, frame, &discarded_samples);
if (ret < 0) return ret;
}
return 0;
}
----------------------------------------
第8层 decode_receive_frame_internal
----------------------------------------
包装函数,继续向下调用
static int decode_receive_frame_internal(AVCodecContext *avctx, AVFrame *frame)
{
AVCodecInternal *avci = avctx->internal;
if (avctx->codec->receive_frame) {
ret = avctx->codec->receive_frame(avctx, frame);
} else
ret = decode_simple_receive_frame(avctx, frame); //通过该函数向下调用
if (!ret) { //调用成功后的处理
frame->best_effort_timestamp = guess_correct_pts(avctx,
frame->pts,
frame->pkt_dts);
if (frame->private_ref) { // 私有参考的后处理
FrameDecodeData *fdd = (FrameDecodeData*)frame->private_ref->data;
if (fdd->post_process) {
ret = fdd->post_process(avctx, frame);
if (ret < 0) {
av_frame_unref(frame);
return ret;
}
}
}
}
/* free the per-frame decode data */
av_buffer_unref(&frame->private_ref);
return ret;
}
----------------------------------------
第9层 avcodec_send_packet
----------------------------------------
int avcodec_send_packet(AVCodecContext *avctx, const AVPacket *avpkt)
{
AVCodecInternal *avci = avctx->internal;
av_packet_unref(avci->buffer_pkt); //先释放掉旧的包
if (avpkt && (avpkt->data || avpkt->side_data_elems)) {
ret = av_packet_ref(avci->buffer_pkt, avpkt); //avpkt 转移到avci->buffer_pkt
if (ret < 0) return ret;
}
ret = av_bsf_send_packet(avci->bsf, avci->buffer_pkt); //把avci->buffer_pkt 转移到avci->bsf
if (!avci->buffer_frame->buf[0]) {
ret = decode_receive_frame_internal(avctx, avci->buffer_frame); //调用上面接受frame的过程
if (ret < 0 && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF)
return ret; //返回出错值
}
return 0;
}
----------------------------------------
补充: avcodec_receive_frame
----------------------------------------
它就是调用的decode_receive_frame_internal, 当然,如果avci->buffer_frame->buf 有东西(已经解好了)，
那就不用解了,把它挪过来做为输出就可以了.
int attribute_align_arg avcodec_receive_frame(AVCodecContext *avctx, AVFrame *frame)
{
AVCodecInternal *avci = avctx->internal;
av_frame_unref(frame); // 先释放掉旧的frame
if (avci->buffer_frame->buf[0]) {
av_frame_move_ref(frame, avci->buffer_frame); //send_packet 都已经给准备好了,挪走就行
} else {
int ret = decode_receive_frame_internal(avctx, frame); //要亲自接受.参考上面分析
if (ret < 0) return ret;
}
avctx->frame_number++;
}
小结: 本博客忽略了具体的h264解码算法, 但搞清楚了整个解码流程.