Lowlatency-IJKPlayer: 基于ijkplayer实现低延迟直播播放器，优化RTSP直播的延时为大约120ms

一、编译环境请参考ijkplayer 二、细节如下：

1、修改编译脚本支持RTSP

ijkPlayer默认是没有把RTSP协议编译进去，所以我们得修改编译脚本，原来的disable改为 export COMMON_FF_CFG_FLAGS="$COMMON_FF_CFG_FLAGS --enable-protocol=rtp" export COMMON_FF_CFG_FLAGS="$COMMON_FF_CFG_FLAGS --enable-protocol=tcp" export COMMON_FF_CFG_FLAGS="$COMMON_FF_CFG_FLAGS --enable-demuxer=rtsp" export COMMON_FF_CFG_FLAGS="$COMMON_FF_CFG_FLAGS --enable-demuxer=sdp" export COMMON_FF_CFG_FLAGS="$COMMON_FF_CFG_FLAGS --enable-demuxer=rtp"

2、解码器设为零延时

大家应该听过编码器的零延时（zerolatency），但可能没听过解码器零延时。其实解码器内部默认会缓存几帧数据，用于后续关联帧的解码，大概是3-5帧。经过反复测试，发现解码器的缓存帧会带来100多ms延时。也就是说，假如能够去掉缓存帧，就可以减少100多ms的延时。而在avcodec.h文件的AVCodecContext结构体有一个参数（flags）用来设置解码器延时：

typedef struct AVCodecContext { ...... int flags; ...... } 为了去掉解码器缓存帧，我们可以把flags设置为CODEC_FLAG_LOW_DELAY。在初始化解码器时进行设置：

//set decoder as low deday codec_ctx->flags |= CODEC_FLAG_LOW_DELAY;

3、减少FFmpeg拆帧等待延时

FFmpeg拆帧是根据下一帧的起始码来作为当前帧结束符，起始码一般是：0x00 0x00 0x00 0x01或者0x00 0x00 0x01。这样就会带来一帧的延时，这一帧延时能不能去掉呢？如果有帧结束符，我们以帧结束符来拆帧，这样做就能解决一帧延时。现在，问题变成找到帧结束符，然后替换成下一帧起始码来拆帧。整个调用流程是：read_frame—>read_frame_internal—>parse_packet—>av_parser_parse2—>parser_parse—>ff_combine_frame. 流程图如下：

1、找到当前帧结束符在rtpdec.c文件的rtp_parse_packet_internal方法里，有获取帧结束符，也就是mark标志位，我们在这里设一个全局变量：

static int rtp_parse_packet_internal(RTPDemuxContext *s, AVPacket *pkt, const uint8_t *buf, int len) { ......

if (buf[1] & 0x80)
    flags |= RTP_FLAG_MARKER;
//the end of a frame
mark_flag = flags;

......

}

4、去掉parse_packet的while循环

我们在外部调用libavformat模块的utils.c文件的read_frame读取一帧数据，而read_frame调用内部方法read_frame_internal，read_frame_internal接着调用parse_packet方法，在该方法里有一个while循环体。现在把循环体去掉，并且释放申请的内存：

static int parse_packet(AVFormatContext *s, AVPacket *pkt, int stream_index) { ......

// while (size > 0 || (pkt == &flush_pkt && got_output)) { int len; int64_t next_pts = pkt->pts; int64_t next_dts = pkt->dts;

    av_init_packet(&out_pkt);
    len = av_parser_parse2(st->parser, st->internal->avctx,
                           &out_pkt.data, &out_pkt.size, data, size,
                           pkt->pts, pkt->dts, pkt->pos);
    pkt->pts = pkt->dts = AV_NOPTS_VALUE;
    pkt->pos = -1;
    /* increment read pointer */
    data += len;
    size -= len;

    got_output = !!out_pkt.size;

    if (!out_pkt.size){
        av_packet_unref(&out_pkt);//release current packet
        av_packet_unref(pkt);//release current packet
        return 0;

// continue; } ......

    ret = add_to_pktbuf(&s->internal->parse_queue, &out_pkt,
                        &s->internal->parse_queue_end, 1);
    av_packet_unref(&out_pkt);
    if (ret < 0)
        goto fail;

// }

/* end of the stream => close and free the parser */
if (pkt == &flush_pkt) {
    av_parser_close(st->parser);
    st->parser = NULL;
}

fail: av_packet_unref(pkt); return ret; }

5、修改av_parser_parse2的帧偏移量

在libavcodec模块的parser.c文件中，parse_packet调用到av_parser_parse2来解释数据包，该方法内部有记录帧偏移量。原先是等待下一帧的起始码，现在改为当前帧结束符，所以要把下一帧起始码这个偏移量长度去掉：

int av_parser_parse2(AVCodecParserContext *s, AVCodecContext *avctx, uint8_t **poutbuf, int *poutbuf_size, const uint8_t *buf, int buf_size, int64_t pts, int64_t dts, int64_t pos) { ......

/* WARNING: the returned index can be negative */
index = s->parser->parser_parse(s, avctx, (const uint8_t **) poutbuf,
                                poutbuf_size, buf, buf_size);
av_assert0(index > -0x20000000); // The API does not allow returning AVERROR codes

#define FILL(name) if(s->name > 0 && avctx->name <= 0) avctx->name = s->name if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) { FILL(field_order); }

/* update the file pointer */
if (*poutbuf_size) {
    /* fill the data for the current frame */
    s->frame_offset = s->next_frame_offset;

    /* offset of the next frame */

// s->next_frame_offset = s->cur_offset + index; //video frame don't plus index if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) { s->next_frame_offset = s->cur_offset; }else{ s->next_frame_offset = s->cur_offset + index; } s->fetch_timestamp = 1; } if (index < 0) index = 0; s->cur_offset += index; return index; }

6、去掉parser_parse的寻找帧起始码

av_parser_parse2调用到parser_parse方法，而我们这里使用的是h264解码，所以在libavcodec模块的h264_parser.c有一个结构体ff_h264_parser，把h264_parse赋值给parser_parse：

AVCodecParser ff_h264_parser = { .codec_ids = { AV_CODEC_ID_H264 }, .priv_data_size = sizeof(H264ParseContext), .parser_init = init, .parser_parse = h264_parse, .parser_close = h264_close, .split = h264_split, }; 现在我们需要h264_parser.c文件的h264_parse方法，去掉寻找下一帧起始码作为当前帧结束符的过程：

static int h264_parse(AVCodecParserContext *s, AVCodecContext *avctx, const uint8_t **poutbuf, int *poutbuf_size, const uint8_t *buf, int buf_size) { ......

if (s->flags & PARSER_FLAG_COMPLETE_FRAMES) {
    next = buf_size;
} else {

//TODO:don't use next frame start code, modify by xufulong // next = h264_find_frame_end(p, buf, buf_size, avctx);

    if (ff_combine_frame(pc, next, &buf, &buf_size) < 0) {
        *poutbuf      = NULL;
        *poutbuf_size = 0;
        return buf_size;
    }

/* if (next < 0 && next != END_NOT_FOUND) { av_assert1(pc->last_index + next >= 0); h264_find_frame_end(p, &pc->buffer[pc->last_index + next], -next, avctx); // update state }*/ }

......

}

7、修改parser.c的组帧方法

h264_parse又调用parser.c的ff_combine_frame组帧方法，我们在这里把mark替换起始码作为帧结束符：

external int mark_flag;//引用全局变量

int ff_combine_frame(ParseContext *pc, int next,const uint8_t **buf, int *buf_size) { ......

/* copy into buffer end return */

// if (next == END_NOT_FOUND) { void *new_buffer = av_fast_realloc(pc->buffer, &pc->buffer_size, *buf_size + pc->index + AV_INPUT_BUFFER_PADDING_SIZE);

    if (!new_buffer) {
      
        pc->index = 0;
        return AVERROR(ENOMEM);
    }
    pc->buffer = new_buffer;
    memcpy(&pc->buffer[pc->index], *buf, *buf_size);
    pc->index += *buf_size;

// return -1; if(!mark_flag) return -1; next = 0; // }

......

}

8、~/ijkmedia/ijkplayer/ff_ffplay.c文件中将这一方法:

static int packet_queue_get_or_buffering(FFPlayer *ffp, PacketQueue *q, AVPacket *pkt, int *serial, int *finished) { assert(finished); if (!ffp->packet_buffering) return packet_queue_get(q, pkt, 1, serial); while (1) { int new_packet = packet_queue_get(q, pkt, 0, serial); if (new_packet < 0) return -1; else if (new_packet == 0) { if (q->is_buffer_indicator && !*finished) ffp_toggle_buffering(ffp, 1); new_packet = packet_queue_get(q, pkt, 1, serial); if (new_packet < 0) return -1; } if (*finished == *serial) { av_packet_unref(pkt); continue; } else break; } return 1; } 替换为 static int packet_queue_get_or_buffering(FFPlayer *ffp, PacketQueue *q, AVPacket *pkt, int *serial, int *finished){ if (!ffp->packet_buffering) return packet_queue_get(q, pkt, 1, serial); while (1) { int new_packet = packet_queue_get(q, pkt, 1, serial); if (new_packet < 0){ new_packet = packet_queue_get(q, pkt, 0, serial); if(new_packet < 0) return -1; }else if (new_packet == 0) { if (q->is_buffer_indicator && !*finished) ffp_toggle_buffering(ffp, 1); new_packet = packet_queue_get(q, pkt, 1, serial); if (new_packet < 0) return -1; } if (*finished == *serial) { av_packet_unref(pkt); continue; } else break; } return 1; }

9、~/ijkmedia/ijkplayer/ff_ffplay.c```文件中修改这一方法为: static double vp_duration(VideoState *is, Frame *vp, Frame *nextvp) { //注释的源代码 /if (vp->serial == nextvp->serial) { double duration = nextvp->pts - vp->pts; if (isnan(duration) || duration <= 0 || duration > is->max_frame_duration) return vp->duration; else return duration; } else { return 0.0; }/ //新增代码 return vp->duration; }

10、~/ijkmedia/ijkplayer/ff_ffplay.c```文件中修改这一方法为:

static int ffplay_video_thread(void *arg) { FFPlayer *ffp = arg; VideoState *is = ffp->is; AVFrame *frame = av_frame_alloc(); double pts; double duration; int ret; AVRational tb = is->video_st->time_base; AVRational frame_rate = av_guess_frame_rate(is->ic, is->video_st, NULL); int64_t dst_pts = -1; int64_t last_dst_pts = -1; int retry_convert_image = 0; int convert_frame_count = 0;

#if CONFIG_AVFILTER AVFilterGraph *graph = avfilter_graph_alloc(); AVFilterContext *filt_out = NULL, *filt_in = NULL; int last_w = 0; int last_h = 0; enum AVPixelFormat last_format = -2; int last_serial = -1; int last_vfilter_idx = 0; if (!graph) { av_frame_free(&frame); return AVERROR(ENOMEM); }

#else ffp_notify_msg2(ffp, FFP_MSG_VIDEO_ROTATION_CHANGED, ffp_get_video_rotate_degrees(ffp)); #endif

if (!frame) {

#if CONFIG_AVFILTER avfilter_graph_free(&graph); #endif return AVERROR(ENOMEM); }

for (;;) {
    ret = get_video_frame(ffp, frame);
    if (ret < 0)
        goto the_end;
    if (!ret)
        continue;

    if (ffp->get_frame_mode) {
        if (!ffp->get_img_info || ffp->get_img_info->count <= 0) {
            av_frame_unref(frame);
            continue;
        }

        last_dst_pts = dst_pts;

        if (dst_pts < 0) {
            dst_pts = ffp->get_img_info->start_time;
        } else {
            dst_pts += (ffp->get_img_info->end_time - ffp->get_img_info->start_time) / (ffp->get_img_info->num - 1);
        }

        pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
        pts = pts * 1000;
        if (pts >= dst_pts) {
            while (retry_convert_image <= MAX_RETRY_CONVERT_IMAGE) {
                ret = convert_image(ffp, frame, (int64_t)pts, frame->width, frame->height);
                if (!ret) {
                    convert_frame_count++;
                    break;
                }
                retry_convert_image++;
                av_log(NULL, AV_LOG_ERROR, "convert image error retry_convert_image = %d\n", retry_convert_image);
            }

            retry_convert_image = 0;
            if (ret || ffp->get_img_info->count <= 0) {
                if (ret) {
                    av_log(NULL, AV_LOG_ERROR, "convert image abort ret = %d\n", ret);
                    ffp_notify_msg3(ffp, FFP_MSG_GET_IMG_STATE, 0, ret);
                } else {
                    av_log(NULL, AV_LOG_INFO, "convert image complete convert_frame_count = %d\n", convert_frame_count);
                }
                goto the_end;
            }
        } else {
            dst_pts = last_dst_pts;
        }
        av_frame_unref(frame);
        continue;
    }

#if CONFIG_AVFILTER if ( last_w != frame->width || last_h != frame->height || last_format != frame->format || last_serial != is->viddec.pkt_serial || ffp->vf_changed || last_vfilter_idx != is->vfilter_idx) { SDL_LockMutex(ffp->vf_mutex); ffp->vf_changed = 0; av_log(NULL, AV_LOG_DEBUG, "Video frame changed from size:%dx%d format:%s serial:%d to size:%dx%d format:%s serial:%d\n", last_w, last_h, (const char *)av_x_if_null(av_get_pix_fmt_name(last_format), "none"), last_serial, frame->width, frame->height, (const char *)av_x_if_null(av_get_pix_fmt_name(frame->format), "none"), is->viddec.pkt_serial); avfilter_graph_free(&graph); graph = avfilter_graph_alloc(); if ((ret = configure_video_filters(ffp, graph, is, ffp->vfilters_list ? ffp->vfilters_list[is->vfilter_idx] : NULL, frame)) < 0) { // FIXME: post error SDL_UnlockMutex(ffp->vf_mutex); goto the_end; } filt_in = is->in_video_filter; filt_out = is->out_video_filter; last_w = frame->width; last_h = frame->height; last_format = frame->format; last_serial = is->viddec.pkt_serial; last_vfilter_idx = is->vfilter_idx; frame_rate = av_buffersink_get_frame_rate(filt_out); SDL_UnlockMutex(ffp->vf_mutex); }

    ret = av_buffersrc_add_frame(filt_in, frame);
    if (ret < 0)
        goto the_end;

    while (ret >= 0) {
        is->frame_last_returned_time = av_gettime_relative() / 1000000.0;

        ret = av_buffersink_get_frame_flags(filt_out, frame, 0);
        if (ret < 0) {
            if (ret == AVERROR_EOF)
                is->viddec.finished = is->viddec.pkt_serial;
            ret = 0;
            break;
        }

        is->frame_last_filter_delay = av_gettime_relative() / 1000000.0 - is->frame_last_returned_time;
        if (fabs(is->frame_last_filter_delay) > AV_NOSYNC_THRESHOLD / 10.0)
            is->frame_last_filter_delay = 0;
        tb = av_buffersink_get_time_base(filt_out);

#endif
//注释的代码 //duration = (frame_rate.num && frame_rate.den ? av_q2d((AVRational){frame_rate.den, frame_rate.num}) : 0); //新增代码 duration=0.01; //////////

        pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
        ret = queue_picture(ffp, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial);
        av_frame_unref(frame);

#if CONFIG_AVFILTER } #endif

    if (ret < 0)
        goto the_end;
}

the_end: #if CONFIG_AVFILTER avfilter_graph_free(&graph); #endif av_log(NULL, AV_LOG_INFO, "convert image convert_frame_count = %d\n", convert_frame_count); av_frame_free(&frame); return 0; }

编译后的Demo工程中setOption

formatMap["probesize"] = 1024 * 10

formatMap["infbuf"] = 1 //清空dns，因为多种协议播放会缓存协议导致播放h264后无法播放h265. formatMap["dns_cache_clear"] = 1

// 设置是否开启环路过滤: 0开启，画面质量高，解码开销大，48关闭，画面质量差点，解码开销小 codecMap["skip_loop_filter"] = 48 //跳帧 codecMap["skip_frame"] = 0 //准备好了就播放.提高首开速度 playerMap["start-on-prepared"] = 1 // 是否开启预缓冲,直接禁用否则会有14s的卡顿缓冲时间. playerMap["packet-buffering"] = 0 // 不额外优化（使能非规范兼容优化，默认值0 ） playerMap["fast"] = 1 // 自动旋屏 playerMap["mediacodec-auto-rotate"] = 0 // 处理分辨率变化 playerMap["mediacodec-handle-resolution-change"] = 0 // 最大缓冲大小,单位kb，IJKPlayer拖动播放进度会导致重新请求数据，未使用已经缓冲好的数据，所以应该尽量控制缓冲区大小，减少不必要的数据损失 playerMap["max-buffer-size"] = 1024 // 视频的话，设置缓冲多少帧即开始播放 playerMap["min-frames"] = 2 // 最大缓存时长 playerMap["max_cached_duration"] = 0 //丢帧 playerMap["framedrop"] = 2 //视频硬解 playerMap["mediacodec"] = 1 //音频硬解，开启后会导致视频播放时间延长2s，一开始声音会延时2s播放（后续会和视频同步） playerMap["opensles"] = 1

// 屏幕常亮 mVideoView.keepScreenOn = true // 设置超时 playerMap["timeout"] = 10000000 playerMap["connect_timeout"] = 10000000 playerMap["listen_timeout"] = 10000000 playerMap["addrinfo_timeout"] = 10000000 playerMap["dns_cache_timeout"] = 10000000

gitsha/Lowlatency-IJKPlayer

1、修改编译脚本支持RTSP

2、解码器设为零延时

3、减少FFmpeg拆帧等待延时

4、去掉parse_packet的while循环

5、修改av_parser_parse2的帧偏移量

6、去掉parser_parse的寻找帧起始码

7、修改parser.c的组帧方法

8、~/ijkmedia/ijkplayer/ff_ffplay.c文件中将这一方法:

10、~/ijkmedia/ijkplayer/ff_ffplay.c```文件中修改这一方法为:

编译后的Demo工程中setOption

简介

发行版

贡献者

近期动态

gitsha/Lowlatency-IJKPlayer .gitee-modal { width: 500px !important; }

1、修改编译脚本支持RTSP

2、解码器设为零延时

3、减少FFmpeg拆帧等待延时

4、去掉parse_packet的while循环

5、 修改av_parser_parse2的帧偏移量

6、去掉parser_parse的寻找帧起始码

7、修改parser.c的组帧方法

8、~/ijkmedia/ijkplayer/ff_ffplay.c文件中 将这一方法:

10、~/ijkmedia/ijkplayer/ff_ffplay.c```文件中 修改这一方法为:

编译后的Demo工程中setOption

简介

发行版

贡献者

近期动态

搜索帮助

gitsha/Lowlatency-IJKPlayer

5、修改av_parser_parse2的帧偏移量

8、~/ijkmedia/ijkplayer/ff_ffplay.c文件中将这一方法:

10、~/ijkmedia/ijkplayer/ff_ffplay.c```文件中修改这一方法为: