# Lowlatency-IJKPlayer

**Repository Path**: tdcoding/lowlatency-ijkplayer

## Basic Information

- **Project Name**: Lowlatency-IJKPlayer
- **Description**: 基于ijkplayer实现低延迟直播播放器，优化RTSP直播的延时为大约120ms

- **Primary Language**: Java
- **License**: GPL-2.0
- **Default Branch**: master
- **Homepage**: None
- **GVP Project**: No

## Statistics

- **Stars**: 0
- **Forks**: 1
- **Created**: 2024-07-17
- **Last Updated**: 2024-09-18

## Categories & Tags

**Categories**: Uncategorized

**Tags**: None

## README

一、编译环境请参考ijkplayer
二、细节如下：
### 1、修改编译脚本支持RTSP

ijkPlayer默认是没有把RTSP协议编译进去，所以我们得修改编译脚本，原来的disable改为
export COMMON_FF_CFG_FLAGS="$COMMON_FF_CFG_FLAGS --enable-protocol=rtp"
export COMMON_FF_CFG_FLAGS="$COMMON_FF_CFG_FLAGS --enable-protocol=tcp"
export COMMON_FF_CFG_FLAGS="$COMMON_FF_CFG_FLAGS --enable-demuxer=rtsp"
export COMMON_FF_CFG_FLAGS="$COMMON_FF_CFG_FLAGS --enable-demuxer=sdp"
export COMMON_FF_CFG_FLAGS="$COMMON_FF_CFG_FLAGS --enable-demuxer=rtp"

### 2、解码器设为零延时

大家应该听过编码器的零延时（zerolatency），但可能没听过解码器零延时。其实解码器内部默认会缓存几帧数据，用于后续关联帧的解码，大概是3-5帧。经过反复测试，发现解码器的缓存帧会带来100多ms延时。也就是说，假如能够去掉缓存帧，就可以减少100多ms的延时。而在avcodec.h文件的AVCodecContext结构体有一个参数（flags）用来设置解码器延时：

typedef struct AVCodecContext {
......
int flags;
......
}
为了去掉解码器缓存帧，我们可以把flags设置为CODEC_FLAG_LOW_DELAY。在初始化解码器时进行设置：

//set decoder as low deday
codec_ctx->flags |= CODEC_FLAG_LOW_DELAY;

### 3、减少FFmpeg拆帧等待延时

FFmpeg拆帧是根据下一帧的起始码来作为当前帧结束符，起始码一般是：0x00 0x00 0x00 0x01或者0x00 0x00 0x01。这样就会带来一帧的延时，这一帧延时能不能去掉呢？如果有帧结束符，我们以帧结束符来拆帧，这样做就能解决一帧延时。现在，问题变成找到帧结束符，然后替换成下一帧起始码来拆帧。整个调用流程是：read_frame—>read_frame_internal—>parse_packet—>av_parser_parse2—>parser_parse—>ff_combine_frame. 流程图如下：


1、找到当前帧结束符
在rtpdec.c文件的rtp_parse_packet_internal方法里，有获取帧结束符，也就是mark标志位，我们在这里设一个全局变量：

static int rtp_parse_packet_internal(RTPDemuxContext *s, AVPacket *pkt,
                                     const uint8_t *buf, int len)
{
    ......
 
    if (buf[1] & 0x80)
        flags |= RTP_FLAG_MARKER;
    //the end of a frame
    mark_flag = flags;
 
    ......
}

### 4、去掉parse_packet的while循环

我们在外部调用libavformat模块的utils.c文件的read_frame读取一帧数据，而read_frame调用内部方法read_frame_internal，read_frame_internal接着调用parse_packet方法，在该方法里有一个while循环体。现在把循环体去掉，并且释放申请的内存：

static int parse_packet(AVFormatContext *s, AVPacket *pkt, int stream_index)
{
    ......
 
//    while (size > 0 || (pkt == &flush_pkt && got_output)) {
        int len;
        int64_t next_pts = pkt->pts;
        int64_t next_dts = pkt->dts;
 
        av_init_packet(&out_pkt);
        len = av_parser_parse2(st->parser, st->internal->avctx,
                               &out_pkt.data, &out_pkt.size, data, size,
                               pkt->pts, pkt->dts, pkt->pos);
        pkt->pts = pkt->dts = AV_NOPTS_VALUE;
        pkt->pos = -1;
        /* increment read pointer */
        data += len;
        size -= len;
 
        got_output = !!out_pkt.size;
 
        if (!out_pkt.size){
            av_packet_unref(&out_pkt);//release current packet
            av_packet_unref(pkt);//release current packet
            return 0;
//            continue;
        }
    ......        
   
        ret = add_to_pktbuf(&s->internal->parse_queue, &out_pkt,
                            &s->internal->parse_queue_end, 1);
        av_packet_unref(&out_pkt);
        if (ret < 0)
            goto fail;
//    }
 
    /* end of the stream => close and free the parser */
    if (pkt == &flush_pkt) {
        av_parser_close(st->parser);
        st->parser = NULL;
    }
 
fail:
    av_packet_unref(pkt);
    return ret;
}

### 5、 修改av_parser_parse2的帧偏移量

在libavcodec模块的parser.c文件中，parse_packet调用到av_parser_parse2来解释数据包，该方法内部有记录帧偏移量。原先是等待下一帧的起始码，现在改为当前帧结束符，所以要把下一帧起始码这个偏移量长度去掉：

int av_parser_parse2(AVCodecParserContext *s, AVCodecContext *avctx,
                     uint8_t **poutbuf, int *poutbuf_size,
                     const uint8_t *buf, int buf_size,
                     int64_t pts, int64_t dts, int64_t pos)
{
    ......
 
    /* WARNING: the returned index can be negative */
    index = s->parser->parser_parse(s, avctx, (const uint8_t **) poutbuf,
                                    poutbuf_size, buf, buf_size);
    av_assert0(index > -0x20000000); // The API does not allow returning AVERROR codes
#define FILL(name) if(s->name > 0 && avctx->name <= 0) avctx->name = s->name
    if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
        FILL(field_order);
    }
 
    /* update the file pointer */
    if (*poutbuf_size) {
        /* fill the data for the current frame */
        s->frame_offset = s->next_frame_offset;
 
        /* offset of the next frame */
//        s->next_frame_offset = s->cur_offset + index;
        //video frame don't plus index
        if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
            s->next_frame_offset = s->cur_offset;
        }else{
            s->next_frame_offset = s->cur_offset + index;
        }
        s->fetch_timestamp   = 1;
    }
    if (index < 0)
        index = 0;
    s->cur_offset += index;
    return index;
}

### 6、去掉parser_parse的寻找帧起始码


av_parser_parse2调用到parser_parse方法，而我们这里使用的是h264解码，所以在libavcodec模块的h264_parser.c有一个结构体ff_h264_parser，把h264_parse赋值给parser_parse：

AVCodecParser ff_h264_parser = {
    .codec_ids      = { AV_CODEC_ID_H264 },
    .priv_data_size = sizeof(H264ParseContext),
    .parser_init    = init,
    .parser_parse   = h264_parse,
    .parser_close   = h264_close,
    .split          = h264_split,
};
现在我们需要h264_parser.c文件的h264_parse方法，去掉寻找下一帧起始码作为当前帧结束符的过程：

static int h264_parse(AVCodecParserContext *s,
                      AVCodecContext *avctx,
                      const uint8_t **poutbuf, int *poutbuf_size,
                      const uint8_t *buf, int buf_size)
{
    ......
 
    if (s->flags & PARSER_FLAG_COMPLETE_FRAMES) {
        next = buf_size;
    } else {
//TODO:don't use next frame start code, modify by xufulong
//        next = h264_find_frame_end(p, buf, buf_size, avctx);
 
        if (ff_combine_frame(pc, next, &buf, &buf_size) < 0) {
            *poutbuf      = NULL;
            *poutbuf_size = 0;
            return buf_size;
        }
 
/*        if (next < 0 && next != END_NOT_FOUND) {
            av_assert1(pc->last_index + next >= 0);
            h264_find_frame_end(p, &pc->buffer[pc->last_index + next], -next, avctx); // update state
        }*/
    }
 
    ......
}

### 7、修改parser.c的组帧方法


h264_parse又调用parser.c的ff_combine_frame组帧方法，我们在这里把mark替换起始码作为帧结束符：

external int mark_flag;//引用全局变量
 
int ff_combine_frame(ParseContext *pc, int next,const uint8_t **buf, int *buf_size)
{
    ......
 
    /* copy into buffer end return */
//    if (next == END_NOT_FOUND) {
        void *new_buffer = av_fast_realloc(pc->buffer, &pc->buffer_size,
                                           *buf_size + pc->index +
                                           AV_INPUT_BUFFER_PADDING_SIZE);
 
        if (!new_buffer) {
          
            pc->index = 0;
            return AVERROR(ENOMEM);
        }
        pc->buffer = new_buffer;
        memcpy(&pc->buffer[pc->index], *buf, *buf_size);
        pc->index += *buf_size;
//        return -1;
          if(!mark_flag)
            return -1;
        next = 0;
//    }
 
    ......
 
}

### 8、~/ijkmedia/ijkplayer/ff_ffplay.c文件中 将这一方法:
static int packet_queue_get_or_buffering(FFPlayer *ffp, PacketQueue
*q, AVPacket *pkt, int *serial, int *finished)
{
assert(finished);
if (!ffp->packet_buffering)
return packet_queue_get(q, pkt, 1, serial); while (1) {
int new_packet = packet_queue_get(q, pkt, 0, serial); if (new_packet < 0)
return -1;
else if (new_packet == 0) {
if (q->is_buffer_indicator && !*finished) ffp_toggle_buffering(ffp, 1);
new_packet = packet_queue_get(q, pkt, 1, serial); if (new_packet < 0)
return -1; }
if (*finished == *serial) { av_packet_unref(pkt); continue;
}
else break;
}
return 1; }
替换为
static int packet_queue_get_or_buffering(FFPlayer *ffp, PacketQueue
*q, AVPacket *pkt, int *serial, int *finished){
if (!ffp->packet_buffering)
return packet_queue_get(q, pkt, 1, serial);
while (1) {
int new_packet = packet_queue_get(q, pkt, 1, serial); if (new_packet < 0){
new_packet = packet_queue_get(q, pkt, 0, serial); if(new_packet < 0)
return -1;
}else if (new_packet == 0) {
if (q->is_buffer_indicator && !*finished) ffp_toggle_buffering(ffp, 1);
new_packet = packet_queue_get(q, pkt, 1, serial); if (new_packet < 0)
return -1; }
if (*finished == *serial) { av_packet_unref(pkt); continue;
}
else break;
}
return 1; }

### 
9、~/ijkmedia/ijkplayer/ff_ffplay.c```文件中 修改这一方法为:
static double vp_duration(VideoState *is, Frame *vp, Frame *nextvp) {
    //注释的源代码
    /*if (vp->serial == nextvp->serial) {
        double duration = nextvp->pts - vp->pts;
        if (isnan(duration) || duration <= 0 || duration > is->max_frame_duration)
            return vp->duration;
        else
            return duration;
    } else {
        return 0.0;
    }*/
    //新增代码
    return vp->duration;
}

### 10、~/ijkmedia/ijkplayer/ff_ffplay.c```文件中 修改这一方法为:
static int ffplay_video_thread(void *arg)
{
    FFPlayer *ffp = arg;
    VideoState *is = ffp->is;
    AVFrame *frame = av_frame_alloc();
    double pts;
    double duration;
    int ret;
    AVRational tb = is->video_st->time_base;
    AVRational frame_rate = av_guess_frame_rate(is->ic, is->video_st, NULL);
    int64_t dst_pts = -1;
    int64_t last_dst_pts = -1;
    int retry_convert_image = 0;
    int convert_frame_count = 0;

#if CONFIG_AVFILTER
    AVFilterGraph *graph = avfilter_graph_alloc();
    AVFilterContext *filt_out = NULL, *filt_in = NULL;
    int last_w = 0;
    int last_h = 0;
    enum AVPixelFormat last_format = -2;
    int last_serial = -1;
    int last_vfilter_idx = 0;
    if (!graph) {
        av_frame_free(&frame);
        return AVERROR(ENOMEM);
    }

#else
    ffp_notify_msg2(ffp, FFP_MSG_VIDEO_ROTATION_CHANGED, ffp_get_video_rotate_degrees(ffp));
#endif

    if (!frame) {
#if CONFIG_AVFILTER
        avfilter_graph_free(&graph);
#endif
        return AVERROR(ENOMEM);
    }

    for (;;) {
        ret = get_video_frame(ffp, frame);
        if (ret < 0)
            goto the_end;
        if (!ret)
            continue;

        if (ffp->get_frame_mode) {
            if (!ffp->get_img_info || ffp->get_img_info->count <= 0) {
                av_frame_unref(frame);
                continue;
            }

            last_dst_pts = dst_pts;

            if (dst_pts < 0) {
                dst_pts = ffp->get_img_info->start_time;
            } else {
                dst_pts += (ffp->get_img_info->end_time - ffp->get_img_info->start_time) / (ffp->get_img_info->num - 1);
            }

            pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
            pts = pts * 1000;
            if (pts >= dst_pts) {
                while (retry_convert_image <= MAX_RETRY_CONVERT_IMAGE) {
                    ret = convert_image(ffp, frame, (int64_t)pts, frame->width, frame->height);
                    if (!ret) {
                        convert_frame_count++;
                        break;
                    }
                    retry_convert_image++;
                    av_log(NULL, AV_LOG_ERROR, "convert image error retry_convert_image = %d\n", retry_convert_image);
                }

                retry_convert_image = 0;
                if (ret || ffp->get_img_info->count <= 0) {
                    if (ret) {
                        av_log(NULL, AV_LOG_ERROR, "convert image abort ret = %d\n", ret);
                        ffp_notify_msg3(ffp, FFP_MSG_GET_IMG_STATE, 0, ret);
                    } else {
                        av_log(NULL, AV_LOG_INFO, "convert image complete convert_frame_count = %d\n", convert_frame_count);
                    }
                    goto the_end;
                }
            } else {
                dst_pts = last_dst_pts;
            }
            av_frame_unref(frame);
            continue;
        }

#if CONFIG_AVFILTER
        if (   last_w != frame->width
            || last_h != frame->height
            || last_format != frame->format
            || last_serial != is->viddec.pkt_serial
            || ffp->vf_changed
            || last_vfilter_idx != is->vfilter_idx) {
            SDL_LockMutex(ffp->vf_mutex);
            ffp->vf_changed = 0;
            av_log(NULL, AV_LOG_DEBUG,
                   "Video frame changed from size:%dx%d format:%s serial:%d to size:%dx%d format:%s serial:%d\n",
                   last_w, last_h,
                   (const char *)av_x_if_null(av_get_pix_fmt_name(last_format), "none"), last_serial,
                   frame->width, frame->height,
                   (const char *)av_x_if_null(av_get_pix_fmt_name(frame->format), "none"), is->viddec.pkt_serial);
            avfilter_graph_free(&graph);
            graph = avfilter_graph_alloc();
            if ((ret = configure_video_filters(ffp, graph, is, ffp->vfilters_list ? ffp->vfilters_list[is->vfilter_idx] : NULL, frame)) < 0) {
                // FIXME: post error
                SDL_UnlockMutex(ffp->vf_mutex);
                goto the_end;
            }
            filt_in  = is->in_video_filter;
            filt_out = is->out_video_filter;
            last_w = frame->width;
            last_h = frame->height;
            last_format = frame->format;
            last_serial = is->viddec.pkt_serial;
            last_vfilter_idx = is->vfilter_idx;
            frame_rate = av_buffersink_get_frame_rate(filt_out);
            SDL_UnlockMutex(ffp->vf_mutex);
        }

        ret = av_buffersrc_add_frame(filt_in, frame);
        if (ret < 0)
            goto the_end;

        while (ret >= 0) {
            is->frame_last_returned_time = av_gettime_relative() / 1000000.0;

            ret = av_buffersink_get_frame_flags(filt_out, frame, 0);
            if (ret < 0) {
                if (ret == AVERROR_EOF)
                    is->viddec.finished = is->viddec.pkt_serial;
                ret = 0;
                break;
            }

            is->frame_last_filter_delay = av_gettime_relative() / 1000000.0 - is->frame_last_returned_time;
            if (fabs(is->frame_last_filter_delay) > AV_NOSYNC_THRESHOLD / 10.0)
                is->frame_last_filter_delay = 0;
            tb = av_buffersink_get_time_base(filt_out);
#endif   
            //注释的代码
            //duration = (frame_rate.num && frame_rate.den ? av_q2d((AVRational){frame_rate.den, frame_rate.num}) : 0);
            //新增代码
            duration=0.01;
            //////////
            
            pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);
            ret = queue_picture(ffp, frame, pts, duration, frame->pkt_pos, is->viddec.pkt_serial);
            av_frame_unref(frame);
#if CONFIG_AVFILTER
        }
#endif

        if (ret < 0)
            goto the_end;
    }
 the_end:
#if CONFIG_AVFILTER
    avfilter_graph_free(&graph);
#endif
    av_log(NULL, AV_LOG_INFO, "convert image convert_frame_count = %d\n", convert_frame_count);
    av_frame_free(&frame);
    return 0;
}

### 编译后的Demo工程中setOption
formatMap["probesize"] = 1024 * 10

formatMap["infbuf"] = 1
//清空dns，因为多种协议播放会缓存协议导致播放h264后无法播放h265.
formatMap["dns_cache_clear"] = 1

// 设置是否开启环路过滤: 0开启，画面质量高，解码开销大，48关闭，画面质量差点，解码开销小
codecMap["skip_loop_filter"] = 48
//跳帧
codecMap["skip_frame"] = 0
//准备好了就播放.提高首开速度
playerMap["start-on-prepared"] = 1
// 是否开启预缓冲,直接禁用否则会有14s的卡顿缓冲时间.
playerMap["packet-buffering"] = 0
// 不额外优化（使能非规范兼容优化，默认值0 ）
playerMap["fast"] = 1
// 自动旋屏
playerMap["mediacodec-auto-rotate"] = 0
// 处理分辨率变化
playerMap["mediacodec-handle-resolution-change"] = 0
// 最大缓冲大小,单位kb，IJKPlayer拖动播放进度会导致重新请求数据，未使用已经缓冲好的数据，所以应该尽量控制缓冲区大小，减少不必要的数据损失
playerMap["max-buffer-size"] = 1024
// 视频的话，设置缓冲多少帧即开始播放
playerMap["min-frames"] = 2
// 最大缓存时长
playerMap["max_cached_duration"] = 0
//丢帧
playerMap["framedrop"] = 2
//视频硬解
playerMap["mediacodec"] = 1
//音频硬解，开启后会导致视频播放时间延长2s，一开始声音会延时2s播放（后续会和视频同步）
playerMap["opensles"] = 1

// 屏幕常亮
mVideoView.keepScreenOn = true
// 设置超时
playerMap["timeout"] = 10000000
playerMap["connect_timeout"] = 10000000
playerMap["listen_timeout"] = 10000000
playerMap["addrinfo_timeout"] = 10000000
playerMap["dns_cache_timeout"] = 10000000