diff --git a/design/histreamer_avcodec_process_analysis.md b/design/histreamer_avcodec_process_analysis.md new file mode 100644 index 0000000000000000000000000000000000000000..a107edbc3d48651baf6a058961f9817301ecfa8f --- /dev/null +++ b/design/histreamer_avcodec_process_analysis.md @@ -0,0 +1,65 @@ +## 音视频编解码流程分析 +### 背景 +在测试mp3音频文件循环播放时,发现一段时间之后,会在某一次播放结束后卡住,无法继续下一次循环; +问题定位流程参考如下issue: +https://gitee.com/openharmony/multimedia_histreamer/issues/I7A8OT?from=project-issue + +DemuxerFilter读取数据并解封装后,将数据push到DecoderFilter;异步模式的DecoderFilter会开启三个线程来处理数据: +1. HandleFrame线程:获取待解码数据并送入解码插件进行解码; +2. DecodeFrame线程:分配输出buffer并读取解码后的数据; +3. FinishFrame线程:将解码后的数据送入下一个filter节点; +### 音频解码处理中各线程调用时序如下: +```plantuml +participant DemuxerFilter +participant DecoderFilter_AsyncMode +participant DecoderPlugin +participant FfmpegAPI +participant AudioSinkFilter +participant AudioSinkPlugin + +alt#Gold DemuxerLoop 线程 +DemuxerFilter -> DemuxerFilter: ReadFrame() +DemuxerFilter -> DecoderFilter_AsyncMode: HandleFrame() -> PushData() +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: inBufQue_->Push(buffer) +end +alt#Gold #LightBlue HandleFrame 线程 +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: inBufQue_->Pop() +activate DecoderFilter_AsyncMode +DecoderFilter_AsyncMode -> DecoderPlugin: ret = plugin_->QueueInputBuffer()\n +DecoderPlugin -> FfmpegAPI: ret = avcodec_send_packet()\n @return AVERROR(EAGAIN)\ninput is not accepted in the current state\n user must read output with avcodec_receive_frame() +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: if (ret == ERROR_AGAIN) cv_.Wait()\n else continue +deactivate +end +alt#Gold #Pink DecodeFrame 线程 +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: outBufPool_->AllocateBuffer() +activate DecoderFilter_AsyncMode +DecoderFilter_AsyncMode -> DecoderPlugin: ret = plugin_->QueueOutputBuffer() +DecoderPlugin -> FfmpegAPI: ret = avcodec_receive_frame()\n @return AVERROR(EAGAIN)\n output is not available in this state\n user must try to send new input +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: if (ret == AVERROR(EAGAIN))\n log no enough data\n//Here need call cv_.NotifyOne() but not +deactivate +DecoderPlugin -> DecoderFilter_AsyncMode: else CallBack outBufQue_.push()\n cv_.NotifyOne() +end +alt#Gold FinishFrame 线程 +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: outBufQue_.pop() +activate DecoderFilter_AsyncMode +DecoderFilter_AsyncMode -> AudioSinkFilter: oPort->PushData(buffer) +AudioSinkFilter -> AudioSinkPlugin: plugin_->Write(buffer) +AudioSinkFilter -> AudioSinkPlugin: if buffer EOS plugin_->Drain() +AudioSinkFilter -> AudioSinkFilter: CallBack to hiplayer_engine\n EVENT_COMPLETE +AudioSinkFilter -> DecoderFilter_AsyncMode: release buffer +deactivate +end +``` + +### 循环播放时序图 +播放结束后,回调 EVENT_COMPLETE 事件到 histreamer 引擎,histreamer引擎继续回调 INFO_TYPE_EOS 事件到上层 PlayerServer,上层收到事件后,调用 Seek(0) 重新开始播放。 +```plantuml +participant AudioSinkFilter +participant HiPlayerImpl +participant PlayerServer + +AudioSinkFilter -> AudioSinkFilter: if (buffer->flag = EOS) +AudioSinkFilter -> HiPlayerImpl: CallBack OnEvent(EVENT_COMPLETE) +HiPlayerImpl -> PlayerServer: CallBack OnInfo(INFO_TYPE_EOS) +PlayerServer -> HiPlayerImpl: Seek(0) +``` \ No newline at end of file diff --git "a/\351\227\256\351\242\230\345\256\232\344\275\215\346\241\210\344\276\213/20030523_Filter\345\215\225\345\256\236\344\276\213\345\274\225\350\265\267\347\232\204Sniff\345\244\261\350\264\245\351\227\256\351\242\230/\351\227\256\351\242\230\345\210\206\346\236\220\350\277\207\347\250\213.md" "b/\351\227\256\351\242\230\345\256\232\344\275\215\346\241\210\344\276\213/20230523_Filter\345\215\225\345\256\236\344\276\213\345\274\225\350\265\267\347\232\204Sniff\345\244\261\350\264\245\351\227\256\351\242\230/\351\227\256\351\242\230\345\210\206\346\236\220\350\277\207\347\250\213.md" similarity index 100% rename from "\351\227\256\351\242\230\345\256\232\344\275\215\346\241\210\344\276\213/20030523_Filter\345\215\225\345\256\236\344\276\213\345\274\225\350\265\267\347\232\204Sniff\345\244\261\350\264\245\351\227\256\351\242\230/\351\227\256\351\242\230\345\210\206\346\236\220\350\277\207\347\250\213.md" rename to "\351\227\256\351\242\230\345\256\232\344\275\215\346\241\210\344\276\213/20230523_Filter\345\215\225\345\256\236\344\276\213\345\274\225\350\265\267\347\232\204Sniff\345\244\261\350\264\245\351\227\256\351\242\230/\351\227\256\351\242\230\345\210\206\346\236\220\350\277\207\347\250\213.md" diff --git "a/\351\227\256\351\242\230\345\256\232\344\275\215\346\241\210\344\276\213/20230602_\346\227\240\346\263\225\351\225\277\346\227\266\351\227\264\345\215\225\346\233\262\345\276\252\347\216\257\347\232\204\351\227\256\351\242\230/log_13_loop.rar" "b/\351\227\256\351\242\230\345\256\232\344\275\215\346\241\210\344\276\213/20230602_\346\227\240\346\263\225\351\225\277\346\227\266\351\227\264\345\215\225\346\233\262\345\276\252\347\216\257\347\232\204\351\227\256\351\242\230/log_13_loop.rar" new file mode 100644 index 0000000000000000000000000000000000000000..41cc65db21cf081e09b73de1733c7ff2f9e41f6a Binary files /dev/null and "b/\351\227\256\351\242\230\345\256\232\344\275\215\346\241\210\344\276\213/20230602_\346\227\240\346\263\225\351\225\277\346\227\266\351\227\264\345\215\225\346\233\262\345\276\252\347\216\257\347\232\204\351\227\256\351\242\230/log_13_loop.rar" differ diff --git "a/\351\227\256\351\242\230\345\256\232\344\275\215\346\241\210\344\276\213/20230602_\346\227\240\346\263\225\351\225\277\346\227\266\351\227\264\345\215\225\346\233\262\345\276\252\347\216\257\347\232\204\351\227\256\351\242\230/\351\225\277\346\227\266\351\227\264\345\215\225\346\233\262\345\276\252\347\216\257\346\222\255\346\224\276\345\244\261\350\264\245\345\210\206\346\236\220.md" "b/\351\227\256\351\242\230\345\256\232\344\275\215\346\241\210\344\276\213/20230602_\346\227\240\346\263\225\351\225\277\346\227\266\351\227\264\345\215\225\346\233\262\345\276\252\347\216\257\347\232\204\351\227\256\351\242\230/\351\225\277\346\227\266\351\227\264\345\215\225\346\233\262\345\276\252\347\216\257\346\222\255\346\224\276\345\244\261\350\264\245\345\210\206\346\236\220.md" new file mode 100644 index 0000000000000000000000000000000000000000..a107edbc3d48651baf6a058961f9817301ecfa8f --- /dev/null +++ "b/\351\227\256\351\242\230\345\256\232\344\275\215\346\241\210\344\276\213/20230602_\346\227\240\346\263\225\351\225\277\346\227\266\351\227\264\345\215\225\346\233\262\345\276\252\347\216\257\347\232\204\351\227\256\351\242\230/\351\225\277\346\227\266\351\227\264\345\215\225\346\233\262\345\276\252\347\216\257\346\222\255\346\224\276\345\244\261\350\264\245\345\210\206\346\236\220.md" @@ -0,0 +1,65 @@ +## 音视频编解码流程分析 +### 背景 +在测试mp3音频文件循环播放时,发现一段时间之后,会在某一次播放结束后卡住,无法继续下一次循环; +问题定位流程参考如下issue: +https://gitee.com/openharmony/multimedia_histreamer/issues/I7A8OT?from=project-issue + +DemuxerFilter读取数据并解封装后,将数据push到DecoderFilter;异步模式的DecoderFilter会开启三个线程来处理数据: +1. HandleFrame线程:获取待解码数据并送入解码插件进行解码; +2. DecodeFrame线程:分配输出buffer并读取解码后的数据; +3. FinishFrame线程:将解码后的数据送入下一个filter节点; +### 音频解码处理中各线程调用时序如下: +```plantuml +participant DemuxerFilter +participant DecoderFilter_AsyncMode +participant DecoderPlugin +participant FfmpegAPI +participant AudioSinkFilter +participant AudioSinkPlugin + +alt#Gold DemuxerLoop 线程 +DemuxerFilter -> DemuxerFilter: ReadFrame() +DemuxerFilter -> DecoderFilter_AsyncMode: HandleFrame() -> PushData() +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: inBufQue_->Push(buffer) +end +alt#Gold #LightBlue HandleFrame 线程 +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: inBufQue_->Pop() +activate DecoderFilter_AsyncMode +DecoderFilter_AsyncMode -> DecoderPlugin: ret = plugin_->QueueInputBuffer()\n +DecoderPlugin -> FfmpegAPI: ret = avcodec_send_packet()\n @return AVERROR(EAGAIN)\ninput is not accepted in the current state\n user must read output with avcodec_receive_frame() +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: if (ret == ERROR_AGAIN) cv_.Wait()\n else continue +deactivate +end +alt#Gold #Pink DecodeFrame 线程 +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: outBufPool_->AllocateBuffer() +activate DecoderFilter_AsyncMode +DecoderFilter_AsyncMode -> DecoderPlugin: ret = plugin_->QueueOutputBuffer() +DecoderPlugin -> FfmpegAPI: ret = avcodec_receive_frame()\n @return AVERROR(EAGAIN)\n output is not available in this state\n user must try to send new input +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: if (ret == AVERROR(EAGAIN))\n log no enough data\n//Here need call cv_.NotifyOne() but not +deactivate +DecoderPlugin -> DecoderFilter_AsyncMode: else CallBack outBufQue_.push()\n cv_.NotifyOne() +end +alt#Gold FinishFrame 线程 +DecoderFilter_AsyncMode -> DecoderFilter_AsyncMode: outBufQue_.pop() +activate DecoderFilter_AsyncMode +DecoderFilter_AsyncMode -> AudioSinkFilter: oPort->PushData(buffer) +AudioSinkFilter -> AudioSinkPlugin: plugin_->Write(buffer) +AudioSinkFilter -> AudioSinkPlugin: if buffer EOS plugin_->Drain() +AudioSinkFilter -> AudioSinkFilter: CallBack to hiplayer_engine\n EVENT_COMPLETE +AudioSinkFilter -> DecoderFilter_AsyncMode: release buffer +deactivate +end +``` + +### 循环播放时序图 +播放结束后,回调 EVENT_COMPLETE 事件到 histreamer 引擎,histreamer引擎继续回调 INFO_TYPE_EOS 事件到上层 PlayerServer,上层收到事件后,调用 Seek(0) 重新开始播放。 +```plantuml +participant AudioSinkFilter +participant HiPlayerImpl +participant PlayerServer + +AudioSinkFilter -> AudioSinkFilter: if (buffer->flag = EOS) +AudioSinkFilter -> HiPlayerImpl: CallBack OnEvent(EVENT_COMPLETE) +HiPlayerImpl -> PlayerServer: CallBack OnInfo(INFO_TYPE_EOS) +PlayerServer -> HiPlayerImpl: Seek(0) +``` \ No newline at end of file