From c8a4ae0ca793d445a782ce8cc5a7de6016c9509d Mon Sep 17 00:00:00 2001 From: dongyu_dy Date: Tue, 24 Oct 2023 17:45:00 +0800 Subject: [PATCH] add doc for mediafoundation of media_refactor Signed-off-by: dongyu_dy --- .../class\345\256\232\344\271\211.md" | 184 ++++++++++ .../pipeline\346\223\215\344\275\234.md" | 330 ++++++++++++++++++ 2 files changed, 514 insertions(+) create mode 100644 "design/MediaFoundation(new)/class\345\256\232\344\271\211.md" create mode 100644 "design/MediaFoundation(new)/pipeline\346\223\215\344\275\234.md" diff --git "a/design/MediaFoundation(new)/class\345\256\232\344\271\211.md" "b/design/MediaFoundation(new)/class\345\256\232\344\271\211.md" new file mode 100644 index 0000000..b980675 --- /dev/null +++ "b/design/MediaFoundation(new)/class\345\256\232\344\271\211.md" @@ -0,0 +1,184 @@ +# MediaFoundation 主要类定义及关系 + +```plantuml +class AVBufferQueue { + +AVBufferQueue& Create(...) + +BufferQueueProducer* GetProducer() + +BufferQueueConsumer* GetConsumer() +} + +class AVBuffer + +enum FilterType { + FILTETYPE_SOURCE + FILTETYPE_DEMUXER + FILTETYPE_AENC + FILTETYPE_ADEC + FILTETYPE_VENC + FILTETYPE_VDEC + FILTETYPE_MUXER + FILTETYPE_ASINK + FILTETYPE_FSINK + FILTETYPE_MAX +} + +enum StreamType { + STREAMTYPE_PACKED + STREAMTYPE_ENCODEC_AUDIO + STREAMTYPE_ENCODEC_VIDEO + STREAMTYPE_RAW_AUDIO + STREAMTYPE_RAW_VIDEO + STREAMTYPE_SUBTITLE + STREAMTYPE_MAX +} + +enum FilterCallbackCommand { + NEXT_FILTER_NEEDED + NEXT_FILTER_REMOVED + NEXT_FILTER_UPDATED + FILTER_CALLBACK_COMMAND_MAX +} + +Interface FilterCallback { + +OnCallback(Filter* filter, FilterCallbackCommand cmd, StreamType outType) +} + +Interface EventReceiver { + +OnEvent(Event event) +} + +class Meta + +Interface Filter { + +Filter(FilterType type) + +void Init(EventReceiver* receiver, FilterCallback* callback) + +Status SetParameter(shared_ptr meta) + +Status GetParameter(shared_ptr meta) + +Status LinkNext(Filter* filter, FilterType outType) + +Status OnLinked(Filter* filter, FilterType inType, shared_ptr meta) + +Status UpdateNext(Filter* filter, FilterType outType) + +Status OnUpdated(shared_ptr meta) + +Status UnlinkNext(Filter* filter, FilterType outType) + +Status OnUnlinked(Filter* filter, FilterType inType) + +Status Prepare() + +Status Start() + +Status Pause() + +Status Stop() + +Status Flush() + +Status Release() + +BufferQueueProducer* GetInputBufferQueue() + +FilterType GetFitlerType() + -SetOutputBufferQueue(BufferQueueProducer* queue) +} + +class SomeFilter { + -MediaSome ability_ + -vector supportedInStreams_ + -vector supportedOutStreams_ + -map> outStreams_ + -FilterCallback* callback_ + -EventReceiver* receiver_ + +PublicFunctions() + -PrivateFunctions() +} + +class FilterFactory { + +FilterFactory& Instance() + +void Init() + +void RegisterFilter(FilterType type, InstanceGenerator& generator) + +Filter* CreateFilter(FilterType type) + -FilterFactorty() + -map generatorMap_ +} + +class MediaSome { + +Status Init() + +Status DeInit() + +Status Prepare() + +Status Start() + +Status Reset() + +Status Stop() + -SomePlugin plugin_ +} + +Interface Plugin { + +string GetName() + +Status Prepare() + +Status Start() + +Status Reset() + +Status Stop() + +Status SetParameter(shared_ptr meta) + +Status GetParameter(shared_ptr meta) +} + +Interface SomePlugin { + +BufferQueueProducer* GetInputBufferQueue() + +BufferQueueConsumer* GetOutBufferQueue() + +SetInputBufferQueue(BufferQueueConsumer* queue) + +SetOutputBufferQueue(BufferQueueProducer* queue) + +void SetCallback(SomeCallback* callback) +} + +class PluginManager { + +PluginManager& Instance() + +void RegisterGenericPlugin(const GenericPluginDef& pluginDef) + +vector ListPlugins(PluginType pluginType) + shared_ptr GetPluginInfo(PluginType type, const string& name) + shared_ptr CreatePlugin(PluginType type, const string& name) +} + +class Pipeline { + +void Init(EventReceiver* receiver, FilterCallback* callback) + +Status AddFilters(vector filters) + +Status RemoveFilter(FilterType type, Filter* filter) + +Status LinkFilters(Filter* filter, vector filters, StreamType type) + +Status UnlinkFilters(Filter* filter, vector filters, StreamType type) + +Status UpdateFilters(Filter* filter, vector filters, StreamType type) + +Status Prepare() + +Status Start() + +Status Pause() + +Status Resume() + +Status Stop() + +Status Flush() + +Status Release() + -map> filters_ +} + +Filter .up.> Meta +Filter --* Pipeline + +Pipeline ..> FilterType +Pipeline .up.|> EventReceiver + +SomeFilter .up.|> Filter +SomeFilter ..> StreamType +SomeFilter ..> FilterType +SomeFilter --left> MediaSome +SomeFilter --> AVBufferQueue +SomeFilter -down-o FilterFactory + +MediaSome .up.> PluginManager +MediaSome --left> SomePlugin +MediaSome --> AVBufferQueue + +SomePluginImpl1 .up.|> SomePlugin +SomePluginImpl2 .up.|> SomePlugin +SomePluginImpl3 .up.|> SomePlugin + +SomePlugin -up-|> Plugin +SomePlugin ..> AVBuffer + +Plugin -right-o PluginManager +Plugin .up.> Meta + +AVBuffer -right-* AVBufferQueue + +HiPlayer -up-> Pipeline +HiPlayer .up.> FilterFactory +HiPlayer .right.|> FilterCallback + +FilterCallback .up.> FilterCallbackCommand + + + +``` diff --git "a/design/MediaFoundation(new)/pipeline\346\223\215\344\275\234.md" "b/design/MediaFoundation(new)/pipeline\346\223\215\344\275\234.md" new file mode 100644 index 0000000..ebefaf7 --- /dev/null +++ "b/design/MediaFoundation(new)/pipeline\346\223\215\344\275\234.md" @@ -0,0 +1,330 @@ +# MediaFoundation Pipeline操作 + +## A. 播放流程概览 + +```plantuml +App -> HiPlayer: new +HiPlayer -> Pipeline: new +HiPlayer -> SourceFilter: create + +App -> HiPlayer: SetSource(url) +HiPlayer -> SourceFilter: SetParameter({SOURCE, url}) + +App -> HiPlayer: Prepare +HiPlayer -> Pipeline: Prepare +activate Pipeline + Pipeline -> SourceFilter: Prepare + activate SourceFilter + SourceFilter -> CodecFilter: create, link, prepare + activate CodecFilter + CodecFilter -> SinkFilter: create, link, prepare + activate SinkFilter + return + return + return +return + +App -> HiPlayer: Play/Pause/Stop/Release +HiPlayer -> Pipeline: Start/Pause/Stop/Release +activate Pipeline + Pipeline -> SourceFilter: Start/Pause/Stop/Release + activate SourceFilter + SourceFilter -> CodecFilter: Start/Pause/Stop/Release + activate CodecFilter + CodecFilter -> SinkFilter: Start/Pause/Stop/Release + activate SinkFilter + return + return + return +return +``` + +## A. Pipeline动态构建 + +### 1. 创建关键filter，并配置给pipeline管理 + +- 哪些filter是需要pipeline管理的关键filter，由业务方（player/recorder...）决定。 +- 后续prepare/start/stop等操作时，pipeline只会调用关键filter的对应方法 +- 而与关键filter相连的其他filter都由关键filter链式调用触发。 + +#### 1.1 player + +```plantuml +HiPlayer -> Pipeline: Init +HiPlayer -> FilterFactory: CreateFilter +activate FilterFactory + FilterFactory -> SourceFilter: Create +return sourceFilter + +HiPlayer -> SourceFilter: SetParameter({SOURCE, url}) + +HiPlayer -> Pipeline: AddFilters({sourceFilter}) +``` + +#### 1.2 recorder + +```plantuml +HiPlayer -> Pipeline: Init +HiPlayer -> FilterFactory: CreateFilter +activate FilterFactory + FilterFactory -> SourceFilter: Create +return audioSourceFilter + +HiPlayer -> FilterFactory: CreateFilter +activate FilterFactory + FilterFactory -> CodecFilter: Create +return videoDecoderFilter + +HiPlayer -> FilterFactory: CreateFilter +activate FilterFactory + FilterFactory -> MuxerFilter: Create +return muxerFilter + +HiPlayer -> SourceFilter: SetParameter({SOURCE, builtin_mic}) +HiPlayer -> CodecFilter: SetParameter({SOURCE, builtin_camera}) +HiPlayer -> MuxerFilter: SetParameter({FORMAT, ...}) + +HiPlayer -> Pipeline: AddFilters({audioSourceFilter, videoDecoderFilter}) +``` + +### 2. 实现FilterCallback接口，定义业务相关的filter组装逻辑 + +当前FilterCallback根据FilterCallbackCommand，需要做3种类型的响应。 + +```cpp +// sample callback流程 +FilterCallback::OnCallback(Filter* filter, FilterCallbackCommand cmd, StreamType outType) +{ + switch(cmd) { + case NEXT_FILTER_NEEDED: { + // step1: determine which filter should be linked to input filter + // step2: create target filter, or reuse existing filter + // step3: link current filter to target filter + pipeline_->LinkFilters(filter, {targetFilter}, outType); + break; + } + case NEXT_FILTER_REMOVED:{ + // step1: determine which filter should be unlinked + // step2: choose target filter + // step3: do unlink + pipeline_->UnlinkFilters(filter, {targetFilter}, outType); + break; + } + case NEXT_FILTER_UPDATED:{ + // step1: determine which filter should be updated + // step2: choose target filter + // step3: do update + pipeline_->UpdateFilters(filter, {targetFilter}, outType); + break; + } + default: + break; + }； +} +``` + +#### 2.1 player处理callback的filter选择逻辑 + +- FILTERTPYE_SOURCE + STREAMTYPE_ENCODED_AUDIO = FILTERTYPE_ADEC +- FILTERTPYE_SOURCE + STREAMTYPE_ENCODED_VIDEO = FILTERTYPE_VDEC +- FILTERTYPE_ADEC + STREAMTYPE_RAW_AUDIO = FILTERTYPE_ASINK + +#### 2.2 recorder处理callback逻辑 + +- FILTERTPYE_SOURCE + STREAMTYPE_RAW_AUDIO = FILTERTYPE_AENC +- FILTERTYPE_AENC + STREAMTYPE_ENCODED_AUDIO = FILTERTYPE_MUXER +- FILTERTYPE_VENC + STREAMTYPE_ENCODED_VIDEO = FILTERTYPE_MUXER + +### 3. 通过Pipeline控制业务状态 + +以音频播放为例 + +#### 3.1 业务首次Prepare + +- HiPlayer已经创建好Pipeline和SourceFilter，并通过AddFilters加入到Pipeline的关键filter内 +- HiPlayer已完成向SourceFilter设置播放的url +- 对pipeline执行Prepare，完成filter动态创建与连接 + +```plantuml +HiPlayer -> Pipeline: Prepare +activate Pipeline + Pipeline -> SourceFilter: Prepare + activate SourceFilter + SourceFilter -> SourceFilter: PrePrepare + note right: 完成原子能力创建\n数据提取、格式嗅探、\n插件选择、文件头解析，\n确定mediainfo + + SourceFilter -> HiPlayer: OnCallback(FILTER_NEEDED, STREAM_ENC_A) + activate HiPlayer + HiPlayer -> HiPlayer: GetTargetFilterType + note right: 确定需要连接\n的Filter类型\n如没有可用filter\n就创建一个新的 + + HiPlayer -> FilterFactory: CreateFilter(FILTERTYPE_ADEC) + activate FilterFactory + FilterFactory -> CodecFilter: Create + return aDecFilter + + HiPlayer -> Pipeline: LinkFilters(sourceFilter, {aDecFilter}, STREAM_ENC_A) + activate Pipeline + Pipeline -> SourceFilter: LinkNext(aDecFilter, STREAM_ENC_A) + activate SourceFilter + SourceFilter -> CodecFilter: OnLinked(sourceFilter, STREAM_ENC_A, meta) + note left: 将码流信息通过\nmeta配置给audio decoder + return + return + return + + SourceFilter -> CodecFilter: Prepare + activate CodecFilter + CodecFilter -> CodecFilter: PrePrepare + note left: 完成原子能力创建 + CodecFilter -> HiPlayer: OnCallback + activate HiPlayer + HiPlayer -> FilterFactory: CreateFilter + activate FilterFactory + FilterFactory -> AudioSinkFilter + return aSinkFilter + + HiPlayer -> Pipeline: LinkFilters + activate Pipeline + Pipeline -> CodecFilter: LinkNext + activate CodecFilter + CodecFilter -> AudioSinkFilter: OnLinked + note left: 将需要的mediainfo\n传递给aSinkFilter + return + return + return + CodecFilter -> AudioSinkFilter: Prepare + activate AudioSinkFilter + AudioSinkFilter -> AudioSinkFilter: PrePrepare + note left: 完成原子能力创建 + AudioSinkFilter -> AudioSinkFilter: PostPrepare + note left: 完成插件选择、\n初始化、配置 + AudioSinkFilter -> Pipeline: OnEvent + return + + CodecFilter -> AudioSinkFilter: GetInputBufferQueue + CodecFilter -> CodecFilter: PostPrepare + note left: 完成插件选择、初始化\n连接到下游filter + CodecFilter -> Pipeline: OnEvent + return + + SourceFilter -> CodecFilter: GetInputBufferQueue + SourceFilter -> SourceFilter: PostPrepare + note left: 连接到下游filter + SourceFilter -> Pipeline: OnEvent + return +return +``` + +#### 3.2 业务Start/Pause/Stop等 + +- 业务状态改变流程中，Filter会首先将接口调用透传到后置filter，等待后置filter执行完毕后再完成改变操作，实现逆拓扑序操作。 +- 实际业务场景下，不相连的filter的状态改变可以并行完成 + +```plantuml +App -> HiPlayer: Play/Pause/Stop/Release +HiPlayer -> Pipeline: Start/Pause/Stop/Release +activate Pipeline + Pipeline -> SourceFilter: Start/Pause/Stop/Release + activate SourceFilter + SourceFilter -> CodecFilter: Start/Pause/Stop/Release + activate CodecFilter + CodecFilter -> AudioSinkFilter: Start/Pause/Stop/Release + activate AudioSinkFilter + AudioSinkFilter -> AudioSinkFilter: DoStart/DoPause/DoStop/DoRelease + note left: 实际完成状态改变操作 + return + CodecFilter -> CodecFilter: DoStart/DoPause/DoStop/DoRelease + note left: 实际完成状态改变操作 + return + SourceFilter -> SourceFilter: DoStart/DoPause/DoStop/DoRelease + note left: 实际完成状态改变操作 + return +return +``` + +#### 3.3 复用Pipeline + +播放过程中，业务调用方可能会切换播放源，并复用现有pipeline。此时，便存在FilterCallbackCommand的3种情况： + +- 当前filter需要连接新的filter。比如播放音乐后，复用pipeline开始播放视频，需要demuxer新增对接视频解码器。 +- 当前filter需要断开与部分filter的连接。比如播放正常视频后，复用pipeline播放一段没有音轨的视频，此时需要端口demuxer与音频解码器的连接。 +- 当前filter参数改变。比如播放两段格式不同的视频，demuxer，decoder都需要重新配置和选择插件。 + +```plantuml +App -> HiPlayer +HiPlayer -> Pipeline +Pipeline -> SourceFilter +SourceFilter -> CodecFilter +CodecFilter -> AudioSinkFilter + +App -> HiPlayer: SetSource(url) +HiPlayer -> SourceFilter: SetParameter(SOURCE, url) +note left: 复用pipeline\n设置新的source + +App -> HiPlayer: Prepare +HiPlayer -> Pipeline: Prepare +activate Pipeline + Pipeline -> SourceFilter:Prepare + activate SourceFilter + SourceFilter -> SourceFilter: PrePrepare + note right: 重新选择插件、解析码流\n获取更新后的mediainfo + + SourceFilter -> HiPlayer: OnCallback(FILTER_UPDATED, STREAM_ENC_A) + activate HiPlayer + HiPlayer -> HiPlayer: GetTargetFilterType + HiPlayer -> Pipeline: UpdateFilters(sourceFilter, {aDecFilter}, STREAM_ENC_A) + activate Pipeline + Pipeline -> SourceFilter: UpdateNext(aDecFilter, STREAM_ENC_A) + activate SourceFilter + SourceFilter -> CodecFilter: OnUpdated(meta) + note left: 传递需要更新的meta + return + return + return + + SourceFilter -> CodecFilter: Prepare + activate CodecFilter + CodecFilter -> CodecFilter: PrePrepare + note left: 根据新的meta\n更新原子能力参数 + + CodecFilter -> HiPlayer: OnCallback + activate HiPlayer + HiPlayer -> HiPlayer: GetTargetFilterType + HiPlayer -> Pipeline: UpdataFilters + activate Pipeline + Pipeline -> CodecFilter: UpdateNext + activate CodecFilter + CodecFilter -> AudioSinkFilter: OnUpdated(meta) + note left: 传递需要更新的meta + return + return + return + + CodecFilter -> AudioSinkFilter: Prepare + activate AudioSinkFilter + AudioSinkFilter -> AudioSinkFilter: PrePrepare + note left: 更新原子能力参数 + AudioSinkFilter -> AudioSinkFilter: PostPrepare + note left: 更新插件 + AudioSinkFilter -> Pipeline: OnEvent + + return + + CodecFilter -> AudioSinkFilter: GetInputBufferQueue + CodecFilter -> CodecFilter: PostPrepare + note left: 更新解码插件\n更新filter数据通路 + + CodecFilter -> Pipeline: OnEvent + return + + SourceFilter -> CodecFilter: GetInputBufferQueue + SourceFilter -> SourceFilter: PostPrepare + note left: 更新filter数据通路 + + SourceFilter -> Pipeline: OnEvent + return +return +``` + +需要移除后置Filter的流程与上述流程类似。 -- Gitee