diff --git a/interfaces/inner_api/native/audio_sink.h b/interfaces/inner_api/native/audio_sink.h index 60cd548fb04fbfb028a06c46e72aa639c02970b3..dc12cf42f503b2bab7c1859d024b066e60245860 100644 --- a/interfaces/inner_api/native/audio_sink.h +++ b/interfaces/inner_api/native/audio_sink.h @@ -88,6 +88,20 @@ private: void UpdateAudioWriteTimeMayWait(); void DrainAndReportEosEvent(); void HandleEosInner(bool drain); + class UnderrunDetector { + public: + void DetectAudioUnderrun(int64_t clkTime, int64_t latency); + void SetEventReceiver(std::weak_ptr eventReceiver); + void UpdateBufferTimeNoLock(int64_t clkTime, int64_t latency); + void SetLastAudioBufferDuration(int64_t durationUs); + void Reset(); + private: + std::weak_ptr eventReceiver_; + Mutex mutex_ {}; + int64_t lastClkTime_ {HST_TIME_NONE}; + int64_t lastLatency_ {HST_TIME_NONE}; + int64_t lastBufferDuration_ {HST_TIME_NONE}; + }; std::shared_ptr plugin_ {}; std::shared_ptr playerEventReceiver_; int32_t appUid_{0}; @@ -132,6 +146,7 @@ private: int64_t lastBufferWriteTime_ {0}; bool lastBufferWriteSuccess_ {true}; bool isMuted_ = false; + UnderrunDetector underrunDetector_; }; } } diff --git a/interfaces/inner_api/native/media_sync_manager.h b/interfaces/inner_api/native/media_sync_manager.h index 25e0b8c5417ae91b83afcd3985d50cfad24653f3..8207343d5ad3d46260a1c9f70bb6a7bdf127e361 100644 --- a/interfaces/inner_api/native/media_sync_manager.h +++ b/interfaces/inner_api/native/media_sync_manager.h @@ -24,6 +24,7 @@ #include "osal/task/mutex.h" #include "osal/task/autolock.h" #include "osal/utils/steady_clock.h" +#include "filter/filter.h" #include "common/status.h" #include "plugin/plugin_time.h" @@ -45,6 +46,7 @@ public: Status Seek(int64_t mediaTime, bool isClosest = false); Status Reset() override; bool InSeeking(); + void SetEventReceiver(std::weak_ptr eventReceiver); std::condition_variable seekCond_; // interfaces from IMediaSyncCenter @@ -126,6 +128,7 @@ private: void SetAllSyncShouldWaitNoLock(); int64_t BoundMediaProgress(int64_t newMediaProgressTime); void UpdateFirstPtsAfterSeek(int64_t mediaTime); + void ReportLagEvent(int64_t lagDurationMs); int64_t ClipMediaTime(int64_t inTime); OHOS::Media::Mutex clockMutex_ {}; @@ -163,6 +166,7 @@ private: std::atomic lastAudioBufferDuration_ {0}; std::atomic lastReportMediaTime_ {HST_TIME_NONE}; std::atomic frameAfterSeeked_ {false}; + std::weak_ptr eventReceiver_; }; } // namespace Pipeline } // namespace Media diff --git a/services/media_engine/filters/video_decoder_adapter.cpp b/services/media_engine/filters/video_decoder_adapter.cpp index 2a6f2e9749b3d98d036c3a333c05ac19d65958ba..68db9e862b28a180ce6f581268f81bb85990d966 100644 --- a/services/media_engine/filters/video_decoder_adapter.cpp +++ b/services/media_engine/filters/video_decoder_adapter.cpp @@ -159,7 +159,6 @@ Status VideoDecoderAdapter::Start() videoCodecFaultInfo.errMsg = "mediaCodec_ start failed"; FaultVideoCodecEventWrite(videoCodecFaultInfo); } - currentTime_ = GetCurrentMillisecond(); return ret == AVCodecServiceErrCode::AVCS_ERR_OK ? Status::OK : Status::ERROR_INVALID_STATE; } @@ -187,6 +186,7 @@ Status VideoDecoderAdapter::Flush() bufferVector_.clear(); inputBufferQueueConsumer_->SetQueueSize(0); } + currentTime_ = -1; return ret == AVCodecServiceErrCode::AVCS_ERR_OK ? Status::OK : Status::ERROR_INVALID_STATE; } @@ -204,6 +204,7 @@ Status VideoDecoderAdapter::Reset() bufferVector_.clear(); inputBufferQueueConsumer_->SetQueueSize(0); } + currentTime_ = -1; return Status::OK; } @@ -212,6 +213,7 @@ Status VideoDecoderAdapter::Release() MEDIA_LOG_I_SHORT("Release enter."); FALSE_RETURN_V_MSG(mediaCodec_ != nullptr, Status::ERROR_INVALID_STATE, "mediaCodec_ is nullptr"); int32_t ret = mediaCodec_->Release(); + currentTime_ = -1; return ret == AVCodecServiceErrCode::AVCS_ERR_OK ? Status::OK : Status::ERROR_INVALID_STATE; } @@ -354,13 +356,19 @@ int32_t VideoDecoderAdapter::ReleaseOutputBuffer(uint32_t index, bool render) { MEDIA_LOG_I_SHORT("VideoDecoderAdapter::ReleaseOutputBuffer"); mediaCodec_->ReleaseOutputBuffer(index, render); - if (render && currentTime_ != -1) { + FALSE_RETURN_V(render, 0); + if (currentTime_ == -1) { + currentTime_ = GetCurrentMillisecond(); + } else { int64_t currentTime = GetCurrentMillisecond(); int64_t diffTime = currentTime - currentTime_; if (diffTime > LAG_LIMIT_TIME) { lagTimes_++; maxLagDuration_ = maxLagDuration_ > diffTime ? maxLagDuration_ : diffTime; totalLagDuration_ += diffTime; + if (eventReceiver_ != nullptr) { + eventReceiver_->OnEvent({"VideoDecoderAdapter", EventType::EVENT_VIDEO_LAG, diffTime}); + } } currentTime_ = currentTime; } diff --git a/services/media_engine/modules/sink/audio_sink.cpp b/services/media_engine/modules/sink/audio_sink.cpp index ef33b279aefb100ea2d6358dc980131e77e0d44e..79c81d27322f5b2e4ae854aa7d660d6ca52095fd 100644 --- a/services/media_engine/modules/sink/audio_sink.cpp +++ b/services/media_engine/modules/sink/audio_sink.cpp @@ -21,6 +21,7 @@ namespace { constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_ONLY_PRERELEASE, LOG_DOMAIN_SYSTEM_PLAYER, "HiStreamer" }; constexpr int64_t MAX_BUFFER_DURATION_US = 200000; // Max buffer duration is 200 ms +constexpr int64_t US_TO_MS = 1000; // 1000 us per ms } namespace OHOS { @@ -148,6 +149,7 @@ Status AudioSink::Stop() playRangeStartTime_ = DEFAULT_PLAY_RANGE_VALUE; playRangeEndTime_ = DEFAULT_PLAY_RANGE_VALUE; Status ret = plugin_->Stop(); + underrunDetector_.Reset(); if (ret != Status::OK) { return ret; } @@ -162,6 +164,7 @@ Status AudioSink::Stop() Status AudioSink::Pause() { Status ret = Status::OK; + underrunDetector_.Reset(); if (isTransitent_ || isEos_) { ret = plugin_->PauseTransitent(); } else { @@ -201,6 +204,7 @@ Status AudioSink::Resume() Status AudioSink::Flush() { Status ret = Status::OK; + underrunDetector_.Reset(); ret = plugin_->Flush(); { AutoLock lock(eosMutex_); @@ -212,6 +216,7 @@ Status AudioSink::Flush() Status AudioSink::Release() { + underrunDetector_.Reset(); return plugin_->Deinit(); } @@ -409,6 +414,53 @@ void AudioSink::ResetSyncInfo() forceUpdateTimeAnchorNextTime_ = false; } +void AudioSink::UnderrunDetector::Reset() +{ + AutoLock lock(mutex_); + lastClkTime_ = HST_TIME_NONE; + lastLatency_ = HST_TIME_NONE; + lastBufferDuration_ = HST_TIME_NONE; +} + +void AudioSink::UnderrunDetector::SetEventReceiver(std::weak_ptr eventReceiver) +{ + eventReceiver_ = eventReceiver; +} + +void AudioSink::UnderrunDetector::UpdateBufferTimeNoLock(int64_t clkTime, int64_t latency) +{ + lastClkTime_ = clkTime; + lastLatency_ = latency; +} + +void AudioSink::UnderrunDetector::SetLastAudioBufferDuration(int64_t durationUs) +{ + AutoLock lock(mutex_); + lastBufferDuration_ = durationUs; +} + +void AudioSink::UnderrunDetector::DetectAudioUnderrun(int64_t clkTime, int64_t latency) +{ + if (lastClkTime_ == HST_TIME_NONE) { + AutoLock lock(mutex_); + UpdateBufferTimeNoLock(clkTime, latency); + return; + } + int64_t underrunTimeUs = 0; + { + AutoLock lock(mutex_); + int64_t elapsedClk = clkTime - lastClkTime_; + underrunTimeUs = elapsedClk - (lastLatency_ + lastBufferDuration_); + UpdateBufferTimeNoLock(clkTime, latency); + } + if (underrunTimeUs > 0) { + MEDIA_LOG_D("AudioSink maybe underrun, underrunTimeUs=" PUBLIC_LOG_D64, underrunTimeUs); + auto eventReceiver = eventReceiver_.lock(); + FALSE_RETURN(eventReceiver != nullptr); + eventReceiver->OnEvent({"AudioSink", EventType::EVENT_AUDIO_LAG, underrunTimeUs / US_TO_MS}); + } +} + int64_t AudioSink::DoSyncWrite(const std::shared_ptr& buffer) { bool render = true; // audio sink always report time anchor and do not drop @@ -430,6 +482,7 @@ int64_t AudioSink::DoSyncWrite(const std::shared_ptr& buf if (plugin_->GetLatency(latency) != Status::OK) { MEDIA_LOG_W("failed to get latency"); } + underrunDetector_.DetectAudioUnderrun(nowCt, latency); if (syncCenter) { render = syncCenter->UpdateTimeAnchor(nowCt, latency + fixDelay_, buffer->pts_ - firstPts_, buffer->pts_, buffer->duration_, this); @@ -449,6 +502,7 @@ int64_t AudioSink::DoSyncWrite(const std::shared_ptr& buf } else { latestBufferDuration_ = buffer->duration_ / speed_; } + underrunDetector_.SetLastAudioBufferDuration(latestBufferDuration_); if (syncCenter) { syncCenter->SetLastAudioBufferDuration(latestBufferDuration_); } diff --git a/services/media_engine/modules/sink/media_sync_manager.cpp b/services/media_engine/modules/sink/media_sync_manager.cpp index 0091187f4638515e7e46ab1d51bf1f292f668d80..d05d90b9e4456a7df5401eaa4d6da155829425a8 100644 --- a/services/media_engine/modules/sink/media_sync_manager.cpp +++ b/services/media_engine/modules/sink/media_sync_manager.cpp @@ -24,6 +24,7 @@ namespace { constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "MediaSyncManager" }; +constexpr int64_t US_TO_MS = 1000; // 1000 us per ms } namespace OHOS { @@ -395,6 +396,13 @@ void MediaSyncManager::SetLastAudioBufferDuration(int64_t durationUs) } } +void MediaSyncManager::ReportLagEvent(int64_t lagDurationMs) +{ + auto eventReceiver = eventReceiver_.lock(); + FALSE_RETURN(eventReceiver != nullptr); + eventReceiver->OnEvent({"SyncManager", EventType::EVENT_STREAM_LAG, lagDurationMs}); +} + int64_t MediaSyncManager::BoundMediaProgress(int64_t newMediaProgressTime) { int64_t maxMediaProgress; @@ -404,6 +412,7 @@ int64_t MediaSyncManager::BoundMediaProgress(int64_t newMediaProgressTime) maxMediaProgress = currentAnchorMediaTime_; } if (newMediaProgressTime > maxMediaProgress) { + ReportLagEvent((newMediaProgressTime - maxMediaProgress) / US_TO_MS); lastReportMediaTime_ = maxMediaProgress; // Avoid media progress go too far when data underrun. MEDIA_LOG_W("Data underrun for %{public}" PRId64 " us, currentSyncerPriority_ is %{public}" PRId32, newMediaProgressTime - maxMediaProgress, currentSyncerPriority_); @@ -550,6 +559,11 @@ void MediaSyncManager::ReportEos(IMediaSynchronizer* supplier) } } } + +void MediaSyncManager::SetEventReceiver(std::weak_ptr eventReceiver) +{ + eventReceiver_ = eventReceiver; +} } // namespace Pipeline } // namespace Media } // namespace OHOS \ No newline at end of file