diff --git a/interfaces/inner_api/native/audio_sink.h b/interfaces/inner_api/native/audio_sink.h index 0f86ecc4755ea966a41a35e470dd12dd8b8151a2..ada807e9c0f06399ae90ac7bab681165f3d7ffce 100644 --- a/interfaces/inner_api/native/audio_sink.h +++ b/interfaces/inner_api/native/audio_sink.h @@ -73,6 +73,8 @@ private: bool OnNewAudioMediaTime(int64_t mediaTimeUs); int64_t getPendingAudioPlayoutDurationUs(int64_t nowUs); int64_t getDurationUsPlayedAtSampleRate(uint32_t numFrames); + void UpdateAudioWriteTimeMayWait(); + void ReportEosEventAndDrain(); std::shared_ptr plugin_ {}; std::shared_ptr playerEventReceiver_; int32_t appUid_{0}; @@ -96,9 +98,13 @@ private: bool isEos_ {false}; std::mutex pluginMutex_; float volume_ {-1.0f}; - float speed_ {-1.0f}; + float speed_ {1.0f}; int32_t effectMode_ {-1}; bool isApe_ {false}; + // vars for audio progress optimization + int64_t playingBufferDurationUs_ {0}; + int64_t lastBufferWriteTime_ {0}; + bool lastBufferWriteSuccess_ {true}; }; } } diff --git a/services/media_engine/modules/sink/audio_sink.cpp b/services/media_engine/modules/sink/audio_sink.cpp index 766fdf527f4adc19a7f8557a6bd952523f420f76..d1e2517108f4f4ed65db6cbfd3f04eebddd3dede 100644 --- a/services/media_engine/modules/sink/audio_sink.cpp +++ b/services/media_engine/modules/sink/audio_sink.cpp @@ -58,6 +58,10 @@ Status AudioSink::Init(std::shared_ptr& meta, const std::shared_ptrPrepare(); meta->GetData(Tag::AUDIO_SAMPLE_RATE, sampleRate_); meta->GetData(Tag::AUDIO_SAMPLE_PER_FRAME, samplePerFrame_); + if (samplePerFrame_ > 0 && sampleRate_ > 0) { + playingBufferDurationUs_ = samplePerFrame_ * 1000000 / sampleRate_; // 1000000 usec per sec + } + MEDIA_LOG_I("Audiosink playingBufferDurationUs_ = " PUBLIC_LOG_D64, playingBufferDurationUs_); int64_t startTime = 0; if (!meta->GetData(Tag::MEDIA_START_TIME, startTime)) { startTime = 0; @@ -232,6 +236,36 @@ std::shared_ptr AudioSink::CreatePlugin() return std::reinterpret_pointer_cast(plugin); } +void AudioSink::UpdateAudioWriteTimeMayWait() +{ + if (latestBufferDuration_ <= 0) { + return; + } + int64_t timeNow = Plugins::HstTime2Us(SteadyClock::GetCurrentTimeNanoSec()); + if (!lastBufferWriteSuccess_) { + int64_t writeSleepTime = latestBufferDuration_ - (timeNow - lastBufferWriteTime_); + MEDIA_LOG_W("Last buffer write fail, sleep time is " PUBLIC_LOG_D64 "us", writeSleepTime); + if (writeSleepTime > 0) { + usleep(writeSleepTime); + timeNow = Plugins::HstTime2Us(SteadyClock::GetCurrentTimeNanoSec()); + } + } + lastBufferWriteTime_ = timeNow; +} + +void AudioSink::ReportEosEventAndDrain() +{ + isEos_ = true; + Event event { + .srcFilter = "AudioSink", + .type = EventType::EVENT_COMPLETE, + }; + FALSE_RETURN(playerEventReceiver_ != nullptr); + playerEventReceiver_->OnEvent(event); + plugin_->Drain(); + plugin_->PauseTransitent(); +} + void AudioSink::DrainOutputBuffer() { std::lock_guard lock(pluginMutex_); @@ -249,21 +283,13 @@ void AudioSink::DrainOutputBuffer() return; } if (filledOutputBuffer->flag_ & BUFFER_FLAG_EOS) { - isEos_ = true; - Event event { - .srcFilter = "AudioSink", - .type = EventType::EVENT_COMPLETE, - }; - FALSE_RETURN(playerEventReceiver_ != nullptr); - playerEventReceiver_->OnEvent(event); + ReportEosEventAndDrain(); inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer); - plugin_->Drain(); - plugin_->PauseTransitent(); return; } - + UpdateAudioWriteTimeMayWait(); DoSyncWrite(filledOutputBuffer); - plugin_->Write(filledOutputBuffer); + lastBufferWriteSuccess_ = (plugin_->Write(filledOutputBuffer) == Status::OK); MEDIA_LOG_D("audio DrainOutputBuffer pts = " PUBLIC_LOG_D64, filledOutputBuffer->pts_); numFramesWritten_++; inputBufferQueueConsumer_->ReleaseBuffer(filledOutputBuffer); @@ -313,7 +339,7 @@ int64_t AudioSink::DoSyncWrite(const std::shared_ptr& buf forceUpdateTimeAnchorNextTime_ = true; } latestBufferPts_ = buffer->pts_ - firstPts_; - latestBufferDuration_ = buffer->duration_; + latestBufferDuration_ = playingBufferDurationUs_ / speed_; return render ? 0 : -1; } @@ -322,11 +348,14 @@ Status AudioSink::SetSpeed(float speed) if (plugin_ == nullptr) { return Status::ERROR_NULL_POINTER; } - if (speed < 0) { + if (speed <= 0) { return Status::ERROR_INVALID_PARAMETER; } - speed_ = speed; - return plugin_->SetSpeed(speed); + auto ret = plugin_->SetSpeed(speed); + if (ret == Status::OK) { + speed_ = speed; + } + return ret; } Status AudioSink::SetAudioEffectMode(int32_t effectMode) diff --git a/services/media_engine/plugins/sink/audio_server_sink_plugin.cpp b/services/media_engine/plugins/sink/audio_server_sink_plugin.cpp index 02a63152a6cb2246a464a969e0006045c568fc82..848ef7685f40ca6958a1c8f2eb643313abe1ff3e 100644 --- a/services/media_engine/plugins/sink/audio_server_sink_plugin.cpp +++ b/services/media_engine/plugins/sink/audio_server_sink_plugin.cpp @@ -37,6 +37,7 @@ namespace { using namespace OHOS::Media::Plugins; constexpr int TUPLE_SECOND_ITEM_INDEX = 2; +constexpr int32_t DEFAULT_BUFFER_NUM = 8; const std::pair g_auInterruptMap[] = { {AudioInterruptMode::SHARE_MODE, OHOS::AudioStandard::InterruptMode::SHARE_MODE}, @@ -859,6 +860,99 @@ Status AudioServerSinkPlugin::GetLatency(uint64_t &hstTime) return Status::OK; } +Status AudioServerSinkPlugin::DrainCacheData(bool render) +{ + MediaAVCodec::AVCodecTrace trace("AudioServerSinkPlugin::DrainCacheData " + std::to_string(render)); + if (!render) { + MEDIA_LOG_D("Drop cache buffer because render=false"); + cachedBuffers_.clear(); + return Status::OK; + } + if (cachedBuffers_.empty()) { + return Status::OK; + } + AudioStandard::RendererState rendererState = (audioRenderer_ != nullptr) ? + audioRenderer_->GetStatus() : AudioStandard::RendererState::RENDERER_INVALID; + if (rendererState == AudioStandard::RendererState::RENDERER_PAUSED) { + MEDIA_LOG_W("audioRenderer_ is still paused, try again later"); + return Status::ERROR_AGAIN; + } + if (rendererState != AudioStandard::RendererState::RENDERER_RUNNING) { + cachedBuffers_.clear(); + MEDIA_LOG_W("Drop cache buffer because audioRenderer_ state invalid"); + return Status::ERROR_UNKNOWN; + } + while (cachedBuffers_.size() > 0) { // do drain cache buffers + auto currBuffer = cachedBuffers_.front(); + uint8_t* destBuffer = currBuffer.data(); + size_t destLength = currBuffer.size(); + bool shouldDrop = false; + size_t remained = WriteAudioBuffer(destBuffer, destLength, shouldDrop); + if (remained == 0) { // write ok + cachedBuffers_.pop_front(); + continue; + } + if (shouldDrop) { // write error and drop buffer + cachedBuffers_.clear(); + MEDIA_LOG_W("Drop cache buffer, error happens during drain"); + return Status::ERROR_UNKNOWN; + } + if (remained < destLength) { // some data written, then audioRender paused again, update cache + std::vector tmpBuffer(destBuffer + destLength - remained, destBuffer + destLength); + cachedBuffers_.pop_front(); + cachedBuffers_.emplace_front(std::move(tmpBuffer)); + } // else no data written, no need to update front cache + MEDIA_LOG_W("Audiorender pause again during drain cache buffers"); + return Status::ERROR_AGAIN; + } + MEDIA_LOG_I("Drain cache buffer success"); + return Status::OK; +} + +void AudioServerSinkPlugin::CacheData(uint8_t* inputBuffer, size_t bufferSize) +{ + MediaAVCodec::AVCodecTrace trace("AudioServerSinkPlugin::CacheData " + std::to_string(bufferSize)); + std::vector tmpBuffer(inputBuffer, inputBuffer + bufferSize); + cachedBuffers_.emplace_back(std::move(tmpBuffer)); + MEDIA_LOG_I("Cache one audio buffer, data size is " PUBLIC_LOG_U64, bufferSize); + while (cachedBuffers_.size() > DEFAULT_BUFFER_NUM) { + auto dropSize = cachedBuffers_.front().size(); + MEDIA_LOG_W("Drop one cached buffer size " PUBLIC_LOG_U64 " because max cache size reached.", dropSize); + cachedBuffers_.pop_front(); + } +} + +size_t AudioServerSinkPlugin::WriteAudioBuffer(uint8_t* inputBuffer, size_t bufferSize, bool& shouldDrop) +{ + uint8_t* destBuffer = inputBuffer; + size_t destLength = bufferSize; + while (destLength > 0) { + MediaAVCodec::AVCodecTrace trace("AudioServerSinkPlugin::WriteAudioBuffer: " + std::to_string(destLength)); + int32_t ret = audioRenderer_->Write(destBuffer, destLength); + if (ret < 0) { + if (audioRenderer_->GetStatus() == AudioStandard::RendererState::RENDERER_PAUSED) { + MEDIA_LOG_W("WriteAudioBuffer error because audioRenderer_ paused, cache data."); + shouldDrop = false; + } else { + MEDIA_LOG_W("WriteAudioBuffer error because audioRenderer_ error, drop data."); + shouldDrop = true; + } + break; + } else if (static_cast(ret) < destLength) { + OHOS::Media::SleepInJob(5); // 5ms + } + if (static_cast(ret) > destLength) { + MEDIA_LOG_W("audioRenderer_ return ret " PUBLIC_LOG_D32 "> destLength " PUBLIC_LOG_U64, + ret, destLength); + ret = destLength; + } + destBuffer += ret; + destLength -= ret; + MEDIA_LOG_D("Written data size " PUBLIC_LOG_D32 ", bufferSize " PUBLIC_LOG_U64, ret, bufferSize); + } + return destLength; +} + Status AudioServerSinkPlugin::Write(const std::shared_ptr &inputBuffer) { MEDIA_LOG_D_T("Write buffer to audio framework"); @@ -879,24 +973,26 @@ Status AudioServerSinkPlugin::Write(const std::shared_ptr while (isForcePaused_ && seekable_ == Seekable::SEEKABLE) { OHOS::Media::SleepInJob(5); // 5ms } - - FALSE_RETURN_V(audioRenderer_ != nullptr, Status::ERROR_NULL_POINTER); - for (; destLength > 0;) { - MediaAVCodec::AVCodecTrace trace("AudioServerSinkPlugin::To be written: " + std::to_string(destLength)); - ret = audioRenderer_->Write(destBuffer, destLength); - if (ret < 0) { - MEDIA_LOG_E_T("Write data error ret is: " PUBLIC_LOG_D32, ret); - break; - } else if (static_cast(ret) < destLength) { - OHOS::Media::SleepInJob(5); // 5ms + if (audioRenderer_ == nullptr) { + DrainCacheData(false); + return Status::ERROR_NULL_POINTER; + } + auto drainCacheRet = DrainCacheData(true); + if (drainCacheRet != Status::OK) { + if (drainCacheRet == Status::ERROR_AGAIN) { + CacheData(destBuffer, destLength); } - DumpEntireAudioBuffer(destBuffer, static_cast(ret)); - DumpSliceAudioBuffer(destBuffer, static_cast(ret)); - destBuffer += ret; - destLength -= ret; - MEDIA_LOG_DD("written data size " PUBLIC_LOG_D32, ret); + return Status::ERROR_UNKNOWN; + } + bool shouldDrop = false; + size_t remained = WriteAudioBuffer(destBuffer, destLength, shouldDrop); + if (remained == 0) { + return Status::OK; + } + if (!shouldDrop) { + CacheData(destBuffer + destLength - remained, remained); } - return ret >= 0 ? Status::OK : Status::ERROR_UNKNOWN; + return Status::ERROR_UNKNOWN; } int32_t AudioServerSinkPlugin::WriteAudioVivid(const std::shared_ptr &inputBuffer) @@ -915,6 +1011,7 @@ int32_t AudioServerSinkPlugin::WriteAudioVivid(const std::shared_ptrDrain()) { uint64_t latency = 0; audioRenderer_->GetLatency(latency); diff --git a/services/media_engine/plugins/sink/audio_server_sink_plugin.h b/services/media_engine/plugins/sink/audio_server_sink_plugin.h index 08d8296fe2c350305c6ace0fe43ae43d44d05b12..838e07976624068e217ace55b47c2ac19b8f29fd 100644 --- a/services/media_engine/plugins/sink/audio_server_sink_plugin.h +++ b/services/media_engine/plugins/sink/audio_server_sink_plugin.h @@ -171,6 +171,10 @@ private: void SetAudioDumpBySysParam(); void DumpEntireAudioBuffer(uint8_t* buffer, const size_t& bytesSingle); void DumpSliceAudioBuffer(uint8_t* buffer, const size_t& bytesSingle); + void CacheData(uint8_t* inputBuffer, size_t bufferSize); + Status DrainCacheData(bool render); + //return value is the remained buffer size + size_t WriteAudioBuffer(uint8_t* inputBuffer, size_t bufferSize, bool& shouldDrop); OHOS::Media::Mutex renderMutex_{}; Callback *callback_{}; @@ -212,6 +216,7 @@ private: bool enableEntireDump_ {false}; bool enableDumpSlice_ {false}; bool audioRenderSetFlag_ {false}; + std::list> cachedBuffers_; }; } // namespace Plugin } // namespace Media