From 0d4f92fbacdc1f0d605b27b8a325a4477060e41c Mon Sep 17 00:00:00 2001 From: FileStream Date: Sat, 31 Aug 2024 07:54:23 +0000 Subject: [PATCH 1/5] =?UTF-8?q?=E8=AE=B0=E5=BD=95=E5=85=B3=E9=94=AE?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=88=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: FileStream --- .../inner_api/native/surface_encoder_filter.h | 1 + .../filters/surface_encoder_adapter.cpp | 82 ++++++++++++++++++- .../filters/surface_encoder_adapter.h | 19 +++++ .../filters/surface_encoder_filter.cpp | 44 ++++++++++ 4 files changed, 145 insertions(+), 1 deletion(-) diff --git a/interfaces/inner_api/native/surface_encoder_filter.h b/interfaces/inner_api/native/surface_encoder_filter.h index d1c283d04..f75d1eb05 100644 --- a/interfaces/inner_api/native/surface_encoder_filter.h +++ b/interfaces/inner_api/native/surface_encoder_filter.h @@ -64,6 +64,7 @@ public: void OnUnlinkedResult(std::shared_ptr &meta); void SetCallingInfo(int32_t appUid, int32_t appPid, const std::string &bundleName, uint64_t instanceId); void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode); + void OnReportKeyFramePts(std::string KeyFramePts); protected: Status OnLinked(StreamType inType, const std::shared_ptr &meta, diff --git a/services/media_engine/filters/surface_encoder_adapter.cpp b/services/media_engine/filters/surface_encoder_adapter.cpp index 8e2b4090e..b15160b24 100644 --- a/services/media_engine/filters/surface_encoder_adapter.cpp +++ b/services/media_engine/filters/surface_encoder_adapter.cpp @@ -263,6 +263,13 @@ Status SurfaceEncoderAdapter::SetEncoderAdapterCallback( } } +Status SurfaceEncoderAdapter::SetEncoderAdapterKeyFramePtsCallback( + const std::shared_ptr &encoderAdapterKeyFramePtsCallback) +{ + MEDIA_LOG_I("SetEncoderAdapterKeyFramePtsCallback"); + encoderAdapterKeyFramePtsCallback_ = encoderAdapterKeyFramePtsCallback; + return Status::OK; + Status SurfaceEncoderAdapter::SetInputSurface(sptr surface) { MEDIA_LOG_I("GetInputSurface"); @@ -307,6 +314,7 @@ Status SurfaceEncoderAdapter::Start() } ret = codecServer_->Start(); isStart_ = true; + isStartKeyFramePts_= true; if (ret == 0) { return Status::OK; } else { @@ -320,11 +328,14 @@ Status SurfaceEncoderAdapter::Stop() MEDIA_LOG_I("Stop"); MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::Stop"); GetCurrentTime(stopTime_); + isStopKeyFramePts_ = true; + isStartKeyFramePts_ = false; MEDIA_LOG_I("Stop time: " PUBLIC_LOG_D64, stopTime_); if (isStart_ && !isTransCoderMode) { std::unique_lock lock(stopMutex_); stopCondition_.wait_for(lock, std::chrono::milliseconds(TIME_OUT_MS)); + AddStopPts(); } if (releaseBufferTask_) { isThreadExit_ = true; @@ -361,6 +372,8 @@ Status SurfaceEncoderAdapter::Pause() (pauseResumeQueue_.back().second == StateCode::RESUME && pauseResumeQueue_.back().first <= pauseTime)) { pauseResumeQueue_.push_back({pauseTime, StateCode::PAUSE}); pauseResumeQueue_.push_back({std::numeric_limits::max(), StateCode::RESUME}); + pauseResumePts_.push_back({pauseTime, StateCode::PAUSE}); + pauseResumePts_.push_back({std::numeric_limits::max(), StateCode::RESUME}); } return Status::OK; } @@ -383,6 +396,7 @@ Status SurfaceEncoderAdapter::Resume() } if (pauseResumeQueue_.back().second == StateCode::RESUME) { pauseResumeQueue_.back().first = std::min(resumeTime, pauseResumeQueue_.back().first); + pauseResumePts_.back().first = std::min(resumeTime, pauseResumePts_.back().first); } return Status::OK; } @@ -415,8 +429,10 @@ Status SurfaceEncoderAdapter::Reset() stopTime_ = -1; totalPauseTime_ = 0; isStart_ = false; + isStartKeyFramePts_ = false; mappingTimeQueue_.clear(); pauseResumeQueue_.clear(); + pauseResumePts_.clear(); if (ret == 0) { return Status::OK; } else { @@ -564,6 +580,11 @@ void SurfaceEncoderAdapter::OnOutputBufferAvailable(uint32_t index, std::shared_ if (startBufferTime_ == -1) { startBufferTime_ = buffer->pts_; } + // cache recent 2 pts + preKeyFramePts_ = currentKeyFramePts_; + currentKeyFramePts_ = buffer->pts_; + AddStartPts(buffer->pts_); + AddPauseResumePts(buffer->pts_); } else { MEDIA_LOG_D("OnOutputBufferAvailable buffer->flag_" PUBLIC_LOG_U32, buffer->flag_); mappingTime = startBufferTime_ + buffer->pts_; @@ -603,7 +624,7 @@ void SurfaceEncoderAdapter::ReleaseBuffer() { std::unique_lock lock(releaseBufferMutex_); releaseBufferCondition_.wait(lock, [this] { - return isThreadExit_||!indexs_.empty(); + return isThreadExit_ || !indexs_.empty(); }); indexs = indexs_; indexs_.clear(); @@ -741,5 +762,64 @@ void SurfaceEncoderAdapter::GetCurrentTime(int64_t ¤tTime) clock_gettime(CLOCK_MONOTONIC, ×tamp); currentTime = static_cast(timestamp.tv_sec) * SEC_TO_NS + static_cast(timestamp.tv_nsec); } + +void SurfaceEncoderAdapter::AddStartPts(int64_t currentPts) +{ + // start time + if (isStartKeyFramePts_) { + + keyFramePts_ += std::to_string(currentPts) + ","; + isStartKeyFramePts_ = false; + MEDIA_LOG_I("lwj AddStartPts success %{public}s end", keyFramePts_.c_str()); + } +} + +void SurfaceEncoderAdapter::AddStopPts() +{ + // stop time + MEDIA_LOG_D("AddStopPts enter"); + if (isStopKeyFramePts_) { + if (currentKeyFramePts_ > stopTime_) { + keyFramePts_ += std::to_string(preKeyFramePts_); + MEDIA_LOG_I("AddStopPts preKeyFramePts_ %{public}s end", keyFramePts_.c_str()); + } else { + keyFramePts_ += std::to_string(currentKeyFramePts_); + MEDIA_LOG_I("AddStopPts currentKeyFramePts_ %{public}s end", keyFramePts_.c_str()); + } + isStopKeyFramePts_ = false; + encoderAdapterKeyFramePtsCallback_->OnReportKeyFramePts(keyFramePts_); + keyFramePts_.clear(); + } +} + +bool SurfaceEncoderAdapter::AddPauseResumePts(int64_t currentPts) +{ + if (pauseResumePts_.empty()) { + return false; + } + auto stateCode = pauseResumePts_[0].second; + MEDIA_LOG_D("CheckFrames stateCode: " PUBLIC_LOG_D32 + " time:" PUBLIC_LOG_D64, static_cast(stateCode), pauseResumePts_[0].first); + // means not dropped frames when less than pause time + if (stateCode == StateCode::PAUSE && currentPts < pauseResumePts_[0].first) { + return false; + } + // means dropped frames when less than resume time + if (stateCode == StateCode::RESUME && currentPts < pauseResumePts_[0].first) { + return true; + } + if (stateCode == StateCode::PAUSE) { + MEDIA_LOG_D("AddPausePts %{public}s start", keyFramePts_.c_str()); + keyFramePts_ += std::to_string(preKeyFramePts_) + ","; + MEDIA_LOG_D("AddPausePts %{public}s end", keyFramePts_.c_str()); + } + if (stateCode == StateCode::RESUME) { + MEDIA_LOG_D("AddResumePts %{public}s start", keyFramePts_.c_str()); + keyFramePts_ += std::to_string(currentKeyFramePts_) + ","; + MEDIA_LOG_D("AddResumePts %{public}s end", keyFramePts_.c_str()); + } + pauseResumePts_.pop_front(); + return AddPauseResumePts(currentPts); +} } // namespace MEDIA } // namespace OHOS diff --git a/services/media_engine/filters/surface_encoder_adapter.h b/services/media_engine/filters/surface_encoder_adapter.h index 9d191e50c..6095a0fb5 100644 --- a/services/media_engine/filters/surface_encoder_adapter.h +++ b/services/media_engine/filters/surface_encoder_adapter.h @@ -50,6 +50,12 @@ public: virtual void OnOutputFormatChanged(const std::shared_ptr &format) = 0; }; +class EncoderAdapterKeyFramePtsCallback { +public: + virtual ~EncoderAdapterKeyFramePtsCallback() = default; + virtual void OnReportKeyFramePts(std::string KeyFramePts) = 0; +}; + class SurfaceEncoderAdapter : public std::enable_shared_from_this { public: explicit SurfaceEncoderAdapter(); @@ -60,6 +66,8 @@ public: Status SetWatermark(std::shared_ptr &waterMarkBuffer); Status SetOutputBufferQueue(const sptr &bufferQueueProducer); Status SetEncoderAdapterCallback(const std::shared_ptr &encoderAdapterCallback); + Status SetEncoderAdapterKeyFramePtsCallback( + const std::shared_ptr &encoderAdapterKeyFramePtsCallback); Status SetInputSurface(sptr surface); Status SetTransCoderMode(); sptr GetInputSurface(); @@ -82,6 +90,7 @@ public: std::shared_ptr ¶meter); std::shared_ptr encoderAdapterCallback_; + std::shared_ptr encoderAdapterKeyFramePtsCallback_; private: void ReleaseBuffer(); @@ -91,6 +100,9 @@ private: void ConfigureAboutEnableTemporalScale(MediaAVCodec::Format &format, const std::shared_ptr &meta); bool CheckFrames(int64_t currentPts, int64_t &checkFramesPauseTime); void GetCurrentTime(int64_t ¤tTime); + void AddStartPts(int64_t currentPts); + void AddStopPts(); + bool AddPauseResumePts(int64_t currentPts); std::shared_ptr codecServer_; sptr outputBufferQueueProducer_; @@ -121,6 +133,13 @@ private: uint64_t instanceId_{0}; int32_t appUid_ {0}; int32_t appPid_ {0}; + + std::string keyFramePts_; + bool isStartKeyFramePts_ = false; + bool isStopKeyFramePts_ = false; + int64_t currentKeyFramePts_{-1}; + int64_t preKeyFramePts_{-1}; + std::deque> pauseResumePts_; }; } // namespace MediaAVCodec } // namespace OHOS diff --git a/services/media_engine/filters/surface_encoder_filter.cpp b/services/media_engine/filters/surface_encoder_filter.cpp index bbf45a880..b9d623ceb 100644 --- a/services/media_engine/filters/surface_encoder_filter.cpp +++ b/services/media_engine/filters/surface_encoder_filter.cpp @@ -22,6 +22,7 @@ #include "common/media_core.h" #include "filter/filter_factory.h" #include "surface_encoder_adapter.h" +#include "muxer_filter.h" namespace { constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "HiStreamer" }; @@ -42,6 +43,7 @@ public: : surfaceEncoderFilter_(std::move(surfaceEncoderFilter)) { } + void OnLinkedResult(const sptr &queue, std::shared_ptr &meta) override { if (auto surfaceEncoderFilter = surfaceEncoderFilter_.lock()) { @@ -50,6 +52,7 @@ public: MEDIA_LOG_I("invalid surfaceEncoderFilter"); } } + void OnUnlinkedResult(std::shared_ptr &meta) override { if (auto surfaceEncoderFilter = surfaceEncoderFilter_.lock()) { @@ -58,6 +61,7 @@ public: MEDIA_LOG_I("invalid surfaceEncoderFilter"); } } + void OnUpdatedResult(std::shared_ptr &meta) override { if (auto surfaceEncoderFilter = surfaceEncoderFilter_.lock()) { @@ -66,6 +70,7 @@ public: MEDIA_LOG_I("invalid surfaceEncoderFilter"); } } + private: std::weak_ptr surfaceEncoderFilter_; }; @@ -94,6 +99,28 @@ private: std::weak_ptr surfaceEncoderFilter_; }; +class SurfaceEncoderAdapterKeyFramePtsCallback : public EncoderAdapterKeyFramePtsCallback { +public: + explicit SurfaceEncoderAdapterKeyFramePtsCallback(std::shared_ptr surfaceEncoderFilter) + : surfaceEncoderFilter_(std::move(surfaceEncoderFilter)) + { + } + + void OnReportKeyFramePts(std::string KeyFramePts) override + { + if (auto surfaceEncoderFilter = surfaceEncoderFilter_.lock()) { + MEDIA_LOG_D("SurfaceEncoderAdapterKeyFramePtsCallback OnReportKeyFramePts start"); + surfaceEncoderFilter->OnReportKeyFramePts(KeyFramePts); + MEDIA_LOG_D("SurfaceEncoderAdapterKeyFramePtsCallback OnReportKeyFramePts end"); + } else { + MEDIA_LOG_I("invalid surfaceEncoderFilter"); + } + } + +private: + std::weak_ptr surfaceEncoderFilter_; +}; + SurfaceEncoderFilter::SurfaceEncoderFilter(std::string name, FilterType type): Filter(name, type) { MEDIA_LOG_I("encoder filter create"); @@ -142,6 +169,9 @@ void SurfaceEncoderFilter::Init(const std::shared_ptr &receiver, std::shared_ptr encoderAdapterCallback = std::make_shared(shared_from_this()); mediaCodec_->SetEncoderAdapterCallback(encoderAdapterCallback); + std::shared_ptr encoderAdapterKeyFramePtsCallback = + std::make_shared(shared_from_this()); + mediaCodec_->SetEncoderAdapterKeyFramePtsCallback(encoderAdapterKeyFramePtsCallback); } else { MEDIA_LOG_I("Init mediaCodec fail"); eventReceiver_->OnEvent({"surface_encoder_filter", EventType::EVENT_ERROR, Status::ERROR_UNKNOWN}); @@ -378,6 +408,20 @@ void SurfaceEncoderFilter::SetCallingInfo(int32_t appUid, int32_t appPid, mediaCodec_->SetCallingInfo(appUid, appPid, bundleName, instanceId); } } + +void SurfaceEncoderFilter::OnReportKeyFramePts(std::string KeyFramePts) +{ + MEDIA_LOG_I("OnReportKeyFramePts %{public}s enter", KeyFramePts.c_str()); + std::shared_ptr userMeta = std::make_shared(); + userMeta->SetData("com.openharmony.recorder.timestamp", KeyFramePts.data()); + std::shared_ptr muxerFilter = std::static_pointer_cast(nextFilter_); + if (muxerFilter != nullptr) { + muxerFilter->SetUserMeta(userMeta); + MEDIA_LOG_I("SetUserMeta %{public}s", KeyFramePts.c_str()); + } else { + MEDIA_LOG_E("muxerFilter is null"); + } +} } // namespace Pipeline } // namespace MEDIA } // namespace OHOS \ No newline at end of file -- Gitee From c4109e19435be84adbfd7bcbed1f11962cac23a2 Mon Sep 17 00:00:00 2001 From: FileStream Date: Sat, 31 Aug 2024 07:59:48 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=E8=AE=B0=E5=BD=95=E5=85=B3=E9=94=AE?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=88=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: FileStream --- services/media_engine/filters/surface_encoder_adapter.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/media_engine/filters/surface_encoder_adapter.cpp b/services/media_engine/filters/surface_encoder_adapter.cpp index b15160b24..2a22bc95f 100644 --- a/services/media_engine/filters/surface_encoder_adapter.cpp +++ b/services/media_engine/filters/surface_encoder_adapter.cpp @@ -767,10 +767,9 @@ void SurfaceEncoderAdapter::AddStartPts(int64_t currentPts) { // start time if (isStartKeyFramePts_) { - keyFramePts_ += std::to_string(currentPts) + ","; isStartKeyFramePts_ = false; - MEDIA_LOG_I("lwj AddStartPts success %{public}s end", keyFramePts_.c_str()); + MEDIA_LOG_I("AddStartPts success %{public}s end", keyFramePts_.c_str()); } } -- Gitee From 69470b0d461ef97a8dacea6fde3df519b3f2c425 Mon Sep 17 00:00:00 2001 From: FileStream Date: Sat, 31 Aug 2024 09:14:54 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=E8=AE=B0=E5=BD=95=E5=85=B3=E9=94=AE?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=88=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: FileStream --- services/media_engine/filters/surface_encoder_adapter.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/services/media_engine/filters/surface_encoder_adapter.cpp b/services/media_engine/filters/surface_encoder_adapter.cpp index 2a22bc95f..5631a6d6a 100644 --- a/services/media_engine/filters/surface_encoder_adapter.cpp +++ b/services/media_engine/filters/surface_encoder_adapter.cpp @@ -269,6 +269,7 @@ Status SurfaceEncoderAdapter::SetEncoderAdapterKeyFramePtsCallback( MEDIA_LOG_I("SetEncoderAdapterKeyFramePtsCallback"); encoderAdapterKeyFramePtsCallback_ = encoderAdapterKeyFramePtsCallback; return Status::OK; +} Status SurfaceEncoderAdapter::SetInputSurface(sptr surface) { -- Gitee From 1acbf5479d55389fa07714dab8361ec400eb7a0e Mon Sep 17 00:00:00 2001 From: FileStream Date: Sat, 31 Aug 2024 14:01:39 +0000 Subject: [PATCH 4/5] =?UTF-8?q?=E8=AE=B0=E5=BD=95=E5=85=B3=E9=94=AE?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=88=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: FileStream --- test/unittest/decoder_filter_test/BUILD.gn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/unittest/decoder_filter_test/BUILD.gn b/test/unittest/decoder_filter_test/BUILD.gn index ddbef31c1..3ee23afb6 100644 --- a/test/unittest/decoder_filter_test/BUILD.gn +++ b/test/unittest/decoder_filter_test/BUILD.gn @@ -84,6 +84,9 @@ ohos_unittest("decoder_filter_unit_test") { "metadata_filter_unit_test.cpp", "surface_encoder_filter_unit_test.cpp", ] + + public_deps = [ "$av_codec_root_dir/services/media_engine/filters:av_codec_media_engine_filters" ] + deps = [ "$av_codec_root_dir/interfaces/inner_api/native:av_codec_client", "$av_codec_root_dir/interfaces/kits/c:native_media_codecbase", -- Gitee From 550a44d597b45c3a308f3a64c06808b388b3418e Mon Sep 17 00:00:00 2001 From: FileStream Date: Sat, 31 Aug 2024 14:19:23 +0000 Subject: [PATCH 5/5] =?UTF-8?q?=E8=AE=B0=E5=BD=95=E5=85=B3=E9=94=AE?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E6=88=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: FileStream --- test/unittest/decoder_filter_test/BUILD.gn | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unittest/decoder_filter_test/BUILD.gn b/test/unittest/decoder_filter_test/BUILD.gn index 3ee23afb6..1c0dfb5db 100644 --- a/test/unittest/decoder_filter_test/BUILD.gn +++ b/test/unittest/decoder_filter_test/BUILD.gn @@ -78,11 +78,11 @@ ohos_unittest("decoder_filter_unit_test") { "$av_codec_root_dir/services/dfx:av_codec_service_log_dfx_public_config", ] sources = decoder_filter_test_sources + [ - "video_resize_filter_unit_test.cpp", "audio_decoder_filter_unit_test.cpp", - "video_decoder_adapter_unit_test.cpp", "metadata_filter_unit_test.cpp", "surface_encoder_filter_unit_test.cpp", + "video_decoder_adapter_unit_test.cpp", + "video_resize_filter_unit_test.cpp", ] public_deps = [ "$av_codec_root_dir/services/media_engine/filters:av_codec_media_engine_filters" ] -- Gitee