diff --git a/interfaces/inner_api/native/surface_encoder_filter.h b/interfaces/inner_api/native/surface_encoder_filter.h index d1c283d045598a517aca913bee34631f5efa6034..f75d1eb054ece41fee1549c82ee1f3fc562efb0f 100644 --- a/interfaces/inner_api/native/surface_encoder_filter.h +++ b/interfaces/inner_api/native/surface_encoder_filter.h @@ -64,6 +64,7 @@ public: void OnUnlinkedResult(std::shared_ptr &meta); void SetCallingInfo(int32_t appUid, int32_t appPid, const std::string &bundleName, uint64_t instanceId); void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode); + void OnReportKeyFramePts(std::string KeyFramePts); protected: Status OnLinked(StreamType inType, const std::shared_ptr &meta, diff --git a/services/media_engine/filters/surface_encoder_adapter.cpp b/services/media_engine/filters/surface_encoder_adapter.cpp index c26851793240efbfd14cceb845a18682060c09e9..1aa101b0058f0176acfbd30c86c7bcfc7fea8b55 100644 --- a/services/media_engine/filters/surface_encoder_adapter.cpp +++ b/services/media_engine/filters/surface_encoder_adapter.cpp @@ -263,6 +263,14 @@ Status SurfaceEncoderAdapter::SetEncoderAdapterCallback( } } +Status SurfaceEncoderAdapter::SetEncoderAdapterKeyFramePtsCallback( + const std::shared_ptr &encoderAdapterKeyFramePtsCallback) +{ + MEDIA_LOG_I("SetEncoderAdapterKeyFramePtsCallback"); + encoderAdapterKeyFramePtsCallback_ = encoderAdapterKeyFramePtsCallback; + return Status::OK; +} + Status SurfaceEncoderAdapter::SetInputSurface(sptr surface) { MEDIA_LOG_I("GetInputSurface"); @@ -307,6 +315,7 @@ Status SurfaceEncoderAdapter::Start() } ret = codecServer_->Start(); isStart_ = true; + isStartKeyFramePts_= true; if (ret == 0) { return Status::OK; } else { @@ -320,11 +329,13 @@ Status SurfaceEncoderAdapter::Stop() MEDIA_LOG_I("Stop"); MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::Stop"); GetCurrentTime(stopTime_); + isStopKeyFramePts_ = true; MEDIA_LOG_I("Stop time: " PUBLIC_LOG_D64, stopTime_); if (isStart_ && !isTransCoderMode) { std::unique_lock lock(stopMutex_); stopCondition_.wait_for(lock, std::chrono::milliseconds(TIME_OUT_MS)); + AddStopPts(); } if (releaseBufferTask_) { isThreadExit_ = true; @@ -361,6 +372,8 @@ Status SurfaceEncoderAdapter::Pause() (pauseResumeQueue_.back().second == StateCode::RESUME && pauseResumeQueue_.back().first <= pauseTime)) { pauseResumeQueue_.push_back({pauseTime, StateCode::PAUSE}); pauseResumeQueue_.push_back({std::numeric_limits::max(), StateCode::RESUME}); + pauseResumePts_.push_back({pauseTime, StateCode::PAUSE}); + pauseResumePts_.push_back({std::numeric_limits::max(), StateCode::RESUME}); } return Status::OK; } @@ -383,6 +396,7 @@ Status SurfaceEncoderAdapter::Resume() } if (pauseResumeQueue_.back().second == StateCode::RESUME) { pauseResumeQueue_.back().first = std::min(resumeTime, pauseResumeQueue_.back().first); + pauseResumePts_.back().first = std::min(resumeTime, pauseResumePts_.back().first); } return Status::OK; } @@ -415,8 +429,10 @@ Status SurfaceEncoderAdapter::Reset() stopTime_ = -1; totalPauseTime_ = 0; isStart_ = false; + isStartKeyFramePts_ = false; mappingTimeQueue_.clear(); pauseResumeQueue_.clear(); + pauseResumePts_.clear(); if (ret == 0) { return Status::OK; } else { @@ -564,6 +580,11 @@ void SurfaceEncoderAdapter::OnOutputBufferAvailable(uint32_t index, std::shared_ } mappingTime = mappingTimeQueue_.front().second; mappingTimeQueue_.pop_front(); + // cache recent 2 pts + preKeyFramePts_ = currentKeyFramePts_; + currentKeyFramePts_ = buffer->pts_; + AddStartPts(buffer->pts_); + AddPauseResumePts(buffer->pts_); } int32_t size = buffer->memory_->GetSize(); std::shared_ptr emptyOutputBuffer; @@ -741,5 +762,63 @@ void SurfaceEncoderAdapter::GetCurrentTime(int64_t ¤tTime) clock_gettime(CLOCK_MONOTONIC, ×tamp); currentTime = static_cast(timestamp.tv_sec) * SEC_TO_NS + static_cast(timestamp.tv_nsec); } + +void SurfaceEncoderAdapter::AddStartPts(int64_t currentPts) +{ + // start time + if (isStartKeyFramePts_) { + keyFramePts_ += std::to_string(currentPts / NS_PER_US) + ","; + isStartKeyFramePts_ = false; + MEDIA_LOG_I("AddStartPts success %{public}s end", keyFramePts_.c_str()); + } +} + +void SurfaceEncoderAdapter::AddStopPts() +{ + // stop time + MEDIA_LOG_D("AddStopPts enter"); + if (isStopKeyFramePts_) { + if (currentKeyFramePts_ > stopTime_) { + keyFramePts_ += std::to_string(preKeyFramePts_ / NS_PER_US); + MEDIA_LOG_I("AddStopPts preKeyFramePts_ %{public}s end", keyFramePts_.c_str()); + } else { + keyFramePts_ += std::to_string(currentKeyFramePts_ / NS_PER_US); + MEDIA_LOG_I("AddStopPts currentKeyFramePts_ %{public}s end", keyFramePts_.c_str()); + } + isStopKeyFramePts_ = false; + encoderAdapterKeyFramePtsCallback_->OnReportKeyFramePts(keyFramePts_); + keyFramePts_.clear(); + } +} + +bool SurfaceEncoderAdapter::AddPauseResumePts(int64_t currentPts) +{ + if (pauseResumePts_.empty()) { + return false; + } + auto stateCode = pauseResumePts_[0].second; + MEDIA_LOG_D("CheckFrames stateCode: " PUBLIC_LOG_D32 + " time:" PUBLIC_LOG_D64, static_cast(stateCode), pauseResumePts_[0].first); + // means not dropped frames when less than pause time + if (stateCode == StateCode::PAUSE && currentPts < pauseResumePts_[0].first) { + return false; + } + // means dropped frames when less than resume time + if (stateCode == StateCode::RESUME && currentPts < pauseResumePts_[0].first) { + return true; + } + if (stateCode == StateCode::PAUSE) { + MEDIA_LOG_D("AddPausePts %{public}s start", keyFramePts_.c_str()); + keyFramePts_ += std::to_string(preKeyFramePts_ / NS_PER_US) + ","; + MEDIA_LOG_D("AddPausePts %{public}s end", keyFramePts_.c_str()); + } + if (stateCode == StateCode::RESUME) { + MEDIA_LOG_D("AddResumePts %{public}s start", keyFramePts_.c_str()); + keyFramePts_ += std::to_string(currentKeyFramePts_ / NS_PER_US) + ","; + MEDIA_LOG_D("AddResumePts %{public}s end", keyFramePts_.c_str()); + } + pauseResumePts_.pop_front(); + return AddPauseResumePts(currentPts); +} } // namespace MEDIA } // namespace OHOS diff --git a/services/media_engine/filters/surface_encoder_adapter.h b/services/media_engine/filters/surface_encoder_adapter.h index 9c7330e4f08cd7b4558b5a95e14958a6e1ed2de8..85e5015b497441170f73f2f7d5afde909332ffae 100644 --- a/services/media_engine/filters/surface_encoder_adapter.h +++ b/services/media_engine/filters/surface_encoder_adapter.h @@ -50,6 +50,12 @@ public: virtual void OnOutputFormatChanged(const std::shared_ptr &format) = 0; }; +class EncoderAdapterKeyFramePtsCallback { +public: + virtual ~EncoderAdapterKeyFramePtsCallback() = default; + virtual void OnReportKeyFramePts(std::string KeyFramePts) = 0; +}; + class SurfaceEncoderAdapter : public std::enable_shared_from_this { public: explicit SurfaceEncoderAdapter(); @@ -60,6 +66,8 @@ public: Status SetWatermark(std::shared_ptr &waterMarkBuffer); Status SetOutputBufferQueue(const sptr &bufferQueueProducer); Status SetEncoderAdapterCallback(const std::shared_ptr &encoderAdapterCallback); + Status SetEncoderAdapterKeyFramePtsCallback( + const std::shared_ptr &encoderAdapterKeyFramePtsCallback); Status SetInputSurface(sptr surface); Status SetTransCoderMode(); sptr GetInputSurface(); @@ -82,6 +90,7 @@ public: std::shared_ptr ¶meter); std::shared_ptr encoderAdapterCallback_; + std::shared_ptr encoderAdapterKeyFramePtsCallback_; private: void ReleaseBuffer(); @@ -91,6 +100,9 @@ private: void ConfigureAboutEnableTemporalScale(MediaAVCodec::Format &format, const std::shared_ptr &meta); bool CheckFrames(int64_t currentPts, int64_t &checkFramesPauseTime); void GetCurrentTime(int64_t ¤tTime); + void AddStartPts(int64_t currentPts); + void AddStopPts(); + bool AddPauseResumePts(int64_t currentPts); std::shared_ptr codecServer_; sptr outputBufferQueueProducer_; @@ -122,6 +134,13 @@ private: uint64_t instanceId_{0}; int32_t appUid_ {0}; int32_t appPid_ {0}; + + std::string keyFramePts_; + bool isStartKeyFramePts_ = false; + bool isStopKeyFramePts_ = false; + int64_t currentKeyFramePts_{-1}; + int64_t preKeyFramePts_{-1}; + std::deque> pauseResumePts_; }; } // namespace MediaAVCodec } // namespace OHOS diff --git a/services/media_engine/filters/surface_encoder_filter.cpp b/services/media_engine/filters/surface_encoder_filter.cpp index bbf45a880e2642fb24ad930bf546e3181424ed48..2ef2947fc367af75d1f2115824bb39a2d92bc5ea 100644 --- a/services/media_engine/filters/surface_encoder_filter.cpp +++ b/services/media_engine/filters/surface_encoder_filter.cpp @@ -22,6 +22,7 @@ #include "common/media_core.h" #include "filter/filter_factory.h" #include "surface_encoder_adapter.h" +#include "muxer_filter.h" namespace { constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "HiStreamer" }; @@ -94,6 +95,28 @@ private: std::weak_ptr surfaceEncoderFilter_; }; +class SurfaceEncoderAdapterKeyFramePtsCallback : public EncoderAdapterKeyFramePtsCallback { +public: + explicit SurfaceEncoderAdapterKeyFramePtsCallback(std::shared_ptr surfaceEncoderFilter) + : surfaceEncoderFilter_(std::move(surfaceEncoderFilter)) + { + } + + void OnReportKeyFramePts(std::string KeyFramePts) override + { + if (auto surfaceEncoderFilter = surfaceEncoderFilter_.lock()) { + MEDIA_LOG_D("SurfaceEncoderAdapterKeyFramePtsCallback OnReportKeyFramePts start"); + surfaceEncoderFilter->OnReportKeyFramePts(KeyFramePts); + MEDIA_LOG_D("SurfaceEncoderAdapterKeyFramePtsCallback OnReportKeyFramePts end"); + } else { + MEDIA_LOG_I("invalid surfaceEncoderFilter"); + } + } + +private: + std::weak_ptr surfaceEncoderFilter_; +}; + SurfaceEncoderFilter::SurfaceEncoderFilter(std::string name, FilterType type): Filter(name, type) { MEDIA_LOG_I("encoder filter create"); @@ -142,6 +165,9 @@ void SurfaceEncoderFilter::Init(const std::shared_ptr &receiver, std::shared_ptr encoderAdapterCallback = std::make_shared(shared_from_this()); mediaCodec_->SetEncoderAdapterCallback(encoderAdapterCallback); + std::shared_ptr encoderAdapterKeyFramePtsCallback = + std::make_shared(shared_from_this()); + mediaCodec_->SetEncoderAdapterKeyFramePtsCallback(encoderAdapterKeyFramePtsCallback); } else { MEDIA_LOG_I("Init mediaCodec fail"); eventReceiver_->OnEvent({"surface_encoder_filter", EventType::EVENT_ERROR, Status::ERROR_UNKNOWN}); @@ -378,6 +404,20 @@ void SurfaceEncoderFilter::SetCallingInfo(int32_t appUid, int32_t appPid, mediaCodec_->SetCallingInfo(appUid, appPid, bundleName, instanceId); } } + +void SurfaceEncoderFilter::OnReportKeyFramePts(std::string KeyFramePts) +{ + MEDIA_LOG_I("OnReportKeyFramePts %{public}s enter", KeyFramePts.c_str()); + const std::shared_ptr userMeta = std::make_shared(); + userMeta->SetData("com.openharmony.recorder.timestamp", KeyFramePts); + std::shared_ptr muxerFilter = std::static_pointer_cast(nextFilter_); + if (muxerFilter != nullptr) { + muxerFilter->SetUserMeta(userMeta); + MEDIA_LOG_I("SetUserMeta %{public}s", KeyFramePts.c_str()); + } else { + MEDIA_LOG_E("muxerFilter is null"); + } +} } // namespace Pipeline } // namespace MEDIA } // namespace OHOS \ No newline at end of file diff --git a/services/media_engine/modules/muxer/media_muxer.cpp b/services/media_engine/modules/muxer/media_muxer.cpp index 2fd7d1a1191e6d9e802033ebecc6d16eb4f90876..0710c28de8b6f1db67b4e94dc63de8a018115997 100644 --- a/services/media_engine/modules/muxer/media_muxer.cpp +++ b/services/media_engine/modules/muxer/media_muxer.cpp @@ -157,8 +157,16 @@ Status MediaMuxer::SetUserMeta(const std::shared_ptr &userMeta) { MEDIA_LOG_I("SetUserMeta"); std::lock_guard lock(mutex_); + std::vector keys; + userMeta->GetKeys(keys); + for (auto& k: keys) { + if (k.compare("com.openharmony.recorder.timestamp") == 0) { + MEDIA_LOG_I("set com.openharmony.recorder.timestamp"); + return muxer_->SetUserMeta(userMeta); + } + } FALSE_RETURN_V_MSG_E(state_ == State::INITIALIZED || state_ == State::STARTED, Status::ERROR_WRONG_STATE, - "The state is not INITIALIZED, the interface must be called after constructor and before Start(). " + "The state is not INITIALIZED, the interface must be called at initialized or started state. " "The current state is %{public}s.", StateConvert(state_).c_str()); return muxer_->SetUserMeta(userMeta); }