diff --git a/interfaces/inner_api/native/surface_encoder_filter.h b/interfaces/inner_api/native/surface_encoder_filter.h
index d1c283d045598a517aca913bee34631f5efa6034..f75d1eb054ece41fee1549c82ee1f3fc562efb0f 100644
--- a/interfaces/inner_api/native/surface_encoder_filter.h
+++ b/interfaces/inner_api/native/surface_encoder_filter.h
@@ -64,6 +64,7 @@ public:
void OnUnlinkedResult(std::shared_ptr &meta);
void SetCallingInfo(int32_t appUid, int32_t appPid, const std::string &bundleName, uint64_t instanceId);
void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode);
+ void OnReportKeyFramePts(std::string KeyFramePts);
protected:
Status OnLinked(StreamType inType, const std::shared_ptr &meta,
diff --git a/services/media_engine/filters/surface_encoder_adapter.cpp b/services/media_engine/filters/surface_encoder_adapter.cpp
index c26851793240efbfd14cceb845a18682060c09e9..1aa101b0058f0176acfbd30c86c7bcfc7fea8b55 100644
--- a/services/media_engine/filters/surface_encoder_adapter.cpp
+++ b/services/media_engine/filters/surface_encoder_adapter.cpp
@@ -263,6 +263,14 @@ Status SurfaceEncoderAdapter::SetEncoderAdapterCallback(
}
}
+Status SurfaceEncoderAdapter::SetEncoderAdapterKeyFramePtsCallback(
+ const std::shared_ptr &encoderAdapterKeyFramePtsCallback)
+{
+ MEDIA_LOG_I("SetEncoderAdapterKeyFramePtsCallback");
+ encoderAdapterKeyFramePtsCallback_ = encoderAdapterKeyFramePtsCallback;
+ return Status::OK;
+}
+
Status SurfaceEncoderAdapter::SetInputSurface(sptr surface)
{
MEDIA_LOG_I("GetInputSurface");
@@ -307,6 +315,7 @@ Status SurfaceEncoderAdapter::Start()
}
ret = codecServer_->Start();
isStart_ = true;
+ isStartKeyFramePts_= true;
if (ret == 0) {
return Status::OK;
} else {
@@ -320,11 +329,13 @@ Status SurfaceEncoderAdapter::Stop()
MEDIA_LOG_I("Stop");
MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::Stop");
GetCurrentTime(stopTime_);
+ isStopKeyFramePts_ = true;
MEDIA_LOG_I("Stop time: " PUBLIC_LOG_D64, stopTime_);
if (isStart_ && !isTransCoderMode) {
std::unique_lock lock(stopMutex_);
stopCondition_.wait_for(lock, std::chrono::milliseconds(TIME_OUT_MS));
+ AddStopPts();
}
if (releaseBufferTask_) {
isThreadExit_ = true;
@@ -361,6 +372,8 @@ Status SurfaceEncoderAdapter::Pause()
(pauseResumeQueue_.back().second == StateCode::RESUME && pauseResumeQueue_.back().first <= pauseTime)) {
pauseResumeQueue_.push_back({pauseTime, StateCode::PAUSE});
pauseResumeQueue_.push_back({std::numeric_limits::max(), StateCode::RESUME});
+ pauseResumePts_.push_back({pauseTime, StateCode::PAUSE});
+ pauseResumePts_.push_back({std::numeric_limits::max(), StateCode::RESUME});
}
return Status::OK;
}
@@ -383,6 +396,7 @@ Status SurfaceEncoderAdapter::Resume()
}
if (pauseResumeQueue_.back().second == StateCode::RESUME) {
pauseResumeQueue_.back().first = std::min(resumeTime, pauseResumeQueue_.back().first);
+ pauseResumePts_.back().first = std::min(resumeTime, pauseResumePts_.back().first);
}
return Status::OK;
}
@@ -415,8 +429,10 @@ Status SurfaceEncoderAdapter::Reset()
stopTime_ = -1;
totalPauseTime_ = 0;
isStart_ = false;
+ isStartKeyFramePts_ = false;
mappingTimeQueue_.clear();
pauseResumeQueue_.clear();
+ pauseResumePts_.clear();
if (ret == 0) {
return Status::OK;
} else {
@@ -564,6 +580,11 @@ void SurfaceEncoderAdapter::OnOutputBufferAvailable(uint32_t index, std::shared_
}
mappingTime = mappingTimeQueue_.front().second;
mappingTimeQueue_.pop_front();
+ // cache recent 2 pts
+ preKeyFramePts_ = currentKeyFramePts_;
+ currentKeyFramePts_ = buffer->pts_;
+ AddStartPts(buffer->pts_);
+ AddPauseResumePts(buffer->pts_);
}
int32_t size = buffer->memory_->GetSize();
std::shared_ptr emptyOutputBuffer;
@@ -741,5 +762,63 @@ void SurfaceEncoderAdapter::GetCurrentTime(int64_t ¤tTime)
clock_gettime(CLOCK_MONOTONIC, ×tamp);
currentTime = static_cast(timestamp.tv_sec) * SEC_TO_NS + static_cast(timestamp.tv_nsec);
}
+
+void SurfaceEncoderAdapter::AddStartPts(int64_t currentPts)
+{
+ // start time
+ if (isStartKeyFramePts_) {
+ keyFramePts_ += std::to_string(currentPts / NS_PER_US) + ",";
+ isStartKeyFramePts_ = false;
+ MEDIA_LOG_I("AddStartPts success %{public}s end", keyFramePts_.c_str());
+ }
+}
+
+void SurfaceEncoderAdapter::AddStopPts()
+{
+ // stop time
+ MEDIA_LOG_D("AddStopPts enter");
+ if (isStopKeyFramePts_) {
+ if (currentKeyFramePts_ > stopTime_) {
+ keyFramePts_ += std::to_string(preKeyFramePts_ / NS_PER_US);
+ MEDIA_LOG_I("AddStopPts preKeyFramePts_ %{public}s end", keyFramePts_.c_str());
+ } else {
+ keyFramePts_ += std::to_string(currentKeyFramePts_ / NS_PER_US);
+ MEDIA_LOG_I("AddStopPts currentKeyFramePts_ %{public}s end", keyFramePts_.c_str());
+ }
+ isStopKeyFramePts_ = false;
+ encoderAdapterKeyFramePtsCallback_->OnReportKeyFramePts(keyFramePts_);
+ keyFramePts_.clear();
+ }
+}
+
+bool SurfaceEncoderAdapter::AddPauseResumePts(int64_t currentPts)
+{
+ if (pauseResumePts_.empty()) {
+ return false;
+ }
+ auto stateCode = pauseResumePts_[0].second;
+ MEDIA_LOG_D("CheckFrames stateCode: " PUBLIC_LOG_D32
+ " time:" PUBLIC_LOG_D64, static_cast(stateCode), pauseResumePts_[0].first);
+ // means not dropped frames when less than pause time
+ if (stateCode == StateCode::PAUSE && currentPts < pauseResumePts_[0].first) {
+ return false;
+ }
+ // means dropped frames when less than resume time
+ if (stateCode == StateCode::RESUME && currentPts < pauseResumePts_[0].first) {
+ return true;
+ }
+ if (stateCode == StateCode::PAUSE) {
+ MEDIA_LOG_D("AddPausePts %{public}s start", keyFramePts_.c_str());
+ keyFramePts_ += std::to_string(preKeyFramePts_ / NS_PER_US) + ",";
+ MEDIA_LOG_D("AddPausePts %{public}s end", keyFramePts_.c_str());
+ }
+ if (stateCode == StateCode::RESUME) {
+ MEDIA_LOG_D("AddResumePts %{public}s start", keyFramePts_.c_str());
+ keyFramePts_ += std::to_string(currentKeyFramePts_ / NS_PER_US) + ",";
+ MEDIA_LOG_D("AddResumePts %{public}s end", keyFramePts_.c_str());
+ }
+ pauseResumePts_.pop_front();
+ return AddPauseResumePts(currentPts);
+}
} // namespace MEDIA
} // namespace OHOS
diff --git a/services/media_engine/filters/surface_encoder_adapter.h b/services/media_engine/filters/surface_encoder_adapter.h
index 9c7330e4f08cd7b4558b5a95e14958a6e1ed2de8..85e5015b497441170f73f2f7d5afde909332ffae 100644
--- a/services/media_engine/filters/surface_encoder_adapter.h
+++ b/services/media_engine/filters/surface_encoder_adapter.h
@@ -50,6 +50,12 @@ public:
virtual void OnOutputFormatChanged(const std::shared_ptr &format) = 0;
};
+class EncoderAdapterKeyFramePtsCallback {
+public:
+ virtual ~EncoderAdapterKeyFramePtsCallback() = default;
+ virtual void OnReportKeyFramePts(std::string KeyFramePts) = 0;
+};
+
class SurfaceEncoderAdapter : public std::enable_shared_from_this {
public:
explicit SurfaceEncoderAdapter();
@@ -60,6 +66,8 @@ public:
Status SetWatermark(std::shared_ptr &waterMarkBuffer);
Status SetOutputBufferQueue(const sptr &bufferQueueProducer);
Status SetEncoderAdapterCallback(const std::shared_ptr &encoderAdapterCallback);
+ Status SetEncoderAdapterKeyFramePtsCallback(
+ const std::shared_ptr &encoderAdapterKeyFramePtsCallback);
Status SetInputSurface(sptr surface);
Status SetTransCoderMode();
sptr GetInputSurface();
@@ -82,6 +90,7 @@ public:
std::shared_ptr ¶meter);
std::shared_ptr encoderAdapterCallback_;
+ std::shared_ptr encoderAdapterKeyFramePtsCallback_;
private:
void ReleaseBuffer();
@@ -91,6 +100,9 @@ private:
void ConfigureAboutEnableTemporalScale(MediaAVCodec::Format &format, const std::shared_ptr &meta);
bool CheckFrames(int64_t currentPts, int64_t &checkFramesPauseTime);
void GetCurrentTime(int64_t ¤tTime);
+ void AddStartPts(int64_t currentPts);
+ void AddStopPts();
+ bool AddPauseResumePts(int64_t currentPts);
std::shared_ptr codecServer_;
sptr outputBufferQueueProducer_;
@@ -122,6 +134,13 @@ private:
uint64_t instanceId_{0};
int32_t appUid_ {0};
int32_t appPid_ {0};
+
+ std::string keyFramePts_;
+ bool isStartKeyFramePts_ = false;
+ bool isStopKeyFramePts_ = false;
+ int64_t currentKeyFramePts_{-1};
+ int64_t preKeyFramePts_{-1};
+ std::deque> pauseResumePts_;
};
} // namespace MediaAVCodec
} // namespace OHOS
diff --git a/services/media_engine/filters/surface_encoder_filter.cpp b/services/media_engine/filters/surface_encoder_filter.cpp
index bbf45a880e2642fb24ad930bf546e3181424ed48..2ef2947fc367af75d1f2115824bb39a2d92bc5ea 100644
--- a/services/media_engine/filters/surface_encoder_filter.cpp
+++ b/services/media_engine/filters/surface_encoder_filter.cpp
@@ -22,6 +22,7 @@
#include "common/media_core.h"
#include "filter/filter_factory.h"
#include "surface_encoder_adapter.h"
+#include "muxer_filter.h"
namespace {
constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "HiStreamer" };
@@ -94,6 +95,28 @@ private:
std::weak_ptr surfaceEncoderFilter_;
};
+class SurfaceEncoderAdapterKeyFramePtsCallback : public EncoderAdapterKeyFramePtsCallback {
+public:
+ explicit SurfaceEncoderAdapterKeyFramePtsCallback(std::shared_ptr surfaceEncoderFilter)
+ : surfaceEncoderFilter_(std::move(surfaceEncoderFilter))
+ {
+ }
+
+ void OnReportKeyFramePts(std::string KeyFramePts) override
+ {
+ if (auto surfaceEncoderFilter = surfaceEncoderFilter_.lock()) {
+ MEDIA_LOG_D("SurfaceEncoderAdapterKeyFramePtsCallback OnReportKeyFramePts start");
+ surfaceEncoderFilter->OnReportKeyFramePts(KeyFramePts);
+ MEDIA_LOG_D("SurfaceEncoderAdapterKeyFramePtsCallback OnReportKeyFramePts end");
+ } else {
+ MEDIA_LOG_I("invalid surfaceEncoderFilter");
+ }
+ }
+
+private:
+ std::weak_ptr surfaceEncoderFilter_;
+};
+
SurfaceEncoderFilter::SurfaceEncoderFilter(std::string name, FilterType type): Filter(name, type)
{
MEDIA_LOG_I("encoder filter create");
@@ -142,6 +165,9 @@ void SurfaceEncoderFilter::Init(const std::shared_ptr &receiver,
std::shared_ptr encoderAdapterCallback =
std::make_shared(shared_from_this());
mediaCodec_->SetEncoderAdapterCallback(encoderAdapterCallback);
+ std::shared_ptr encoderAdapterKeyFramePtsCallback =
+ std::make_shared(shared_from_this());
+ mediaCodec_->SetEncoderAdapterKeyFramePtsCallback(encoderAdapterKeyFramePtsCallback);
} else {
MEDIA_LOG_I("Init mediaCodec fail");
eventReceiver_->OnEvent({"surface_encoder_filter", EventType::EVENT_ERROR, Status::ERROR_UNKNOWN});
@@ -378,6 +404,20 @@ void SurfaceEncoderFilter::SetCallingInfo(int32_t appUid, int32_t appPid,
mediaCodec_->SetCallingInfo(appUid, appPid, bundleName, instanceId);
}
}
+
+void SurfaceEncoderFilter::OnReportKeyFramePts(std::string KeyFramePts)
+{
+ MEDIA_LOG_I("OnReportKeyFramePts %{public}s enter", KeyFramePts.c_str());
+ const std::shared_ptr userMeta = std::make_shared();
+ userMeta->SetData("com.openharmony.recorder.timestamp", KeyFramePts);
+ std::shared_ptr muxerFilter = std::static_pointer_cast(nextFilter_);
+ if (muxerFilter != nullptr) {
+ muxerFilter->SetUserMeta(userMeta);
+ MEDIA_LOG_I("SetUserMeta %{public}s", KeyFramePts.c_str());
+ } else {
+ MEDIA_LOG_E("muxerFilter is null");
+ }
+}
} // namespace Pipeline
} // namespace MEDIA
} // namespace OHOS
\ No newline at end of file
diff --git a/services/media_engine/modules/muxer/media_muxer.cpp b/services/media_engine/modules/muxer/media_muxer.cpp
index 2fd7d1a1191e6d9e802033ebecc6d16eb4f90876..0710c28de8b6f1db67b4e94dc63de8a018115997 100644
--- a/services/media_engine/modules/muxer/media_muxer.cpp
+++ b/services/media_engine/modules/muxer/media_muxer.cpp
@@ -157,8 +157,16 @@ Status MediaMuxer::SetUserMeta(const std::shared_ptr &userMeta)
{
MEDIA_LOG_I("SetUserMeta");
std::lock_guard lock(mutex_);
+ std::vector keys;
+ userMeta->GetKeys(keys);
+ for (auto& k: keys) {
+ if (k.compare("com.openharmony.recorder.timestamp") == 0) {
+ MEDIA_LOG_I("set com.openharmony.recorder.timestamp");
+ return muxer_->SetUserMeta(userMeta);
+ }
+ }
FALSE_RETURN_V_MSG_E(state_ == State::INITIALIZED || state_ == State::STARTED, Status::ERROR_WRONG_STATE,
- "The state is not INITIALIZED, the interface must be called after constructor and before Start(). "
+ "The state is not INITIALIZED, the interface must be called at initialized or started state. "
"The current state is %{public}s.", StateConvert(state_).c_str());
return muxer_->SetUserMeta(userMeta);
}