diff --git a/interfaces/inner_api/native/metadata_filter.h b/interfaces/inner_api/native/metadata_filter.h new file mode 100644 index 0000000000000000000000000000000000000000..846bb4ce474d4cd87694cee913ebd445b384e698 --- /dev/null +++ b/interfaces/inner_api/native/metadata_filter.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023-2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FILTERS_METADATA_FILTER_H +#define FILTERS_METADATA_FILTER_H + +#include +#include "filter/filter.h" +#include "surface.h" +#include "meta/meta.h" +#include "buffer/avbuffer.h" +#include "buffer/avallocator.h" +#include "buffer/avbuffer_queue.h" +#include "buffer/avbuffer_queue_producer.h" +#include "buffer/avbuffer_queue_consumer.h" +#include "common/status.h" + +#define TIME_NONE ((int64_t) -1) + +namespace OHOS { +namespace Media { +namespace Pipeline { +class MetaDataFilter : public Filter, public std::enable_shared_from_this { +public: + explicit MetaDataFilter(std::string name, FilterType type); + ~MetaDataFilter() override; + Status SetCodecFormat(const std::shared_ptr &format); + void Init(const std::shared_ptr &receiver, + const std::shared_ptr &callback) override; + Status Configure(const std::shared_ptr ¶meter); + Status SetInputMetaSurface(sptr surface); + sptr GetInputMetaSurface(); + Status DoPrepare() override; + Status DoStart() override; + Status DoPause() override; + Status DoResume() override; + Status DoStop() override; + Status DoFlush() override; + Status DoRelease() override; + Status NotifyEos(); + void SetParameter(const std::shared_ptr ¶meter) override; + void GetParameter(std::shared_ptr ¶meter) override; + Status LinkNext(const std::shared_ptr &nextFilter, StreamType outType) override; + Status UpdateNext(const std::shared_ptr &nextFilter, StreamType outType) override; + Status UnLinkNext(const std::shared_ptr &nextFilter, StreamType outType) override; + FilterType GetFilterType(); + void OnLinkedResult(const sptr &outputBufferQueue, std::shared_ptr &meta); + void OnUpdatedResult(std::shared_ptr &meta); + void OnUnlinkedResult(std::shared_ptr &meta); + void OnBufferAvailable(); + +protected: + Status OnLinked(StreamType inType, const std::shared_ptr &meta, + const std::shared_ptr &callback) override; + Status OnUpdated(StreamType inType, const std::shared_ptr &meta, + const std::shared_ptr &callback) override; + Status OnUnLinked(StreamType inType, const std::shared_ptr& callback) override; + +private: + void UpdateBufferConfig(std::shared_ptr buffer, int64_t timestamp); + static constexpr uint32_t METASURFACE_USAGE = BUFFER_USAGE_CPU_READ | BUFFER_USAGE_CPU_WRITE | + BUFFER_USAGE_MEM_DMA; + std::string name_; + FilterType filterType_ = FilterType::TIMED_METADATA; + + std::shared_ptr eventReceiver_; + std::shared_ptr filterCallback_; + + std::shared_ptr onLinkedResultCallback_; + std::shared_ptr configureParameter_; + std::shared_ptr nextFilter_; + + sptr outputBufferQueueProducer_; + + sptr inputSurface_; + + bool isStop_{false}; + bool refreshTotalPauseTime_{false}; + int64_t startBufferTime_{TIME_NONE}; + int64_t latestBufferTime_{TIME_NONE}; + int64_t latestPausedTime_{TIME_NONE}; + int64_t totalPausedTime_{0}; +}; +} // namespace Pipeline +} // namespace MEDIA +} // namespace OHOS +#endif // FILTERS_METADATA_FILTER_H \ No newline at end of file diff --git a/interfaces/inner_api/native/muxer_filter.h b/interfaces/inner_api/native/muxer_filter.h index f3904219ba10e90662826226d959e41fa22c7862..0446e530c1974da4f4100dacc2de2c552c6df940 100644 --- a/interfaces/inner_api/native/muxer_filter.h +++ b/interfaces/inner_api/native/muxer_filter.h @@ -77,8 +77,10 @@ private: int32_t stopCount_{0}; int32_t eosCount_{0}; std::map bufferPtsMap_; + std::map trackIndexMap_; std::string videoCodecMimeType_; std::string audioCodecMimeType_; + std::string metaDataCodecMimeType_; std::string bundleName_; uint64_t instanceId_{0}; int32_t appUid_ {0}; diff --git a/services/dfx/include/avcodec_sysevent.h b/services/dfx/include/avcodec_sysevent.h index 94c870ff39dee7c36011ab9a2be782a575131d98..4145c94a3f2297b9f9a3ce2849c1ddeb643006a5 100644 --- a/services/dfx/include/avcodec_sysevent.h +++ b/services/dfx/include/avcodec_sysevent.h @@ -63,6 +63,7 @@ struct MuxerFaultInfo { std::string callerType; std::string videoCodec; std::string audioCodec; + std::string metaCodec; std::string containerFormat; std::string errMsg; }; diff --git a/services/media_engine/filters/BUILD.gn b/services/media_engine/filters/BUILD.gn index d07f8355da22c48bb51423b89e9326326e2d7dae..d6aa458750aa94c3ef5fb4aef8c76a99a035e010 100644 --- a/services/media_engine/filters/BUILD.gn +++ b/services/media_engine/filters/BUILD.gn @@ -74,6 +74,7 @@ ohos_shared_library("av_codec_media_engine_filters") { "codec_capability_adapter.cpp", "decoder_surface_filter.cpp", "demuxer_filter.cpp", + "metadata_filter.cpp", "muxer_filter.cpp", "subtitle_sink_filter.cpp", "surface_decoder_adapter.cpp", diff --git a/services/media_engine/filters/demuxer_filter.cpp b/services/media_engine/filters/demuxer_filter.cpp index 6cbbd161b53ab6d911ff5f2be4cfbc01a2f96367..6dbb11ed3d5ab1b7b405a0477f20ad210b0d8380 100644 --- a/services/media_engine/filters/demuxer_filter.cpp +++ b/services/media_engine/filters/demuxer_filter.cpp @@ -589,6 +589,8 @@ bool DemuxerFilter::ShouldTrackSkipped(Plugins::MediaType mediaType, std::string } else if (!disabledMediaTracks_.empty() && disabledMediaTracks_.find(mediaType) != disabledMediaTracks_.end()) { MEDIA_LOG_W_SHORT("mediaType disabled, index: %zu", index); return true; + } else if (mediaType == Plugins::MediaType::TIMEDMETA) { + return true; } return false; } diff --git a/services/media_engine/filters/metadata_filter.cpp b/services/media_engine/filters/metadata_filter.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5ea0f6fa2a1f4004635b1c6021b55fccc3038b26 --- /dev/null +++ b/services/media_engine/filters/metadata_filter.cpp @@ -0,0 +1,393 @@ +/* + * Copyright (c) 2023-2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "metadata_filter.h" +#include +#include +#include "sync_fence.h" +#include "filter/filter_factory.h" +#include "avcodec_info.h" +#include "avcodec_common.h" +#include "avcodec_trace.h" +#include "common/log.h" + +namespace { +constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_METADATA, "HiStreamer" }; +} + +namespace OHOS { +namespace Media { +namespace Pipeline { +static AutoRegisterFilter g_registerTimedMetaSurfaceFilter("builtin.recorder.timed_metadata", + FilterType::TIMED_METADATA, + [](const std::string& name, const FilterType type) { + return std::make_shared(name, FilterType::TIMED_METADATA); + }); + +class MetaDataFilterLinkCallback : public FilterLinkCallback { +public: + explicit MetaDataFilterLinkCallback(std::shared_ptr metaDataFilter) + : metaDataFilter_(std::move(metaDataFilter)) + { + } + + ~MetaDataFilterLinkCallback() = default; + + void OnLinkedResult(const sptr &queue, std::shared_ptr &meta) override + { + if (auto metaDataFilter = metaDataFilter_.lock()) { + metaDataFilter->OnLinkedResult(queue, meta); + } else { + MEDIA_LOG_I("invalid metaDataFilter"); + } + } + + void OnUnlinkedResult(std::shared_ptr &meta) override + { + if (auto metaDataFilter = metaDataFilter_.lock()) { + metaDataFilter->OnUnlinkedResult(meta); + } else { + MEDIA_LOG_I("invalid metaDataFilter"); + } + } + + void OnUpdatedResult(std::shared_ptr &meta) override + { + if (auto metaDataFilter = metaDataFilter_.lock()) { + metaDataFilter->OnUpdatedResult(meta); + } else { + MEDIA_LOG_I("invalid metaDataFilter"); + } + } +private: + std::weak_ptr metaDataFilter_; +}; + +class MetaDataSurfaceBufferListener : public IBufferConsumerListener { +public: + explicit MetaDataSurfaceBufferListener(std::shared_ptr metaDataFilter) + : metaDataFilter_(std::move(metaDataFilter)) + { + } + + void OnBufferAvailable() + { + if (auto metaDataFilter = metaDataFilter_.lock()) { + metaDataFilter->OnBufferAvailable(); + } else { + MEDIA_LOG_I("invalid metaDataFilter"); + } + } + +private: + std::weak_ptr metaDataFilter_; +}; + +MetaDataFilter::MetaDataFilter(std::string name, FilterType type): Filter(name, type) +{ + MEDIA_LOG_I("timed meta data filter create"); +} + +MetaDataFilter::~MetaDataFilter() +{ + MEDIA_LOG_I("timed meta data filter destroy"); +} + +Status MetaDataFilter::SetCodecFormat(const std::shared_ptr &format) +{ + MEDIA_LOG_I("SetCodecFormat"); + return Status::OK; +} + +void MetaDataFilter::Init(const std::shared_ptr &receiver, + const std::shared_ptr &callback) +{ + MEDIA_LOG_I("Init"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::Init"); + eventReceiver_ = receiver; + filterCallback_ = callback; +} + +Status MetaDataFilter::Configure(const std::shared_ptr ¶meter) +{ + MEDIA_LOG_I("Configure"); + configureParameter_ = parameter; + return Status::OK; +} + +Status MetaDataFilter::SetInputMetaSurface(sptr surface) +{ + MEDIA_LOG_I("SetInputMetaSurface"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::SetInputMetaSurface"); + if (surface == nullptr) { + MEDIA_LOG_E("surface is nullptr"); + return Status::ERROR_UNKNOWN; + } + inputSurface_ = surface; + sptr listener = new MetaDataSurfaceBufferListener(shared_from_this()); + inputSurface_->RegisterConsumerListener(listener); + return Status::OK; +} + +sptr MetaDataFilter::GetInputMetaSurface() +{ + MEDIA_LOG_I("GetInputMetaSurface"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::GetInputMetaSurface"); + sptr consumerSurface = Surface::CreateSurfaceAsConsumer("MetadataSurface"); + if (consumerSurface == nullptr) { + MEDIA_LOG_E("Create the surface consumer fail"); + return nullptr; + } + GSError err = consumerSurface->SetDefaultUsage(METASURFACE_USAGE); + if (err == GSERROR_OK) { + MEDIA_LOG_I("set consumer usage 0x%{public}x succ", METASURFACE_USAGE); + } else { + MEDIA_LOG_E("set consumer usage 0x%{public}x fail", METASURFACE_USAGE); + } + sptr producer = consumerSurface->GetProducer(); + if (producer == nullptr) { + MEDIA_LOG_E("Get the surface producer fail"); + return nullptr; + } + sptr producerSurface = Surface::CreateSurfaceAsProducer(producer); + if (producerSurface == nullptr) { + MEDIA_LOG_E("CreateSurfaceAsProducer fail"); + return nullptr; + } + inputSurface_ = consumerSurface; + sptr listener = new MetaDataSurfaceBufferListener(shared_from_this()); + inputSurface_->RegisterConsumerListener(listener); + return producerSurface; +} + +Status MetaDataFilter::DoPrepare() +{ + MEDIA_LOG_I("Prepare"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::Prepare"); + filterCallback_->OnCallback(shared_from_this(), FilterCallBackCommand::NEXT_FILTER_NEEDED, + StreamType::STREAMTYPE_ENCODED_VIDEO); + return Status::OK; +} + +Status MetaDataFilter::DoStart() +{ + MEDIA_LOG_I("Start"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::Start"); + isStop_ = false; + return Status::OK; +} + +Status MetaDataFilter::DoPause() +{ + MEDIA_LOG_I("Pause"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::Pause"); + isStop_ = true; + latestPausedTime_ = latestBufferTime_; + return Status::OK; +} + +Status MetaDataFilter::DoResume() +{ + MEDIA_LOG_I("Resume"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::Resume"); + isStop_ = false; + refreshTotalPauseTime_ = true; + return Status::OK; +} + +Status MetaDataFilter::DoStop() +{ + MEDIA_LOG_I("Stop"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::Stop"); + isStop_ = true; + latestBufferTime_ = TIME_NONE; + latestPausedTime_ = TIME_NONE; + totalPausedTime_ = 0; + refreshTotalPauseTime_ = false; + return Status::OK; +} + +Status MetaDataFilter::DoFlush() +{ + MEDIA_LOG_I("Flush"); + return Status::OK; +} + +Status MetaDataFilter::DoRelease() +{ + MEDIA_LOG_I("Release"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::Release"); + return Status::OK; +} + +Status MetaDataFilter::NotifyEos() +{ + MEDIA_LOG_I("NotifyEos"); + return Status::OK; +} + +void MetaDataFilter::SetParameter(const std::shared_ptr ¶meter) +{ + MEDIA_LOG_I("SetParameter"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::SetParameter"); +} + +void MetaDataFilter::GetParameter(std::shared_ptr ¶meter) +{ + MEDIA_LOG_I("GetParameter"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::GetParameter"); +} + +Status MetaDataFilter::LinkNext(const std::shared_ptr &nextFilter, StreamType outType) +{ + MEDIA_LOG_I("LinkNext"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::LinkNext"); + nextFilter_ = nextFilter; + nextFiltersMap_[outType].push_back(nextFilter_); + std::shared_ptr filterLinkCallback = + std::make_shared(shared_from_this()); + nextFilter->OnLinked(outType, configureParameter_, filterLinkCallback); + return Status::OK; +} + +Status MetaDataFilter::UpdateNext(const std::shared_ptr &nextFilter, StreamType outType) +{ + MEDIA_LOG_I("UpdateNext"); + return Status::OK; +} + +Status MetaDataFilter::UnLinkNext(const std::shared_ptr &nextFilter, StreamType outType) +{ + MEDIA_LOG_I("UnLinkNext"); + return Status::OK; +} + +FilterType MetaDataFilter::GetFilterType() +{ + MEDIA_LOG_I("GetFilterType"); + return filterType_; +} + +Status MetaDataFilter::OnLinked(StreamType inType, const std::shared_ptr &meta, + const std::shared_ptr &callback) +{ + MEDIA_LOG_I("OnLinked"); + return Status::OK; +} + +Status MetaDataFilter::OnUpdated(StreamType inType, const std::shared_ptr &meta, + const std::shared_ptr &callback) +{ + MEDIA_LOG_I("OnUpdated"); + return Status::OK; +} + +Status MetaDataFilter::OnUnLinked(StreamType inType, const std::shared_ptr& callback) +{ + MEDIA_LOG_I("OnUnLinked"); + return Status::OK; +} + +void MetaDataFilter::OnLinkedResult(const sptr &outputBufferQueue, + std::shared_ptr &meta) +{ + MEDIA_LOG_I("OnLinkedResult"); + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::OnLinkedResult"); + outputBufferQueueProducer_ = outputBufferQueue; +} + +void MetaDataFilter::OnUpdatedResult(std::shared_ptr &meta) +{ + MEDIA_LOG_I("OnUpdatedResult"); +} + +void MetaDataFilter::OnUnlinkedResult(std::shared_ptr &meta) +{ + MEDIA_LOG_I("OnUnlinkedResult"); +} + +void MetaDataFilter::OnBufferAvailable() +{ + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::OnBufferAvailable"); + sptr buffer; + sptr fence; + int64_t timestamp; + int32_t bufferSize = 0; + OHOS::Rect damage; + GSError ret = inputSurface_->AcquireBuffer(buffer, fence, timestamp, damage); + FALSE_RETURN(ret == GSERROR_OK && buffer != nullptr); + constexpr uint32_t waitForEver = -1; + (void)fence->Wait(waitForEver); + if (isStop_) { + inputSurface_->ReleaseBuffer(buffer, -1); + return; + } + auto extraData = buffer->GetExtraData(); + if (extraData) { + extraData->ExtraGet("timeStamp", timestamp); + extraData->ExtraGet("dataSize", bufferSize); + } + MEDIA_LOG_D("timestamp:%{public}ld, dataSize:%{public}d", timestamp, bufferSize); + if (timestamp == 0) { + MEDIA_LOG_E("timestamp invalid."); + inputSurface_->ReleaseBuffer(buffer, -1); + return; + } + + std::shared_ptr emptyOutputBuffer; + AVBufferConfig avBufferConfig; + avBufferConfig.size = bufferSize; + avBufferConfig.memoryType = MemoryType::SHARED_MEMORY; + avBufferConfig.memoryFlag = MemoryFlag::MEMORY_READ_WRITE; + int32_t timeOutMs = 100; + Status status = outputBufferQueueProducer_->RequestBuffer(emptyOutputBuffer, avBufferConfig, timeOutMs); + if (status != Status::OK) { + MEDIA_LOG_E("RequestBuffer fail."); + inputSurface_->ReleaseBuffer(buffer, -1); + return; + } + std::shared_ptr &bufferMem = emptyOutputBuffer->memory_; + if (emptyOutputBuffer->memory_ == nullptr) { + MEDIA_LOG_I("emptyOutputBuffer->memory_ is nullptr."); + inputSurface_->ReleaseBuffer(buffer, -1); + return; + } + bufferMem->Write((const uint8_t *)buffer->GetVirAddr(), bufferSize, 0); + UpdateBufferConfig(emptyOutputBuffer, timestamp); + status = outputBufferQueueProducer_->PushBuffer(emptyOutputBuffer, true); + FALSE_LOG_MSG(status == Status::OK, "PushBuffer fail"); + inputSurface_->ReleaseBuffer(buffer, -1); +} + +void MetaDataFilter::UpdateBufferConfig(std::shared_ptr buffer, int64_t timestamp) +{ + if (startBufferTime_ == TIME_NONE) { + startBufferTime_ = timestamp; + } + latestBufferTime_ = timestamp; + if (refreshTotalPauseTime_) { + if (latestPausedTime_ != TIME_NONE && latestBufferTime_ > latestPausedTime_) { + totalPausedTime_ += latestBufferTime_ - latestPausedTime_; + } + refreshTotalPauseTime_ = false; + } + buffer->pts_ = timestamp - startBufferTime_ - totalPausedTime_; + MediaAVCodec::AVCodecTrace trace("MetaDataFilter::UpdateBufferConfig"); + MEDIA_LOG_D("UpdateBufferConfig buffer->pts" PUBLIC_LOG_D64, buffer->pts_); +} + +} // namespace Pipeline +} // namespace MEDIA +} // namespace OHOS diff --git a/services/media_engine/filters/muxer_filter.cpp b/services/media_engine/filters/muxer_filter.cpp index ac4f45fccbe47ff80775cfeda389afba1311d706..6a453fe737525e07b2e4e48ac62fd5e4b81c78ff 100644 --- a/services/media_engine/filters/muxer_filter.cpp +++ b/services/media_engine/filters/muxer_filter.cpp @@ -251,6 +251,14 @@ Status MuxerFilter::OnLinked(StreamType inType, const std::shared_ptr &met audioCodecMimeType_ = mimeType; } else if (mimeType.find("video/") == 0) { videoCodecMimeType_ = mimeType; + } else if (mimeType.find("meta/") == 0) { + metaDataCodecMimeType_ = mimeType; + std::string srcMimeType; + meta->Get(srcMimeType); + if (trackIndexMap_.find(srcMimeType) != trackIndexMap_.end()) { + auto sourceTrackIndex = trackIndexMap_.at(videoCodecMimeType_); + meta->Set(sourceTrackIndex); + } } auto ret = mediaMuxer_->AddTrack(trackIndex, meta); if (ret != Status::OK) { @@ -258,6 +266,7 @@ Status MuxerFilter::OnLinked(StreamType inType, const std::shared_ptr &met SetFaultEvent("MuxerFilter::OnLinked error", (int32_t)ret); return ret; } + trackIndexMap_.emplace(std::make_pair(mimeType, trackIndex)); sptr inputBufferQueue = mediaMuxer_->GetInputBufferQueue(trackIndex); callback->OnLinkedResult(inputBufferQueue, const_cast &>(meta)); sptr listener = new MuxerBrokerListener(shared_from_this(), trackIndex, @@ -357,6 +366,7 @@ void MuxerFilter::SetFaultEvent(const std::string &errMsg) muxerFaultInfo.callerType = "player_framework"; muxerFaultInfo.videoCodec = videoCodecMimeType_; muxerFaultInfo.audioCodec = audioCodecMimeType_; + muxerFaultInfo.metaCodec = metaDataCodecMimeType_; muxerFaultInfo.containerFormat = GetContainerFormat(outputFormat_); muxerFaultInfo.errMsg = errMsg; FaultMuxerEventWrite(muxerFaultInfo);