From 17eefa184012c0383715ce21f55aa5d0898e84ab Mon Sep 17 00:00:00 2001 From: yyf Date: Tue, 9 Jan 2024 11:50:33 +0800 Subject: [PATCH] avmetadatahelper support fetch video thumbnail Signed-off-by: yyf Change-Id: I8cf761840e2575282f11ebcec60a63685a666a91 --- .../histreamer/avmetadatahelper/BUILD.gn | 2 + .../avmetadatahelper/avmetadata_collector.cpp | 39 +- .../avmetadatahelper_impl.cpp | 434 +++++++++++++++--- .../avmetadatahelper/avmetadatahelper_impl.h | 65 ++- 4 files changed, 435 insertions(+), 105 deletions(-) diff --git a/services/engine/histreamer/avmetadatahelper/BUILD.gn b/services/engine/histreamer/avmetadatahelper/BUILD.gn index 0d9dc8726..d94ad80ee 100644 --- a/services/engine/histreamer/avmetadatahelper/BUILD.gn +++ b/services/engine/histreamer/avmetadatahelper/BUILD.gn @@ -60,11 +60,13 @@ ohos_static_library("media_engine_histreamer_avmetadatahelper") { ] deps = [ + "$MEDIA_PLAYER_ROOT_DIR/services/dfx:media_service_dfx", "$MEDIA_PLAYER_ROOT_DIR/services/dfx:media_service_log_dfx", "$MEDIA_PLAYER_ROOT_DIR/services/utils:media_service_utils", ] external_deps = [ + "av_codec:av_codec_client", "av_codec:av_codec_media_engine_filters", "av_codec:av_codec_media_engine_modules", "c_utils:utils", diff --git a/services/engine/histreamer/avmetadatahelper/avmetadata_collector.cpp b/services/engine/histreamer/avmetadatahelper/avmetadata_collector.cpp index 412b8d851..3ec1be14f 100644 --- a/services/engine/histreamer/avmetadatahelper/avmetadata_collector.cpp +++ b/services/engine/histreamer/avmetadatahelper/avmetadata_collector.cpp @@ -16,11 +16,15 @@ #include "avmetadata_collector.h" #include #include "ctime" -#include "common/log.h" +#include "media_log.h" #include "meta/video_types.h" #include "buffer/avsharedmemorybase.h" #include "meta/any.h" +namespace { + constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AVMetaDataCollector"}; +} + namespace OHOS { namespace Media { static const std::unordered_map AVMETA_KEY_TO_X_MAP = { @@ -46,18 +50,19 @@ static const std::unordered_map AVMETA_KEY_TO_X_MAP = { AVMetaDataCollector::AVMetaDataCollector() { - MEDIA_LOG_D("enter ctor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this)); + MEDIA_LOGD("enter ctor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this)); } AVMetaDataCollector::~AVMetaDataCollector() { - MEDIA_LOG_D("enter dtor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this)); + MEDIA_LOGD("enter dtor, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this)); } std::unordered_map AVMetaDataCollector::GetMetadata(const std::shared_ptr &globalInfo, const std::vector> &trackInfos) { - FALSE_RETURN_V_MSG_E(globalInfo != nullptr && trackInfos.size() != 0, {}, "globalInfo or trackInfos are invalid."); + CHECK_AND_RETURN_RET_LOG(globalInfo != nullptr && trackInfos.size() != 0, + {}, "globalInfo or trackInfos are invalid."); std::vector keys; globalInfo->GetKeys(keys); for (int32_t i = 0; i < static_cast(keys.size()); i++) { @@ -75,13 +80,13 @@ std::unordered_map AVMetaDataCollector::GetMetadata(const for (size_t index = 0; index < trackCount; index++) { std::shared_ptr meta = trackInfos[index]; if (meta == nullptr) { - MEDIA_LOG_E("meta is invalid, index: %zu", index); + MEDIA_LOGE("meta is invalid, index: %zu", index); return metadata.tbl_; } Plugins::MediaType mediaType; if (!meta->GetData(Tag::MEDIA_TYPE, mediaType)) { - MEDIA_LOG_E("mediaType not found, index: %zu", index); + MEDIA_LOGE("mediaType not found, index: %zu", index); return metadata.tbl_; } @@ -89,7 +94,7 @@ std::unordered_map AVMetaDataCollector::GetMetadata(const } auto it = metadata.tbl_.begin(); while (it != metadata.tbl_.end()) { - MEDIA_LOG_I("metadata tbl, key: %{public}d, keyName: %{public}s, val: %{public}s", it->first, + MEDIA_LOGI("metadata tbl, key: %{public}d, keyName: %{public}s, val: %{public}s", it->first, AVMETA_KEY_TO_X_MAP.find(it->first)->second.c_str(), it->second.c_str()); it++; } @@ -98,12 +103,12 @@ std::unordered_map AVMetaDataCollector::GetMetadata(const std::shared_ptr AVMetaDataCollector::GetArtPicture(const std::vector> &trackInfos) { - MEDIA_LOG_I("GetArtPicture In"); + MEDIA_LOGI("GetArtPicture In"); size_t trackCount = trackInfos.size(); for (size_t index = 0; index < trackCount; index++) { std::shared_ptr meta = trackInfos[index]; if (meta == nullptr) { - MEDIA_LOG_W("meta is invalid, index: %zu", index); + MEDIA_LOGW("meta is invalid, index: %zu", index); continue; } @@ -116,7 +121,7 @@ std::shared_ptr AVMetaDataCollector::GetArtPicture(const std::ve coverAddr = AnyCast>(mapIt->second); } if (coverAddr.size() == 0 || static_cast(coverAddr.size()) > artPictureMaxSize) { - MEDIA_LOG_E("InvalidArtPictureSize %d", coverAddr.size()); + MEDIA_LOGE("InvalidArtPictureSize %d", coverAddr.size()); return nullptr; } uint8_t *addr = coverAddr.data(); @@ -125,14 +130,14 @@ std::shared_ptr AVMetaDataCollector::GetArtPicture(const std::ve static_cast(size), AVSharedMemory::FLAGS_READ_ONLY, "artpic"); errno_t rc = memcpy_s(artPicMem->GetBase(), static_cast(artPicMem->GetSize()), addr, size); if (rc != EOK) { - MEDIA_LOG_E("memcpy_s failed, trackCount no %{public}d", index); + MEDIA_LOGE("memcpy_s failed, trackCount no %{public}d", index); return nullptr; } - MEDIA_LOG_I("GetArtPicture Out"); + MEDIA_LOGI("GetArtPicture Out"); return artPicMem; } - MEDIA_LOG_E("GetArtPicture Failed"); + MEDIA_LOGE("GetArtPicture Failed"); return nullptr; } @@ -152,12 +157,14 @@ void AVMetaDataCollector::ConvertToAVMeta(const std::shared_ptr &innerMeta int32_t intVal; if (innerMeta->GetData(innerKey, intVal) && intVal != 0) { avmeta.SetMeta(avKey, std::to_string(intVal)); + MEDIA_LOGI("found innerKey: %{public}d, val: %{public}d", avKey, intVal); } SetEmptyStringIfNoData(avmeta, avKey); } else if (Any::IsSameTypeWith(type)) { std::string strVal; if (innerMeta->GetData(innerKey, strVal)) { avmeta.SetMeta(avKey, strVal); + MEDIA_LOGI("found innerKey: %{public}d, val: %{public}s", avKey, strVal.c_str()); } SetEmptyStringIfNoData(avmeta, avKey); } else if (Any::IsSameTypeWith(type)) { @@ -179,7 +186,7 @@ void AVMetaDataCollector::ConvertToAVMeta(const std::shared_ptr &innerMeta } SetEmptyStringIfNoData(avmeta, avKey); } else { - MEDIA_LOG_E("not found type matched with innerKey: %{public}s", innerKey.c_str()); + MEDIA_LOGE("not found type matched with innerKey: %{public}s", innerKey.c_str()); } } } @@ -194,7 +201,7 @@ void AVMetaDataCollector::SetEmptyStringIfNoData(Metadata &avmeta, int32_t avKey std::string AVMetaDataCollector::ConvertTimestampToDatetime(const std::string ×tamp) const { if (timestamp.empty()) { - MEDIA_LOG_E("datetime is empty, format failed"); + MEDIA_LOGE("datetime is empty, format failed"); return ""; } @@ -206,7 +213,7 @@ std::string AVMetaDataCollector::ConvertTimestampToDatetime(const std::string &t size_t sizeDateStr = strftime(date, maxDateTimeSize, "%Y-%m-%d", pTime); size_t sizeTimeStr = strftime(time, maxDateTimeSize, "%H:%M:%S", pTime); if (sizeDateStr != standardDateStrSize || sizeTimeStr != standardTimeStrSize) { - MEDIA_LOG_E("datetime is invalid, format failed"); + MEDIA_LOGE("datetime is invalid, format failed"); return ""; } diff --git a/services/engine/histreamer/avmetadatahelper/avmetadatahelper_impl.cpp b/services/engine/histreamer/avmetadatahelper/avmetadatahelper_impl.cpp index 3192184c0..0b3905d10 100644 --- a/services/engine/histreamer/avmetadatahelper/avmetadatahelper_impl.cpp +++ b/services/engine/histreamer/avmetadatahelper/avmetadatahelper_impl.cpp @@ -14,17 +14,80 @@ */ #include "avmetadatahelper_impl.h" -#include "media_errors.h" -#include "common/log.h" + +#include "buffer/avbuffer_common.h" #include "common/media_source.h" -#include "uri_helper.h" +#include "ibuffer_consumer_listener.h" +#include "image_source.h" +#include "graphic_common_c.h" +#include "media_errors.h" +#include "media_log.h" +#include "media_description.h" #include "meta/meta.h" #include "meta/meta_key.h" +#include "sync_fence.h" +#include "uri_helper.h" + +namespace { + constexpr OHOS::HiviewDFX::HiLogLabel LABEL = {LOG_CORE, LOG_DOMAIN, "AVMetadataHelperImpl"}; +} namespace OHOS { namespace Media { -static const std::set SUPPORTED_PIXELFORMAT = { - PixelFormat::RGB_565, PixelFormat::RGB_888, PixelFormat::RGBA_8888 +namespace { +const std::set SUPPORTED_PIXELFORMAT = { + PixelFormat::RGB_565, + PixelFormat::RGB_888, + PixelFormat::RGBA_8888 +}; + +constexpr uint64_t DECODER_USAGE = BUFFER_USAGE_CPU_READ | BUFFER_USAGE_CPU_WRITE | BUFFER_USAGE_MEM_DMA + | BUFFER_USAGE_VIDEO_DECODER; +} + +class FetchedFrameConsumerListener : public IBufferConsumerListener { +public: + explicit FetchedFrameConsumerListener(AVMetadataHelperImpl *helperImpl) : helperImpl_(helperImpl) {} + ~FetchedFrameConsumerListener() override = default; + + void OnBufferAvailable() override + { + MEDIA_LOGI("FetchedFrameConsumerListener OnBufferAvailable"); + helperImpl_->OnFetchedFrameBufferAvailable(); + } + +private: + AVMetadataHelperImpl *helperImpl_; +}; + +class MetadataHelperCodecCallback : public OHOS::MediaAVCodec::MediaCodecCallback { +public: + explicit MetadataHelperCodecCallback(AVMetadataHelperImpl *helperImpl) : helperImpl_(helperImpl) {} + + ~MetadataHelperCodecCallback() = default; + + void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode) override + { + helperImpl_->OnError(errorType, errorCode); + } + + void OnOutputFormatChanged(const MediaAVCodec::Format &format) override + { + helperImpl_->OnOutputFormatChanged(format); + } + + void OnInputBufferAvailable(uint32_t index, std::shared_ptr buffer) override + { + helperImpl_->OnInputBufferAvailable(index, buffer); + } + + void OnOutputBufferAvailable(uint32_t index, std::shared_ptr buffer) override + { + helperImpl_->OnOutputBufferAvailable(index, buffer); + } + +private: + AVMetadataHelperImpl *helperImpl_; }; class HelperEventReceiver : public Pipeline::EventReceiver { @@ -67,147 +130,311 @@ void AVMetadataHelperImpl::OnCallback(std::shared_ptr filter, { } +void AVMetadataHelperImpl::OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode) +{ + MEDIA_LOGI("OnError errorType:%{public}d, errorCode:%{public}d", + static_cast(errorType), errorCode); + Destroy(); +} + +void AVMetadataHelperImpl::OnOutputFormatChanged(const MediaAVCodec::Format &format) +{ + MEDIA_LOGI("OnOutputFormatChanged"); + outputFormat_ = format; +} + +void AVMetadataHelperImpl::OnInputBufferAvailable(uint32_t index, std::shared_ptr buffer) +{ + MEDIA_LOGI("OnInputBufferAvailable"); + CHECK_AND_RETURN_LOG(mediaDemuxer_ != nullptr, "OnInputBufferAvailable demuxer is nullptr."); + mediaDemuxer_->ReadSample(trackIndex_, buffer); + CHECK_AND_RETURN_LOG(videoDecoder_ != nullptr, "OnInputBufferAvailable decoder is nullptr."); + videoDecoder_->QueueInputBuffer(index); +} + +void AVMetadataHelperImpl::OnFetchedFrameBufferAvailable() +{ + MEDIA_LOGI("OnFetchedFrameBufferAvailable"); + CHECK_AND_RETURN_LOG(consumerSurface_ != nullptr, "consumerSurface_ is nullptr"); + sptr surfaceBuffer; + sptr fence; + int64_t timestamp; + OHOS::Rect damage; + GSError err = consumerSurface_->AcquireBuffer(surfaceBuffer, fence, timestamp, damage); + if (err != GSERROR_OK || surfaceBuffer == nullptr) { + MEDIA_LOGW("OnFetchedFrameBufferAvailable AcquireBuffer failed, err:%{public}d", err); + return; + } + MEDIA_LOGI("OnFetchedFrameBufferAvailable AcquireBuffer OK timestamp:%{public}" PRId64"", timestamp); + + if (timestamp >= seekTime_ && !hasFetchedFrame_) { + ConvertToAVSharedMemory(surfaceBuffer); + hasFetchedFrame_ = true; + cond_.notify_all(); + } + err = consumerSurface_->ReleaseBuffer(surfaceBuffer, -1); + CHECK_AND_RETURN_LOG(err == GSERROR_OK, "ReleaseBuffer failed, err:%{public}d", err); +} + +std::unique_ptr AVMetadataHelperImpl::GetYuvDataAlignStride(const sptr &surfaceBuffer) +{ + int32_t width = surfaceBuffer->GetWidth(); + int32_t height = surfaceBuffer->GetHeight(); + int32_t outputWidth; + int32_t outputHeight; + outputFormat_.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, outputWidth); + outputFormat_.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, outputHeight); + MEDIA_LOGI("GetYuvDataAlignStride outputWidth:%{public}d, outputHeight:%{public}d", + outputWidth, outputHeight); + + InitializationOptions initOpts; + initOpts.size = {width, height}; + initOpts.srcPixelFormat = PixelFormat::NV12; + std::unique_ptr yuvPixelMap = PixelMap::Create(initOpts); + uint8_t *dstData = static_cast(yuvPixelMap->GetWritablePixels()); + uint8_t *srcPtr = static_cast(surfaceBuffer->GetVirAddr()); + uint8_t *dstPtr = dstData; + + // copy src Y component to dst + for (int32_t y = 0; y < height; y++) { + memcpy_s(dstPtr, width, srcPtr, width); + srcPtr += outputWidth; + dstPtr += width; + } + + srcPtr = static_cast(surfaceBuffer->GetVirAddr()) + outputWidth * outputHeight; + + // copy src UV component to dst, height(UV) = height(Y) / 2 + for (int32_t uv = 0; uv < height / 2; uv++) { + memcpy_s(dstPtr, width, srcPtr, width); + srcPtr += outputWidth; + dstPtr += width; + } + + return yuvPixelMap; +} + +bool AVMetadataHelperImpl::ConvertToAVSharedMemory(const sptr &surfaceBuffer) +{ + int32_t format = surfaceBuffer->GetFormat(); + int32_t size = surfaceBuffer->GetSize(); + int32_t stride = surfaceBuffer->GetStride(); + int32_t width = surfaceBuffer->GetWidth(); + int32_t height = surfaceBuffer->GetHeight(); + MEDIA_LOGI("ConvertToAVSharedMemory format:%{public}d, size:%{public}d, stride:%{public}d, " + "width:%{public}d, height:%{public}d", format, size, stride, width, height); + + std::unique_ptr yuvPixelMap = GetYuvDataAlignStride(surfaceBuffer); + SourceOptions srcOpts; + srcOpts.pixelFormat = PixelFormat::NV12; + srcOpts.size = {width, height}; + uint32_t errorCode; + std::unique_ptr imageSource = + ImageSource::CreateImageSource(yuvPixelMap->GetPixels(), size, srcOpts, errorCode); + CHECK_AND_RETURN_RET_LOG(errorCode == 0, false, "CreateImageSource failed:%{public}d", errorCode); + + DecodeOptions decodeOpts; + decodeOpts.desiredSize = { + outputConfig_.dstWidth > 0 ? outputConfig_.dstWidth : width, + outputConfig_.dstHeight > 0 ? outputConfig_.dstHeight : height + }; + decodeOpts.desiredPixelFormat = outputConfig_.colorFormat; + std::unique_ptr pixelMap = imageSource->CreatePixelMapEx(0, decodeOpts, errorCode); + CHECK_AND_RETURN_RET_LOG(errorCode == 0, false, "CreatePixelMapEx failed:%{public}d", errorCode); + + fetchedFrameAtTime_ = std::make_shared(sizeof(OutputFrame) + pixelMap->GetRowStride() * + pixelMap->GetHeight(), AVSharedMemory::Flags::FLAGS_READ_WRITE, "FetchedFrameMemory"); + + int32_t ret = fetchedFrameAtTime_->Init(); + CHECK_AND_RETURN_RET_LOG(ret == static_cast(Status::OK), false, + "Create AVSharedmemory failed, ret:%{public}d", ret); + OutputFrame *frame = reinterpret_cast(fetchedFrameAtTime_->GetBase()); + frame->width_ = pixelMap->GetWidth(); + frame->height_ = pixelMap->GetHeight(); + frame->stride_ = pixelMap->GetRowStride(); + frame->bytesPerPixel_ = pixelMap->GetPixelBytes(); + frame->size_ = pixelMap->GetRowStride() * pixelMap->GetHeight(); + fetchedFrameAtTime_->Write(pixelMap->GetPixels(), frame->size_, sizeof(OutputFrame)); + return true; +} + +void AVMetadataHelperImpl::OnOutputBufferAvailable(uint32_t index, std::shared_ptr buffer) +{ + MEDIA_LOGI("OnOutputBufferAvailable."); + CHECK_AND_RETURN_LOG(videoDecoder_ != nullptr, "OnOutputBufferAvailable videoDecoder_ is nullptr."); + videoDecoder_->ReleaseOutputBuffer(index, true); +} + AVMetadataHelperImpl::AVMetadataHelperImpl() { - MEDIA_LOG_I("enter AVMetadataHelperImpl, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this)); + MEDIA_LOGI("Enter AVMetadataHelperImpl, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this)); metaCollector_ = std::make_shared(); } AVMetadataHelperImpl::~AVMetadataHelperImpl() { - MEDIA_LOG_I("enter ~AVMetadataHelperImpl, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this)); - Reset(); + MEDIA_LOGI("Enter ~AVMetadataHelperImpl, instance: 0x%{public}06" PRIXPTR "", FAKE_POINTER(this)); + Destroy(); } int32_t AVMetadataHelperImpl::SetSource(const std::string &uri, int32_t usage) { if ((usage != AVMetadataUsage::AV_META_USAGE_META_ONLY) && (usage != AVMetadataUsage::AV_META_USAGE_PIXEL_MAP)) { - MEDIA_LOG_E("Invalid avmetadatahelper usage: %{public}d", usage); + MEDIA_LOGE("Invalid avmetadatahelper usage: %{public}d", usage); return MSERR_INVALID_VAL; } UriHelper uriHelper(uri); if (uriHelper.UriType() != UriHelper::URI_TYPE_FILE && uriHelper.UriType() != UriHelper::URI_TYPE_FD) { - MEDIA_LOG_E("Unsupported uri type : %{public}s", uri.c_str()); + MEDIA_LOGE("Unsupported uri type : %{private}s", uri.c_str()); return MSERR_UNSUPPORT; } usage_ = usage; - MEDIA_LOG_I("uri: %{public}s, usage: %{public}d", uri.c_str(), usage); - - if (usage == AVMetadataUsage::AV_META_USAGE_PIXEL_MAP) { - pipeline_ = std::make_shared(); - demuxerFilter_ = Pipeline::FilterFactory::Instance().CreateFilter( - "builtin.player.demuxer", Pipeline::FilterType::FILTERTYPE_DEMUXER); - FALSE_RETURN_V(demuxerFilter_ != nullptr, MSERR_INVALID_VAL); - - auto eventReceiver = std::make_shared(this); - auto filterCallback = std::make_shared(this); - pipeline_->Init(eventReceiver, filterCallback); - } else { - mediaDemuxer_ = std::make_shared(); - } + MEDIA_LOGI("SetSource uri: %{private}s, type:%{public}d, usage: %{public}d", uri.c_str(), + uriHelper.UriType(), usage); Status ret = SetSourceInternel(uri, usage); - FALSE_RETURN_V_MSG_E(ret == Status::OK, MSERR_INVALID_VAL, "Failed to call SetSourceInternel"); - - MEDIA_LOG_I("set source success"); + CHECK_AND_RETURN_RET_LOG(ret == Status::OK, MSERR_INVALID_VAL, "Failed to call SetSourceInternel"); return MSERR_OK; } int32_t AVMetadataHelperImpl::SetSource(const std::shared_ptr &dataSrc) { - MEDIA_LOG_I("SetSource"); + MEDIA_LOGI("SetSource"); return MSERR_OK; } std::string AVMetadataHelperImpl::ResolveMetadata(int32_t key) { - MEDIA_LOG_I("enter ResolveMetadata with key: %{public}d", key); + MEDIA_LOGI("enter ResolveMetadata with key: %{public}d", key); std::string result; int32_t ret = ExtractMetadata(); - FALSE_RETURN_V_MSG_E(ret == MSERR_OK, result, "Failed to call ExtractMetadata"); + CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, result, "Failed to call ExtractMetadata"); auto it = collectedMeta_.find(key); if (it == collectedMeta_.end() || it->second.empty()) { - MEDIA_LOG_E("The specified metadata %{public}d cannot be obtained from the specified stream.", key); + MEDIA_LOGE("The specified metadata %{public}d cannot be obtained from the specified stream.", key); return result; } - MEDIA_LOG_I("exit ResolveMetadata with key"); + MEDIA_LOGI("exit ResolveMetadata with key"); result = collectedMeta_[key]; return result; } std::unordered_map AVMetadataHelperImpl::ResolveMetadata() { - MEDIA_LOG_I("enter ResolveMetadata"); + MEDIA_LOGI("enter ResolveMetadata"); int32_t ret = ExtractMetadata(); - FALSE_RETURN_V_MSG_E(ret == MSERR_OK, {}, "Failed to call ExtractMetadata"); + CHECK_AND_RETURN_RET_LOG(ret == MSERR_OK, {}, "Failed to call ExtractMetadata"); - MEDIA_LOG_I("exit ResolveMetadata"); + MEDIA_LOGI("exit ResolveMetadata"); return collectedMeta_; } std::shared_ptr AVMetadataHelperImpl::FetchArtPicture() { - MEDIA_LOG_I("enter FetchArtPicture"); + MEDIA_LOGI("enter FetchArtPicture"); if (collectedArtPicture_ != nullptr) { - MEDIA_LOG_I("Repeated ExtractArtPicture"); + MEDIA_LOGI("Repeated ExtractArtPicture"); return collectedArtPicture_; } if (mediaDemuxer_ == nullptr) { - MEDIA_LOG_E("ExtractArtPicutre Failed : mediaDemuxer_ is nullptr"); + MEDIA_LOGE("ExtractArtPicutre Failed : mediaDemuxer_ is nullptr"); return nullptr; } const std::vector> trackInfos = mediaDemuxer_->GetStreamMetaInfo(); collectedArtPicture_ = metaCollector_->GetArtPicture(trackInfos); if (collectedArtPicture_ == nullptr) { - MEDIA_LOG_E("ExtractArtPicutre Failed"); + MEDIA_LOGE("ExtractArtPicutre Failed"); } return collectedArtPicture_; } -std::shared_ptr AVMetadataHelperImpl::FetchFrameAtTime( - int64_t timeUs, int32_t option, const OutputConfiguration ¶m) +std::shared_ptr AVMetadataHelperImpl::GetTargetTrackInfo() { - MEDIA_LOG_I("enter FetchFrameAtTime"); + CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, nullptr, "GetTargetTrackInfo demuxer is nullptr"); + std::vector> trackInfos = mediaDemuxer_->GetStreamMetaInfo(); + size_t trackCount = trackInfos.size(); + CHECK_AND_RETURN_RET_LOG(trackCount > 0, nullptr, "GetTargetTrackInfo trackCount is invalid"); + for (size_t index = 0; index < trackCount; index++) { + if (!(trackInfos[index]->GetData(Tag::MIME_TYPE, trackMime_))) { + MEDIA_LOGW("GetTargetTrackInfo get mime type failed %{public}s", trackMime_.c_str()); + continue; + } + if (trackMime_.find("video/") == 0) { + Plugins::MediaType mediaType; + CHECK_AND_RETURN_RET_LOG(trackInfos[index]->GetData(Tag::MEDIA_TYPE, mediaType), nullptr, + "GetTargetTrackInfo failed to get mediaType, index:%{public}d", index); + MEDIA_LOGI("GetData mediaType:%{public}d", static_cast(mediaType)); + CHECK_AND_RETURN_RET_LOG(mediaType == Plugins::MediaType::VIDEO, nullptr, + "GetTargetTrackInfo mediaType is not video, index:%{public}d", index); + trackIndex_ = index; + MEDIA_LOGI("GetTargetTrackInfo success trackIndex_:%{public}d, trackMime_:%{public}s", + trackIndex_, trackMime_.c_str()); + return trackInfos[trackIndex_]; + } + } + MEDIA_LOGW("GetTargetTrackInfo FAILED."); return nullptr; } -Status AVMetadataHelperImpl::SetSourceInternel(const std::string &uri, int32_t usage) +std::shared_ptr AVMetadataHelperImpl::FetchFrameAtTime( + int64_t timeUs, int32_t option, const OutputConfiguration ¶m) { - Status ret; - if (usage_ == AVMetadataUsage::AV_META_USAGE_PIXEL_MAP) { - ret = demuxerFilter_->SetDataSource(std::make_shared(uri)); - } else { - ret = mediaDemuxer_->SetDataSource(std::make_shared(uri)); - } - FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Failed to call SetDataSource"); + MEDIA_LOGI("FetchFrameAtTime 0x%{public}06" PRIXPTR " timeUs:%{public}" PRId64", option:%{public}d," + "dstWidth:%{public}d, dstHeight:%{public}d, colorFormat:%{public}d", FAKE_POINTER(this), + timeUs, option, param.dstWidth, param.dstHeight, static_cast(param.colorFormat)); + CHECK_AND_RETURN_RET_LOG(usage_ == AVMetadataUsage::AV_META_USAGE_PIXEL_MAP, nullptr, + "FetchFrameAtTime usage invalid"); + CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, nullptr, "FetchFrameAtTime demuxer is nullptr"); - ret = PrepareInternel(); - FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "Failed to call PrepareInternel"); + hasFetchedFrame_ = false; + outputConfig_ = param; + seekTime_ = timeUs; - return Status::OK; + trackInfo_ = GetTargetTrackInfo(); + CHECK_AND_RETURN_RET_LOG(trackInfo_ != nullptr, nullptr, "FetchFrameAtTime trackInfo_ is nullptr."); + + CHECK_AND_RETURN_RET_LOG(InitSurface() == Status::OK, nullptr, "FetchFrameAtTime InitSurface failed."); + CHECK_AND_RETURN_RET_LOG(InitDecoder() == Status::OK, nullptr, "FetchFrameAtTime InitDecoder failed."); + + mediaDemuxer_->SelectTrack(trackIndex_); + int64_t realSeekTime = timeUs; + mediaDemuxer_->SeekTo(timeUs, static_cast(option), realSeekTime); + MEDIA_LOGI("FetchFrameAtTime realSeekTime:%{public}" PRId64"", realSeekTime); + + std::unique_lock lock(mutex_); + + // wait up to 3s to fetch frame AVSharedMemory at time. + if (cond_.wait_for(lock, std::chrono::seconds(3), [this] {return hasFetchedFrame_.load();})) { + MEDIA_LOGI("Fetch target frame OK."); + } else { + hasFetchedFrame_ = true; + MEDIA_LOGI("Fetch target frame timeout."); + } + return fetchedFrameAtTime_; } -Status AVMetadataHelperImpl::PrepareInternel() +Status AVMetadataHelperImpl::SetSourceInternel(const std::string &uri, int32_t usage) { - Status ret = Status::OK; - if (usage_ == AVMetadataUsage::AV_META_USAGE_PIXEL_MAP) { - FALSE_RETURN_V_MSG_E(demuxerFilter_ != nullptr, Status::ERROR_INVALID_OPERATION, "set source firstly"); - ret = demuxerFilter_->Prepare(); - } - FALSE_RETURN_V_MSG_E(ret == Status::OK, Status::ERROR_INVALID_DATA, "prepare failed"); + Reset(); + srcUri_ = uri; + mediaDemuxer_ = std::shared_ptr(new(std::nothrow) MediaDemuxer()); + CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, Status::ERROR_INVALID_DATA, + "SetSourceInternel demuxer is nullptr"); + mediaDemuxer_->SetEventReceiver(std::make_shared(this)); + Status ret = mediaDemuxer_->SetDataSource(std::make_shared(srcUri_)); + CHECK_AND_RETURN_RET_LOG(ret == Status::OK, ret, "SetSourceInternel demuxer failed to call SetDataSource"); return Status::OK; } int32_t AVMetadataHelperImpl::ExtractMetadata() { - FALSE_RETURN_V_MSG_E(usage_ != AVMetadataUsage::AV_META_USAGE_PIXEL_MAP, MSERR_INVALID_OPERATION, "usage error"); - FALSE_RETURN_V_MSG_E(mediaDemuxer_ != nullptr, MSERR_INVALID_OPERATION, "mediaDemuxer_ is nullptr"); + CHECK_AND_RETURN_RET_LOG(mediaDemuxer_ != nullptr, MSERR_INVALID_OPERATION, "mediaDemuxer_ is nullptr"); if (!hasCollectMeta_) { const std::shared_ptr globalInfo = mediaDemuxer_->GetGlobalMetaInfo(); @@ -220,22 +447,87 @@ int32_t AVMetadataHelperImpl::ExtractMetadata() void AVMetadataHelperImpl::Reset() { - if (demuxerFilter_ != nullptr) { - demuxerFilter_->Stop(); - hasCollectMeta_ = false; - collectedArtPicture_ = nullptr; + if (mediaDemuxer_ != nullptr) { + mediaDemuxer_->Reset(); } - if (demuxerFilter_ != nullptr) { - demuxerFilter_->Reset(); + if (videoDecoder_ != nullptr) { + videoDecoder_->Reset(); } - if (mediaDemuxer_ != nullptr) { - mediaDemuxer_->Reset(); - } + hasFetchedFrame_ = false; + hasCollectMeta_ = false; + collectedArtPicture_ = nullptr; errHappened_ = false; firstFetch_ = true; } + +void AVMetadataHelperImpl::Destroy() +{ + if (mediaDemuxer_ != nullptr) { + mediaDemuxer_->Stop(); + } + + if (videoDecoder_ != nullptr) { + videoDecoder_->Stop(); + videoDecoder_->Release(); + } +} + +Status AVMetadataHelperImpl::InitDecoder() +{ + if (videoDecoder_ != nullptr) { + MEDIA_LOGD("InitDecoder already."); + return Status::OK; + } + MEDIA_LOGD("AVMetadataHelperImpl InitDecoder."); + videoDecoder_ = MediaAVCodec::VideoDecoderFactory::CreateByMime(trackMime_); + CHECK_AND_RETURN_RET_LOG(videoDecoder_ != nullptr, Status::ERROR_NO_MEMORY, + "Create videoDecoder_ is nullptr"); + Format trackFormat {}; + trackFormat.SetMeta(trackInfo_); + int32_t width; + int32_t height; + trackFormat.GetIntValue(MediaDescriptionKey::MD_KEY_WIDTH, width); + trackFormat.GetIntValue(MediaDescriptionKey::MD_KEY_HEIGHT, height); + MEDIA_LOGI("InitDecoder trackFormat width:%{public}d, height:%{public}d", width, height); + trackFormat.PutIntValue(MediaDescriptionKey::MD_KEY_PIXEL_FORMAT, + static_cast(Plugins::VideoPixelFormat::NV12)); + videoDecoder_->Configure(trackFormat); + std::shared_ptr mediaCodecCallback = + std::make_shared(this); + videoDecoder_->SetCallback(mediaCodecCallback); + videoDecoder_->SetOutputSurface(producerSurface_); + videoDecoder_->Prepare(); + videoDecoder_->Start(); + return Status::OK; +} + +Status AVMetadataHelperImpl::InitSurface() +{ + if (producerSurface_ != nullptr) { + MEDIA_LOGD("InitSurface already."); + return Status::OK; + } + MEDIA_LOGD("AVMetadataHelperImpl InitSurface enter"); + consumerSurface_ = Surface::CreateSurfaceAsConsumer("AVMetadataHelperSurface"); + CHECK_AND_RETURN_RET_LOG(consumerSurface_ != nullptr, Status::ERROR_NO_MEMORY, + "InitSurface create consumer surface failed."); + + GSError err = consumerSurface_->SetDefaultUsage(DECODER_USAGE); + CHECK_AND_RETURN_RET_LOG(err == GSERROR_OK, Status::ERROR_INVALID_OPERATION, + "InitSurface SetDefaultUsage failed."); + sptr listener = new FetchedFrameConsumerListener(this); + err = consumerSurface_->RegisterConsumerListener(listener); + CHECK_AND_RETURN_RET_LOG(err == GSERROR_OK, Status::ERROR_INVALID_OPERATION, + "InitSurface RegisterConsumerListener failed."); + + sptr producer = consumerSurface_->GetProducer(); + producerSurface_ = Surface::CreateSurfaceAsProducer(producer); + CHECK_AND_RETURN_RET_LOG(producerSurface_ != nullptr, Status::ERROR_NO_MEMORY, + "InitSurface create producer surface failed."); + return Status::OK; +} } // namespace Media } // namespace OHOS \ No newline at end of file diff --git a/services/engine/histreamer/avmetadatahelper/avmetadatahelper_impl.h b/services/engine/histreamer/avmetadatahelper/avmetadatahelper_impl.h index c7d73f22c..e68ea2241 100644 --- a/services/engine/histreamer/avmetadatahelper/avmetadatahelper_impl.h +++ b/services/engine/histreamer/avmetadatahelper/avmetadatahelper_impl.h @@ -16,21 +16,25 @@ #ifndef AVMETADATAHELPER_HST_IMPL_H #define AVMETADATAHELPER_HST_IMPL_H -#include #include -#include "nocopyable.h" -#include "i_avmetadatahelper_service.h" -#include "i_avmetadatahelper_engine.h" +#include + +#include "avmetadata_collector.h" +#include "buffer/avsharedmemorybase.h" #include "common/status.h" -#include "pipeline/pipeline.h" -#include "filter/filter_factory.h" #include "demuxer_filter.h" +#include "filter/filter_factory.h" +#include "i_avmetadatahelper_engine.h" +#include "i_avmetadatahelper_service.h" #include "media_demuxer.h" -#include "avmetadata_collector.h" +#include "nocopyable.h" +#include "pipeline/pipeline.h" +#include "video_decoder_adapter.h" namespace OHOS { namespace Media { -class AVMetadataHelperImpl : public IAVMetadataHelperEngine, public NoCopyable { +class AVMetadataHelperImpl : public IAVMetadataHelperEngine, public std::enable_shared_from_this, + public NoCopyable { public: AVMetadataHelperImpl(); ~AVMetadataHelperImpl(); @@ -46,28 +50,53 @@ public: void OnEvent(const Event &event); void OnCallback(std::shared_ptr filter, const OHOS::Media::Pipeline::FilterCallBackCommand cmd, OHOS::Media::Pipeline::StreamType outType); + void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode); + void OnOutputFormatChanged(const MediaAVCodec::Format &format); + void OnInputBufferAvailable(uint32_t index, std::shared_ptr buffer); + void OnOutputBufferAvailable(uint32_t index, std::shared_ptr buffer); + void OnFetchedFrameBufferAvailable(); private: Status SetSourceInternel(const std::string &uri, int32_t usage); - Status PrepareInternel(); int32_t ExtractMetadata(); void Reset(); + void Destroy(); + Status InitSurface(); + Status InitDecoder(); + + std::shared_ptr GetTargetTrackInfo(); + std::unique_ptr GetYuvDataAlignStride(const sptr &surfaceBuffer); + bool ConvertToAVSharedMemory(const sptr &surfaceBuffer); std::unordered_map collectedMeta_; - bool hasCollectMeta_ = false; - int32_t usage_ = AVMetadataUsage::AV_META_USAGE_PIXEL_MAP; + bool hasCollectMeta_{false}; + int32_t usage_{AVMetadataUsage::AV_META_USAGE_PIXEL_MAP}; - std::shared_ptr collectedArtPicture_; - + std::string srcUri_; std::mutex mutex_; std::condition_variable cond_; - bool errHappened_ = false; - bool firstFetch_ = true; + bool errHappened_{false}; + bool firstFetch_{true}; + + std::shared_ptr collectedArtPicture_; + std::shared_ptr fetchedFrameAtTime_; - std::shared_ptr pipeline_; - std::shared_ptr demuxerFilter_; - std::shared_ptr mediaDemuxer_; std::shared_ptr metaCollector_; + std::shared_ptr mediaDemuxer_; + std::shared_ptr videoDecoder_; + + BufferRequestConfig requestCfg_; + OutputConfiguration outputConfig_; + Format outputFormat_; + + int64_t seekTime_{0}; + std::atomic_bool hasFetchedFrame_{false}; + std::string trackMime_; + std::shared_ptr trackInfo_; + size_t trackIndex_{0}; + + sptr producerSurface_{nullptr}; + sptr consumerSurface_{nullptr}; }; } // namespace Media } // namespace OHOS -- Gitee