diff --git a/interfaces/inner_api/native/media_demuxer.h b/interfaces/inner_api/native/media_demuxer.h index a3ee0f340e8ae83ee27920d1d91021c5df0d9dad..99f17c0f0e67c8ec02d4f83021f17611617de18d 100644 --- a/interfaces/inner_api/native/media_demuxer.h +++ b/interfaces/inner_api/native/media_demuxer.h @@ -17,22 +17,28 @@ #define MEDIA_DEMUXER_H #include +#include #include + #include "avcodec_common.h" -#include "osal/task/task.h" -#include "meta/media_types.h" #include "buffer/avbuffer_queue_producer.h" #include "buffer/avbuffer.h" -#include "plugin/plugin_base.h" +#include "common/media_source.h" #include "demuxer/data_packer.h" #include "demuxer/type_finder.h" -#include "common/media_source.h" +#include "filter/filter.h" +#include "meta/media_types.h" +#include "osal/task/task.h" +#include "plugin/plugin_base.h" #include "plugin/plugin_info.h" +#include "plugin/plugin_time.h" #include "plugin/demuxer_plugin.h" -#include "filter/filter.h" namespace OHOS { namespace Media { +namespace { + constexpr uint32_t TRACK_ID_DUMMY = std::numeric_limits::max(); +} using MediaSource = OHOS::Media::Plugins::MediaSource; class DataPacker; class TypeFinder; @@ -89,6 +95,8 @@ private: }; bool isHttpSource_ = false; + std::string videoMime_{}; + bool IsContainIdrFrame(const uint8_t* buff, size_t bufSize); void InitTypeFinder(); bool CreatePlugin(std::string pluginName); @@ -118,7 +126,7 @@ private: std::string pluginName_; std::shared_ptr plugin_; - std::atomic pluginState_; + std::atomic pluginState_{DemuxerState::DEMUXER_STATE_NULL}; std::shared_ptr dataSource_; std::shared_ptr mediaSource_; std::shared_ptr source_; @@ -128,13 +136,15 @@ private: std::function&)> peekRange_; std::function&)> getRange_; + bool PullDataWithCache(uint64_t offset, size_t size, std::shared_ptr& bufferPtr); + bool PullDataWithoutCache(uint64_t offset, size_t size, std::shared_ptr& bufferPtr); void ReadLoop(uint32_t trackId); Status CopyFrameToUserQueue(uint32_t trackId); - bool GetBufferFromUserQueue(uint32_t queueIndex, int32_t size = 0); + bool GetBufferFromUserQueue(uint32_t queueIndex, uint32_t size = 0); Status InnerReadSample(uint32_t trackId, std::shared_ptr); Status InnerSelectTrack(int32_t trackId); - std::mutex mutex_ {}; + std::mutex mutex_{}; std::map> bufferQueueMap_; std::map> bufferMap_; std::map eosMap_; @@ -146,6 +156,15 @@ private: std::map> threadMap_; std::shared_ptr eventReceiver_; + int64_t lastSeekTime_{Plugins::HST_TIME_NONE}; + struct CacheData { + std::shared_ptr data = nullptr; + uint64_t offset = 0; + }; + + CacheData cacheData_; + bool isSeeked_{false}; + uint32_t videoTrackId_{TRACK_ID_DUMMY}; }; } // namespace Media } // namespace OHOS diff --git a/interfaces/plugin/source_plugin.h b/interfaces/plugin/source_plugin.h index bd79a4279610249de019a1a917637049643a4a31..1a0c0e1915bf96e31dd518aaa3b3f083c8cbc391 100644 --- a/interfaces/plugin/source_plugin.h +++ b/interfaces/plugin/source_plugin.h @@ -18,11 +18,13 @@ #include #include + #include "common/media_source.h" -#include "plugin/plugin_caps.h" #include "plugin/plugin_base.h" -#include "plugin/plugin_definition.h" #include "plugin/plugin_buffer.h" +#include "plugin/plugin_caps.h" +#include "plugin/plugin_definition.h" +#include "plugin/plugin_time.h" namespace OHOS { namespace Media { @@ -35,8 +37,9 @@ namespace Plugins { * @since 1.0 * @version 1.0 */ -struct SourcePlugin : public PluginBase { +class SourcePlugin : public PluginBase { /// constructor +public: explicit SourcePlugin(std::string name): PluginBase(std::move(name)) {} /** * @brief Set the data source to source plugin. @@ -113,6 +116,22 @@ struct SourcePlugin : public PluginBase { { return Status::OK; } + + virtual bool IsSeekToTimeSupported() + { + return false; + } + + virtual Status SeekToTime(int64_t seekTime) + { + return Status::OK; + } + + virtual Status GetDuration(int64_t& duration) + { + duration = Plugins::HST_TIME_NONE; + return Status::OK; + } }; /// Source plugin api major number. diff --git a/services/media_engine/filters/decoder_surface_filter.cpp b/services/media_engine/filters/decoder_surface_filter.cpp index 16fe58294a367da6da4cf4e6fc95a01b84b61299..d238984faa9738327b790f828f0658497ebdeaca 100644 --- a/services/media_engine/filters/decoder_surface_filter.cpp +++ b/services/media_engine/filters/decoder_surface_filter.cpp @@ -98,7 +98,9 @@ DecoderSurfaceFilter::DecoderSurfaceFilter(const std::string& name, FilterType t DecoderSurfaceFilter::~DecoderSurfaceFilter() { + MEDIA_LOG_I("~DecoderSurfaceFilter() enter."); videoDecoder_->Release(); + MEDIA_LOG_I("~DecoderSurfaceFilter() exit."); } void DecoderSurfaceFilter::Init(const std::shared_ptr &receiver, diff --git a/services/media_engine/filters/video_decoder_adapter.cpp b/services/media_engine/filters/video_decoder_adapter.cpp index c44ea6a4ca59dcc06f4d10ab87af4ba35d64842b..48de83205b12afeb3802cf5e9898253f4c5b590b 100644 --- a/services/media_engine/filters/video_decoder_adapter.cpp +++ b/services/media_engine/filters/video_decoder_adapter.cpp @@ -32,11 +32,13 @@ using namespace MediaAVCodec; const std::string VIDEO_INPUT_BUFFER_QUEUE_NAME = "VideoDecoderInputBufferQueue"; AVBufferAvailableListener::AVBufferAvailableListener(std::shared_ptr videoDecoder) { + MEDIA_LOG_I("AVBufferAvailableListener instances create."); videoDecoder_ = videoDecoder; } AVBufferAvailableListener::~AVBufferAvailableListener() { + MEDIA_LOG_I("~AVBufferAvailableListener()"); } void AVBufferAvailableListener::OnBufferAvailable() @@ -46,12 +48,13 @@ void AVBufferAvailableListener::OnBufferAvailable() VideoDecoderCallback::VideoDecoderCallback(std::shared_ptr videoDecoder) { - MEDIA_LOG_I("VideoDecoderCallback::VideoDecoderCallback"); + MEDIA_LOG_I("VideoDecoderCallback instances create."); videoDecoderAdapter_ = videoDecoder; } VideoDecoderCallback::~VideoDecoderCallback() { + MEDIA_LOG_I("~VideoDecoderCallback()"); } void VideoDecoderCallback::OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode) @@ -85,6 +88,7 @@ VideoDecoderAdapter::VideoDecoderAdapter() VideoDecoderAdapter::~VideoDecoderAdapter() { + MEDIA_LOG_I("~VideoDecoderAdapter()"); mediaCodec_->Release(); if (!isThreadExit_) { Stop(); @@ -198,15 +202,20 @@ void VideoDecoderAdapter::AquireAvailableInputBuffer() uint32_t index; FALSE_RETURN_MSG(tmpBuffer->meta_->GetData(Tag::REGULAR_TRACK_ID, index), "get index failed."); if (tmpBuffer->flag_ & (uint32_t)(Plugins::AVBufferFlag::EOS)) { + MEDIA_LOG_I("ReleaseBuffer for eos, index: %{public}u, bufferid: %{public}" PRIu64 + ", pts: %{public}" PRIu64", flag: %{public}u", index, tmpBuffer->GetUniqueId(), + tmpBuffer->pts_, tmpBuffer->flag_); inputBufferQueueConsumer_->ReleaseBuffer(tmpBuffer); return; } if (mediaCodec_->QueueInputBuffer(index) != ERR_OK) { - MEDIA_LOG_E("QueueInputBuffer failed index: %{public}u, bufferid: %{public}" PRIu64, - index, tmpBuffer->GetUniqueId()); + MEDIA_LOG_E("QueueInputBuffer failed, index: %{public}u, bufferid: %{public}" PRIu64 + ", pts: %{public}" PRIu64", flag: %{public}u", index, tmpBuffer->GetUniqueId(), + tmpBuffer->pts_, tmpBuffer->flag_); } else { - MEDIA_LOG_D("QueueInputBuffer success index: %{public}u, bufferid: %{public}" PRIu64, - index, tmpBuffer->GetUniqueId()); + MEDIA_LOG_D("QueueInputBuffer success, index: %{public}u, bufferid: %{public}" PRIu64 + ", pts: %{public}" PRIu64", flag: %{public}u", index, tmpBuffer->GetUniqueId(), + tmpBuffer->pts_, tmpBuffer->flag_); } } else { MEDIA_LOG_E("AcquireBuffer failed."); @@ -223,8 +232,13 @@ void VideoDecoderAdapter::OnInputBufferAvailable(uint32_t index, std::shared_ptr } if (inputBufferQueueConsumer_->IsBufferInQueue(buffer)) { if (inputBufferQueueConsumer_->ReleaseBuffer(buffer) != Status::OK) { - MEDIA_LOG_E("IsBufferInQueue ReleaseBuffer failed. index: %{public}u, bufferid: %{public}" PRIu64, - index, buffer->GetUniqueId()); + MEDIA_LOG_E("IsBufferInQueue ReleaseBuffer failed. index: %{public}u, bufferid: %{public}" PRIu64 + ", pts: %{public}" PRIu64", flag: %{public}u", index, buffer->GetUniqueId(), + buffer->pts_, buffer->flag_); + } else { + MEDIA_LOG_D("IsBufferInQueue ReleaseBuffer success. index: %{public}u, bufferid: %{public}" PRIu64 + ", pts: %{public}" PRIu64", flag: %{public}u", index, buffer->GetUniqueId(), + buffer->pts_, buffer->flag_); } } else { uint32_t size = inputBufferQueueConsumer_->GetQueueSize() + 1; @@ -249,6 +263,7 @@ void VideoDecoderAdapter::RenderLoop() index = indexs_[0]; indexs_.erase(indexs_.begin()); } + MEDIA_LOG_I("RenderLoop ReleaseOutputBuffer start, index: %{public}u", index); mediaCodec_->ReleaseOutputBuffer(index, true); } } @@ -267,7 +282,10 @@ void VideoDecoderAdapter::OnOutputFormatChanged(const MediaAVCodec::Format &form void VideoDecoderAdapter::OnOutputBufferAvailable(uint32_t index, std::shared_ptr buffer) { + FALSE_RETURN_MSG(buffer != nullptr, "buffer is nullptr"); FALSE_RETURN_MSG(callback_ != nullptr, "callback_ is nullptr"); + MEDIA_LOG_D("OnOutputBufferAvailable start. index: %{public}u, bufferid: %{public}" PRIu64 ", pts: %{public}" PRIu64 + ", flag: %{public}u", index, buffer->GetUniqueId(), buffer->pts_, buffer->flag_); callback_->OnOutputBufferAvailable(index, buffer); } diff --git a/services/media_engine/modules/BUILD.gn b/services/media_engine/modules/BUILD.gn index b72c77d0a5416e088c92e30adc2b9b4c289440ce..27e91808a06ae5067f02c64460491df13fe28235 100644 --- a/services/media_engine/modules/BUILD.gn +++ b/services/media_engine/modules/BUILD.gn @@ -66,6 +66,7 @@ ohos_shared_library("av_codec_media_engine_modules") { sources = [ "demuxer/data_packer.cpp", + "demuxer/frame_detector.cpp", "demuxer/media_demuxer.cpp", "demuxer/type_finder.cpp", "media_codec/media_codec.cpp", diff --git a/services/media_engine/modules/demuxer/frame_detector.cpp b/services/media_engine/modules/demuxer/frame_detector.cpp new file mode 100644 index 0000000000000000000000000000000000000000..dc3ccce33bd66e1297861891a6fbbe7d28ed90f1 --- /dev/null +++ b/services/media_engine/modules/demuxer/frame_detector.cpp @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "frame_detector.h" + +#include +#include + +#include "common/log.h" + +namespace OHOS { +namespace Media { +using namespace std; + +std::shared_ptr FrameDetector::GetFrameDetector(CodeType type) +{ + switch (type) { + case H264: + static std::shared_ptr frameDetectorH264 = make_shared(); + return frameDetectorH264; + case H265: + static std::shared_ptr frameDetectorH265 = make_shared(); + return frameDetectorH265; + default: + return nullptr; + } +} + +bool FrameDetector::IsContainIdrFrame(const uint8_t* buff, size_t bufSize) +{ + if (buff == nullptr) { + return false; + } + using FirstByteInNalu = uint8_t; + list> posOfNalUnit; + size_t pos = 0; + while (pos < bufSize) { + auto pFound = search(buff + pos, buff + bufSize, begin(START_CODE), end(START_CODE)); + pos = distance(buff, pFound); + if (pos == bufSize || pos + START_CODE_LEN >= bufSize) { // fail to find startCode or just at the end + break; + } + posOfNalUnit.emplace_back(pos, buff[pos + START_CODE_LEN]); + pos += START_CODE_LEN; + } + for (auto it = posOfNalUnit.begin(); it != posOfNalUnit.end(); ++it) { + auto nex = next(it); + NALUInfo nal { + .startPos = it->first, + .endPos = (nex == posOfNalUnit.end()) ? (bufSize) : (nex->first), + .nalType = GetNalType(it->second), + }; + if (IsIDR(nal.nalType)) { + return true; + } + } + return false; +} + +uint8_t FrameDetectorH264::GetNalType(uint8_t byte) +{ + return byte & 0b0001'1111; +} + +bool FrameDetectorH264::IsPPS(uint8_t nalType) +{ + return nalType == H264NalType::PPS; +} + +bool FrameDetectorH264::IsVCL(uint8_t nalType) +{ + return nalType >= H264NalType::NON_IDR && nalType <= H264NalType::IDR; +} + +bool FrameDetectorH264::IsIDR(uint8_t nalType) +{ + return nalType == H264NalType::IDR; +} + +uint8_t FrameDetectorH265::GetNalType(uint8_t byte) +{ + return (byte & 0b0111'1110) >> 1; +} + +bool FrameDetectorH265::IsPPS(uint8_t nalType) +{ + return nalType == H265NalType::HEVC_PPS_NUT; +} + +bool FrameDetectorH265::IsVCL(uint8_t nalType) +{ + return nalType >= H265NalType::HEVC_TRAIL_N && nalType <= H265NalType::HEVC_CRA_NUT; +} + +bool FrameDetectorH265::IsIDR(uint8_t nalType) +{ + return nalType == H265NalType::HEVC_IDR_W_RADL || + nalType == H265NalType::HEVC_IDR_N_LP || + nalType == H265NalType::HEVC_CRA_NUT; +} + +bool FrameDetectorH265::IsPrefixSEI(uint8_t nalType) +{ + return nalType == H265NalType::HEVC_PREFIX_SEI_NUT; +} +} // namespace Media +} // namespace OHOS \ No newline at end of file diff --git a/services/media_engine/modules/demuxer/frame_detector.h b/services/media_engine/modules/demuxer/frame_detector.h new file mode 100644 index 0000000000000000000000000000000000000000..84e7af8cbdd622680aa30e1fbeacfe5aa0f8eba5 --- /dev/null +++ b/services/media_engine/modules/demuxer/frame_detector.h @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OHOS_MEDIA_FRAME_DETECTOR_H +#define OHOS_MEDIA_FRAME_DETECTOR_H + +#include +#include + +namespace OHOS { +namespace Media { +enum CodeType { + H264, + H265 +}; + +class FrameDetector { +public: + static std::shared_ptr GetFrameDetector(CodeType type); + bool IsContainIdrFrame(const uint8_t* buff, size_t bufSize); + +protected: + FrameDetector() = default; + virtual ~FrameDetector() = default; + +private: + struct NALUInfo { + size_t startPos; + size_t endPos; + uint8_t nalType; + }; + + virtual uint8_t GetNalType(uint8_t byte) = 0; + virtual bool IsPPS(uint8_t nalType) = 0; + virtual bool IsVCL(uint8_t nalType) = 0; + virtual bool IsIDR(uint8_t nalType) = 0; + virtual bool IsPrefixSEI(uint8_t nalType) { return false; } + + static constexpr uint8_t START_CODE[] = {0, 0, 1}; + static constexpr size_t START_CODE_LEN = sizeof(START_CODE); +}; + +class FrameDetectorH264 : public FrameDetector { +public: + FrameDetectorH264() = default; + ~FrameDetectorH264() override = default; + +private: + enum H264NalType : uint8_t { + UNSPECIFIED = 0, + NON_IDR = 1, + PARTITION_A = 2, + PARTITION_B = 3, + PARTITION_C = 4, + IDR = 5, + SEI = 6, + SPS = 7, + PPS = 8, + AU_DELIMITER = 9, + END_OF_SEQUENCE = 10, + END_OF_STREAM = 11, + FILLER_DATA = 12, + SPS_EXT = 13, + PREFIX = 14, + SUB_SPS = 15, + DPS = 16, + }; + uint8_t GetNalType(uint8_t byte) override; + bool IsPPS(uint8_t nalType) override; + bool IsVCL(uint8_t nalType) override; + bool IsIDR(uint8_t nalType) override; +}; + +class FrameDetectorH265 : public FrameDetector { +public: + FrameDetectorH265() = default; + ~FrameDetectorH265() override = default; + +private: + enum H265NalType : uint8_t { + HEVC_TRAIL_N = 0, + HEVC_TRAIL_R = 1, + HEVC_TSA_N = 2, + HEVC_TSA_R = 3, + HEVC_STSA_N = 4, + HEVC_STSA_R = 5, + HEVC_RADL_N = 6, + HEVC_RADL_R = 7, + HEVC_RASL_N = 8, + HEVC_RASL_R = 9, + HEVC_BLA_W_LP = 16, + HEVC_BLA_W_RADL = 17, + HEVC_BLA_N_LP = 18, + HEVC_IDR_W_RADL = 19, + HEVC_IDR_N_LP = 20, + HEVC_CRA_NUT = 21, + HEVC_VPS_NUT = 32, + HEVC_SPS_NUT = 33, + HEVC_PPS_NUT = 34, + HEVC_AUD_NUT = 35, + HEVC_EOS_NUT = 36, + HEVC_EOB_NUT = 37, + HEVC_FD_NUT = 38, + HEVC_PREFIX_SEI_NUT = 39, + HEVC_SUFFIX_SEI_NUT = 40, + }; + uint8_t GetNalType(uint8_t byte) override; + bool IsPPS(uint8_t nalType) override; + bool IsVCL(uint8_t nalType) override; + bool IsIDR(uint8_t nalType) override; + bool IsPrefixSEI(uint8_t nalType) override; +}; +} // namespace Media +} // namespace OHOS +#endif // OHOS_MEDIA_FRAME_DETECTOR_H \ No newline at end of file diff --git a/services/media_engine/modules/demuxer/media_demuxer.cpp b/services/media_engine/modules/demuxer/media_demuxer.cpp index 4c16c63ec62e1063b72d312a198c53e916d26a5b..8fea16648b8ddb6f6491bcc761b2d1a49d05db72 100644 --- a/services/media_engine/modules/demuxer/media_demuxer.cpp +++ b/services/media_engine/modules/demuxer/media_demuxer.cpp @@ -15,22 +15,25 @@ #define HST_LOG_TAG "MediaDemuxer" +#include "media_demuxer.h" + #include #include #include + #include "avcodec_common.h" -#include "source/source.h" #include "cpp_ext/type_traits_ext.h" #include "buffer/avallocator.h" +#include "common/event.h" #include "common/log.h" -#include "osal/utils/dump_buffer.h" -#include "plugin/plugin_info.h" +#include "frame_detector.h" #include "meta/media_types.h" #include "meta/meta.h" +#include "osal/utils/dump_buffer.h" +#include "plugin/plugin_info.h" #include "plugin/plugin_manager.h" -#include "common/event.h" #include "plugin/plugin_buffer.h" -#include "media_demuxer.h" +#include "source/source.h" namespace OHOS { namespace Media { @@ -141,7 +144,8 @@ MediaDemuxer::MediaDemuxer() mediaMetaData_(), bufferQueueMap_(), bufferMap_(), - eventReceiver_() + eventReceiver_(), + cacheData_() { MEDIA_LOG_I("MediaDemuxer called"); } @@ -168,6 +172,8 @@ MediaDemuxer::~MediaDemuxer() source_ = nullptr; eventReceiver_ = nullptr; eosMap_.clear(); + cacheData_.data = nullptr; + cacheData_.offset = 0; } void MediaDemuxer::PushData(std::shared_ptr& bufferPtr, uint64_t offset) @@ -275,7 +281,7 @@ Status MediaDemuxer::ProcessDrmInfos() Status MediaDemuxer::SetDataSource(const std::shared_ptr &source) { - MEDIA_LOG_I("SetDataSource"); + MEDIA_LOG_I("SetDataSource enter"); FALSE_RETURN_V_MSG_E(isThreadExit_, Status::ERROR_WRONG_STATE, "Process is running, need to stop if first."); source_->SetCallback(this); source_->SetSource(source); @@ -298,19 +304,19 @@ Status MediaDemuxer::SetDataSource(const std::shared_ptr &source) InitMediaMetaData(mediaInfo); pluginState_ = DemuxerState::DEMUXER_STATE_PARSE_FRAME; } else { - MEDIA_LOG_E("demuxer filter parse meta failed, ret=" PUBLIC_LOG_D32, (int32_t)(ret)); + MEDIA_LOG_E("demuxer filter parse meta failed, ret: " PUBLIC_LOG_D32, (int32_t)(ret)); } ProcessDrmInfos(); - + MEDIA_LOG_I("SetDataSource exit"); return ret; } Status MediaDemuxer::SetOutputBufferQueue(int32_t trackId, const sptr& producer) { std::unique_lock lock(mutex_); - FALSE_RETURN_V_MSG_E(trackId >= 0 && trackId < mediaMetaData_.trackMetas.size(), Status::ERROR_INVALID_PARAMETER, - "Set bufferQueue trackId error."); + FALSE_RETURN_V_MSG_E(trackId >= 0 && (uint32_t)trackId < mediaMetaData_.trackMetas.size(), + Status::ERROR_INVALID_PARAMETER, "Set bufferQueue trackId error."); useBufferQueue_ = true; MEDIA_LOG_I("Set bufferQueue for track " PUBLIC_LOG_D32 ".", trackId); FALSE_RETURN_V_MSG_E(isThreadExit_, Status::ERROR_WRONG_STATE, "Process is running, need to stop if first."); @@ -336,8 +342,8 @@ Status MediaDemuxer::InnerSelectTrack(int32_t trackId) Status MediaDemuxer::SelectTrack(int32_t trackId) { - FALSE_RETURN_V_MSG_E(trackId >= 0 && trackId < mediaMetaData_.trackMetas.size(), Status::ERROR_INVALID_PARAMETER, - "Select trackId error."); + FALSE_RETURN_V_MSG_E(trackId >= 0 && (uint32_t)trackId < mediaMetaData_.trackMetas.size(), + Status::ERROR_INVALID_PARAMETER, "Select trackId error."); FALSE_RETURN_V_MSG_E(!useBufferQueue_, Status::ERROR_WRONG_STATE, "Cannot select track when use buffer queue."); return InnerSelectTrack(trackId); } @@ -350,19 +356,24 @@ Status MediaDemuxer::UnselectTrack(int32_t trackId) Status MediaDemuxer::SeekTo(int64_t seekTime, Plugins::SeekMode mode, int64_t& realSeekTime) { - FALSE_RETURN_V_MSG_E(isThreadExit_, Status::ERROR_WRONG_STATE, "Process is running, need to stop if first."); - if (!plugin_) { - MEDIA_LOG_E("SeekTo failed due to no valid plugin"); - return Status::ERROR_INVALID_OPERATION; - } - auto rtv = plugin_->SeekTo(-1, seekTime, mode, realSeekTime); - if (rtv != Status::OK) { - MEDIA_LOG_E("SeekTo failed with return value: " PUBLIC_LOG_D32, static_cast(rtv)); + FALSE_RETURN_V_MSG_E(plugin_ != nullptr, Status::ERROR_INVALID_OPERATION, "SeekTo error seekTime: " PUBLIC_LOG_D64 + ", mode: " PUBLIC_LOG_U32, seekTime, (uint32_t)mode); + lastSeekTime_ = seekTime; + Status ret = Status::ERROR_INVALID_OPERATION; + if (source_ != nullptr && source_->IsSeekToTimeSupported()) { + MEDIA_LOG_I("SeekTo source SeekToTime start"); + ret = source_->SeekToTime(seekTime); + } else { + MEDIA_LOG_I("SeekTo start"); + ret = plugin_->SeekTo(-1, seekTime, mode, realSeekTime); } + isSeeked_ = true; + lastSeekTime_ = Plugins::HST_TIME_NONE; for (auto item : eosMap_) { eosMap_[item.first] = false; } - return rtv; + MEDIA_LOG_I("SeekTo done"); + return ret; } Status MediaDemuxer::SelectBitRate(uint32_t bitRate) @@ -371,6 +382,8 @@ Status MediaDemuxer::SelectBitRate(uint32_t bitRate) MEDIA_LOG_E("SelectBitRate failed, source_ is nullptr"); return Status::ERROR_INVALID_OPERATION; } + cacheData_.data = nullptr; + cacheData_.offset = 0; return source_->SelectBitRate(bitRate); } @@ -495,6 +508,67 @@ bool MediaDemuxer::InitPlugin(std::string pluginName) return st == Status::OK; } +bool MediaDemuxer::PullDataWithCache(uint64_t offset, size_t size, std::shared_ptr& bufferPtr) +{ + if (cacheData_.data == nullptr || cacheData_.data->GetMemory() == nullptr || + offset < cacheData_.offset || offset >= (cacheData_.offset + cacheData_.data->GetMemory()->GetSize())) { + return false; + } + auto memory = cacheData_.data->GetMemory(); + if (memory == nullptr || memory->GetSize() <= 0) { + return false; + } + + MEDIA_LOG_I("PullMode, Read data from cache data."); + uint64_t offsetInCache = offset - cacheData_.offset; + if (size <= memory->GetSize() - offsetInCache) { + bufferPtr->GetMemory()->Write(memory->GetReadOnlyData() + offsetInCache, size, 0); + return true; + } + bufferPtr->GetMemory()->Write(memory->GetReadOnlyData() + offsetInCache, memory->GetSize() - offsetInCache, 0); + uint64_t remainOffset = cacheData_.offset + memory->GetSize(); + uint64_t remainSize = size - (memory->GetSize() - offsetInCache); + std::shared_ptr tempBuffer = Buffer::CreateDefaultBuffer(remainSize); + Status ret = source_->PullData(remainOffset, lastSeekTime_, remainSize, tempBuffer); + if (ret == Status::OK) { + std::shared_ptr mergedBuffer = Buffer::CreateDefaultBuffer( + tempBuffer->GetMemory()->GetSize() + memory->GetSize()); + mergedBuffer->GetMemory()->Write(memory->GetReadOnlyData(), memory->GetSize(), 0); + mergedBuffer->GetMemory()->Write(tempBuffer->GetMemory()->GetReadOnlyData(), + tempBuffer->GetMemory()->GetSize(), memory->GetSize()); + bufferPtr->GetMemory()->Write(tempBuffer->GetMemory()->GetReadOnlyData(), + tempBuffer->GetMemory()->GetSize(), memory->GetSize() - offsetInCache); + cacheData_.data = mergedBuffer; + MEDIA_LOG_I("PullMode, offset: " PUBLIC_LOG_U64 ", cache offset: " PUBLIC_LOG_U64 + ", cache size: " PUBLIC_LOG_ZU, offset, cacheData_.offset, + cacheData_.data->GetMemory()->GetSize()); + } + return ret == Status::OK; +} + +bool MediaDemuxer::PullDataWithoutCache(uint64_t offset, size_t size, std::shared_ptr& bufferPtr) +{ + Status ret = source_->PullData(offset, lastSeekTime_, size, bufferPtr); + if ((cacheData_.data == nullptr || cacheData_.data->GetMemory() == nullptr) && ret == Status::OK) { + MEDIA_LOG_I("PullMode, write cache data."); + if (bufferPtr->GetMemory() == nullptr) { + MEDIA_LOG_W("PullMode, write cache data error. memory is nullptr!"); + } else { + auto buffer = Buffer::CreateDefaultBuffer(bufferPtr->GetMemory()->GetSize()); + if (buffer != nullptr && buffer->GetMemory() != nullptr) { + buffer->GetMemory()->Write(bufferPtr->GetMemory()->GetReadOnlyData(), + bufferPtr->GetMemory()->GetSize(), 0); + cacheData_.data = buffer; + cacheData_.offset = offset; + MEDIA_LOG_I("PullMode, write cache data success."); + } else { + MEDIA_LOG_W("PullMode, write cache data failed. memory is nullptr!"); + } + } + } + return ret == Status::OK; +} + void MediaDemuxer::ActivatePullMode() { MEDIA_LOG_I("ActivatePullMode called"); @@ -503,14 +577,27 @@ void MediaDemuxer::ActivatePullMode() return true; }; peekRange_ = [this](uint64_t offset, size_t size, std::shared_ptr& bufferPtr) -> bool { - auto ret = source_->PullData(offset, size, bufferPtr); - return ret == Status::OK; + if (pluginState_.load() == DemuxerState::DEMUXER_STATE_PARSE_FRAME) { + MEDIA_LOG_D("PullMode, DemuxerState::DEMUXER_STATE_PARSE_FRAME"); + return Status::OK == source_->PullData(offset, lastSeekTime_, size, bufferPtr); + } + MEDIA_LOG_D("PullMode, offset: " PUBLIC_LOG_U64 ", cache offset: " PUBLIC_LOG_U64 + ", cache data: " PUBLIC_LOG_D32, offset, cacheData_.offset, (int32_t)(cacheData_.data != nullptr)); + if (cacheData_.data != nullptr && cacheData_.data->GetMemory() != nullptr && + offset >= cacheData_.offset && offset < (cacheData_.offset + cacheData_.data->GetMemory()->GetSize())) { + auto memory = cacheData_.data->GetMemory(); + if (memory != nullptr && memory->GetSize() > 0) { + MEDIA_LOG_I("PullMode, Read data from cache data."); + return PullDataWithCache(offset, size, bufferPtr); + } + } + return PullDataWithCache(offset, size, bufferPtr); }; getRange_ = peekRange_; typeFinder_->Init(uri_, mediaDataSize_, checkRange_, peekRange_); std::string type = typeFinder_->FindMediaType(); FALSE_RETURN_MSG(!type.empty(), "Find media type failed"); - MEDIA_LOG_I("PullMode FindMediaType result : type : " PUBLIC_LOG_S ", uri_ : %{private}s, mediaDataSize_ : " + MEDIA_LOG_I("PullMode FindMediaType result type: " PUBLIC_LOG_S ", uri: %{private}s, mediaDataSize: " PUBLIC_LOG_U64, type.c_str(), uri_.c_str(), mediaDataSize_); MediaTypeFound(std::move(type)); } @@ -546,10 +633,24 @@ void MediaDemuxer::MediaTypeFound(std::string pluginName) void MediaDemuxer::InitMediaMetaData(const Plugins::MediaInfo& mediaInfo) { mediaMetaData_.globalMeta = std::make_shared(mediaInfo.general); + if (source_ != nullptr && source_->IsSeekToTimeSupported()) { + int64_t duration = source_->GetDuration(); + if (duration != Plugins::HST_TIME_NONE) { + MEDIA_LOG_I("InitMediaMetaData for hls, duration: " PUBLIC_LOG_D64, duration); + mediaMetaData_.globalMeta->Set(Plugins::HstTime2Us(duration)); + } + } mediaMetaData_.trackMetas.clear(); mediaMetaData_.trackMetas.reserve(mediaInfo.tracks.size()); - for (auto& trackMeta : mediaInfo.tracks) { + for (uint32_t index = 0; index < mediaInfo.tracks.size(); index++) { + auto trackMeta = mediaInfo.tracks[index]; mediaMetaData_.trackMetas.emplace_back(std::make_shared(trackMeta)); + std::string mimeType; + if (trackMeta.Get(mimeType) && mimeType.find("video") == 0) { + MEDIA_LOG_I("Found video track, id: " PUBLIC_LOG_U32 ", mimeType: " PUBLIC_LOG_S, index, mimeType.c_str()); + videoMime_ = mimeType; + videoTrackId_ = index; + } } } @@ -561,46 +662,51 @@ bool MediaDemuxer::IsOffsetValid(int64_t offset) const return true; } -bool MediaDemuxer::GetBufferFromUserQueue(uint32_t queueIndex, int32_t size) +bool MediaDemuxer::GetBufferFromUserQueue(uint32_t queueIndex, uint32_t size) { - MEDIA_LOG_I("Get buffer from user queue " PUBLIC_LOG_D32 ".", queueIndex); + MEDIA_LOG_D("Get buffer from user queue: " PUBLIC_LOG_U32 ", size: " PUBLIC_LOG_U32, queueIndex, size); FALSE_RETURN_V_MSG_E(bufferQueueMap_.count(queueIndex) > 0 && bufferQueueMap_[queueIndex] != nullptr, false, "bufferQueue " PUBLIC_LOG_D32 " is nullptr", queueIndex); AVBufferConfig avBufferConfig; avBufferConfig.capacity = size; - Status ret = bufferQueueMap_[queueIndex]->RequestBuffer(bufferMap_[queueIndex], - avBufferConfig, REQUEST_BUFFER_TIMEOUT); - if (ret != Status::OK) { - MEDIA_LOG_I("Get buffer failed due to get buffer from bufferQueue failed, ret=" PUBLIC_LOG_D32, - (int32_t)(ret)); - return false; - } - return true; + Status ret = bufferQueueMap_[queueIndex]->RequestBuffer(bufferMap_[queueIndex], avBufferConfig, + REQUEST_BUFFER_TIMEOUT); + FALSE_LOG_MSG_W(ret == Status::OK, "Get buffer failed due to get buffer from bufferQueue failed, user queue: " + PUBLIC_LOG_U32 ", ret: " PUBLIC_LOG_D32, queueIndex, (int32_t)(ret)); + return ret == Status::OK; } Status MediaDemuxer::CopyFrameToUserQueue(uint32_t trackId) { - MEDIA_LOG_D("Copy one loop, trackId=" PUBLIC_LOG_U32, trackId); + MEDIA_LOG_D("CopyFrameToUserQueue enter, copy frame for track: " PUBLIC_LOG_U32, trackId); Status ret; uint32_t size = plugin_->GetNextSampleSize(trackId); - if (size == 0) { - MEDIA_LOG_D("No more cache in track " PUBLIC_LOG_U32, trackId); - return Status::ERROR_INVALID_PARAMETER; - } - - if (!GetBufferFromUserQueue(trackId, size)) { - MEDIA_LOG_D("Get buffer from queue failed "); - return Status::ERROR_INVALID_PARAMETER; - } + FALSE_RETURN_V_MSG_E(size != 0, Status::ERROR_INVALID_PARAMETER, "No more cache in track " PUBLIC_LOG_U32, trackId); + FALSE_RETURN_V_MSG_E(GetBufferFromUserQueue(trackId, size), Status::ERROR_INVALID_PARAMETER, + "Get buffer from queue failed, trackId: " PUBLIC_LOG_U32, trackId); ret = InnerReadSample(trackId, bufferMap_[trackId]); + if (source_ != nullptr && source_->IsSeekToTimeSupported() && isSeeked_) { + if (trackId != videoTrackId_ || ret != Status::OK || + !IsContainIdrFrame(bufferMap_[trackId]->memory_->GetAddr(), bufferMap_[trackId]->memory_->GetSize())) { + MEDIA_LOG_I("CopyFrameToUserQueue is seeking, not found idr frame. trackId: " PUBLIC_LOG_U32, trackId); + bufferQueueMap_[trackId]->PushBuffer(bufferMap_[trackId], false); + return Status::ERROR_INVALID_PARAMETER; + } + MEDIA_LOG_I("CopyFrameToUserQueue is seeking, found idr frame. trackId: " PUBLIC_LOG_U32, trackId); + isSeeked_ = false; + } if (ret == Status::OK || ret == Status::END_OF_STREAM) { if (bufferMap_[trackId]->flag_ & (uint32_t)(AVBufferFlag::EOS)) { eosMap_[trackId] = true; + MEDIA_LOG_I("CopyFrameToUserQueue track eos, trackId: " PUBLIC_LOG_U32 ", bufferId: " PUBLIC_LOG_U64 + ", pts: " PUBLIC_LOG_U64 ", flag: " PUBLIC_LOG_U32, trackId, bufferMap_[trackId]->GetUniqueId(), + bufferMap_[trackId]->pts_, bufferMap_[trackId]->flag_); } ret = bufferQueueMap_[trackId]->PushBuffer(bufferMap_[trackId], true); } + MEDIA_LOG_D("CopyFrameToUserQueue exit, copy frame for track: " PUBLIC_LOG_U32, trackId); return Status::OK; } @@ -609,11 +715,12 @@ Status MediaDemuxer::InnerReadSample(uint32_t trackId, std::shared_ptr MEDIA_LOG_D("copy frame for track " PUBLIC_LOG_U32, trackId); Status ret = plugin_->ReadSample(trackId, sample); if (ret == Status::END_OF_STREAM) { - MEDIA_LOG_I("Read eos buffer for track " PUBLIC_LOG_U32, trackId); + MEDIA_LOG_I("Read buffer eos for track " PUBLIC_LOG_U32, trackId); } else if (ret != Status::OK) { - MEDIA_LOG_I("Read buffer for track " PUBLIC_LOG_U32 " error, ret=" PUBLIC_LOG_D32, trackId, (uint32_t)(ret)); + MEDIA_LOG_I("Read buffer error for track " PUBLIC_LOG_U32 ", ret: " PUBLIC_LOG_D32, trackId, (int32_t)(ret)); } MEDIA_LOG_D("finish copy frame for track " PUBLIC_LOG_U32, trackId); + // to get DrmInfo ProcessDrmInfos(); return ret; @@ -630,13 +737,33 @@ void MediaDemuxer::ReadLoop(uint32_t trackId) break; } if (eosMap_[trackId]) { - MEDIA_LOG_I("Exit [" PUBLIC_LOG_S "] read thread, all track reach eos.", threadReadName.c_str()); + MEDIA_LOG_I("Exit [" PUBLIC_LOG_S "] read thread, track reach eos, trackId: " PUBLIC_LOG_U32, + threadReadName.c_str(), trackId); break; } (void)CopyFrameToUserQueue(trackId); } } +bool MediaDemuxer::IsContainIdrFrame(const uint8_t* buff, size_t bufSize) +{ + if (buff == nullptr) { + return false; + } + std::shared_ptr frameDetector; + if (videoMime_ == std::string(MimeType::VIDEO_AVC)) { + frameDetector = FrameDetector::GetFrameDetector(CodeType::H264); + } else if (videoMime_ == std::string(MimeType::VIDEO_HEVC)) { + frameDetector = FrameDetector::GetFrameDetector(CodeType::H265); + } else { + return true; + } + if (frameDetector == nullptr) { + return true; + } + return frameDetector->IsContainIdrFrame(buff, bufSize); +} + Status MediaDemuxer::ReadSample(uint32_t trackId, std::shared_ptr sample) { FALSE_RETURN_V_MSG_E(!useBufferQueue_, Status::ERROR_WRONG_STATE, diff --git a/services/media_engine/modules/demuxer/type_finder.cpp b/services/media_engine/modules/demuxer/type_finder.cpp index 4e5a12eb3521a99baab4bc21b2a2b6491e5d6dc4..60f73332d46a77306e25459d6de45578d1650b2e 100644 --- a/services/media_engine/modules/demuxer/type_finder.cpp +++ b/services/media_engine/modules/demuxer/type_finder.cpp @@ -15,12 +15,14 @@ #define HST_LOG_TAG "TypeFinder" +#include "type_finder.h" + #include + #include "common/log.h" #include "meta/any.h" #include "osal/utils/util.h" #include "plugin/plugin_info.h" -#include "demuxer/type_finder.h" namespace OHOS { namespace Media { @@ -70,8 +72,7 @@ TypeFinder::TypeFinder() task_(nullptr), checkRange_(), peekRange_(), - typeFound_(), - sniffData_(nullptr) + typeFound_() { MEDIA_LOG_D("TypeFinder ctor called..."); } @@ -82,9 +83,6 @@ TypeFinder::~TypeFinder() if (task_) { task_->Stop(); } - sniffData_.reset(); - sniffData_->Reset(); - sniffData_ = nullptr; } bool TypeFinder::IsSniffNeeded(std::string uri) @@ -149,14 +147,7 @@ Status TypeFinder::ReadAt(int64_t offset, std::shared_ptr& buffer, size_ PUBLIC_LOG_D64, !buffer, expectedLen, offset); return Status::ERROR_INVALID_PARAMETER; } - if (sniffNeeded_ && sniffData_ != nullptr) { - auto memory = sniffData_->GetMemory(); - if (memory != nullptr && memory->GetSize() > 0) { - MEDIA_LOG_I("Has sniff data already."); - buffer->GetMemory()->Write(memory->GetReadOnlyData(), memory->GetSize(), 0); - return Status::OK; - } - } + const int maxTryTimes = 3; int i = 0; while (!checkRange_(offset, expectedLen) && (i++ < maxTryTimes)) { @@ -167,19 +158,6 @@ Status TypeFinder::ReadAt(int64_t offset, std::shared_ptr& buffer, size_ return Status::ERROR_NOT_ENOUGH_DATA; } FALSE_LOG_MSG(peekRange_(static_cast(offset), expectedLen, buffer), "peekRange failed."); - - if (sniffNeeded_ && sniffData_ == nullptr) { - auto memory = buffer->GetMemory(); - if (memory != nullptr) { - MEDIA_LOG_I("Write sniffData_ start."); - sniffData_ = Buffer::CreateDefaultBuffer(memory->GetSize()); - if (sniffData_ != nullptr && sniffData_->GetMemory() != nullptr) { - sniffData_->GetMemory()->Write(memory->GetReadOnlyData(), memory->GetSize(), 0); - } else { - MEDIA_LOG_E("Write sniffData_ failed."); - } - } - } return Status::OK; } @@ -249,7 +227,8 @@ std::string TypeFinder::GuessMediaType() const bool TypeFinder::IsOffsetValid(int64_t offset) const { - return (mediaDataSize_ == -1) || offset < static_cast(mediaDataSize_); + return (mediaDataSize_ == 0) || (static_cast(mediaDataSize_) == -1) || + offset < static_cast(mediaDataSize_); } bool TypeFinder::GetPlugins() diff --git a/services/media_engine/modules/demuxer/type_finder.h b/services/media_engine/modules/demuxer/type_finder.h index f1e28700f1370a1212aa3fdbc7ef510d07977972..d4fe45212df915556a091297153ded0ad180e0c6 100644 --- a/services/media_engine/modules/demuxer/type_finder.h +++ b/services/media_engine/modules/demuxer/type_finder.h @@ -73,7 +73,6 @@ private: std::function checkRange_; std::function&)> peekRange_; std::function typeFound_; - std::shared_ptr sniffData_; }; } // namespace Media } // namespace OHOS diff --git a/services/media_engine/modules/sink/video_sink.cpp b/services/media_engine/modules/sink/video_sink.cpp index 7c50f3aa48dd6cd2ac908f5c2730ad4cbfa5bf19..6120b9cb10d68156340ede02b7e7074ef1dbd42d 100644 --- a/services/media_engine/modules/sink/video_sink.cpp +++ b/services/media_engine/modules/sink/video_sink.cpp @@ -13,15 +13,19 @@ * limitations under the License. */ #include "video_sink.h" -#include "osal/task/jobutils.h" + +#include + #include "common/log.h" #include "media_sync_manager.h" +#include "osal/task/jobutils.h" namespace OHOS { namespace Media { namespace Pipeline { /// Video Key Frame Flag constexpr int BUFFER_FLAG_KEY_FRAME = 0x00000002; +constexpr int64_t WAIT_TIME_MS_THRESHOLD = 80; VideoSink::VideoSink() { @@ -101,7 +105,7 @@ bool VideoSink::CheckBufferLatenessMayWait(const std::shared_ptrGetClockTime(buffer->pts_); - if (ct4Buffer != HST_TIME_NONE) { + if (ct4Buffer != Plugins::HST_TIME_NONE) { auto nowCt = syncCenter->GetClockTimeNow(); uint64_t latency = 0; GetLatency(latency); @@ -111,6 +115,8 @@ bool VideoSink::CheckBufferLatenessMayWait(const std::shared_ptr 0 && Plugins::HstTime2Ms(diff * HST_USECOND) > 40) { // > 40ms // buffer is late diff --git a/services/media_engine/modules/source/source.cpp b/services/media_engine/modules/source/source.cpp index 0d12b4867301dcb1326ded2717175bf4658e0d0f..116c115e49c24fa7355a66d55f6f53f6ead0f940 100644 --- a/services/media_engine/modules/source/source.cpp +++ b/services/media_engine/modules/source/source.cpp @@ -83,25 +83,29 @@ void Source::ClearData() mediaOffset_ = 0; isPluginReady_ = false; isAboveWaterline_ = false; + isHls_ = false; } Status Source::SetSource(const std::shared_ptr& source) { - MEDIA_LOG_I("SetSource entered."); - if (source == nullptr) { - MEDIA_LOG_E("Invalid source"); - return Status::ERROR_INVALID_PARAMETER; - } + MEDIA_LOG_I("SetSource enter."); + FALSE_RETURN_V_MSG_E(source != nullptr, Status::ERROR_INVALID_PARAMETER, "SetSource Invalid source"); + ClearData(); - Status err = FindPlugin(source); - if (err != Status::OK) { - return err; - } - err = InitPlugin(source); - if (err != Status::OK) { - return err; + Status ret = FindPlugin(source); + FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "SetSource FindPlugin failed"); + + ret = InitPlugin(source); + FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "SetSource InitPlugin failed"); + + if (plugin_ != nullptr) { + isHls_ = plugin_->IsSeekToTimeSupported(); } + MEDIA_LOG_I("SetSource isHls: " PUBLIC_LOG_D32, isHls_); + ActivateMode(); + + MEDIA_LOG_I("SetSource exit."); return Status::OK; } @@ -113,13 +117,17 @@ Status Source::SetPushData(const std::shared_ptr& data) Status Source::InitPlugin(const std::shared_ptr& source) { - MEDIA_LOG_D("IN"); - Status err = plugin_->Init(); - if (err != Status::OK) { - return err; - } + MEDIA_LOG_I("InitPlugin enter"); + FALSE_RETURN_V_MSG_E(plugin_ != nullptr, Status::ERROR_INVALID_OPERATION, "InitPlugin, Source plugin is nullptr"); + + Status ret = plugin_->Init(); + FALSE_RETURN_V_MSG_E(ret == Status::OK, ret, "InitPlugin failed"); + plugin_->SetCallback(this); - return plugin_->SetSource(source); + ret = plugin_->SetSource(source); + + MEDIA_LOG_I("InitPlugin exit"); + return ret; } Status Source::Prepare() @@ -142,47 +150,58 @@ Status Source::Prepare() return ret; } +bool Source::IsSeekToTimeSupported() +{ + return isHls_; +} + Status Source::Start() { MEDIA_LOG_I("Start entered."); return plugin_ ? plugin_->Start() : Status::ERROR_INVALID_OPERATION; } -Status Source::PullData(uint64_t offset, size_t size, std::shared_ptr& data) +Status Source::PullData(uint64_t offset, int64_t seekTime, size_t size, std::shared_ptr& data) { - MEDIA_LOG_DD("IN, offset: " PUBLIC_LOG_U64 ", size: " PUBLIC_LOG_ZU ", outPort: " PUBLIC_LOG_S, - offset, size, outPort.c_str()); + MEDIA_LOG_DD("IN, offset: " PUBLIC_LOG_U64 ", size: " PUBLIC_LOG_ZU ", seekTime: " PUBLIC_LOG_D64 + ", position: " PUBLIC_LOG_U64, offset, size, seekTime, position_); if (!plugin_) { return Status::ERROR_INVALID_OPERATION; } Status err; auto readSize = size; - if (seekable_ == Seekable::SEEKABLE) { - uint64_t totalSize = 0; - if ((plugin_->GetSize(totalSize) == Status::OK) && (totalSize != 0)) { - if (offset >= totalSize) { - MEDIA_LOG_W("Offset: " PUBLIC_LOG_U64 " is larger than totalSize: " PUBLIC_LOG_U64, - offset, totalSize); - return Status::END_OF_STREAM; - } - if ((offset + readSize) > totalSize) { - readSize = totalSize - offset; - } - if (data->GetMemory() != nullptr) { - auto realSize = data->GetMemory()->GetCapacity(); - readSize = (readSize > realSize) ? realSize : readSize; - } - MEDIA_LOG_DD("TotalSize_: " PUBLIC_LOG_U64, totalSize); + if (seekable_ != Seekable::SEEKABLE) { + err = plugin_->Read(data, offset, readSize); + FALSE_LOG_MSG(err == Status::OK, "Unseekable, plugin read failed."); + return err; + } + if (isHls_) { + err = plugin_->Read(data, offset, readSize); + FALSE_LOG_MSG(err == Status::OK, "hls, plugin read failed."); + return err; + } + + uint64_t totalSize = 0; + if ((plugin_->GetSize(totalSize) == Status::OK) && (totalSize != 0)) { + if (offset >= totalSize) { + MEDIA_LOG_W("Offset: " PUBLIC_LOG_U64 " is larger than totalSize: " PUBLIC_LOG_U64, offset, totalSize); + return Status::END_OF_STREAM; } - if (position_ != offset) { - err = plugin_->SeekTo(offset); - if (err != Status::OK) { - MEDIA_LOG_E("Seek to " PUBLIC_LOG_U64 " fail", offset); - return err; - } - position_ = offset; + if ((offset + readSize) > totalSize) { + readSize = totalSize - offset; + } + if (data->GetMemory() != nullptr) { + auto realSize = data->GetMemory()->GetCapacity(); + readSize = (readSize > realSize) ? realSize : readSize; } + MEDIA_LOG_DD("TotalSize_: " PUBLIC_LOG_U64, totalSize); + } + if (position_ != offset) { + err = plugin_->SeekTo(offset); + FALSE_RETURN_V_MSG_E(err == Status::OK, err, "Seek to " PUBLIC_LOG_U64 " fail", offset); + position_ = offset; } + err = plugin_->Read(data, offset, readSize); if (err == Status::OK) { position_ += data->GetMemory()->GetSize(); @@ -210,6 +229,15 @@ Status Source::SelectBitRate(uint32_t bitRate) return plugin_->SelectBitRate(bitRate); } +Status Source::SeekToTime(int64_t seekTime) +{ + int64_t timeNs; + if (Plugins::Ms2HstTime(seekTime, timeNs)) { + return plugin_->SeekToTime(timeNs); + } else { + return Status::ERROR_INVALID_PARAMETER; + } +} Status Source::Stop() { MEDIA_LOG_I("Stop entered."); @@ -240,7 +268,7 @@ void Source::OnEvent(const Plugins::PluginEvent& event) void Source::ActivateMode() { - MEDIA_LOG_D("ActivateMode"); + MEDIA_LOG_I("ActivateMode enter"); int32_t retry {0}; do { if (plugin_) { @@ -254,15 +282,17 @@ void Source::ActivateMode() OSAL::SleepFor(10); // 10 means sleep time pre retry } } while (seekable_ == Seekable::INVALID); + FALSE_LOG(seekable_ != Seekable::INVALID); if (seekable_ == Seekable::UNSEEKABLE) { if (taskPtr_ == nullptr) { taskPtr_ = std::make_shared("DataReader"); taskPtr_->RegisterJob([this] { ReadLoop(); }); } - MEDIA_LOG_D("Source task start"); + MEDIA_LOG_I("ActivateMode Source task start"); taskPtr_->Start(); } + MEDIA_LOG_I("ActivateMode exit"); } Plugins::Seekable Source::GetSeekable() @@ -409,12 +439,18 @@ Status Source::FindPlugin(const std::shared_ptr& source) return Status::ERROR_UNSUPPORTED_FORMAT; } +int64_t Source::GetDuration() +{ + FALSE_RETURN_V_MSG_W(isHls_, Plugins::HST_TIME_NONE, "Source GetDuration return -1 for isHls false."); + int64_t duration; + Status ret = plugin_->GetDuration(duration); + FALSE_RETURN_V_MSG_W(ret == Status::OK, Plugins::HST_TIME_NONE, "Source GetDuration from source plugin failed."); + return duration; +} + Status Source::GetSize(uint64_t &fileSize) { - if (plugin_ == nullptr) { - MEDIA_LOG_E("plugin_ is nullptr!"); - return Status::ERROR_INVALID_OPERATION; - } + FALSE_RETURN_V_MSG_W(plugin_ != nullptr, Status::ERROR_INVALID_OPERATION, "GetSize Source plugin is nullptr!"); return plugin_->GetSize(fileSize); } } // namespace Media diff --git a/services/media_engine/modules/source/source.h b/services/media_engine/modules/source/source.h index 0d4416a381b516266a18f024febcaf1f951b3eef..a6f0031d313bd9e554c152a8a19749ad3343546a 100644 --- a/services/media_engine/modules/source/source.h +++ b/services/media_engine/modules/source/source.h @@ -56,7 +56,7 @@ public: explicit Source(); ~Source() override; - Status PullData(uint64_t offset, size_t size, std::shared_ptr& data); + Status PullData(uint64_t offset, int64_t seekTime, size_t size, std::shared_ptr& data); virtual Status SetSource(const std::shared_ptr& source); Status Prepare(); Status Start(); @@ -66,8 +66,10 @@ public: Status GetSize(uint64_t &fileSize); void OnEvent(const Plugins::PluginEvent &event) override; - + bool IsSeekToTimeSupported(); Status SetPushData(const std::shared_ptr& data); + int64_t GetDuration(); + Status SeekToTime(int64_t seekTime); Status GetBitRates(std::vector& bitRates); Status SelectBitRate(uint32_t bitRate); void SetCallback(Callback* callback); @@ -87,6 +89,7 @@ private: std::shared_ptr taskPtr_; std::string protocol_; + bool isHls_{false}; std::string uri_; Plugins::Seekable seekable_; uint64_t position_; diff --git a/services/media_engine/plugins/ffmpeg_adapter/demuxer/ffmpeg_demuxer_plugin.cpp b/services/media_engine/plugins/ffmpeg_adapter/demuxer/ffmpeg_demuxer_plugin.cpp index e1495a33657b3acfd628ddb311da8d49058117c2..007070341a743f79a3e631b7d5007fd89360098c 100644 --- a/services/media_engine/plugins/ffmpeg_adapter/demuxer/ffmpeg_demuxer_plugin.cpp +++ b/services/media_engine/plugins/ffmpeg_adapter/demuxer/ffmpeg_demuxer_plugin.cpp @@ -453,6 +453,7 @@ void FFmpegDemuxerPlugin::PushEOSToAllCache() Status FFmpegDemuxerPlugin::ReadPacketToCacheQueue() { + MEDIA_LOG_D("Read next frame enter."); std::unique_lock lock(mutex_); MEDIA_LOG_D("Read next frame."); int ffmpegRet = 0; @@ -990,7 +991,7 @@ Status FFmpegDemuxerPlugin::ReadSample(uint32_t trackId, std::shared_ptr samplePacket = cacheQueue_.Front(trackId); FALSE_RETURN_V_MSG_E(samplePacket != nullptr, 0, "Get next sample size failed due to cache sample is nullptr"); if (samplePacket->isEOS) { + MEDIA_LOG_I("Get size for track " PUBLIC_LOG_D32 " EOS.", trackId); return -1; } - FALSE_RETURN_V_MSG_E(samplePacket->pkt != nullptr, 0, - "Get next sample size failed due to cache sample is nullptr"); + FALSE_RETURN_V_MSG_E(samplePacket->pkt != nullptr, 0, "Get next sample size failed due to cache sample is nullptr"); return samplePacket->pkt->size; } diff --git a/services/media_engine/plugins/ffmpeg_adapter/demuxer/ffmpeg_demuxer_plugin.h b/services/media_engine/plugins/ffmpeg_adapter/demuxer/ffmpeg_demuxer_plugin.h index ef2f68e20b2d66df419f3cfc47f36a55dec4c771..0088cab67cc0244d36c409ae20ff5eb57a37141b 100644 --- a/services/media_engine/plugins/ffmpeg_adapter/demuxer/ffmpeg_demuxer_plugin.h +++ b/services/media_engine/plugins/ffmpeg_adapter/demuxer/ffmpeg_demuxer_plugin.h @@ -76,8 +76,8 @@ private: Status SetDrmCencInfo(std::shared_ptr sample, std::shared_ptr samplePacket); Status ConvertAVPacketToSample(std::shared_ptr sample, std::shared_ptr samplePacket); Status ReadEosSample(std::shared_ptr sample); - Status WriteBuffer( - std::shared_ptr outBuffer, int64_t pts, uint32_t flag, const uint8_t *writeData, int32_t writeSize); + Status WriteBuffer(std::shared_ptr outBuffer, int64_t pts, uint32_t flag, const uint8_t *writeData, + int32_t writeSize); void ParseDrmInfo(const MetaDrmInfo *const metaDrmInfo, int32_t drmInfoSize, std::multimap>& drmInfo); diff --git a/services/media_engine/plugins/source/http_source/hls/hls_media_downloader.cpp b/services/media_engine/plugins/source/http_source/hls/hls_media_downloader.cpp index a6ca1bddd07c255ddee203d0185752cd2b8dd364..391de8f62b9d5d255c33b0601a2eb454aec400b4 100644 --- a/services/media_engine/plugins/source/http_source/hls/hls_media_downloader.cpp +++ b/services/media_engine/plugins/source/http_source/hls/hls_media_downloader.cpp @@ -39,7 +39,7 @@ HlsMediaDownloader::HlsMediaDownloader() noexcept buffer_->Init(); downloader_ = std::make_shared("hlsMedia"); - playList_ = std::make_shared>("PlayList", 5000); // 5000 + playList_ = std::make_shared>("PlayList", 5000); // 5000 to prevent blocking download dataSave_ = [this] (uint8_t*&& data, uint32_t&& len) { return SaveData(std::forward(data), std::forward(len)); @@ -115,7 +115,7 @@ bool HlsMediaDownloader::Read(unsigned char* buff, unsigned int wantReadLength, bool HlsMediaDownloader::SeekToTime(int64_t seekTime) { FALSE_RETURN_V(buffer_ != nullptr, false); - MEDIA_LOG_I("Seek: buffer size " PUBLIC_LOG_ZU ", offset " PUBLIC_LOG_D64, buffer_->GetSize(), seekTime); + MEDIA_LOG_I("Seek: buffer size " PUBLIC_LOG_ZU ", seekTime " PUBLIC_LOG_D64, buffer_->GetSize(), seekTime); buffer_->Clear(); // First clear buffer, avoid no available buffer then task pause never exit. downloader_->Cancle(); buffer_->Clear(); @@ -308,7 +308,7 @@ void HlsMediaDownloader::UpdateDownloadFinished(std::string url) { uint32_t bitRate = downloadRequest_->GetBitRate(); if ((curUrl_ == url) && (bitRate > 0)) { - SelectBitRate(bitRate); + MEDIA_LOG_I("SelectBitRate(bitRate) not support for now"); } if (!playList_->Empty()) { auto playInfo = playList_->Pop(); diff --git a/services/media_engine/plugins/source/http_source/http_source_plugin.cpp b/services/media_engine/plugins/source/http_source/http_source_plugin.cpp index 9283cd6f365470b62dc507366d81eb57141e0560..9790e046a85d5f5aff36a194e3480edb82c89e5a 100644 --- a/services/media_engine/plugins/source/http_source/http_source_plugin.cpp +++ b/services/media_engine/plugins/source/http_source/http_source_plugin.cpp @@ -139,11 +139,11 @@ Status HttpSourcePlugin::SetSource(std::shared_ptr source) MEDIA_LOG_D("SetSource enter."); AutoLock lock(mutex_); FALSE_RETURN_V(downloader_ == nullptr, Status::ERROR_INVALID_OPERATION); // not allowed set again - auto uri = source->GetSourceUri(); - if (uri.find(".m3u8") != std::string::npos) { + uri_ = source->GetSourceUri(); + if (IsSeekToTimeSupported()) { downloader_ = std::make_shared(std::make_shared()); delayReady = false; - } else if (uri.compare(0, 4, "http") == 0) { // 0 : position, 4: count + } else if (uri_.compare(0, 4, "http") == 0) { // 0 : position, 4: count downloader_ = std::make_shared(std::make_shared()); } FALSE_RETURN_V(downloader_ != nullptr, Status::ERROR_NULL_POINTER); @@ -152,11 +152,16 @@ Status HttpSourcePlugin::SetSource(std::shared_ptr source) downloader_->SetCallback(callback_); } - MEDIA_LOG_I("SetSource: " PUBLIC_LOG_S, uri.c_str()); - FALSE_RETURN_V(downloader_->Open(uri), Status::ERROR_UNKNOWN); + MEDIA_LOG_I("SetSource: " PUBLIC_LOG_S, uri_.c_str()); + FALSE_RETURN_V(downloader_->Open(uri_), Status::ERROR_UNKNOWN); return Status::OK; } +bool HttpSourcePlugin::IsSeekToTimeSupported() +{ + return uri_.find(".m3u8") != std::string::npos; +} + Status HttpSourcePlugin::Read(std::shared_ptr& buffer, uint64_t offset, size_t expectedLen) { MEDIA_LOG_D("Read enter."); @@ -230,6 +235,7 @@ void HttpSourcePlugin::CloseUri() downloader_->Close(false); downloader_ = nullptr; } + uri_.clear(); } Status HttpSourcePlugin::GetDuration(int64_t& duration) diff --git a/services/media_engine/plugins/source/http_source/http_source_plugin.h b/services/media_engine/plugins/source/http_source/http_source_plugin.h index d06350ecccc32522810c36265cd58b6023a599bf..27789ee2779fbb13e2f1a1326d897677f1d7e041 100644 --- a/services/media_engine/plugins/source/http_source/http_source_plugin.h +++ b/services/media_engine/plugins/source/http_source/http_source_plugin.h @@ -43,8 +43,9 @@ public: Status GetSize(uint64_t& size) override; Seekable GetSeekable() override; Status SeekTo(uint64_t offset) override; - Status SeekToTime(int64_t seekTime); - Status GetDuration(int64_t& duration); + Status SeekToTime(int64_t seekTime) override; + Status GetDuration(int64_t& duration) override; + bool IsSeekToTimeSupported() override; Status GetBitRates(std::vector& bitRates) override; Status SelectBitRate(uint32_t bitRate) override; @@ -57,6 +58,7 @@ private: std::shared_ptr downloader_; Mutex mutex_ {}; bool delayReady {true}; + std::string uri_ {}; }; } // namespace HttpPluginLite } // namespace Plugin