diff --git a/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.cpp b/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.cpp index 42d89d5da1dbe5f8a8ec25e90e60713717558c51..f93dcd9d1e81afba592f9c944de4aaae267dd78a 100644 --- a/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.cpp +++ b/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.cpp @@ -253,6 +253,10 @@ Status FFmpegAACEncoderPlugin::QueueInputBuffer(const std::shared_ptr return Status::ERROR_INVALID_DATA; } bufferMeta_ = inputBuffer->meta_; + if (ptsMode_ == GENERATE_ENCODE_PTS_BY_INPUT_MODE && isFirstInputPts_) { + prevPts_ = inputBuffer->pts_; + isFirstInputPts_ = false; + } dataCallback_->OnInputBufferDone(inputBuffer); ret = Status::OK; } @@ -305,9 +309,16 @@ Status FFmpegAACEncoderPlugin::ReceivePacketSucc(std::shared_ptr &outB // how get perfect pts with upstream pts(us) outBuffer->duration_ = ConvertTimeFromFFmpeg(avPacket_->duration, avCodecContext_->time_base) / NS_PER_US; // adjust ffmpeg duration with sample rate - outBuffer->pts_ = ((INT64_MAX - prevPts_) < avPacket_->duration) - ? (outBuffer->duration_ - (INT64_MAX - prevPts_)) - : (prevPts_ + outBuffer->duration_); + if (ptsMode_ == GENERATE_ENCODE_PTS_BY_INPUT_MODE && isFirstOutputPts_) { + outBuffer->pts_ = prevPts_; + isFirstOutputPts_ = false; + } else if (prevPts_ < 0) { + outBuffer->pts_ = prevPts_ + outBuffer->duration_; + } else { + outBuffer->pts_ = ((INT64_MAX - prevPts_) < outBuffer->duration_) + ? (outBuffer->duration_ - (INT64_MAX - prevPts_)) + : (prevPts_ + outBuffer->duration_); + } prevPts_ = outBuffer->pts_; return Status::OK; } @@ -380,6 +391,9 @@ Status FFmpegAACEncoderPlugin::Reset() std::lock_guard lock(avMutex_); auto ret = CloseCtxLocked(); prevPts_ = 0; + ptsMode_ = DEFAULT_ENCODE_PTS_MODE; + isFirstInputPts_ = true; + isFirstOutputPts_ = true; return ret; } @@ -388,6 +402,10 @@ Status FFmpegAACEncoderPlugin::Release() MEDIA_LOG_I("Release enter"); std::lock_guard lock(avMutex_); auto ret = CloseCtxLocked(); + prevPts_ = 0; + ptsMode_ = DEFAULT_ENCODE_PTS_MODE; + isFirstInputPts_ = true; + isFirstOutputPts_ = true; return ret; } @@ -399,6 +417,8 @@ Status FFmpegAACEncoderPlugin::Flush() avcodec_flush_buffers(avCodecContext_.get()); } prevPts_ = 0; + isFirstInputPts_ = true; + isFirstOutputPts_ = true; if (fifo_) { av_audio_fifo_reset(fifo_); } @@ -562,6 +582,9 @@ Status FFmpegAACEncoderPlugin::GetMetaData(const std::shared_ptr &meta) return Status::ERROR_INVALID_PARAMETER; } } + if (meta->Get(ptsMode_)) { + MEDIA_LOG_I("ptsMode_:%{public}d", static_cast(ptsMode_)); + } if (meta->Get(type)) { aacName_ = (type == 1 ? "aac" : "aac_latm"); @@ -736,6 +759,7 @@ Status FFmpegAACEncoderPlugin::PcmFillFrame(const std::shared_ptr &inp uint8_t *destBuffer = const_cast(srcBuffer); size_t srcBufferSize = static_cast(memory->GetSize()); size_t destBufferSize = srcBufferSize; + uint32_t destSamplesPerFrame; if (needResample_ && resample_ != nullptr) { if (resample_->Convert(srcBuffer, srcBufferSize, destBuffer, destBufferSize) != Status::OK) { MEDIA_LOG_E("Convert sample format failed"); @@ -748,8 +772,11 @@ Status FFmpegAACEncoderPlugin::PcmFillFrame(const std::shared_ptr &inp "frame_size: %{public}d", cachedFrame_->nb_samples, avCodecContext_->frame_size); } - int32_t destSamplesPerFrame = (avCodecContext_->frame_size > (avCodecContext_->sample_rate / FRAMES_PER_SECOND)) ? - avCodecContext_->frame_size : (avCodecContext_->sample_rate / FRAMES_PER_SECOND); + if (needResample_ && resample_ != nullptr) { + destSamplesPerFrame = resample_->GetSampleOffset(); + } else { + destSamplesPerFrame = cachedFrame_->nb_samples; + } cachedFrame_->extended_data = cachedFrame_->data; cachedFrame_->extended_data[0] = destBuffer; cachedFrame_->linesize[0] = cachedFrame_->nb_samples * bytesPerSample; diff --git a/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.h b/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.h index 10595e792f5eac6719ce883187bb678c9a0fccdb..ad79c1e0123c56739775a6a9b5ca841bb22f7cd3 100644 --- a/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.h +++ b/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.h @@ -135,6 +135,9 @@ private: AudioChannelLayout srcLayout_; uint32_t fullInputFrameSize_{0}; uint32_t srcBytesPerSample_{0}; + AudioEncodePtsMode ptsMode_ = DEFAULT_ENCODE_PTS_MODE; + bool isFirstInputPts_ = true; + bool isFirstOutputPts_ = true; std::string aacName_; int32_t channels_; diff --git a/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_convert.cpp b/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_convert.cpp index e7325262dc8790d3fba966362f2344cdf999a6db..cd3eba4a3d2f0fd19d74c74b84e854cc23654e48 100644 --- a/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_convert.cpp +++ b/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_convert.cpp @@ -31,9 +31,11 @@ Status Resample::Init(const ResamplePara &resamplePara) if (resamplePara_.bitsPerSample != 8 && resamplePara_.bitsPerSample != 24) { // 8 24 auto destFrameSize = av_samples_get_buffer_size(nullptr, resamplePara_.channels, resamplePara_.destSamplesPerFrame, resamplePara_.destFmt, 0); + cacheSize_ = destFrameSize; resampleCache_.reserve(destFrameSize); resampleChannelAddr_.reserve(resamplePara_.channels); auto tmp = resampleChannelAddr_.data(); + sampleOffset_ = resamplePara_.destSamplesPerFrame; av_samples_fill_arrays(tmp, nullptr, resampleCache_.data(), resamplePara_.channels, resamplePara_.destSamplesPerFrame, resamplePara_.destFmt, 0); auto swrContext = swr_alloc(); @@ -93,6 +95,41 @@ Status Resample::InitSwrContext(const ResamplePara &resamplePara) return Status::OK; } +#if defined(_WIN32) || !defined(OHOS_LITE) +void Resample::ConvertCommon(const uint8_t *srcBuffer, const size_t srcLength, + uint8_t *&destBuffer, size_t &destLength) +{ + size_t lineSize = srcLength / resamplePara_.channels; + std::vector tmpInput(resamplePara_.channels); + tmpInput[0] = srcBuffer; + if (av_sample_fmt_is_planar(resamplePara_.srcFfFmt)) { + for (size_t i = 1; i < tmpInput.size(); ++i) { + tmpInput[i] = tmpInput[i - 1] + lineSize; + } + } + auto samples = lineSize / static_cast(av_get_bytes_per_sample(resamplePara_.srcFfFmt)); + size_t destSize = samples * av_get_bytes_per_sample(resamplePara_.destFmt) * resamplePara_.channels; + if (cacheSize_ < destSize) { + cacheSize_ = destSize; + resampleCache_.reserve(cacheSize_); + resampleCache_.assign(cacheSize_, 0); + auto tmp = resampleChannelAddr_.data(); + sampleOffset_ = samples; + av_samples_fill_arrays(tmp, nullptr, resampleCache_.data(), resamplePara_.channels, + samples, resamplePara_.destFmt, 1); + } + auto res = swr_convert(swrCtx_.get(), resampleChannelAddr_.data(), samples, tmpInput.data(), samples); + if (res < 0) { + MEDIA_LOG_E("resample input failed"); + destLength = 0; + } else { + destBuffer = resampleCache_.data(); + size_t bytesPerSample = static_cast(av_get_bytes_per_sample(resamplePara_.destFmt)); + destLength = static_cast(res) * bytesPerSample * resamplePara_.channels; + } +} +#endif + Status Resample::Convert(const uint8_t *srcBuffer, const size_t srcLength, uint8_t *&destBuffer, size_t &destLength) { #if defined(_WIN32) || !defined(OHOS_LITE) @@ -100,6 +137,7 @@ Status Resample::Convert(const uint8_t *srcBuffer, const size_t srcLength, uint8 FALSE_RETURN_V_MSG(resamplePara_.destFmt == AV_SAMPLE_FMT_S16, Status::ERROR_UNIMPLEMENTED, "resample 8bit to other format can not support"); destLength = srcLength * 2; // 2 + cacheSize_ = destLength; resampleCache_.reserve(destLength); resampleCache_.assign(destLength, 0); for (size_t i{0}; i < destLength / 2; i++) { // 2 @@ -112,6 +150,7 @@ Status Resample::Convert(const uint8_t *srcBuffer, const size_t srcLength, uint8 FALSE_RETURN_V_MSG(resamplePara_.destFmt == AV_SAMPLE_FMT_S16, Status::ERROR_UNIMPLEMENTED, "resample 24bit to other format can not support"); destLength = srcLength / 3 * 2; // 3 2 + cacheSize_ = destLength; resampleCache_.reserve(destLength); resampleCache_.assign(destLength, 0); for (size_t i = 0; i < destLength / 2; i++) { // 2 @@ -120,25 +159,7 @@ Status Resample::Convert(const uint8_t *srcBuffer, const size_t srcLength, uint8 } destBuffer = resampleCache_.data(); } else { - size_t lineSize = srcLength / resamplePara_.channels; - std::vector tmpInput(resamplePara_.channels); - tmpInput[0] = srcBuffer; - if (av_sample_fmt_is_planar(resamplePara_.srcFfFmt)) { - for (size_t i = 1; i < tmpInput.size(); ++i) { - tmpInput[i] = tmpInput[i - 1] + lineSize; - } - } - auto samples = lineSize / static_cast(av_get_bytes_per_sample(resamplePara_.srcFfFmt)); - auto res = swr_convert(swrCtx_.get(), resampleChannelAddr_.data(), resamplePara_.destSamplesPerFrame, - tmpInput.data(), samples); - if (res < 0) { - MEDIA_LOG_E("resample input failed"); - destLength = 0; - } else { - destBuffer = resampleCache_.data(); - size_t bytesPerSample = static_cast(av_get_bytes_per_sample(resamplePara_.destFmt)); - destLength = static_cast(res) * bytesPerSample * resamplePara_.channels; - } + ConvertCommon(srcBuffer, srcLength, destBuffer, destLength); } #endif return Status::OK; @@ -186,6 +207,11 @@ Status Resample::ConvertFrame(AVFrame *outputFrame, const AVFrame *inputFrame) return Status::OK; } +uint32_t Resample::GetSampleOffset() +{ + return sampleOffset_; +} + #if defined(VIDEO_SUPPORT) Status Scale::Init(const ScalePara &scalePara, uint8_t **dstData, int32_t *dstLineSize) { diff --git a/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_convert.h b/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_convert.h index 239258188b2766ced7fe22cfb870549b455a9728..79038c00583ccc540fc43137dab2b245df5ad584 100644 --- a/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_convert.h +++ b/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_convert.h @@ -57,10 +57,13 @@ public: Status InitSwrContext(const ResamplePara &resamplePara); Status Convert(const uint8_t *srcBuffer, const size_t srcLength, uint8_t *&destBuffer, size_t &destLength); Status ConvertFrame(AVFrame *outputFrame, const AVFrame *inputFrame); - + uint32_t GetSampleOffset(); private: ResamplePara resamplePara_{}; + uint32_t sampleOffset_ = 0; #if defined(_WIN32) || !defined(OHOS_LITE) + void ConvertCommon(const uint8_t *srcBuffer, const size_t srcLength, uint8_t *&destBuffer, size_t &destLength); + size_t cacheSize_ = 0; std::vector resampleCache_{}; std::vector resampleChannelAddr_{}; std::shared_ptr swrCtx_{nullptr}; diff --git a/test/unittest/audio_test/audio_encoder_avbuffer_capi_unit_test.cpp b/test/unittest/audio_test/audio_encoder_avbuffer_capi_unit_test.cpp index 5ae47c0e02dd478ae9223b8a4c0503eb3cf0d733..da20feffa30b910185908f23961a0d12f8b4807d 100644 --- a/test/unittest/audio_test/audio_encoder_avbuffer_capi_unit_test.cpp +++ b/test/unittest/audio_test/audio_encoder_avbuffer_capi_unit_test.cpp @@ -192,6 +192,7 @@ protected: std::unique_ptr soFile_; int32_t fileSize_ = 0; uint32_t frameBytes_ = FLAC_DEFAULT_FRAME_BYTES; // default for flac + int32_t outputFrameCnt_ = 0; }; void AudioEncoderBufferCapiUnitTest::SetUpTestCase(void) @@ -270,8 +271,11 @@ void AudioEncoderBufferCapiUnitTest::InputFunc() } if (!inputFile_->eof()) { inputFile_->read((char *)OH_AVBuffer_GetAddr(buffer), frameBytes_); - buffer->buffer_->memory_->SetSize(frameBytes_); - fileSize_ -= frameBytes_; + int32_t readSize = inputFile_->gcount(); + if (readSize == 0) { + continue; + } + buffer->buffer_->memory_->SetSize(readSize); } else { buffer->buffer_->memory_->SetSize(1); buffer->buffer_->flag_ = AVCODEC_BUFFER_FLAGS_EOS; @@ -304,6 +308,7 @@ void AudioEncoderBufferCapiUnitTest::OutputFunc() cout << "Fatal: open output file fail" << endl; return; } + outputFrameCnt_ = 0; while (isRunning_.load()) { unique_lock lock(signal_->outMutex_); signal_->outCond_.wait(lock, [this]() { return (signal_->outQueue_.size() > 0 || !isRunning_.load()); }); @@ -323,6 +328,7 @@ void AudioEncoderBufferCapiUnitTest::OutputFunc() << index << ", data size:" << avBuffer->buffer_->memory_->GetSize() << endl; outputFile_->write(reinterpret_cast(OH_AVBuffer_GetAddr(avBuffer)), avBuffer->buffer_->memory_->GetSize()); + outputFrameCnt_++; } if (avBuffer != nullptr && (avBuffer->buffer_->flag_ == AVCODEC_BUFFER_FLAGS_EOS || avBuffer->buffer_->memory_->GetSize()== 0)) { @@ -566,6 +572,7 @@ HWTEST_F(AudioEncoderBufferCapiUnitTest, encodeTest_02, TestSize.Level1) { InitFile(AudioBufferFormatType::TYPE_AAC); CreateCodecFunc(AudioBufferFormatType::TYPE_AAC); + int32_t needOutputCnt = fileSize_ / AAC_DEFAULT_FRAME_BYTES; format = OH_AVFormat_Create(); EXPECT_NE(nullptr, format); OH_AVFormat_SetIntValue(format, MediaDescriptionKey::MD_KEY_SAMPLE_RATE.data(), SAMPLE_RATE); @@ -583,6 +590,7 @@ HWTEST_F(AudioEncoderBufferCapiUnitTest, encodeTest_02, TestSize.Level1) } JoinThread(); + EXPECT_EQ(outputFrameCnt_, needOutputCnt); EXPECT_EQ(OH_AVErrCode::AV_ERR_OK, OH_AudioCodec_Flush(audioEnc_)); EXPECT_EQ(OH_AVErrCode::AV_ERR_OK, OH_AudioCodec_Destroy(audioEnc_)); }