From 32e44b86ec9c982f7363ee1a60a3d11a9f9d720a Mon Sep 17 00:00:00 2001 From: SuRuoyan Date: Sun, 10 Nov 2024 01:02:19 +0800 Subject: [PATCH] add vendor aac encoder, support HE-AAC mux Signed-off-by: SuRuoyan --- interfaces/inner_api/native/BUILD.gn | 3 ++ .../inner_api/native/avcodec_codec_name.h | 1 + interfaces/kits/c/native_avcodec_base.h | 2 ++ services/engine/codeclist/BUILD.gn | 6 ++++ .../engine/codeclist/audio_codeclist_info.cpp | 33 ++++++++++++++++++- .../engine/codeclist/audio_codeclist_info.h | 1 + services/engine/codeclist/codeclist_core.cpp | 6 +++- .../aac/ffmpeg_aac_encoder_plugin.cpp | 29 +++++++++++++++- .../ffmpeg_adapter/common/ffmpeg_utils.cpp | 23 +++++++++++-- .../muxer/ffmpeg_muxer_plugin.cpp | 6 ++++ .../coverage_unit_test/BUILD.gn | 6 ++++ .../video_test/vcodec_server_test/BUILD.gn | 6 ++++ 12 files changed, 116 insertions(+), 6 deletions(-) diff --git a/interfaces/inner_api/native/BUILD.gn b/interfaces/inner_api/native/BUILD.gn index 2aa7b5b89..1306ea55e 100644 --- a/interfaces/inner_api/native/BUILD.gn +++ b/interfaces/inner_api/native/BUILD.gn @@ -101,13 +101,16 @@ config("av_codec_client_public_config") { if (target_cpu == "arm64" || is_emulator) { av_codec_plugin_path = "\"/system/lib64/media/av_codec_plugins\"" + av_codec_path = "\"/system/lib64\"" } else { av_codec_plugin_path = "\"/system/lib/media/av_codec_plugins\"" + av_codec_path = "\"/system/lib\"" } defines += [ "AV_CODEC_PLUGIN_PATH=${av_codec_plugin_path}", "AV_CODEC_PLUGIN_FILE_TAIL=\".z.so\"", + "AV_CODEC_PATH=${av_codec_path}", ] if (av_codec_client_support_codec) { diff --git a/interfaces/inner_api/native/avcodec_codec_name.h b/interfaces/inner_api/native/avcodec_codec_name.h index 1188e50f1..df0a71e72 100644 --- a/interfaces/inner_api/native/avcodec_codec_name.h +++ b/interfaces/inner_api/native/avcodec_codec_name.h @@ -38,6 +38,7 @@ public: static constexpr std::string_view AUDIO_ENCODER_OPUS_NAME = "OH.Media.Codec.Encoder.Audio.Opus"; static constexpr std::string_view AUDIO_ENCODER_G711MU_NAME = "OH.Media.Codec.Encoder.Audio.G711mu"; static constexpr std::string_view AUDIO_ENCODER_AAC_NAME = "OH.Media.Codec.Encoder.Audio.AAC"; + static constexpr std::string_view AUDIO_ENCODER_VENDOR_AAC_NAME = "OH.Media.Codec.Encoder.Audio.Vendor.AAC"; static constexpr std::string_view AUDIO_ENCODER_LBVC_NAME = "OH.Media.Codec.Encoder.Audio.LBVC"; static constexpr std::string_view AUDIO_ENCODER_AMRNB_NAME = "OH.Media.Codec.Encoder.Audio.Amrnb"; static constexpr std::string_view AUDIO_ENCODER_AMRWB_NAME = "OH.Media.Codec.Encoder.Audio.Amrwb"; diff --git a/interfaces/kits/c/native_avcodec_base.h b/interfaces/kits/c/native_avcodec_base.h index 6df8e9242..af4456f14 100644 --- a/interfaces/kits/c/native_avcodec_base.h +++ b/interfaces/kits/c/native_avcodec_base.h @@ -681,6 +681,8 @@ typedef enum OH_MediaType { */ typedef enum OH_AACProfile { AAC_PROFILE_LC = 0, + AAC_PROFILE_HE = 3, + AAC_PROFILE_HE_V2 = 4, } OH_AACProfile; /** diff --git a/services/engine/codeclist/BUILD.gn b/services/engine/codeclist/BUILD.gn index 026788a30..06163e5b7 100644 --- a/services/engine/codeclist/BUILD.gn +++ b/services/engine/codeclist/BUILD.gn @@ -35,6 +35,12 @@ ohos_static_library("av_codec_engine_codeclist") { if (av_codec_enable_special_codec) { defines += [ "AV_CODEC_AUDIO_VIVID_CAPACITY" ] } + if (target_cpu == "arm64") { + av_codec_path = "\"/system/lib64\"" + } else { + av_codec_path = "\"/system/lib\"" + } + defines += [ "AV_CODEC_PATH=${av_codec_path}" ] sources = [ "$av_codec_root_dir/services/media_engine/plugins/ffmpeg_adapter/common/hdi_codec.cpp", diff --git a/services/engine/codeclist/audio_codeclist_info.cpp b/services/engine/codeclist/audio_codeclist_info.cpp index f56058f62..3b0ee68db 100644 --- a/services/engine/codeclist/audio_codeclist_info.cpp +++ b/services/engine/codeclist/audio_codeclist_info.cpp @@ -17,6 +17,7 @@ #include "avcodec_mime_type.h" #include "avcodec_codec_name.h" #include "hdi_codec.h" +#include namespace OHOS { namespace MediaAVCodec { @@ -67,6 +68,8 @@ constexpr int MAX_CHANNEL_COUNT_VIVID = 16; constexpr int MAX_BIT_RATE_G711MU_DECODER = 64000; constexpr int MAX_BIT_RATE_G711MU_ENCODER = 64000; +const std::string VENDOR_AAC_LIB_PATH = std::string(AV_CODEC_PATH) + "/libaac_enc.z.so"; + CapabilityData AudioCodeclistInfo::GetMP3DecoderCapability() { CapabilityData audioMp3Capability; @@ -283,6 +286,34 @@ CapabilityData AudioCodeclistInfo::GetLbvcEncoderCapability() audioLbvcCapability.isVendor = true; return audioLbvcCapability; } + +CapabilityData AudioCodeclistInfo::GetVendorAacEncoderCapability() +{ + std::unique_ptr libFile = std::make_unique(VENDOR_AAC_LIB_PATH, std::ios::binary); + CapabilityData audioAacCapability; + if (!libFile->is_open()) { + audioAacCapability.codecName = ""; + audioAacCapability.mimeType = ""; + audioAacCapability.maxInstance = 0; + audioAacCapability.codecType = AVCODEC_TYPE_NONE; + audioAacCapability.isVendor = false; + audioAacCapability.bitrate = Range(0, 0); + audioAacCapability.channels = Range(0, 0); + audioAacCapability.sampleRate = {0}; + return audioAacCapability; + } + libFile->close(); + audioAacCapability.codecName = AVCodecCodecName::AUDIO_ENCODER_VENDOR_AAC_NAME; + audioAacCapability.codecType = AVCODEC_TYPE_AUDIO_ENCODER; + audioAacCapability.mimeType = AVCodecMimeType::MEDIA_MIMETYPE_AUDIO_AAC; + audioAacCapability.isVendor = false; + audioAacCapability.bitrate = Range(MIN_BIT_RATE_AAC_ENCODER, MAX_BIT_RATE_AAC_ENCODER); + audioAacCapability.channels = Range(1, MAX_AUDIO_CHANNEL_COUNT); + audioAacCapability.sampleRate = AUDIO_SAMPLE_RATE; + audioAacCapability.maxInstance = MAX_SUPPORT_AUDIO_INSTANCE; + audioAacCapability.profiles = { AAC_PROFILE_LC, AAC_PROFILE_HE, AAC_PROFILE_HE_V2 }; + return audioAacCapability; +} #endif CapabilityData AudioCodeclistInfo::GetAacEncoderCapability() @@ -365,7 +396,7 @@ AudioCodeclistInfo::AudioCodeclistInfo() GetAPEDecoderCapability(), GetMP3EncoderCapability(), #ifdef AV_CODEC_AUDIO_VIVID_CAPACITY GetVividDecoderCapability(), GetAmrnbEncoderCapability(), GetAmrwbEncoderCapability(), - GetLbvcDecoderCapability(), GetLbvcEncoderCapability(), + GetLbvcDecoderCapability(), GetLbvcEncoderCapability(), GetVendorAacEncoderCapability(), #endif }; } diff --git a/services/engine/codeclist/audio_codeclist_info.h b/services/engine/codeclist/audio_codeclist_info.h index 74864d328..20e4b37ad 100644 --- a/services/engine/codeclist/audio_codeclist_info.h +++ b/services/engine/codeclist/audio_codeclist_info.h @@ -40,6 +40,7 @@ public: CapabilityData GetMP3EncoderCapability(); CapabilityData GetLbvcDecoderCapability(); CapabilityData GetLbvcEncoderCapability(); + CapabilityData GetVendorAacEncoderCapability(); #ifdef AV_CODEC_AUDIO_VIVID_CAPACITY CapabilityData GetVividDecoderCapability(); CapabilityData GetAmrnbEncoderCapability(); diff --git a/services/engine/codeclist/codeclist_core.cpp b/services/engine/codeclist/codeclist_core.cpp index 0ef72b421..dd663148c 100644 --- a/services/engine/codeclist/codeclist_core.cpp +++ b/services/engine/codeclist/codeclist_core.cpp @@ -284,7 +284,11 @@ std::vector CodecListCore::FindCodecNameArray(const std::string &mi for (auto index : iter->second) { if (capabilityArray[index].codecType == codecType) { - nameArray.push_back(capabilityArray[index].codecName); + if (capabilityArray[index].codecName == std::string("OH.Media.Codec.Encoder.Audio.Vendor.AAC")) { + nameArray.insert(nameArray.begin(), capabilityArray[index].codecName); + } else { + nameArray.push_back(capabilityArray[index].codecName); + } } } return nameArray; diff --git a/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.cpp b/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.cpp index 0629f7e0e..5383a4024 100644 --- a/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.cpp +++ b/services/media_engine/plugins/ffmpeg_adapter/audio_encoder/aac/ffmpeg_aac_encoder_plugin.cpp @@ -806,4 +806,31 @@ Status FFmpegAACEncoderPlugin::CloseCtxLocked() } // namespace Ffmpeg } // namespace Plugins } // namespace Media -} // namespace OHOS \ No newline at end of file +} // namespace OHOS + +#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) +#define FFMPEG_AAC_ENCODER_EXPORT extern "C" __declspec(dllexport) +#else +#if defined(__GNUC__) || (defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590)) +#define FFMPEG_AAC_ENCODER_EXPORT extern "C" __attribute__((visibility("default"))) +#else +#define FFMPEG_AAC_ENCODER_EXPORT +#endif +#endif + +namespace { +using namespace Ffmpeg; +FFMPEG_AAC_ENCODER_EXPORT CodecPlugin *CreateFFmpegAacEncoderPluginObject() +{ + const std::string name = std::string(OHOS::MediaAVCodec::AVCodecCodecName::AUDIO_ENCODER_AAC_NAME); + CodecPlugin *obj = new FFmpegAACEncoderPlugin(name); + return obj; +} + +FFMPEG_AAC_ENCODER_EXPORT void DestroyFFmpegAacEncoderPluginObject(CodecPlugin *obj) +{ + if (obj != nullptr) { + delete obj; + } +} +} \ No newline at end of file diff --git a/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_utils.cpp b/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_utils.cpp index 7d71924f3..f2c76a51e 100644 --- a/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_utils.cpp +++ b/services/media_engine/plugins/ffmpeg_adapter/common/ffmpeg_utils.cpp @@ -339,13 +339,30 @@ std::vector GenerateAACCodecConfig(int32_t profile, int32_t sampleRate, profileVal = it1->second; } int32_t sampleRateIndex = 0x10; + int32_t baseIndex = 0xF; auto it2 = sampleRates.find(sampleRate); if (it2 != sampleRates.end()) { sampleRateIndex = it2->second; } - std::vector codecConfig = {0, 0, 0x56, 0xE5, 0}; - codecConfig[0] = ((profileVal + 1) << 0x03) | ((sampleRateIndex & 0x0F) >> 0x01); - codecConfig[1] = ((sampleRateIndex & 0x01) << 0x07) | ((channels & 0x0F) << 0x03); + it2 = sampleRates.find(sampleRate / 2); // 2: HE-AAC require divide base sample rate + if (it2 != sampleRates.end()) { + baseIndex = it2->second; + } + std::vector codecConfig; + if (profile == AAC_PROFILE_HE || profile == AAC_PROFILE_HE_V2) { + codecConfig = {0, 0, 0, 0, 0}; + // 5 bit AOT(0x03:left 3 bits for sample rate) + 4 bit sample rate idx(0x01: 4 - 0x03) + codecConfig[0] = ((profileVal + 1) << 0x03) | ((baseIndex & 0x0F) >> 0x01); + // 0x07: left 7bits for other, 4 bit channel cfg,0x03:left for other + codecConfig[1] = ((baseIndex & 0x01) << 0x07) | ((channels & 0x0F) << 0x03) | ((sampleRateIndex & 0x0F) >> 1) ; + // 4 bit ext sample rate idx(0x07: left 7 bits for other) + 4 bit aot(2: LC-AAC, 0x02: left for other) + codecConfig[2] = ((sampleRateIndex & 0x01) << 0x07) | (2 << 0x02); + } else { + codecConfig = {0, 0, 0x56, 0xE5, 0}; + codecConfig[0] = ((profileVal + 1) << 0x03) | ((sampleRateIndex & 0x0F) >> 0x01); + codecConfig[1] = ((sampleRateIndex & 0x01) << 0x07) | ((channels & 0x0F) << 0x03); + } + return codecConfig; } diff --git a/services/media_engine/plugins/ffmpeg_adapter/muxer/ffmpeg_muxer_plugin.cpp b/services/media_engine/plugins/ffmpeg_adapter/muxer/ffmpeg_muxer_plugin.cpp index 01062e88c..40dde0185 100644 --- a/services/media_engine/plugins/ffmpeg_adapter/muxer/ffmpeg_muxer_plugin.cpp +++ b/services/media_engine/plugins/ffmpeg_adapter/muxer/ffmpeg_muxer_plugin.cpp @@ -41,6 +41,7 @@ constexpr float LATITUDE_MIN = -90.0f; constexpr float LATITUDE_MAX = 90.0f; constexpr float LONGITUDE_MIN = -180.0f; constexpr float LONGITUDE_MAX = 180.0f; +constexpr int32_t MIN_HE_AAC_SAMPLE_RATE = 16000; const std::string TIMED_METADATA_HANDLER_NAME = "timed_metadata"; bool IsMuxerSupported(const char *name) @@ -424,6 +425,11 @@ Status FFmpegMuxerPlugin::SetCodecParameterOfTrack(AVStream *stream, const std:: int32_t channels; trackDesc->Get(profile); trackDesc->Get(sampleRate); + if ((profile == AAC_PROFILE_HE || profile == AAC_PROFILE_HE_V2) && + sampleRate < MIN_HE_AAC_SAMPLE_RATE) { + MEDIA_LOG_E("HE-AAC only support sample rate >= 16k, input rate:%{public}d", sampleRate); + return Status::ERROR_INVALID_PARAMETER; + } trackDesc->Get(channels); codecConfig = GenerateAACCodecConfig(profile, sampleRate, channels); return SetCodecParameterExtra(stream, codecConfig.data(), codecConfig.size()); diff --git a/test/unittest/codeclist_test/coverage_unit_test/BUILD.gn b/test/unittest/codeclist_test/coverage_unit_test/BUILD.gn index 2c67bccee..a8795273c 100644 --- a/test/unittest/codeclist_test/coverage_unit_test/BUILD.gn +++ b/test/unittest/codeclist_test/coverage_unit_test/BUILD.gn @@ -150,6 +150,12 @@ ohos_static_library("av_codec_engine_codeclist_mock") { if (av_codec_enable_special_codec) { defines += [ "AV_CODEC_AUDIO_VIVID_CAPACITY" ] } + if (target_cpu == "arm64") { + av_codec_path = "\"/system/lib64\"" + } else { + av_codec_path = "\"/system/lib\"" + } + defines += [ "AV_CODEC_PATH=${av_codec_path}" ] sanitize = av_codec_sanitize include_dirs = codeclist_coverage_include_dirs diff --git a/test/unittest/video_test/vcodec_server_test/BUILD.gn b/test/unittest/video_test/vcodec_server_test/BUILD.gn index a993b57c5..3dc98f187 100644 --- a/test/unittest/video_test/vcodec_server_test/BUILD.gn +++ b/test/unittest/video_test/vcodec_server_test/BUILD.gn @@ -142,6 +142,12 @@ ohos_static_library("av_codec_engine_codeclist_mock") { if (av_codec_enable_special_codec) { defines += [ "AV_CODEC_AUDIO_VIVID_CAPACITY" ] } + if (target_cpu == "arm64") { + av_codec_path = "\"/system/lib64\"" + } else { + av_codec_path = "\"/system/lib\"" + } + defines += [ "AV_CODEC_PATH=${av_codec_path}" ] sanitize = av_codec_sanitize include_dirs = codec_server_coverage_include_dirs -- Gitee