diff --git a/README.md b/README.md index 4006af364f470af89d6fb459405b8aa75d1e78c7..d959103c15908526c9434c4460ab6e6fca362f4a 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,12 @@ VPE引擎的主要结构如下图所示： ![VPE引擎架构图](./figures/videoProcessingEngine_architecture.png) + +#### 各模块功能说明 +
层级模块功能描述
Interface视频色彩空间 API提供用于视频场景色彩空间转换相关接口
图片色彩空间 API提供用于图片场景色彩空间转换相关接口
视频细节增强 API提供视频超分算法、锐化算法的相关接口
图片细节增强 API提供图片超分算法、锐化算法的相关接口
视频动态元数据API提供接口可用于视频内容动态元数据生成算法调用
图片动态元数据API提供接口可用于图片内容动态元数据生成算法调用
framework视频色彩空间原子能力实现视频场景色彩空间转换软件通路调度及上下文管理，实现视频流过程控制
图片色彩空间原子能力实现图片场景色彩空间转换软件通路调度
视频细节增强原子能力实现视频场景清晰度增强及缩放算法软件通路调度及上下文管理，实现视频流过程控制
图片细节增强原子能力实现图片场景清晰度增强及缩放算法软件通路调度
视频动态元数据原子能力实现视频场景动态元数据生成软件通路调度
图片动态元数据原子能力实现图片场景动态元数据生成软件通路调度
插件层视频色彩空间处理算法插件实现视频色彩空间转换算法功能，具体包括SDR2SDR、HDR2SDR、HDR2HDR场景的色域转换
图片色彩空间算法插件实现图片色彩空间转换算法功能，具体包括SDR2SDR、HDR2SDR、HDR2HDR场景的色域转换
视频细节增强算法插件实现视频缩放、画质增强算法
图片细节增强算法插件实现图片缩放、画质增强算法
视频动态元数据算法插件实现视频源动态元数生成据算法
图片动态元数据算法插件实现图片源动态元数生成据算法
+ + + ## 目录 仓目录结构如下： @@ -23,6 +29,7 @@ VPE引擎的主要结构如下图所示： │ ├── detail_enhancer # 图像细节增强算法框架 │ ├── detail_enhancer_video # 视频细节增强算法框架 │ ├── extension_manager # 插件管理 +│ ├── extensions # 插件算法 │ ├── metadata_generator # 图像元数据生成算法框架 │ ├── metadata_generator_video # 视频元数据生成算法框架 │ ├── video_variable_refresh_rate # 视频可变帧率算法框架 @@ -56,7 +63,7 @@ VPE引擎的主要结构如下图所示： ### 使用说明 VPE引擎作为OpenHarmony的组件，提供系统的视频图像能力，包含视频处理算法框架，以及色彩空间转换、动态元数据生成以及细节增强等插件，支持开发者再插件中注册自定义算法，实现更多高阶图像和视频处理操作。 -#### 图像缩放 +#### 应用开发者调用图像缩放示例 以下步骤描述了具体开发步骤。 1. 添加头文件。 ```cpp @@ -105,7 +112,7 @@ VPE引擎作为OpenHarmony的组件，提供系统的视频图像能力，包含 ```cpp OH_ImageProcessing_DeinitializeEnvironment(); ``` -#### 视频缩放 +#### 应用开发者调用视频缩放示例 可以使用VIDEO_PROCESSING_TYPE_DETAIL_ENHANCER视频的缩放，以下步骤描述了具体开发步骤。 1. 添加头文件。 ```cpp @@ -218,47 +225,165 @@ VPE引擎作为OpenHarmony的组件，提供系统的视频图像能力，包含 ```cpp VideoProcessing_ErrorCode ret = OH_VideoProcessing_DeinitializeEnvironment(); ``` -#### 自定义算法插件注册 -以动态元数据生成为例，以下步骤描述了具体开发步骤。 -1. 实现插件注册函数 + +#### 系统开发者实现自定义算法插件注册示例 +自定义算法插件功能与基类对应关系如下： +| 功能 | 基类 | 头文件 | +| :--: | :--: | :--: | +| 视频色彩空间 | ColorSpaceConverterBase | colorspace_converter_base.h | +| 图片色彩空间 | ColorSpaceConverterBase | colorspace_converter_base.h | +| 视频细节增强 | DetailEnhancerBase | detail_enhancer_base.h | +| 图片细节增强 | DetailEnhancerBase | detail_enhancer_base.h | +| 视频动态元数据 | MetadataGeneratorBase | metadata_generator_base.h | +| 图片动态元数据 | MetadataGeneratorBase | metadata_generator_base.h | +以图像动态元数据生成为例，以下步骤描述了具体开发步骤。 +1. 在framework/algorithm/extensions目录下新建一个文件夹，例如image_metadata_generator，以及对应的cpp和h文件 +``` + /foundation/multimedia/video_processing_engine/ + ├── framework # 框架代码 + │ ├── algorithm # 算法框架 + │ ├── extensions # 插件算法 + │ ├── image_metadata_generator # 图像动态元数据生成 + │ ├── image_metadata_gen_impl.h + │ ├── image_metadata_gen_impl.cpp +``` +1. 图像动态元数据生成头文件实现 + ```cpp - std::vector ImageMetadataGen::BuildCapabilities() + #ifndef IMAGE_METADATA_GEN_IMPL_H + #define IMAGE_METADATA_GEN_IMPL_H + + #include "metadata_generator_base.h" + #include "metadata_generator_capability.h" + + namespace OHOS { + namespace Media { + namespace VideoProcessingEngine { + // 由基类MetadataGeneratorBase创建一个自定义元数据生成类，例如ImageMetadataGen + // 其余算法对应关系如上表所示 + + class ImageMetadataGen : public MetadataGeneratorBase { + public: + // 定义Create函数，函数名可以自定义，这里以Create()为例，用于实例创建, 返回值必须是MetadataGeneratorBase指针。 + static std::unique_ptr Create(); + // 定义能力构建函数，函数名可以自定义，这里以BuildCapabilities()为例，用于定义算法所支持的能力，返回值必须是vector。 + static std::vector BuildCapabilities(); + // 基类虚函数必须实现 + VPEAlgoErrCode Init(VPEContext context) override; + VPEAlgoErrCode Deinit() override; + VPEAlgoErrCode SetParameter(const MetadataGeneratorParameter ¶meter) override; + VPEAlgoErrCode GetParameter(MetadataGeneratorParameter ¶meter) override; + VPEAlgoErrCode Process(const sptr &input) override; + + private: + MetadataGeneratorParameter parameter_; + + }; + // 自定义注册函数 + void RegisterImageMetadataGeneratorExtensions(uintptr_t extensionListAddr); + } // namespace VideoProcessingEngine + } // namespace Media + } // namespace OHOS + #endif // IMAGE_METADATA_GEN_IMPL_H + + ``` +3. 插件算法能力注册与插件算法实现 + ```cpp + #include "image_metadata_gen_impl.h" + #include "metadata_generator_extension.h" + + namespace OHOS { + namespace Media { + namespace VideoProcessingEngine { + // 实现create函数，创建实例 + std::unique_ptr ImageMetadataGen::Create() { - // 自定义支持的输入和输出格式以及优先级 + return std::make_unique(); + } + + // 算法能力注册与功能实现 + std::vector ImageMetadataGen::BuildCapabilities() + { + // 设置算法支持的输入色彩空间，具体可参考framework/capi/image_processing/include/image_processing_capi_capability.h中定义 + std::vector inColorspaceList = { + { GetColorSpaceInfo(CM_BT2020_PQ_LIMIT), CM_IMAGE_HDR_VIVID_SINGLE }}; + // 设置算法支持的pixelmap + std::vector pixelFormatMap; + pixelFormatMap.emplace_back(GRAPHIC_PIXEL_FMT_YCBCR_P010); // NV12 + pixelFormatMap.emplace_back(GRAPHIC_PIXEL_FMT_YCRCB_P010); // NV21 + pixelFormatMap.emplace_back(GRAPHIC_PIXEL_FMT_RGBA_1010102); // rgba1010102 + + // 遍历色彩空间和pixelmap格式的所有组合，表示算法支持的所有能力， std::vector capabilities; for (const auto &inColorspace : inColorspaceList) { - MetadataGeneratorCapability capability = { inColorspace, pixelFormatMap, RANK, IMAGEMETAGENVERSION }; + // RANK_HIGH表示优先级高，默认为RANK_DEFAULT + MetadataGeneratorCapability capability = { inColorspace, pixelFormatMap, Extension::Rank::RANK_HIGH, 0 }; capabilities.emplace_back(capability); } return capabilities; } + + // 实现Init函数，可以根据实际算法，用于一些资源的初始化 + VPEAlgoErrCode ImageMetadataGen::Init(VPEContext context) + { + return VPE_ALGO_ERR_OK; + } + + // 实现Deinit函数，可以根据实际算法，释放初始化的资源 + VPEAlgoErrCode ImageMetadataGen::Deinit() + { + return VPE_ALGO_ERR_OK; + } + // 实现参数设置函数 + VPEAlgoErrCode ImageMetadataGen::SetParameter(const MetadataGeneratorParameter ¶meter) + { + parameter_ = parameter; + return VPE_ALGO_ERR_OK; + } + // 实现参数获取函数 + VPEAlgoErrCode ImageMetadataGen::GetParameter(MetadataGeneratorParameter ¶meter) + { + parameter = parameter_; + return VPE_ALGO_ERR_OK; + } + // 算法功能实现 + VPEAlgoErrCode ImageMetadataGen::Process(const sptr &input) + { + return VPE_ALGO_ERR_OK; + } + // 实现注册函数，注册函数名可自定义修改，需与下面DoRegisterExtensions中的注册函数名一致 static std::vector> RegisterExtensions() { std::vector> extensions; + auto extension = std::make_shared(); + // 填写插件算法信息，当前算法为图像元数据生成，类型则填写METADATA_GENERATOR，后面两个字符串分别为算法名称以及版本号，可自定义修改。 extension->info = { OHOS::Media::VideoProcessingEngine::Extension::ExtensionType::METADATA_GENERATOR, - "ImageMetadataGen", "v1" }; + "ImageMetadataGen", "V1.0" }; + extension->creator = OHOS::Media::VideoProcessingEngine::ImageMetadataGen::Create; extension->capabilitiesBuilder = OHOS::Media::VideoProcessingEngine::ImageMetadataGen::BuildCapabilities; extensions.push_back( std::static_pointer_cast(extension)); return extensions; } - + + // 自定义注册函数 void RegisterImageMetadataGeneratorExtensions(uintptr_t extensionListAddr) { OHOS::Media::VideoProcessingEngine::Extension::DoRegisterExtensions(extensionListAddr, RegisterExtensions); } + } // namespace VideoProcessingEngine + } // namespace Media + } // namespace OHOS + } ``` -2. 实现算法 - ```cpp - ImageMetadataGen::Process(const sptr &input) - ``` -3. 添加注册插件回调函数。 - 在staticExtensionsRegisterMap中添加算法插件。 +3. 添加注册插件回调函数，VPE插件管理会遍历所有插件注册函数。 + 在framework/algorithm/extension_manager/include/static_extension_list.h文件staticExtensionsRegisterMap中添加算法插件。 ```cpp const std::unordered_map staticExtensionsRegisterMap = { - {"ImageMetadataGeneratorExtensions", RegisterImageMetadataGeneratorExtensions} + // 添加自定义插件算法名和注册函数，注册函数需与cpp中定义的注册函数同名 + {"ImgMetadataGeneratorExtensions", RegisterImageMetadataGeneratorExtensions} }; ``` ## 相关仓 @@ -267,7 +392,4 @@ VPE引擎作为OpenHarmony的组件，提供系统的视频图像能力，包含 - [graphic_graphic_surface](https://gitee.com/openharmony/graphic_graphic_surface) - [multimedia_image_framework](https://gitee.com/openharmony/multimedia_image_framework) - [multimedia_media_foundation](https://gitee.com/openharmony/multimedia_media_foundation) -- [third_party_egl](https://gitee.com/openharmony/third_party_egl) -- [third_party_opengles](https://gitee.com/openharmony/third_party_opengles) -- [third_party_opencl-headers](https://gitee.com/openharmony/third_party_opencl-headers) - [third_party_skia](https://gitee.com/openharmony/third_party_skia) \ No newline at end of file diff --git a/config.gni b/config.gni index 943ea6c647fac8ff29ce16c038d9f8d6efb0c17b..993a1cf21b8daddd4175e0434475c8f13ca042ac 100644 --- a/config.gni +++ b/config.gni @@ -28,6 +28,7 @@ COLORSPACE_CONVERTER_VIDEO_DIR = "$ALGORITHM_DIR/colorspace_converter_video METADATA_GENERATOR_DIR = "$ALGORITHM_DIR/metadata_generator" METADATA_GENERATOR_VIDEO_DIR = "$ALGORITHM_DIR/metadata_generator_video" ALGORITHM_EXTENSION_MANAGER_DIR = "$ALGORITHM_DIR/extension_manager" +ALGORITHM_EXTENSION_SKIA_DIR = "$ALGORITHM_DIR/extensions/skia" ALGORITHM_COMMON_DIR = "$ALGORITHM_DIR/common" DETAIL_ENHANCER_DIR = "$ALGORITHM_DIR/detail_enhancer" DETAIL_ENHANCER_VIDEO_DIR = "$ALGORITHM_DIR/detail_enhancer_video" diff --git a/framework/BUILD.gn b/framework/BUILD.gn index 5cdf17fdbfce8f71236aad41b815b03eb2818f73..cb7e23b3cdfe84056cfe781b1299f5aa91da5616 100644 --- a/framework/BUILD.gn +++ b/framework/BUILD.gn @@ -30,6 +30,7 @@ config("export_config") { "$ALGORITHM_DIR/common/include", "$METADATA_GENERATOR_VIDEO_DIR/include", "$ALGORITHM_EXTENSION_MANAGER_DIR/include", + "$ALGORITHM_EXTENSION_SKIA_DIR/include", ] } config("video_process_config") { @@ -186,7 +187,8 @@ ohos_shared_library("videoprocessingengine") { "${target_gen_dir}/../services/video_processing_service_manager_proxy.cpp", "${target_gen_dir}/../services/video_processing_service_manager_stub.cpp", "$ALGORITHM_COMMON_DIR/image_opencl_wrapper.cpp", - "$ALGORITHM_COMMON_DIR/image_openclsetup.cpp" + "$ALGORITHM_COMMON_DIR/image_openclsetup.cpp", + "$ALGORITHM_EXTENSION_SKIA_DIR/skia_impl.cpp", ] deps = [ @@ -336,8 +338,8 @@ ohos_shared_library("image_processing") { if (has_skia) { defines += [ "SKIA_ENABLE" ] external_deps += [ "skia:skia_canvaskit" ] - include_dirs += [ "$ALGORITHM_EXTENSION_MANAGER_DIR/include" ] - sources += [ "$ALGORITHM_EXTENSION_MANAGER_DIR/skia_impl.cpp" ] + include_dirs += [ "$ALGORITHM_EXTENSION_SKIA_DIR/include" ] + sources += [ "$ALGORITHM_EXTENSION_SKIA_DIR/skia_impl.cpp" ] } innerapi_tags = [ "ndk" ] @@ -418,8 +420,8 @@ ohos_shared_library("video_processing") { if (has_skia) { defines += [ "SKIA_ENABLE" ] external_deps += [ "skia:skia_canvaskit" ] - include_dirs += [ "$$ALGORITHM_EXTENSION_MANAGER_DIR/include" ] - sources += [ "$ALGORITHM_EXTENSION_MANAGER_DIR/skia_impl.cpp" ] + include_dirs += [ "$ALGORITHM_EXTENSION_SKIA_DIR/include" ] + sources += [ "$ALGORITHM_EXTENSION_SKIA_DIR/skia_impl.cpp" ] } innerapi_tags = [ "ndk" ] diff --git a/framework/algorithm/aihdr_enhancer/include/aihdr_enhancer_capability.h b/framework/algorithm/aihdr_enhancer/include/aihdr_enhancer_capability.h index 1e50523483593883d16ff935dcb161c37ecba34e..aad36d4b6e53efb3bce132163748093e35e19089 100644 --- a/framework/algorithm/aihdr_enhancer/include/aihdr_enhancer_capability.h +++ b/framework/algorithm/aihdr_enhancer/include/aihdr_enhancer_capability.h @@ -20,6 +20,7 @@ #include #include "frame_info.h" +#include "extension_base.h" namespace OHOS { namespace Media { @@ -27,7 +28,7 @@ namespace VideoProcessingEngine { struct AihdrEnhancerCapability { ColorSpaceDescription colorspaceDesc; std::vector pixelFormats; - uint32_t rank; + Extension::Rank rank; int32_t version; }; diff --git a/framework/algorithm/colorspace_converter/include/colorspace_converter_capability.h b/framework/algorithm/colorspace_converter/include/colorspace_converter_capability.h index cb07b544aedf7bcff0d2ca3e97aa84836b847735..43baa35d96c81e6fab047f579c486c4c9af6f4ee 100644 --- a/framework/algorithm/colorspace_converter/include/colorspace_converter_capability.h +++ b/framework/algorithm/colorspace_converter/include/colorspace_converter_capability.h @@ -19,6 +19,7 @@ #include #include #include "frame_info.h" +#include "extension_base.h" namespace OHOS { namespace Media { @@ -50,7 +51,7 @@ struct ColorSpaceConverterCapability { } */ std::map> pixelFormatMap; - uint32_t rank; + Extension::Rank rank; int32_t version; }; diff --git a/framework/algorithm/colorspace_converter_display/include/colorspace_converter_display_capability.h b/framework/algorithm/colorspace_converter_display/include/colorspace_converter_display_capability.h index 9141281cb068cf5ce802b956313b86a62ba65990..be6f6c9f463ac3ac28494958fc7674919f56fdf6 100644 --- a/framework/algorithm/colorspace_converter_display/include/colorspace_converter_display_capability.h +++ b/framework/algorithm/colorspace_converter_display/include/colorspace_converter_display_capability.h @@ -19,6 +19,7 @@ #include #include #include "algorithm_common.h" +#include "extension_base.h" namespace OHOS { namespace Media { @@ -27,7 +28,7 @@ namespace VideoProcessingEngine { struct ColorSpaceConverterDisplayCapability { ColorSpaceDescription inputColorspaceDesc; ColorSpaceDescription outputColorSpaceDesc; - uint32_t rank; + Extension::Rank rank; int32_t version; }; diff --git a/framework/algorithm/contrast_enhancer/include/contrast_enhancer_capability.h b/framework/algorithm/contrast_enhancer/include/contrast_enhancer_capability.h index 0641a6afb437954da487c34e3e1c2501a8c57776..58a9e23e79e1ac7abcc4fb3131db74d7ed9f26bb 100644 --- a/framework/algorithm/contrast_enhancer/include/contrast_enhancer_capability.h +++ b/framework/algorithm/contrast_enhancer/include/contrast_enhancer_capability.h @@ -20,13 +20,14 @@ #include #include "algorithm_common.h" +#include "extension_base.h" namespace OHOS { namespace Media { namespace VideoProcessingEngine { struct ContrastEnhancerCapability { std::vector levels; - uint32_t rank; + Extension::Rank rank; int32_t version; }; diff --git a/framework/algorithm/detail_enhancer/include/detail_enhancer_capability.h b/framework/algorithm/detail_enhancer/include/detail_enhancer_capability.h index 186a8d56fb5aa6871ea5464b351934907307639e..9f22220d8d8ca76863697d9c71cacbd6536a3bf4 100644 --- a/framework/algorithm/detail_enhancer/include/detail_enhancer_capability.h +++ b/framework/algorithm/detail_enhancer/include/detail_enhancer_capability.h @@ -20,13 +20,14 @@ #include #include "algorithm_common.h" +#include "extension_base.h" namespace OHOS { namespace Media { namespace VideoProcessingEngine { struct DetailEnhancerCapability { std::vector levels; - uint32_t rank; + Extension::Rank rank; int32_t version; }; diff --git a/framework/algorithm/extension_manager/extension_manager.cpp b/framework/algorithm/extension_manager/extension_manager.cpp index 29ea8e2d91480c568e6a04930d8659e3dd5fcb68..63514a669644aa64b8df556205bbf2c140560f4f 100644 --- a/framework/algorithm/extension_manager/extension_manager.cpp +++ b/framework/algorithm/extension_manager/extension_manager.cpp @@ -21,6 +21,7 @@ #include #include "static_extension_list.h" #include "vpe_log.h" +#include "extension_base.h" namespace { using LibFunctionGetRegisters = std::unordered_map* (*)(); @@ -215,9 +216,9 @@ std::shared_ptr ExtensionManager::CreateDetailEnhancer(uint3 ExtensionList ExtensionManager::LoadExtensions() const { ExtensionList extensionList {}; + LoadDynamicExtensions(extensionList); VPEAlgoErrCode ret = LoadStaticExtensions(extensionList); CHECK_AND_RETURN_RET_LOG(ret == VPE_ALGO_ERR_OK, {}, "Load extension failed"); - LoadDynamicExtensions(extensionList); return extensionList; } @@ -525,7 +526,13 @@ std::shared_ptr ExtensionManager::FindColorSpaceCo (CSDesc).range, outputInfo.pixelFormat, outputInfo.colorSpace.metadataType); return nullptr; } - size_t idx = std::get<2>(*(iter->second.cbegin())); + size_t idx = std::get<2>(*(iter->second.cbegin())); // 2 + for (const auto &cap : iter->second) { + if (std::get<0>(cap) == Rank::RANK_HIGH) { + idx = std::get<2>(cap); // 2 + break; + } + } return std::static_pointer_cast(extensionList[idx]); } @@ -556,7 +563,13 @@ std::shared_ptr ExtensionManager::FindMetadataGenera const auto iter = metadataGeneratorCapabilityMap.find(key); CHECK_AND_RETURN_RET_LOG(iter != metadataGeneratorCapabilityMap.cend() && !iter->second.empty(), nullptr, "CSC metadata generator extension is not found"); - size_t idx = std::get<2>(*(iter->second.cbegin())); + size_t idx = std::get<2>(*(iter->second.cbegin())); // 2 + for (const auto &cap : iter->second) { + if (std::get<0>(cap) == Rank::RANK_HIGH) { + idx = std::get<2>(cap); // 2 + break; + } + } return std::static_pointer_cast(extensionList[idx]); } @@ -595,7 +608,13 @@ std::shared_ptr ExtensionManager::FindAihdrEnhancerExten const auto iter = aihdrEnhancerCapabilityMap.find(key); CHECK_AND_RETURN_RET_LOG(iter != aihdrEnhancerCapabilityMap.cend() && !iter->second.empty(), nullptr, "Aihdr enhancer extension is not found"); - size_t idx = std::get<2>(*(iter->second.cbegin())); + size_t idx = std::get<2>(*(iter->second.cbegin())); // 2 + for (const auto &cap : iter->second) { + if (std::get<0>(cap) == Rank::RANK_HIGH) { + idx = std::get<2>(cap); // 2 + break; + } + } return std::static_pointer_cast(extensionList[idx]); } @@ -604,7 +623,7 @@ VPEAlgoErrCode ExtensionManager::ExtractColorSpaceConverterCap(const ColorSpaceC { auto inputColorSpaceDesc = cap.inputColorSpaceDesc; auto outputColorSpaceDesc = cap.outputColorSpaceDesc; - uint32_t rank = cap.rank; + Rank rank = cap.rank; int32_t version = cap.version; for (const auto &[inputPixelFormat, outputPixelFormats] : cap.pixelFormatMap) { for (const auto &outputPixelFormat : outputPixelFormats) { @@ -631,7 +650,7 @@ VPEAlgoErrCode ExtensionManager::ExtractMetadataGeneratorCap(const MetadataGener MetadataGeneratorAlgoType algoType, MetadataGeneratorCapabilityMap& metadataGeneratorCapabilityMap) const { auto colorSpaceDesc = cap.colorspaceDesc; - uint32_t rank = cap.rank; + Rank rank = cap.rank; int32_t version = cap.version; for (const auto &pixelFormat : cap.pixelFormats) { auto key = std::make_tuple(colorSpaceDesc, pixelFormat, algoType); @@ -664,7 +683,14 @@ VPEAlgoErrCode ExtensionManager::BuildDetailEnhancerCaps(const std::shared_ptr(ext); auto capabilities = realExtension->capabilitiesBuilder(); for (const auto &level : capabilities.levels) { - detailEnhancerCapabilityMap.emplace(level, idx); + auto itr = detailEnhancerCapabilityMap.find(level); + if (itr == detailEnhancerCapabilityMap.end()) { + detailEnhancerCapabilityMap.emplace(level, idx); + } else { + if (capabilities.rank == Rank::RANK_HIGH) { + itr->second = idx; + } + } } return VPE_ALGO_ERR_OK; } @@ -673,7 +699,7 @@ VPEAlgoErrCode ExtensionManager::ExtractAihdrEnhancerCap(const AihdrEnhancerCapa AihdrEnhancerCapabilityMap& aihdrEnhancerCapabilityMap) const { auto colorSpaceDesc = cap.colorspaceDesc; - uint32_t rank = cap.rank; + Rank rank = cap.rank; int32_t version = cap.version; for (const auto &pixelFormat : cap.pixelFormats) { auto key = std::make_tuple(colorSpaceDesc, pixelFormat); diff --git a/framework/algorithm/extension_manager/include/extension_base.h b/framework/algorithm/extension_manager/include/extension_base.h index 0bf31e56feb29c59b048eda8914dfde86a07ab15..1e231580a51d86f13d3e5347a4872f9f1c756142 100644 --- a/framework/algorithm/extension_manager/include/extension_base.h +++ b/framework/algorithm/extension_manager/include/extension_base.h @@ -31,6 +31,11 @@ enum class ExtensionType { AIHDR_ENHANCER }; +enum class Rank : uint32_t { + RANK_DEFAULT = 0, + RANK_HIGH, +}; + struct ExtensionInfo { ExtensionType type {ExtensionType::COLORSPACE_CONVERTER}; std::string name; diff --git a/framework/algorithm/extension_manager/include/extension_manager.h b/framework/algorithm/extension_manager/include/extension_manager.h index 840cf15d2dd987fe9422f768202bc14e9d521f32..ee10cce5ab8ebac934ccf6aa1132795f3f90d806 100644 --- a/framework/algorithm/extension_manager/include/extension_manager.h +++ b/framework/algorithm/extension_manager/include/extension_manager.h @@ -62,7 +62,7 @@ namespace { using ColorSpaceConverterCapabilityMap = std::map< std::tuple, - std::vector>>; + std::vector>>; using ColorSpaceConverterDisplayCapabilityMap = ColorSpaceConverterCapabilityMap; /* { @@ -76,12 +76,12 @@ using ColorSpaceConverterDisplayCapabilityMap = ColorSpaceConverterCapabilityMap using MetadataGeneratorCapabilityMap = std::map< std::tuple, - std::vector>>; + std::vector>>; using DetailEnhancerCapabilityMap = std::map; using AihdrEnhancerCapabilityMap = std::map< std::tuple, - std::vector>>; + std::vector>>; using ColorSpaceConverterDisplaySet = std::set>; using ColorSpaceConverterDisplayExtensionSet = std::set>; diff --git a/framework/algorithm/extension_manager/include/static_extension_list.h b/framework/algorithm/extension_manager/include/static_extension_list.h index 5af2ea4f8ff0a512fc34b827d2f84be194624a11..924e79024d5df096e8dcd9658683abe7fd4eddea 100644 --- a/framework/algorithm/extension_manager/include/static_extension_list.h +++ b/framework/algorithm/extension_manager/include/static_extension_list.h @@ -14,9 +14,12 @@ #ifndef VPE_FRAMEWORK_ALGORITHM_EXTENSION_MANAGER_STATIC_EXTENSION_LIST_H #define VPE_FRAMEWORK_ALGORITHM_EXTENSION_MANAGER_STATIC_EXTENSION_LIST_H +#include "skia_impl.h" + namespace OHOS::Media::VideoProcessingEngine::Extension { using RegisterExtensionFunc = void (*)(uintptr_t extensionListAddr); const std::unordered_map staticExtensionsRegisterMap = { + { "Skia", RegisterSkiaExtensions }, }; } // namespace OHOS::Media::VideoProcessingEngine::Extension diff --git a/framework/algorithm/extensions/skia/include/skia_impl.h b/framework/algorithm/extensions/skia/include/skia_impl.h index 49434888b1895126f97bf3b32c54218637e9083c..c15fd688c5440f4681f8c72b091c58684f24cac5 100644 --- a/framework/algorithm/extensions/skia/include/skia_impl.h +++ b/framework/algorithm/extensions/skia/include/skia_impl.h @@ -19,23 +19,31 @@ #include "surface_buffer.h" #include "include/core/SkYUVAPixmaps.h" -#include "algorithm_types.h" +#include "algorithm_errors.h" +#include "detail_enhancer_base.h" +#include "detail_enhancer_capability.h" namespace OHOS { namespace Media { namespace VideoProcessingEngine { -class Skia { +class Skia : public DetailEnhancerBase { public: - static AlgoErrorCode Process(const sptr& input, sptr& output); - -private: Skia() = default; virtual ~Skia() = default; Skia(const Skia&) = delete; Skia& operator=(const Skia&) = delete; Skia(Skia&&) = delete; Skia& operator=(Skia&&) = delete; + + static std::unique_ptr Create(); + static DetailEnhancerCapability BuildCapabilities(); + VPEAlgoErrCode Init() override; + VPEAlgoErrCode Deinit() override; + VPEAlgoErrCode SetParameter(const DetailEnhancerParameters& parameter, int type, bool flag) override; + VPEAlgoErrCode Process(const sptr& input, const sptr& output) override; }; + +void RegisterSkiaExtensions(uintptr_t extensionListAddr); } // VideoProcessingEngine } // Media } // OHOS diff --git a/framework/algorithm/extensions/skia/skia_impl.cpp b/framework/algorithm/extensions/skia/skia_impl.cpp index c83467aaf9926d1156ddfeeaf3f59494f820b4fc..70999981af28cf8ae50fd4de3e61565d3ba7d2bf 100644 --- a/framework/algorithm/extensions/skia/skia_impl.cpp +++ b/framework/algorithm/extensions/skia/skia_impl.cpp @@ -19,10 +19,14 @@ #include #include +#include "detail_enhancer_extension.h" +#include "utils.h" + #include "vpe_log.h" -using namespace OHOS; -using namespace OHOS::Media::VideoProcessingEngine; +namespace OHOS { +namespace Media { +namespace VideoProcessingEngine { namespace { enum ImageFormatType { @@ -80,19 +84,19 @@ SkColorType GetRGBImageFormat(const sptr& surfaceBuffer) return imageFormat; } -AlgoErrorCode PixmapScale(const SkPixmap& inputPixmap, SkPixmap& outputPixmap, SkSamplingOptions options) +VPEAlgoErrCode PixmapScale(const SkPixmap& inputPixmap, SkPixmap& outputPixmap, SkSamplingOptions options) { if (!inputPixmap.scalePixels(outputPixmap, options)) { - return ALGO_ERROR_PROCESS_FAILED; + return VPE_ALGO_ERR_EXTENSION_PROCESS_FAILED; } - return ALGO_SUCCESS; + return VPE_ALGO_ERR_OK; } -AlgoErrorCode RGBScale(const sptr& input, sptr& output) +VPEAlgoErrCode RGBScale(const sptr& input, const sptr& output) { if (input->GetWidth() <= 0 || input->GetHeight() <= 0 || output->GetWidth() <= 0 || output->GetHeight() <= 0) { VPE_LOGE("Invalid input or output size!"); - return ALGO_ERROR_INVALID_VALUE; + return VPE_ALGO_ERR_INVALID_VAL; } SkImageInfo inputInfo = SkImageInfo::Make(input->GetWidth(), input->GetHeight(), GetRGBImageFormat(input), kPremul_SkAlphaType); @@ -199,19 +203,19 @@ int CreateYUVPixmap(const sptr& buffer, std::array& inputPixmap, +VPEAlgoErrCode YUVPixmapScale(const std::array& inputPixmap, std::array& outputPixmap, SkSamplingOptions opt, int numPlanes) { for (int i = 0; i < numPlanes; i++) { if (!inputPixmap[i].scalePixels(outputPixmap[i], opt)) { VPE_LOGE("YUV scale failed!"); - return ALGO_ERROR_PROCESS_FAILED; + return VPE_ALGO_ERR_EXTENSION_PROCESS_FAILED; } } - return ALGO_SUCCESS; + return VPE_ALGO_ERR_OK; } -AlgoErrorCode YUVScale(const sptr& input, sptr& output) +VPEAlgoErrCode YUVScale(const sptr& input, const sptr& output) { std::array inputPixmap; std::array outputPixmap; @@ -219,16 +223,48 @@ AlgoErrorCode YUVScale(const sptr& input, sptr& ou int numPlanesOutput = CreateYUVPixmap(output, outputPixmap); if (numPlanesInput != numPlanesOutput || numPlanesInput * numPlanesOutput == 0) { VPE_LOGE("Wrong YUV settings!"); - return ALGO_ERROR_INVALID_VALUE; + return VPE_ALGO_ERR_INVALID_VAL; } SkSamplingOptions scaleOption(SkFilterMode::kNearest); return YUVPixmapScale(inputPixmap, outputPixmap, scaleOption, numPlanesInput); } + +constexpr Extension::Rank RANK = Extension::Rank::RANK_DEFAULT; +constexpr uint32_t VERSION = 0; +} // namespace + +std::unique_ptr Skia::Create() +{ + return std::make_unique(); +} + +DetailEnhancerCapability Skia::BuildCapabilities() +{ + std::vector levels = { DETAIL_ENH_LEVEL_NONE, DETAIL_ENH_LEVEL_LOW, DETAIL_ENH_LEVEL_MEDIUM, + DETAIL_ENH_LEVEL_HIGH_EVE, DETAIL_ENH_LEVEL_HIGH_AISR, DETAIL_ENH_LEVEL_VIDEO}; + DetailEnhancerCapability capability = { levels, RANK, VERSION }; + return capability; +} + +VPEAlgoErrCode Skia::Init() +{ + return VPE_ALGO_ERR_OK; +} + +VPEAlgoErrCode Skia::Deinit() +{ + return VPE_ALGO_ERR_OK; } -AlgoErrorCode Skia::Process(const sptr& input, sptr& output) +VPEAlgoErrCode Skia::SetParameter([[maybe_unused]] const DetailEnhancerParameters& parameter, + [[maybe_unused]] int type, [[maybe_unused]] bool flag) { - AlgoErrorCode errCode; + return VPE_ALGO_ERR_OK; +} + +VPEAlgoErrCode Skia::Process(const sptr& input, const sptr& output) +{ + VPEAlgoErrCode errCode; ImageFormatType imageType = GetImageType(input, output); if (imageType == IMAGE_FORMAT_TYPE_RGB) { errCode = RGBScale(input, output); @@ -236,7 +272,29 @@ AlgoErrorCode Skia::Process(const sptr& input, sptr> RegisterExtensions() +{ + std::vector> extensions; + + auto extension = std::make_shared(); + CHECK_AND_RETURN_RET_LOG(extension != nullptr, extensions, "null pointer"); + extension->info = { Extension::ExtensionType::DETAIL_ENHANCER, "SKIA", "0.0.1" }; + extension->creator = Skia::Create; + extension->capabilitiesBuilder = Skia::BuildCapabilities; + extensions.push_back(std::static_pointer_cast(extension)); + + return extensions; +} + +void RegisterSkiaExtensions(uintptr_t extensionListAddr) +{ + Extension::DoRegisterExtensions(extensionListAddr, RegisterExtensions); +} +} // VideoProcessingEngine +} // Media +} // OHOS \ No newline at end of file diff --git a/framework/algorithm/metadata_generator/include/metadata_generator_capability.h b/framework/algorithm/metadata_generator/include/metadata_generator_capability.h index a74807a45bb1b2c6b1dac2cc3bc7770f9b21ee84..4623798c81bf2f8f977546b590e88dbe36843695 100644 --- a/framework/algorithm/metadata_generator/include/metadata_generator_capability.h +++ b/framework/algorithm/metadata_generator/include/metadata_generator_capability.h @@ -19,6 +19,7 @@ #include #include #include "frame_info.h" +#include "extension_base.h" namespace OHOS { namespace Media { @@ -26,7 +27,7 @@ namespace VideoProcessingEngine { struct MetadataGeneratorCapability { ColorSpaceDescription colorspaceDesc; std::vector pixelFormats; - uint32_t rank; + Extension::Rank rank; int32_t version; };